Add sliding-sync support

This allows people to try out the new Element X clients, which need to
run against the sliding-sync proxy (https://github.com/matrix-org/sliding-sync).

Supersedes https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/2515

The code is based on the existing PR (#2515), but heavily reworked. Major changes:

- lots of internal refactoring and variable renaming

- fixed self-building to support non-amd64 architectures

- changed to talk to the homeserver locally, over the container network (not
  publicly)

- no more matrix-nginx-proxy support due to complexity (see below)

- no more `matrix_server_fqn_sliding_sync_proxy` in favor of
  `matrix_sliding_sync_hostname` and `matrix_sliding_sync_path_prefix`

- runs on `matrix.DOMAIN/sliding-sync` by default, so it can tried
  easily without having to create new DNS records

group_vars/matrix_servers

@@ -330,6 +330,8 @@ devture_systemd_service_manager_services_list_auto: |
     +
     ([{'name': 'matrix-registration.service', 'priority': 4000, 'groups': ['matrix', 'registration']}] if matrix_registration_enabled else [])
     +
+    ([{'name': 'matrix-sliding-sync.service', 'priority': 4000, 'groups': ['matrix', 'sliding-sync']}] if matrix_sliding_sync_enabled else [])
+    +
     ([{'name': 'matrix-sygnal.service', 'priority': 800, 'groups': ['matrix', 'sygnal']}] if matrix_sygnal_enabled else [])
     +
     ([{'name': 'matrix-goofys.service', 'priority': 800, 'groups': ['matrix', 'goofys']}] if matrix_s3_media_store_enabled else [])
@@ -2612,6 +2614,8 @@ matrix_nginx_proxy_systemd_wanted_services_list: |
     +
     (['matrix-jitsi.service'] if matrix_jitsi_enabled else [])
     +
+    (['matrix-sliding-sync-proxy.service'] if matrix_sliding_sync_enabled else [])
+    +
     (['matrix-bot-go-neb.service'] if matrix_bot_go_neb_enabled else [])
     +
     ([etherpad_identifier + '.service'] if etherpad_enabled else [])
@@ -2745,6 +2749,12 @@ devture_postgres_managed_databases_auto: |
       'password': matrix_dendrite_database_password,
     }] if (matrix_dendrite_enabled and matrix_dendrite_database_hostname == devture_postgres_connection_hostname) else [])
     +
+    ([{
+      'name': matrix_sliding_sync_database_name,
+      'username': matrix_sliding_sync_database_username,
+      'password': matrix_sliding_sync_database_password,
+    }] if (matrix_sliding_sync_enabled) else [])
+    +
     ([{
       'name': matrix_ma1sd_database_name,
       'username': matrix_ma1sd_database_username,
@@ -3639,6 +3649,63 @@ matrix_registration_database_password: "{{ '%s' | format(matrix_homeserver_gener
 ######################################################################
 
 
+
+######################################################################
+#
+# matrix-sliding-sync
+#
+######################################################################
+
+# We don't enable the sliding sync proxy by default.
+matrix_sliding_sync_enabled: false
+
+matrix_sliding_sync_scheme: "{{ 'https' if matrix_playbook_ssl_enabled else 'http' }}"
+
+matrix_sliding_sync_hostname: "{{ matrix_server_fqn_matrix }}"
+
+matrix_sliding_sync_path_prefix: /sliding-sync
+
+matrix_sliding_sync_container_image_self_build: "{{ matrix_architecture not in ['amd64'] }}"
+
+matrix_sliding_sync_container_additional_networks: |
+  {{
+    (
+      ([matrix_playbook_reverse_proxyable_services_additional_network] if matrix_playbook_reverse_proxyable_services_additional_network else [])
+      +
+      ([matrix_nginx_proxy_container_network] if matrix_nginx_proxy_enabled and matrix_nginx_proxy_container_network != matrix_sliding_sync_container_network else [])
+      +
+      ([devture_postgres_container_network] if devture_postgres_enabled and devture_postgres_container_network != matrix_sliding_sync_container_network else [])
+    ) | unique
+  }}
+
+matrix_sliding_sync_container_labels_traefik_enabled: "{{ matrix_playbook_reverse_proxy_type in ['playbook-managed-traefik', 'other-traefik-container'] }}"
+matrix_sliding_sync_container_labels_traefik_docker_network: "{{ matrix_playbook_reverse_proxyable_services_additional_network }}"
+matrix_sliding_sync_container_labels_traefik_entrypoints: "{{ devture_traefik_entrypoint_primary }}"
+matrix_sliding_sync_container_labels_traefik_tls_certResolver: "{{ devture_traefik_certResolver_primary }}"
+
+matrix_sliding_sync_systemd_required_services_list: |
+  {{
+    ['docker.service']
+    +
+    ['matrix-' + matrix_homeserver_implementation + '.service']
+    +
+    ([devture_postgres_identifier ~ '.service'] if devture_postgres_enabled else [])
+    +
+    (['matrix-nginx-proxy.service'] if matrix_nginx_proxy_enabled else [])
+  }}
+
+matrix_sliding_sync_environment_variable_syncv3_secret: "{{ '%s' | format(matrix_homeserver_generic_secret_key) | password_hash('sha512', 'ss.secret', rounds=655555) | to_uuid }}"
+
+matrix_sliding_sync_database_hostname: "{{ devture_postgres_connection_hostname if devture_postgres_enabled else '' }}"
+matrix_sliding_sync_database_password: "{{ '%s' | format(matrix_homeserver_generic_secret_key) | password_hash('sha512', 'ss.db', rounds=655555) | to_uuid }}"
+
+######################################################################
+#
+# /matrix-sliding-sync
+#
+######################################################################
+
+
 ######################################################################
 #
 # matrix-dendrite

playbooks/matrix.yml

@@ -105,6 +105,7 @@
     - custom/matrix-dimension
     - galaxy/etherpad
     - custom/etherpad-proxy-connect
+    - custom/matrix-sliding-sync
     - custom/matrix-email2matrix
     - custom/matrix-sygnal
     - galaxy/ntfy

roles/custom/matrix-base/templates/static-files/well-known/matrix-client.j2

@@ -26,6 +26,11 @@
 		"preferredDomain": {{ matrix_client_element_jitsi_preferredDomain|to_json }}
 	}
 	{% endif %}
+	{% if matrix_sliding_sync_enabled %},
+	"org.matrix.msc3575.proxy": {
+		"url": "{{ matrix_sliding_sync_base_url }}"
+	}
+	{% endif %}
 	{% if matrix_client_element_location_sharing_enabled %},
 	"m.tile_server": {
         "map_style_url": "https://{{ matrix_server_fqn_element }}/map_style.json"

roles/custom/matrix-sliding-sync/defaults/main.yml

@@ -0,0 +1,91 @@
+---
+
+# Sliding Sync Proxy is an implementation of MSC3575 for the new sliding sync
+
+matrix_sliding_sync_enabled: true
+
+matrix_sliding_sync_version: v0.99.1
+
+matrix_sliding_sync_scheme: https
+
+# The hostname at which sliding-sync is served.
+matrix_sliding_sync_hostname: ''
+
+# The path at which sliding-sync is served.
+# This value must either be `/` or not end with a slash (e.g. `/sliding-sync`).
+matrix_sliding_sync_path_prefix: /
+
+matrix_sliding_sync_base_path: "{{ matrix_base_data_path }}/sliding-sync"
+
+matrix_sliding_sync_container_image_self_build: false
+matrix_sliding_sync_container_image_self_build_repo: https://github.com/matrix-org/sliding-sync
+matrix_sliding_sync_container_image_self_build_repo_version: "{{ 'main' if matrix_sliding_sync_version == 'main' else matrix_sliding_sync_version }}"
+matrix_sliding_sync_container_src_path: "{{ matrix_sliding_sync_base_path }}/container-src"
+
+matrix_sliding_sync_container_image: "{{ matrix_sliding_sync_container_image_name_prefix }}matrix-org/sliding-sync:{{ matrix_sliding_sync_container_image_tag }}"
+matrix_sliding_sync_container_image_name_prefix: "{{ 'localhost/' if matrix_sliding_sync_container_image_self_build else matrix_sliding_sync_container_image_registry_prefix }}"
+matrix_sliding_sync_container_image_tag: "{{ matrix_sliding_sync_version }}"
+matrix_sliding_sync_container_image_force_pull: "{{ matrix_sliding_sync_container_image.endswith(':main') }}"
+matrix_sliding_sync_container_image_registry_prefix: ghcr.io/
+
+# The base container network. It will be auto-created by this role if it doesn't exist already.
+matrix_sliding_sync_container_network: matrix-sliding-sync
+
+# A list of additional container networks that the container would be connected to.
+# The role does not create these networks, so make sure they already exist.
+# Use this to expose this container to another reverse proxy, which runs in a different container network.
+matrix_sliding_sync_container_additional_networks: []
+
+# matrix_sliding_sync_container_labels_traefik_enabled controls whether labels to assist a Traefik reverse-proxy will be attached to the container.
+# See `../templates/labels.j2` for details.
+#
+# To inject your own other container labels, see `matrix_sliding_sync_container_labels_additional_labels`.
+matrix_sliding_sync_container_labels_traefik_enabled: true
+matrix_sliding_sync_container_labels_traefik_docker_network: "{{ matrix_sliding_sync_container_network }}"
+matrix_sliding_sync_container_labels_traefik_hostname: "{{ matrix_sliding_sync_hostname }}"
+# The path prefix must either be `/` or not end with a slash (e.g. `/sliding-sync`).
+matrix_sliding_sync_container_labels_traefik_path_prefix: "{{ matrix_sliding_sync_path_prefix }}"
+matrix_sliding_sync_container_labels_traefik_rule: "Host(`{{ matrix_sliding_sync_container_labels_traefik_hostname }}`){% if matrix_sliding_sync_container_labels_traefik_path_prefix != '/' %} && PathPrefix(`{{ matrix_sliding_sync_container_labels_traefik_path_prefix }}`){% endif %}"
+matrix_sliding_sync_container_labels_traefik_priority: 0
+matrix_sliding_sync_container_labels_traefik_entrypoints: web-secure
+matrix_sliding_sync_container_labels_traefik_tls: "{{ matrix_sliding_sync_container_labels_traefik_entrypoints != 'web' }}"
+matrix_sliding_sync_container_labels_traefik_tls_certResolver: default  # noqa var-naming
+
+# Controls which additional headers to attach to all HTTP responses.
+# To add your own headers, use `matrix_sliding_sync_container_labels_traefik_additional_response_headers_custom`
+matrix_sliding_sync_container_labels_traefik_additional_response_headers: "{{ matrix_sliding_sync_container_labels_traefik_additional_response_headers_auto | combine(matrix_sliding_sync_container_labels_traefik_additional_response_headers_custom) }}"
+matrix_sliding_sync_container_labels_traefik_additional_response_headers_auto: {}
+matrix_sliding_sync_container_labels_traefik_additional_response_headers_custom: {}
+
+# matrix_sliding_sync_container_labels_additional_labels contains a multiline string with additional labels to add to the container label file.
+# See `../templates/labels.j2` for details.
+#
+# Example:
+# matrix_sliding_sync_container_labels_additional_labels: |
+#   my.label=1
+#   another.label="here"
+matrix_sliding_sync_container_labels_additional_labels: ''
+
+# A list of extra arguments to pass to the container
+matrix_sliding_sync_container_extra_arguments: []
+
+# List of systemd services that matrix-sliding-sync-proxy.service depends on
+matrix_sliding_sync_systemd_required_services_list: ["docker.service"]
+
+# List of systemd services that matrix-sliding-sync-proxy.service wants
+matrix_sliding_sync_systemd_wanted_services_list: []
+
+# Controls the SYNCV3_SERVER environment variable
+matrix_sliding_sync_environment_variable_syncv3_server: "{{ matrix_homeserver_container_url }}"
+
+# Controls the SYNCV3_SECRET environment variable
+matrix_sliding_sync_environment_variable_syncv3_secret: ''
+
+# Controls the SYNCV3_DB environment variable
+matrix_sliding_sync_environment_variable_syncv3_db: 'user={{ matrix_sliding_sync_database_username }} password={{ matrix_sliding_sync_database_password }} host={{ matrix_sliding_sync_database_hostname }} port={{ matrix_sliding_sync_database_port }} dbname={{ matrix_sliding_sync_database_name }} sslmode=disable'
+
+matrix_sliding_sync_database_username: 'matrix_sliding_sync'
+matrix_sliding_sync_database_password: ''
+matrix_sliding_sync_database_hostname: ''
+matrix_sliding_sync_database_port: 5432
+matrix_sliding_sync_database_name: 'matrix_sliding_sync'

roles/custom/matrix-sliding-sync/tasks/install.yml

@@ -0,0 +1,69 @@
+---
+
+- name: Ensure matrix-sliding-sync paths exist
+  ansible.builtin.file:
+    path: "{{ item.path }}"
+    state: directory
+    mode: 0750
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+  with_items:
+    - path: "{{ matrix_sliding_sync_base_path }}"
+      when: true
+    - path: "{{ matrix_sliding_sync_container_src_path }}"
+      when: "{{ matrix_sliding_sync_container_image_self_build }}"
+  when: item.when | bool
+
+- name: Ensure matrix-sliding-sync support files installed
+  ansible.builtin.template:
+    src: "{{ role_path }}/templates/{{ item }}.j2"
+    dest: "{{ matrix_sliding_sync_base_path }}/{{ item }}"
+    mode: 0640
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+  with_items:
+    - env
+    - labels
+
+- name: Ensure matrix-sliding-sync container image is pulled
+  community.docker.docker_image:
+    name: "{{ matrix_sliding_sync_container_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_sliding_sync_container_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_sliding_sync_container_image_force_pull }}"
+  when: "not matrix_sliding_sync_container_image_self_build | bool"
+  register: result
+  retries: "{{ devture_playbook_help_container_retries_count }}"
+  delay: "{{ devture_playbook_help_container_retries_delay }}"
+  until: result is not failed
+
+- when: matrix_sliding_sync_container_image_self_build | bool
+  block:
+    - name: Ensure matrix-sliding-sync repository is present on self-build
+      ansible.builtin.git:
+        repo: "{{ matrix_sliding_sync_container_image_self_build_repo }}"
+        version: "{{ matrix_sliding_sync_container_image_self_build_repo_version }}"
+        dest: "{{ matrix_sliding_sync_container_src_path }}"
+        force: "yes"
+      become: true
+      become_user: "{{ matrix_user_username }}"
+      register: matrix_sliding_sync_git_pull_results
+
+    - name: Ensure matrix-sliding-sync container image is built
+      ansible.builtin.command:
+        cmd: |-
+          {{ devture_systemd_docker_base_host_command_docker }} buildx build
+          --tag={{ matrix_sliding_sync_container_image }}
+          --file={{ matrix_sliding_sync_container_src_path }}/Dockerfile
+          {{ matrix_sliding_sync_container_src_path }}
+
+- name: Ensure matrix-sliding-sync container network is created
+  community.general.docker_network:
+    name: "{{ matrix_sliding_sync_container_network }}"
+    driver: bridge
+
+- name: Ensure matrix-sliding-sync.service installed
+  ansible.builtin.template:
+    src: "{{ role_path }}/templates/systemd/matrix-sliding-sync.service.j2"
+    dest: "{{ devture_systemd_docker_base_systemd_path }}/matrix-sliding-sync.service"
+    mode: 0644

roles/custom/matrix-sliding-sync/tasks/main.yml

@@ -0,0 +1,19 @@
+---
+- block:
+    - when: matrix_sliding_sync_enabled | bool
+      ansible.builtin.include_tasks: "{{ role_path }}/tasks/validate_config.yml"
+
+    - when: matrix_sliding_sync_enabled | bool
+      ansible.builtin.include_tasks: "{{ role_path }}/tasks/install.yml"
+  tags:
+    - setup-all
+    - setup-sliding-sync
+    - install-all
+    - install-sliding-sync
+
+- block:
+    - when: not matrix_sliding_sync_enabled | bool
+      ansible.builtin.include_tasks: "{{ role_path }}/tasks/uninstall.yml"
+  tags:
+    - setup-all
+    - setup-sliding-sync

roles/custom/matrix-sliding-sync/tasks/uninstall.yml

@@ -0,0 +1,25 @@
+---
+
+- name: Check existence of matrix-sliding-sync service
+  ansible.builtin.stat:
+    path: "{{ devture_systemd_docker_base_systemd_path }}/matrix-sliding-sync.service"
+  register: matrix_sliding_sync_service_stat
+
+- when: matrix_sliding_sync_service_stat.stat.exists | bool
+  block:
+    - name: Ensure matrix-sliding-sync is stopped
+      ansible.builtin.service:
+        name: matrix-sliding-sync
+        state: stopped
+        enabled: false
+        daemon_reload: true
+
+    - name: Ensure matrix-sliding-sync.service doesn't exist
+      ansible.builtin.file:
+        path: "{{ devture_systemd_docker_base_systemd_path }}/matrix-sliding-sync.service"
+        state: absent
+
+    - name: Ensure matrix-sliding-sync paths don't exist
+      ansible.builtin.file:
+        path: "{{ matrix_sliding_sync_base_path }}"
+        state: absent

roles/custom/matrix-sliding-sync/tasks/validate_config.yml

@@ -0,0 +1,11 @@
+---
+- name: Fail if required matrix-sliding-sync settings not defined
+  ansible.builtin.fail:
+    msg: >
+      You need to define a required configuration setting (`{{ item.name }}`).
+  when: "vars[item] == ''"
+  with_items:
+    - matrix_sliding_sync_hostname
+    - matrix_sliding_sync_path_prefix
+    - matrix_sliding_sync_database_hostname
+    - matrix_sliding_sync_environment_variable_syncv3_secret

roles/custom/matrix-sliding-sync/templates/env.j2

@@ -0,0 +1,4 @@
+SYNCV3_SERVER={{ matrix_sliding_sync_environment_variable_syncv3_server }}
+SYNCV3_SECRET={{ matrix_sliding_sync_environment_variable_syncv3_secret }}
+SYNCV3_BINDADDR=:8008
+SYNCV3_DB={{ matrix_sliding_sync_environment_variable_syncv3_db }}

roles/custom/matrix-sliding-sync/templates/labels.j2

@@ -0,0 +1,45 @@
+{% if matrix_sliding_sync_container_labels_traefik_enabled %}
+traefik.enable=true
+
+{% if matrix_sliding_sync_container_labels_traefik_docker_network %}
+traefik.docker.network={{ matrix_sliding_sync_container_labels_traefik_docker_network }}
+{% endif %}
+
+{% set middlewares = [] %}
+
+{% if matrix_sliding_sync_container_labels_traefik_path_prefix != '/' %}
+traefik.http.middlewares.matrix-sliding-sync-slashless-redirect.redirectregex.regex=({{ matrix_sliding_sync_container_labels_traefik_path_prefix | quote }})$
+traefik.http.middlewares.matrix-sliding-sync-slashless-redirect.redirectregex.replacement=${1}/
+{% set middlewares = middlewares + ['matrix-sliding-sync-slashless-redirect'] %}
+{% endif %}
+
+{% if matrix_sliding_sync_container_labels_traefik_path_prefix != '/' %}
+traefik.http.middlewares.matrix-sliding-sync-strip-prefix.stripprefix.prefixes={{ matrix_sliding_sync_container_labels_traefik_path_prefix }}
+{% set middlewares = middlewares + ['matrix-sliding-sync-strip-prefix'] %}
+{% endif %}
+
+{% if matrix_sliding_sync_container_labels_traefik_additional_response_headers.keys() | length > 0 %}
+{% for name, value in matrix_sliding_sync_container_labels_traefik_additional_response_headers.items() %}
+traefik.http.middlewares.matrix-sliding-sync-add-headers.headers.customresponseheaders.{{ name }}={{ value }}
+{% endfor %}
+{% set middlewares = middlewares + ['matrix-sliding-sync-add-headers'] %}
+{% endif %}
+
+traefik.http.routers.matrix-sliding-sync.rule={{ matrix_sliding_sync_container_labels_traefik_rule }}
+{% if matrix_sliding_sync_container_labels_traefik_priority | int > 0 %}
+traefik.http.routers.matrix-sliding-sync.priority={{ matrix_sliding_sync_container_labels_traefik_priority }}
+{% endif %}
+traefik.http.routers.matrix-sliding-sync.service=matrix-sliding-sync
+{% if middlewares | length > 0 %}
+traefik.http.routers.matrix-sliding-sync.middlewares={{ middlewares | join(',') }}
+{% endif %}
+traefik.http.routers.matrix-sliding-sync.entrypoints={{ matrix_sliding_sync_container_labels_traefik_entrypoints }}
+traefik.http.routers.matrix-sliding-sync.tls={{ matrix_sliding_sync_container_labels_traefik_tls | to_json }}
+{% if matrix_sliding_sync_container_labels_traefik_tls %}
+traefik.http.routers.matrix-sliding-sync.tls.certResolver={{ matrix_sliding_sync_container_labels_traefik_tls_certResolver }}
+{% endif %}
+
+traefik.http.services.matrix-sliding-sync.loadbalancer.server.port=8008
+{% endif %}
+
+{{ matrix_sliding_sync_container_labels_additional_labels }}

roles/custom/matrix-sliding-sync/templates/systemd/matrix-sliding-sync.service.j2

@@ -0,0 +1,49 @@
+#jinja2: lstrip_blocks: "True"
+[Unit]
+Description=matrix-sliding-sync
+{% for service in matrix_sliding_sync_systemd_required_services_list %}
+Requires={{ service }}
+After={{ service }}
+{% endfor %}
+{% for service in matrix_sliding_sync_systemd_wanted_services_list %}
+Wants={{ service }}
+{% endfor %}
+DefaultDependencies=no
+
+[Service]
+Type=simple
+Environment="HOME={{ devture_systemd_docker_base_systemd_unit_home_path }}"
+ExecStartPre=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} kill matrix-sliding-sync 2>/dev/null || true'
+ExecStartPre=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} rm matrix-sliding-sync 2>/dev/null || true'
+
+ExecStartPre={{ devture_systemd_docker_base_host_command_docker }} create \
+			--rm \
+			--name=matrix-sliding-sync \
+			--log-driver=none \
+			--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
+			--cap-drop=ALL \
+			--read-only \
+			--tmpfs /tmp \
+			--network={{ matrix_sliding_sync_container_network }} \
+			--env-file={{ matrix_sliding_sync_base_path }}/env \
+			--label-file={{ matrix_sliding_sync_base_path }}/labels \
+			{% for arg in matrix_sliding_sync_container_extra_arguments %}
+			{{ arg }} \
+			{% endfor %}
+			{{ matrix_sliding_sync_container_image }}
+
+{% for network in matrix_sliding_sync_container_additional_networks %}
+ExecStartPre={{ devture_systemd_docker_base_host_command_docker }} network connect {{ network }} matrix-sliding-sync
+{% endfor %}
+
+ExecStart={{ devture_systemd_docker_base_host_command_docker }} start --attach matrix-sliding-sync
+
+ExecStop=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} kill matrix-sliding-sync 2>/dev/null || true'
+ExecStop=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} rm matrix-sliding-sync 2>/dev/null || true'
+
+Restart=always
+RestartSec=30
+SyslogIdentifier=matrix-sliding-sync
+
+[Install]
+WantedBy=multi-user.target

roles/custom/matrix-sliding-sync/vars/main.yml

@@ -0,0 +1,5 @@
+---
+
+# Public facing base URL of the Sliding Sync service.
+# It should not end with a slash.
+matrix_sliding_sync_base_url: "{{ matrix_sliding_sync_scheme }}://{{ matrix_sliding_sync_hostname }}{{ '' if matrix_sliding_sync_path_prefix == '/' else matrix_sliding_sync_path_prefix }}"