add synapse-auto-compressor workaround, fixes #3397 (#3473)

* add synapse-auto-compressor workaround, fixes #3397

* Clarify what the PG-prefixed variables are for

---------

Co-authored-by: Slavi Pantaleev <slavi@devture.com>

group_vars/matrix_servers

@@ -4514,6 +4514,11 @@ matrix_synapse_report_stats_endpoint: "http://{{ matrix_synapse_usage_exporter_i
 # Not enabled by default, for now
 matrix_synapse_auto_compressor_enabled: false
 
+matrix_synapse_auto_compressor_uid: "{{ matrix_user_uid }}"
+matrix_synapse_auto_compressor_gid: "{{ matrix_user_gid }}"
+
+matrix_synapse_auto_compressor_postgres_image: "{{ devture_postgres_container_image_to_use }}"
+
 matrix_synapse_auto_compressor_container_image_self_build: "{{ matrix_architecture not in ['amd64'] }}"
 
 matrix_synapse_auto_compressor_container_network: "{{ (devture_postgres_container_network if (devture_postgres_enabled and matrix_synapse_auto_compressor_database_hostname == matrix_synapse_database_host and matrix_synapse_database_host == devture_postgres_connection_hostname) else 'matrix-synapse-auto-compressor') }}"

roles/custom/matrix-synapse-auto-compressor/defaults/main.yml

@@ -8,6 +8,13 @@ matrix_synapse_auto_compressor_enabled: true
 # renovate: datasource=docker depName=registry.gitlab.com/etke.cc/rust-synapse-compress-state
 matrix_synapse_auto_compressor_version: v0.1.4
 
+# note: UID/GID better to match the UID/GID of the postgres container, but it doesn't really matter, as volumes are not used here
+matrix_synapse_auto_compressor_uid: ''
+matrix_synapse_auto_compressor_gid: ''
+
+# note: the postgres image version doesn't matter, as it is used for a SQL script that doesn't use any specific features
+matrix_synapse_auto_compressor_postgres_image: "postgres"
+
 matrix_synapse_auto_compressor_base_path: "{{ matrix_base_data_path }}/synapse-auto-compressor"
 matrix_synapse_auto_compressor_container_src_files_path: "{{ matrix_synapse_auto_compressor_base_path }}/container-src"
 

roles/custom/matrix-synapse-auto-compressor/tasks/install.yml

@@ -14,7 +14,7 @@
     - path: "{{ matrix_synapse_auto_compressor_container_src_files_path }}"
       when: "{{ matrix_synapse_auto_compressor_container_image_self_build }}"
 
-- name: Ensure synapse-auto-compressor labels installed
+- name: Ensure synapse-auto-compressor env file is installed
   ansible.builtin.template:
     src: "{{ role_path }}/templates/env.j2"
     dest: "{{ matrix_synapse_auto_compressor_base_path }}/env"
@@ -22,6 +22,14 @@
     owner: "{{ matrix_user_username }}"
     group: "{{ matrix_user_groupname }}"
 
+- name: Ensure synapse-auto-compressor workaround script is installed
+  ansible.builtin.template:
+    src: "{{ role_path }}/templates/matrix-synapse-auto-compressor-fix.sh.j2"
+    dest: "{{ matrix_synapse_auto_compressor_base_path }}/matrix-synapse-auto-compressor-fix.sh"
+    mode: 0750
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+
 - name: Ensure synapse-auto-compressor image is pulled
   community.docker.docker_image:
     name: "{{ matrix_synapse_auto_compressor_container_image }}"

roles/custom/matrix-synapse-auto-compressor/templates/env.j2

@@ -1 +1,7 @@
 POSTGRES_LOCATION={{ matrix_synapse_auto_compressor_environment_variable_postgres_location }}
+
+# PG-prefixed variables below are for the matrix-synapse-auto-compressor-fix.sh script
+PGHOST={{ matrix_synapse_auto_compressor_database_hostname }}
+PGDATABASE={{ matrix_synapse_auto_compressor_database_name }}
+PGUSER={{ matrix_synapse_auto_compressor_database_username }}
+PGPASSWORD={{ matrix_synapse_auto_compressor_database_password }}

roles/custom/matrix-synapse-auto-compressor/templates/matrix-synapse-auto-compressor-fix.sh.j2

@@ -0,0 +1,47 @@
+#jinja2: lstrip_blocks: "True"
+#!/bin/bash
+
+# this script is used to workaround the https://github.com/matrix-org/rust-synapse-compress-state/issues/78,
+# and it is based on postgres' cli-non-interactive and https://github.com/matrix-org/rust-synapse-compress-state/issues/78#issuecomment-1409932869
+
+docker run \
+	--rm \
+	--user={{ matrix_synapse_auto_compressor_uid }}:{{ matrix_synapse_auto_compressor_gid }} \
+	--cap-drop=ALL \
+	--interactive \
+	--network={{ matrix_synapse_auto_compressor_container_network }} \
+	--env-file={{ matrix_synapse_auto_compressor_base_path }}/env \
+	{{ matrix_synapse_auto_compressor_postgres_image }} \
+	psql -h {{ matrix_synapse_auto_compressor_database_hostname }} \
+	<<_EOF
+BEGIN;
+
+DELETE
+FROM state_compressor_state AS scs
+WHERE NOT EXISTS
+    (SELECT *
+     FROM rooms AS r
+     WHERE r.room_id = scs.room_id);
+
+DELETE
+FROM state_compressor_state AS scs
+WHERE scs.room_id in
+    (SELECT DISTINCT room_id
+     FROM state_compressor_state AS scs2
+     WHERE scs2.current_head IS NOT NULL
+       AND NOT EXISTS
+         (SELECT *
+          FROM state_groups AS sg
+          WHERE sg.id = scs2.current_head));
+
+DELETE
+FROM state_compressor_progress AS scp
+WHERE NOT EXISTS
+    (SELECT *
+     FROM state_compressor_state AS scs
+     WHERE scs.room_id = scp.room_id);
+
+COMMIT;
+_EOF
+
+# vim: ft=sh

roles/custom/matrix-synapse-auto-compressor/templates/matrix-synapse-auto-compressor.service.j2

@@ -15,6 +15,7 @@ Type=oneshot
 Environment="HOME={{ devture_systemd_docker_base_systemd_unit_home_path }}"
 ExecStartPre=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} stop --time={{ devture_systemd_docker_base_container_stop_grace_time_seconds }} matrix-synapse-auto-compressor 2>/dev/null || true'
 ExecStartPre=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} rm matrix-synapse-auto-compressor 2>/dev/null || true'
+ExecStartPre=-{{ devture_systemd_docker_base_host_command_sh }} {{ matrix_synapse_auto_compressor_base_path }}/matrix-synapse-auto-compressor-fix.sh
 
 ExecStartPre={{ devture_systemd_docker_base_host_command_docker }} create \
 			--rm \
@@ -22,7 +23,7 @@ ExecStartPre={{ devture_systemd_docker_base_host_command_docker }} create \
 			--log-driver=none \
 			--cap-drop=ALL \
 			--read-only \
-			--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
+			--user={{ matrix_synapse_auto_compressor_uid }}:{{ matrix_synapse_auto_compressor_gid }} \
 			--network={{ matrix_synapse_auto_compressor_container_network }} \
 			--env-file={{ matrix_synapse_auto_compressor_base_path }}/env \
 			--entrypoint=/bin/sh \