Switch to using an external Redis role

CHANGELOG.md

@@ -1,5 +1,13 @@
 # 2023-02-17
 
+## The matrix-redis role lives independently now
+
+**TLDR**: the `matrix-redis` role is now included from another repository. Some variables have been renamed. All functionality remains intact.
+
+The `matrix-redis` role (which configures [Redis](https://redis.io/)) has been extracted from the playbook and now lives in its [own repository](https://gitlab.com/etke.cc/roles/redis). This makes it possible to easily use it in other Ansible playbooks.
+
+You need to **update you roles** (`just roles` or `make roles`) regardless of whether you're enabling Ntfy or not. If you're making use of Ntfy via this playbook, you will need to update variable references in your `vars.yml` file (`matrix_redis_` -> `redis_`).
+
 ## The matrix-ntfy role lives independently now
 
 **TLDR**: the `matrix-ntfy` role is now included from another repository. Some variables have been renamed. All functionality remains intact.

group_vars/matrix_servers

@@ -340,7 +340,7 @@ devture_systemd_service_manager_services_list_auto: |
     +
     ([{'name': 'matrix-prometheus-nginxlog-exporter.service', 'priority': 3900, 'groups': ['matrix', 'monitoring', 'prometheus', 'prometheus-exporters']}] if matrix_prometheus_nginxlog_exporter_enabled else [])
     +
-    ([{'name': 'matrix-redis', 'priority': 750, 'groups': ['matrix', 'redis']}] if matrix_redis_enabled else [])
+    ([{'name': (redis_identifier + '.service'), 'priority': 750, 'groups': ['matrix', 'redis']}] if redis_enabled else [])
     +
     ([{'name': 'matrix-registration.service', 'priority': 4000, 'groups': ['matrix', 'registration']}] if matrix_registration_enabled else [])
     +
@@ -2923,15 +2923,22 @@ ntfy_visitor_request_limit_exempt_hosts_hostnames_auto: |
 
 ######################################################################
 #
-# matrix-redis
+# etke/redis
 #
 ######################################################################
 
-matrix_redis_enabled: "{{ matrix_synapse_workers_enabled }}"
+redis_enabled: "{{ matrix_synapse_workers_enabled }}"
+
+redis_identifier: matrix-redis
+
+redis_uid: "{{ matrix_user_uid }}"
+redis_gid: "{{ matrix_user_gid }}"
+
+redis_base_path: "{{ matrix_base_data_path }}/redis"
 
 ######################################################################
 #
-# /matrix-redis
+# /etle/redis
 #
 ######################################################################
 
@@ -3077,7 +3084,12 @@ matrix_synapse_container_metrics_api_host_bind_port: "{{ (matrix_playbook_servic
 #
 # For exposing the Synapse Manhole port (plain HTTP) to the local host.
 matrix_synapse_container_manhole_api_host_bind_port: "{{ (matrix_playbook_service_host_bind_interface_prefix ~ '9000') if matrix_synapse_metrics_enabled and matrix_playbook_service_host_bind_interface_prefix else '' }}"
-#
+
+matrix_synapse_container_additional_networks: |
+  {{
+    ([redis_container_network] if matrix_synapse_redis_enabled and matrix_synapse_redis_host == redis_identifier else [])
+  }}
+
 # For exposing the Synapse worker (and metrics) ports to the local host.
 matrix_synapse_workers_container_host_bind_address: "{{ matrix_playbook_service_host_bind_interface_prefix[0:-1] if (matrix_synapse_workers_enabled and matrix_playbook_service_host_bind_interface_prefix) else '' }}"
 
@@ -3134,6 +3146,8 @@ matrix_synapse_systemd_required_services_list: |
     +
     ([devture_postgres_identifier ~ '.service'] if devture_postgres_enabled else [])
     +
+    ([redis_identifier ~ '.service'] if matrix_synapse_redis_enabled and matrix_synapse_redis_host == redis_identifier else [])
+    +
     (['matrix-goofys.service'] if matrix_s3_media_store_enabled else [])
   }}
 
@@ -3145,9 +3159,9 @@ matrix_synapse_systemd_wanted_services_list: |
   }}
 
 # Synapse workers (used for parallel load-scaling) need Redis for IPC.
-matrix_synapse_redis_enabled: "{{ matrix_redis_enabled }}"
-matrix_synapse_redis_host: "{{ 'matrix-redis' if matrix_redis_enabled else '' }}"
-matrix_synapse_redis_password: "{{ matrix_redis_connection_password if matrix_redis_enabled else '' }}"
+matrix_synapse_redis_enabled: "{{ redis_enabled }}"
+matrix_synapse_redis_host: "{{ redis_identifier if redis_enabled else '' }}"
+matrix_synapse_redis_password: "{{ redis_connection_password if redis_enabled else '' }}"
 
 matrix_synapse_container_extra_arguments_auto: "{{ matrix_homeserver_container_extra_arguments_auto }}"
 matrix_synapse_app_service_config_files_auto: "{{ matrix_homeserver_app_service_config_files_auto }}"

playbooks/matrix.yml

@@ -43,7 +43,7 @@
 
     - role: galaxy/com.devture.ansible.role.postgres
 
-    - custom/matrix-redis
+    - galaxy/redis
     - custom/matrix-corporal
     - custom/matrix-bridge-appservice-discord
     - custom/matrix-bridge-appservice-slack

requirements.yml

@@ -45,6 +45,9 @@
 - src: git+https://gitlab.com/etke.cc/roles/ntfy.git
   version: v2.0.0-0
 
+- src: git+https://gitlab.com/etke.cc/roles/redis.git
+  version: v7.0.7-0
+
 - src: git+https://github.com/devture/com.devture.ansible.role.traefik.git
   version: fb09fd26f877372417d5586f1e79e83f983f0bd6
 

roles/custom/matrix-redis/defaults/main.yml

@@ -1,25 +0,0 @@
----
-# Project source code URL: https://github.com/redis/redis
-
-matrix_redis_enabled: true
-
-matrix_redis_connection_password: ""
-
-matrix_redis_base_path: "{{ matrix_base_data_path }}/redis"
-matrix_redis_data_path: "{{ matrix_redis_base_path }}/data"
-
-matrix_redis_version: 7.0.7-alpine
-matrix_redis_docker_image_v6: "{{ matrix_container_global_registry_prefix }}redis:{{ matrix_redis_version }}"
-matrix_redis_docker_image_latest: "{{ matrix_redis_docker_image_v6 }}"
-matrix_redis_docker_image_to_use: '{{ matrix_redis_docker_image_latest }}'
-
-matrix_redis_docker_image_force_pull: "{{ matrix_redis_docker_image_to_use.endswith(':latest') }}"
-
-# A list of extra arguments to pass to the container
-matrix_redis_container_extra_arguments: []
-
-# Controls whether the matrix-redis container exposes a port (tcp/6379 in the container)
-# that can be used to access redis from outside the container
-#
-# Takes an "<ip>:<port>" or "<port>" value (e.g. "127.0.0.1:6379"), or empty string to not expose.
-matrix_redis_container_redis_bind_port: ""

roles/custom/matrix-redis/tasks/main.yml

@@ -1,17 +0,0 @@
----
-
-- block:
-    - when: matrix_redis_enabled | bool
-      ansible.builtin.include_tasks: "{{ role_path }}/tasks/setup_install.yml"
-  tags:
-    - setup-all
-    - setup-redis
-    - install-all
-    - install-redis
-
-- block:
-    - when: not matrix_redis_enabled | bool
-      ansible.builtin.include_tasks: "{{ role_path }}/tasks/setup_uninstall.yml"
-  tags:
-    - setup-all
-    - setup-redis

roles/custom/matrix-redis/tasks/setup_install.yml

@@ -1,49 +0,0 @@
----
-
-- name: Ensure redis Docker image is pulled
-  community.docker.docker_image:
-    name: "{{ matrix_redis_docker_image_to_use }}"
-    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
-    force_source: "{{ matrix_redis_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
-    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_redis_docker_image_force_pull }}"
-  register: result
-  retries: "{{ devture_playbook_help_container_retries_count }}"
-  delay: "{{ devture_playbook_help_container_retries_delay }}"
-  until: result is not failed
-
-- name: Ensure redis paths exist
-  ansible.builtin.file:
-    path: "{{ item }}"
-    state: directory
-    mode: 0700
-    owner: "{{ matrix_user_username }}"
-    group: "{{ matrix_user_username }}"
-  with_items:
-    - "{{ matrix_redis_base_path }}"
-    - "{{ matrix_redis_data_path }}"
-
-# We do this as a separate task, because:
-# - we'd like to do it for the data path only, not for the base path (which contains root-owned environment variable files we'd like to leave as-is)
-# - we need to do it without `mode`, or we risk making certain `.conf` and other files's executable bit to flip to true
-- name: Ensure redis data path ownership is correct
-  ansible.builtin.file:
-    path: "{{ matrix_redis_data_path }}"
-    state: directory
-    owner: "{{ matrix_user_username }}"
-    group: "{{ matrix_user_username }}"
-    recurse: true
-
-- name: Ensure redis environment variables file created
-  ansible.builtin.template:
-    src: "{{ role_path }}/templates/{{ item }}.j2"
-    dest: "{{ matrix_redis_base_path }}/{{ item }}"
-    mode: 0644
-  with_items:
-    - "redis.conf"
-
-- name: Ensure matrix-redis.service installed
-  ansible.builtin.template:
-    src: "{{ role_path }}/templates/systemd/matrix-redis.service.j2"
-    dest: "{{ devture_systemd_docker_base_systemd_path }}/matrix-redis.service"
-    mode: 0644
-  register: matrix_redis_systemd_service_result

roles/custom/matrix-redis/tasks/setup_uninstall.yml

@@ -1,31 +0,0 @@
----
-
-- name: Check existence of matrix-redis service
-  ansible.builtin.stat:
-    path: "{{ devture_systemd_docker_base_systemd_path }}/matrix-redis.service"
-  register: matrix_redis_service_stat
-
-- when: matrix_redis_service_stat.stat.exists | bool
-  block:
-    - name: Ensure matrix-redis is stopped
-      ansible.builtin.service:
-        name: matrix-redis
-        state: stopped
-        enabled: false
-        daemon_reload: true
-
-    - name: Ensure matrix-redis.service doesn't exist
-      ansible.builtin.file:
-        path: "{{ devture_systemd_docker_base_systemd_path }}/matrix-redis.service"
-        state: absent
-
-- name: Check existence of matrix-redis local data path
-  ansible.builtin.stat:
-    path: "{{ matrix_redis_data_path }}"
-  register: matrix_redis_data_path_stat
-
-# We just want to notify the user. Deleting data is too destructive.
-- name: Notify if matrix-redis local data remains
-  ansible.builtin.debug:
-    msg: "Note: You are not using a local redis instance, but some old data remains from before in `{{ matrix_redis_data_path }}`. Feel free to delete it."
-  when: matrix_redis_data_path_stat.stat.exists | bool

roles/custom/matrix-redis/templates/redis.conf.j2

@@ -1,4 +0,0 @@
-#jinja2: lstrip_blocks: "True"
-{% if matrix_redis_connection_password %}
-requirepass {{ matrix_redis_connection_password }}
-{% endif %}

roles/custom/matrix-redis/templates/systemd/matrix-redis.service.j2

@@ -1,37 +0,0 @@
-#jinja2: lstrip_blocks: "True"
-[Unit]
-Description=Matrix Redis server
-After=docker.service
-Requires=docker.service
-
-[Service]
-Type=simple
-ExecStartPre=-/usr/bin/docker stop matrix-redis
-ExecStartPre=-/usr/bin/docker rm matrix-redis
-
-ExecStart=/usr/bin/docker run --rm --name matrix-redis \
-			--log-driver=none \
-			--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
-			--cap-drop=ALL \
-			--read-only \
-			--tmpfs=/tmp:rw,noexec,nosuid,size=100m \
-			--network={{ matrix_docker_network }} \
-			{% if matrix_redis_container_redis_bind_port %}
-			-p {{ matrix_redis_container_redis_bind_port }}:6379 \
-			{% endif %}
-			--mount type=bind,src={{ matrix_redis_base_path }}/redis.conf,dst=/usr/local/etc/redis/redis.conf,ro \
-			--mount type=bind,src={{ matrix_redis_data_path }},dst=/data \
-			{% for arg in matrix_redis_container_extra_arguments %}
-			{{ arg }} \
-			{% endfor %}
-			{{ matrix_redis_docker_image_to_use }} \
-			redis-server /usr/local/etc/redis/redis.conf
-
-ExecStop=-/usr/bin/docker stop matrix-redis
-ExecStop=-/usr/bin/docker rm matrix-redis
-Restart=always
-RestartSec=30
-SyslogIdentifier=matrix-redis
-
-[Install]
-WantedBy=multi-user.target

roles/custom/matrix_playbook_migration/defaults/main.yml

@@ -14,3 +14,6 @@ matrix_playbook_migration_matrix_grafana_migration_validation_enabled: true
 
 # Controls if (`matrix_ntfy` -> `ntfy`) validation will run.
 matrix_playbook_migration_matrix_ntfy_migration_validation_enabled: true
+
+# Controls if (`matrix_redis` -> `redis`) validation will run.
+matrix_playbook_migration_matrix_redis_migration_validation_enabled: true

roles/custom/matrix_playbook_migration/tasks/validate_config.yml

@@ -125,3 +125,18 @@
           Please change your configuration (vars.yml) to rename all variables (`matrix_ntfy_` -> `ntfy_`).
           We found usage of the following variables: {{ matrix_playbook_migration_ntfy_migration_vars.keys() | join(', ') }}
       when: "matrix_playbook_migration_ntfy_migration_vars | length > 0"
+
+- when: matrix_playbook_migration_matrix_redis_migration_validation_enabled | bool
+  block:
+    - ansible.builtin.set_fact:
+        matrix_playbook_migration_redis_migration_vars: |-
+          {{ vars | dict2items | selectattr('key', 'match', 'matrix_redis_.*') | list | items2dict }}
+
+    - name: (Deprecation) Catch and report matrix_redis variables
+      ansible.builtin.fail:
+        msg: >-
+          The matrix-redis role that used to be part of this playbook has been replaced by https://gitlab.com/etke.cc/roles/redis.
+          The new role is compatible with the old one, but uses different names for its variables.
+          Please change your configuration (vars.yml) to rename all variables (`matrix_redis_` -> `redis_`).
+          We found usage of the following variables: {{ matrix_playbook_migration_redis_migration_vars.keys() | join(', ') }}
+      when: "matrix_playbook_migration_redis_migration_vars | length > 0"