From c2ba5c6412822b9af4bd5c1bdf71f1d6fcc6637e Mon Sep 17 00:00:00 2001 From: Jost Alemann Date: Wed, 24 Jan 2024 19:22:35 +0100 Subject: [PATCH 01/46] add missing ' in config --- docs/configuring-playbook-own-webserver.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuring-playbook-own-webserver.md b/docs/configuring-playbook-own-webserver.md index 74d0d06c6..702103670 100644 --- a/docs/configuring-playbook-own-webserver.md +++ b/docs/configuring-playbook-own-webserver.md @@ -169,7 +169,7 @@ devture_traefik_config_entrypoint_web_forwardedHeaders_insecure: true # If your reverse-proxy runs on another machine, consider: # - using `0.0.0.0:8449`, just `8449` or `SOME_IP_ADDRESS_OF_THIS_MACHINE:8449` below # - adjusting `matrix_playbook_public_matrix_federation_api_traefik_entrypoint_config_custom` (below) - removing `insecure: true` and enabling/configuring `trustedIPs` -matrix_playbook_public_matrix_federation_api_traefik_entrypoint_host_bind_port: 127.0.0.1:8449 +matrix_playbook_public_matrix_federation_api_traefik_entrypoint_host_bind_port: '127.0.0.1:8449' # Depending on the value of `matrix_playbook_public_matrix_federation_api_traefik_entrypoint_host_bind_port` above, # this may need to be reconfigured. See the comments above. From bd027159b10e3e8bd912e2a189a212c9d2ec7ec8 Mon Sep 17 00:00:00 2001 From: Michael Hollister Date: Wed, 24 Jan 2024 13:14:34 -0600 Subject: [PATCH 02/46] Added extra systemd service arguments to synapse workers and proxy companion --- .../defaults/main.yml | 14 +++++++++ ...synapse-reverse-proxy-companion.service.j2 | 3 ++ roles/custom/matrix-synapse/defaults/main.yml | 17 ++++++++++ .../workers/util/setup_files_for_worker.yml | 1 + .../systemd/matrix-synapse-worker.service.j2 | 31 +++++++++++++++++++ .../synapse/systemd/matrix-synapse.service.j2 | 3 ++ 6 files changed, 69 insertions(+) diff --git a/roles/custom/matrix-synapse-reverse-proxy-companion/defaults/main.yml b/roles/custom/matrix-synapse-reverse-proxy-companion/defaults/main.yml index f480f2891..d0638a244 100644 --- a/roles/custom/matrix-synapse-reverse-proxy-companion/defaults/main.yml +++ b/roles/custom/matrix-synapse-reverse-proxy-companion/defaults/main.yml @@ -116,6 +116,20 @@ matrix_synapse_reverse_proxy_companion_container_labels_public_federation_api_tr # another.label="here" matrix_synapse_reverse_proxy_companion_container_labels_additional_labels: '' +# A list of extra arguments to pass to the container +# Also see `matrix_synapse_reverse_proxy_companion_container_arguments` +matrix_synapse_reverse_proxy_companion_container_extra_arguments: [] + +# matrix_synapse_reverse_proxy_companion_container_extra_arguments_auto is a list of extra arguments to pass to the container. +# This list is managed by the playbook. You're not meant to override this variable. +# If you'd like to inject your own arguments, see `matrix_synapse_reverse_proxy_companion_container_extra_arguments`. +matrix_synapse_reverse_proxy_companion_container_extra_arguments_auto: [] + +# matrix_synapse_reverse_proxy_companion_container_arguments holds the final list of extra arguments to pass to the container. +# You're not meant to override this variable. +# If you'd like to inject your own arguments, see `matrix_synapse_reverse_proxy_companion_container_extra_arguments`. +matrix_synapse_reverse_proxy_companion_container_arguments: "{{ matrix_synapse_reverse_proxy_companion_container_extra_arguments + matrix_synapse_reverse_proxy_companion_container_extra_arguments_auto }}" + # The amount of worker processes and connections # Consider increasing these when you are expecting high amounts of traffic # http://nginx.org/en/docs/ngx_core_module.html#worker_connections diff --git a/roles/custom/matrix-synapse-reverse-proxy-companion/templates/systemd/matrix-synapse-reverse-proxy-companion.service.j2 b/roles/custom/matrix-synapse-reverse-proxy-companion/templates/systemd/matrix-synapse-reverse-proxy-companion.service.j2 index 1afb81c89..3e1732369 100755 --- a/roles/custom/matrix-synapse-reverse-proxy-companion/templates/systemd/matrix-synapse-reverse-proxy-companion.service.j2 +++ b/roles/custom/matrix-synapse-reverse-proxy-companion/templates/systemd/matrix-synapse-reverse-proxy-companion.service.j2 @@ -37,6 +37,9 @@ ExecStartPre={{ devture_systemd_docker_base_host_command_docker }} create \ --mount type=bind,src={{ matrix_synapse_reverse_proxy_companion_base_path }}/nginx.conf,dst=/etc/nginx/nginx.conf,ro \ --mount type=bind,src={{ matrix_synapse_reverse_proxy_companion_confd_path }},dst=/etc/nginx/conf.d,ro \ --label-file={{ matrix_synapse_reverse_proxy_companion_base_path }}/labels \ + {% for arg in matrix_synapse_reverse_proxy_companion_container_arguments %} + {{ arg }} \ + {% endfor %} {{ matrix_synapse_reverse_proxy_companion_container_image }} {% for network in matrix_synapse_reverse_proxy_companion_container_additional_networks %} diff --git a/roles/custom/matrix-synapse/defaults/main.yml b/roles/custom/matrix-synapse/defaults/main.yml index dd4ac05a2..eecd061b8 100644 --- a/roles/custom/matrix-synapse/defaults/main.yml +++ b/roles/custom/matrix-synapse/defaults/main.yml @@ -296,6 +296,10 @@ matrix_synapse_container_extra_arguments_auto: [] # If you'd like to inject your own arguments, see `matrix_synapse_container_extra_arguments`. matrix_synapse_container_arguments: "{{ matrix_synapse_container_extra_arguments + matrix_synapse_container_extra_arguments_auto }}" +# matrix_synapse_container_master_extra_arguments contains arguments specific to the master process whereas +# matrix_synapse_container_extra_arguments contains arguments the apply to all synapse worker processes. +matrix_synapse_container_master_extra_arguments: [] + # List of systemd services that matrix-synapse.service depends on matrix_synapse_systemd_required_services_list: "{{ matrix_synapse_systemd_required_services_list_default + matrix_synapse_systemd_required_services_list_auto + matrix_synapse_systemd_required_services_list_custom }}" matrix_synapse_systemd_required_services_list_default: ['docker.service'] @@ -769,26 +773,31 @@ matrix_synapse_worker_container_labels_additional_labels: '' matrix_synapse_workers_room_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['room_workers_count'] }}" matrix_synapse_workers_room_workers_port_range_start: 28111 matrix_synapse_workers_room_workers_metrics_range_start: 29111 +matrix_synapse_workers_room_workers_container_arguments: [] # Sync workers matrix_synapse_workers_sync_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['sync_workers_count'] }}" matrix_synapse_workers_sync_workers_port_range_start: 28211 matrix_synapse_workers_sync_workers_metrics_range_start: 29211 +matrix_synapse_workers_sync_workers_container_arguments: [] # Client reader workers matrix_synapse_workers_client_reader_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['client_reader_workers_count'] }}" matrix_synapse_workers_client_reader_workers_port_range_start: 28311 matrix_synapse_workers_client_reader_workers_metrics_range_start: 29311 +matrix_synapse_workers_client_reader_workers_container_arguments: [] # Federation reader workers matrix_synapse_workers_federation_reader_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['federation_reader_workers_count'] }}" matrix_synapse_workers_federation_reader_workers_port_range_start: 28411 matrix_synapse_workers_federation_reader_workers_metrics_range_start: 29411 +matrix_synapse_workers_federation_reader_workers_container_arguments: [] # Generic workers matrix_synapse_workers_generic_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['generic_workers_count'] }}" matrix_synapse_workers_generic_workers_port_range_start: 18111 matrix_synapse_workers_generic_workers_metrics_range_start: 19111 +matrix_synapse_workers_generic_workers_container_arguments: [] # matrix_synapse_workers_stream_writer_events_stream_workers_count controls how many stream writers that handle the `events` stream to spawn. # More than 1 worker is also supported of this type. @@ -833,6 +842,8 @@ matrix_synapse_workers_stream_writers: | ([{'stream': 'presence'}] * matrix_synapse_workers_stream_writer_presence_stream_workers_count | int) }} +matrix_synapse_workers_stream_writers_container_arguments: [] + # matrix_synapse_stream_writers populates the `stream_writers` Synapse configuration used when Synapse workers are in use (`matrix_synapse_workers_enabled`). # What you see below is an initial default value which will be adjusted at runtime based on the value of `matrix_synapse_workers_stream_writers`. # Adjusting this value manually is generally not necessary. @@ -859,6 +870,7 @@ matrix_synapse_workers_stream_writer_workers_metrics_range_start: 19211 # See https://matrix-org.github.io/synapse/latest/workers.html#synapseapppusher matrix_synapse_workers_pusher_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['pusher_workers_count'] }}" matrix_synapse_workers_pusher_workers_metrics_range_start: 19200 +matrix_synapse_workers_pusher_workers_container_arguments: [] # matrix_synapse_federation_pusher_instances populates the `pusher_instances` Synapse configuration used when Synapse workers are in use (`matrix_synapse_workers_enabled`). # What you see below is an initial default value which will be adjusted at runtime based on the value of `matrix_synapse_workers_pusher_workers_count` or `matrix_synapse_workers_enabled_list`. @@ -869,6 +881,7 @@ matrix_synapse_federation_pusher_instances: [] # See https://matrix-org.github.io/synapse/latest/workers.html#synapseappfederation_sender matrix_synapse_workers_federation_sender_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['federation_sender_workers_count'] }}" matrix_synapse_workers_federation_sender_workers_metrics_range_start: 19400 +matrix_synapse_workers_federation_sender_workers_container_arguments: [] # matrix_synapse_federation_sender_instances populates the `federation_sender_instances` Synapse configuration used when Synapse workers are in use (`matrix_synapse_workers_enabled`). # What you see below is an initial default value which will be adjusted at runtime based on the value of `matrix_synapse_workers_federation_sender_workers_count` or `matrix_synapse_workers_enabled_list`. @@ -878,6 +891,7 @@ matrix_synapse_federation_sender_instances: [] matrix_synapse_workers_media_repository_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['media_repository_workers_count'] if not matrix_synapse_ext_media_repo_enabled else 0 }}" matrix_synapse_workers_media_repository_workers_port_range_start: 18551 matrix_synapse_workers_media_repository_workers_metrics_range_start: 19551 +matrix_synapse_workers_media_repository_workers_container_arguments: [] # matrix_synapse_enable_media_repo controls if the main Synapse process should serve media repository endpoints or if it should be left to media_repository workers (see `matrix_synapse_workers_media_repository_workers_count`). # This is enabled if workers are disabled, or if they are enabled, but there are no media repository workers. @@ -894,6 +908,7 @@ matrix_synapse_media_instance_running_background_jobs: "{{ (matrix_synapse_worke # Our implementation uses generic worker services and assigns them to perform appservice work using the `notify_appservices_from_worker` Synapse option. matrix_synapse_workers_appservice_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['appservice_workers_count'] }}" matrix_synapse_workers_appservice_workers_metrics_range_start: 19300 +matrix_synapse_workers_appservice_workers_container_arguments: [] # matrix_synapse_notify_appservices_from_worker populates the `notify_appservices_from_worker` Synapse configuration used when Synapse workers are in use (`matrix_synapse_workers_enabled`). # `notify_appservices_from_worker` is meant to point to a worker, which is dedicated to sending output traffic to Application Services. @@ -905,6 +920,7 @@ matrix_synapse_notify_appservices_from_worker: "{{ (matrix_synapse_workers_enabl matrix_synapse_workers_user_dir_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['user_dir_workers_count'] }}" matrix_synapse_workers_user_dir_workers_port_range_start: 18661 matrix_synapse_workers_user_dir_workers_metrics_range_start: 19661 +matrix_synapse_workers_user_dir_workers_container_arguments: [] # matrix_synapse_update_user_directory_from_worker populates the `update_user_directory_from_worker` Synapse configuration used when Synapse workers are in use (`matrix_synapse_workers_enabled`). # `update_user_directory_from_worker` is meant to point to a worker, which is dedicated to updating the user directory and servicing some user directory URL endpoints (`matrix_synapse_workers_user_dir_worker_client_server_endpoints`). @@ -914,6 +930,7 @@ matrix_synapse_update_user_directory_from_worker: "{{ (matrix_synapse_workers_en # Our implementation uses a generic worker and assigns Synapse to perform background work on this worker using the `run_background_tasks_on` Synapse option. matrix_synapse_workers_background_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['background_workers_count'] }}" matrix_synapse_workers_background_workers_metrics_range_start: 19700 +matrix_synapse_workers_background_workers_container_arguments: [] # matrix_synapse_run_background_tasks_on populates the `run_background_tasks_on` Synapse configuration used when Synapse workers are in use (`matrix_synapse_workers_enabled`). # `run_background_tasks_on` is meant to point to a worker, which is dedicated to processing background tasks. diff --git a/roles/custom/matrix-synapse/tasks/synapse/workers/util/setup_files_for_worker.yml b/roles/custom/matrix-synapse/tasks/synapse/workers/util/setup_files_for_worker.yml index 5b652c475..480ffba02 100644 --- a/roles/custom/matrix-synapse/tasks/synapse/workers/util/setup_files_for_worker.yml +++ b/roles/custom/matrix-synapse/tasks/synapse/workers/util/setup_files_for_worker.yml @@ -3,6 +3,7 @@ - ansible.builtin.set_fact: matrix_synapse_worker_systemd_service_name: "{{ matrix_synapse_worker_details.name }}" matrix_synapse_worker_container_name: "{{ matrix_synapse_worker_details.name }}" + matrix_synapse_worker_type: "{{ matrix_synapse_worker_details.type }}" matrix_synapse_worker_config_file_name: "worker.{{ matrix_synapse_worker_details.name }}.yaml" matrix_synapse_worker_labels_file_name: "worker.{{ matrix_synapse_worker_details.name }}.labels" diff --git a/roles/custom/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker.service.j2 b/roles/custom/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker.service.j2 index 4163efc3d..21b747b0e 100644 --- a/roles/custom/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker.service.j2 +++ b/roles/custom/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker.service.j2 @@ -5,6 +5,34 @@ AssertPathExists={{ matrix_synapse_config_dir_path }}/{{ matrix_synapse_worker_c After=matrix-synapse.service Requires=matrix-synapse.service +{% set matrix_synapse_container_worker_arguments = [] %} + +{% if matrix_synapse_worker_type == "room_worker" %} + {% set matrix_synapse_container_worker_arguments = matrix_synapse_workers_room_workers_container_arguments %} +{% elif matrix_synapse_worker_type == "sync_worker" %} + {% set matrix_synapse_container_worker_arguments = matrix_synapse_workers_sync_workers_container_arguments %} +{% elif matrix_synapse_worker_type == "client_reader" %} + {% set matrix_synapse_container_worker_arguments = matrix_synapse_workers_client_reader_workers_container_arguments %} +{% elif matrix_synapse_worker_type == "federation_reader" %} + {% set matrix_synapse_container_worker_arguments = matrix_synapse_workers_federation_reader_workers_container_arguments %} +{% elif matrix_synapse_worker_type == "generic_worker" %} + {% set matrix_synapse_container_worker_arguments = matrix_synapse_workers_generic_workers_container_arguments %} +{% elif matrix_synapse_worker_type == "stream_writer" %} + {% set matrix_synapse_container_worker_arguments = matrix_synapse_workers_stream_writers_container_arguments %} +{% elif matrix_synapse_worker_type == "federation_sender" %} + {% set matrix_synapse_container_worker_arguments = matrix_synapse_workers_federation_sender_workers_container_arguments %} +{% elif matrix_synapse_worker_type == "pusher" %} + {% set matrix_synapse_container_worker_arguments = matrix_synapse_workers_pusher_workers_container_arguments %} +{% elif matrix_synapse_worker_type == "appservice" %} + {% set matrix_synapse_container_worker_arguments = matrix_synapse_workers_appservice_workers_container_arguments %} +{% elif matrix_synapse_worker_type == "user_dir" %} + {% set matrix_synapse_container_worker_arguments = matrix_synapse_workers_user_dir_workers_container_arguments %} +{% elif matrix_synapse_worker_type == "background" %} + {% set matrix_synapse_container_worker_arguments = matrix_synapse_workers_background_workers_container_arguments %} +{% elif matrix_synapse_worker_type == "media_repository" %} + {% set matrix_synapse_container_worker_arguments = matrix_synapse_workers_media_repository_workers_container_arguments %} +{% endif %} + [Service] Type=simple Environment="HOME={{ devture_systemd_docker_base_systemd_unit_home_path }}" @@ -43,6 +71,9 @@ ExecStartPre={{ devture_systemd_docker_base_host_command_docker }} create \ {% for arg in matrix_synapse_container_arguments %} {{ arg }} \ {% endfor %} + {% for arg in matrix_synapse_container_worker_arguments %} + {{ arg }} \ + {% endfor %} {{ matrix_synapse_docker_image_final }} \ run -m synapse.app.{{ matrix_synapse_worker_details.app }} -c /data/homeserver.yaml -c /data/{{ matrix_synapse_worker_config_file_name }} diff --git a/roles/custom/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 b/roles/custom/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 index 3d9d5f5af..a22c09b1e 100644 --- a/roles/custom/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 +++ b/roles/custom/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 @@ -57,6 +57,9 @@ ExecStartPre={{ devture_systemd_docker_base_host_command_docker }} create \ {% for arg in matrix_synapse_container_arguments %} {{ arg }} \ {% endfor %} + {% for arg in matrix_synapse_container_master_extra_arguments %} + {{ arg }} \ + {% endfor %} {{ matrix_synapse_docker_image_final }} \ run -m synapse.app.homeserver -c /data/homeserver.yaml From 4ed522e8fefff632523d68c7d2f2f9a6d86e3b1b Mon Sep 17 00:00:00 2001 From: needo37 Date: Wed, 24 Jan 2024 19:41:58 -0600 Subject: [PATCH 03/46] Bring default config inline with upstream --- .../templates/config.yaml.j2 | 84 ++++++++++++++++++- 1 file changed, 81 insertions(+), 3 deletions(-) diff --git a/roles/custom/matrix-bridge-mautrix-discord/templates/config.yaml.j2 b/roles/custom/matrix-bridge-mautrix-discord/templates/config.yaml.j2 index f5ccd4f7e..c54201c9a 100644 --- a/roles/custom/matrix-bridge-mautrix-discord/templates/config.yaml.j2 +++ b/roles/custom/matrix-bridge-mautrix-discord/templates/config.yaml.j2 @@ -72,11 +72,14 @@ bridge: # Displayname template for Discord users. This is also used as the room name in DMs if private_chat_portal_meta is enabled. # Available variables: # {{ '{{.ID}}' }} - Internal user ID - # {{ '{{.Username}}' }} - User's displayname on Discord + # {{ '{{.Username}}' }} - Legacy display/username on Discord + # {{ '{{.GlobalName}}' }} - New displayname on Discord # {{ '{{.Discriminator}}' }} - The 4 numbers after the name on Discord # {{ '{{.Bot}}' }} - Whether the user is a bot # {{ '{{.System}}' }} - Whether the user is an official system user - displayname_template: "{{ '{{.Username}} {{if .Bot}} (bot){{end}}' }}" + # {{ '{{.Webhook}}' }} - Whether the user is a webhook and is not an application + # {{ '{{.Application}}' }} - Whether the user is an application + displayname_template: "{{ '{{or .GlobalName .Username}}{{if .Bot}} (bot){{end}}' }}" # Displayname template for Discord channels (bridged as rooms, or spaces when type=4). # Available variables: # {{ '{{.Name}}' }} - Channel name, or user displayname (pre-formatted with displayname_template) in DMs. @@ -91,7 +94,11 @@ bridge: guild_name_template: "{{ '{{.Name}}' }}" # Should the bridge explicitly set the avatar and room name for DM portal rooms? # This is implicitly enabled in encrypted rooms. - private_chat_portal_meta: false + # Whether to explicitly set the avatar and room name for private chat portal rooms. + # If set to `default`, this will be enabled in encrypted rooms and disabled in unencrypted rooms. + # If set to `always`, all DM rooms will have explicit names and avatars set. + # If set to `never`, DM rooms will never have names and avatars set. + private_chat_portal_meta: default portal_message_buffer: 128 # Number of private channel portals to create on bridge startup. # Other portals will be created when receiving messages. @@ -112,12 +119,59 @@ bridge: # Set this to true to tell the bridge to re-send m.bridge events to all rooms on the next run. # This field will automatically be changed back to false after it, except if the config file is not writable. resend_bridge_info: false + # Should incoming custom emoji reactions be bridged as mxc:// URIs? + # If set to false, custom emoji reactions will be bridged as the shortcode instead, and the image wont be available. + custom_emoji_reactions: true # Should the bridge attempt to completely delete portal rooms when a channel is deleted on Discord? # If true, the bridge will try to kick Matrix users from the room. Otherwise, the bridge only makes ghosts leave. delete_portal_on_channel_delete: false + # Should the bridge delete all portal rooms when you leave a guild on Discord? + # This only applies if the guild has no other Matrix users on this bridge instance. + delete_guild_on_leave: true # Whether or not created rooms should have federation enabled. # If false, created portal rooms will never be federated. federate_rooms: {{ matrix_mautrix_discord_federate_rooms|to_json }} + # Prefix messages from webhooks with the profile info? This can be used along with a custom displayname_template + # to better handle webhooks that change their name all the time (like ones used by bridges). + prefix_webhook_messages: false + # Bridge webhook avatars? + enable_webhook_avatars: true + # Should the bridge upload media to the Discord CDN directly before sending the message when using a user token, + # like the official client does? The other option is sending the media in the message send request as a form part + # (which is always used by bots and webhooks). + use_discord_cdn_upload: true + # Should mxc uris copied from Discord be cached? + # This can be `never` to never cache, `unencrypted` to only cache unencrypted mxc uris, or `always` to cache everything. + # If you have a media repo that generates non-unique mxc uris, you should set this to never. + cache_media: unencrypted + # Patterns for converting Discord media to custom mxc:// URIs instead of reuploading. + # Each of the patterns can be set to null to disable custom URIs for that type of media. + # More details can be found at https://docs.mau.fi/bridges/go/discord/direct-media.html + media_patterns: + # Should custom mxc:// URIs be used instead of reuploading media? + enabled: true + # Pattern for normal message attachments. + attachments: mxc://discord-media.mau.dev/attachments|{{.ChannelID}}|{{.AttachmentID}}|{{.FileName}} + # Pattern for custom emojis. + emojis: mxc://discord-media.mau.dev/emojis|{{.ID}}.{{.Ext}} + # Pattern for stickers. Note that animated lottie stickers will not be converted if this is enabled. + stickers: mxc://discord-media.mau.dev/stickers|{{.ID}}.{{.Ext}} + # Pattern for static user avatars. + avatars: mxc://discord-media.mau.dev/avatars|{{.UserID}}|{{.AvatarID}}.{{.Ext}} + # Settings for converting animated stickers. + animated_sticker: + # Format to which animated stickers should be converted. + # disable - No conversion, send as-is (lottie JSON) + # png - converts to non-animated png (fastest) + # gif - converts to animated gif + # webm - converts to webm video, requires ffmpeg executable with vp9 codec and webm container support + # webp - converts to animated webp, requires ffmpeg executable with webp codec/container support + target: webp + # Arguments for converter. All converters take width and height. + args: + width: 320 + height: 320 + fps: 25 # only for webm, webp and gif (2, 5, 10, 20 or 25 recommended) # Servers to always allow double puppeting from double_puppet_server_map: "{{ matrix_mautrix_discord_homeserver_domain }}": {{ matrix_mautrix_discord_homeserver_address }} @@ -145,6 +199,30 @@ bridge: # Optional extra text sent when joining a management room. additional_help: "" + # Settings for backfilling messages. + backfill: + # Limits for forward backfilling. + forward_limits: + # Initial backfill (when creating portal). 0 means backfill is disabled. + # A special unlimited value is not supported, you must set a limit. Initial backfill will + # fetch all messages first before backfilling anything, so high limits can take a lot of time. + initial: + dm: 0 + channel: 0 + thread: 0 + # Missed message backfill (on startup). + # 0 means backfill is disabled, -1 means fetch all messages since last bridged message. + # When using unlimited backfill (-1), messages are backfilled as they are fetched. + # With limits, all messages up to the limit are fetched first and backfilled afterwards. + missed: + dm: 0 + channel: 0 + thread: 0 + # Maximum members in a guild to enable backfilling. Set to -1 to disable limit. + # This can be used as a rough heuristic to disable backfilling in channels that are too active. + # Currently only applies to missed message backfill. + max_guild_members: -1 + # End-to-bridge encryption support options. # # See https://docs.mau.fi/bridges/general/end-to-bridge-encryption.html for more info. From 243d828e50b17fe35170c1448416e70f6b9ba798 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 25 Jan 2024 07:35:16 +0200 Subject: [PATCH 04/46] Fix mautrix-discord config Jinja2 syntax error Related to https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/3133 Regression since 4ed522e8fefff63252 --- .../templates/config.yaml.j2 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/custom/matrix-bridge-mautrix-discord/templates/config.yaml.j2 b/roles/custom/matrix-bridge-mautrix-discord/templates/config.yaml.j2 index c54201c9a..561a9c3c7 100644 --- a/roles/custom/matrix-bridge-mautrix-discord/templates/config.yaml.j2 +++ b/roles/custom/matrix-bridge-mautrix-discord/templates/config.yaml.j2 @@ -151,13 +151,13 @@ bridge: # Should custom mxc:// URIs be used instead of reuploading media? enabled: true # Pattern for normal message attachments. - attachments: mxc://discord-media.mau.dev/attachments|{{.ChannelID}}|{{.AttachmentID}}|{{.FileName}} + attachments: {% raw %}mxc://discord-media.mau.dev/attachments|{{.ChannelID}}|{{.AttachmentID}}|{{.FileName}}{% endraw %} # Pattern for custom emojis. - emojis: mxc://discord-media.mau.dev/emojis|{{.ID}}.{{.Ext}} + emojis: {% raw %}mxc://discord-media.mau.dev/emojis|{{.ID}}.{{.Ext}}{% endraw %} # Pattern for stickers. Note that animated lottie stickers will not be converted if this is enabled. - stickers: mxc://discord-media.mau.dev/stickers|{{.ID}}.{{.Ext}} + stickers: {% raw %}mxc://discord-media.mau.dev/stickers|{{.ID}}.{{.Ext}}{% endraw %} # Pattern for static user avatars. - avatars: mxc://discord-media.mau.dev/avatars|{{.UserID}}|{{.AvatarID}}.{{.Ext}} + avatars: {% raw %}mxc://discord-media.mau.dev/avatars|{{.UserID}}|{{.AvatarID}}.{{.Ext}}{% endraw %} # Settings for converting animated stickers. animated_sticker: # Format to which animated stickers should be converted. From ad9ba1e2bd67767f85025a58d6c03242ce4417a7 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 25 Jan 2024 07:39:25 +0200 Subject: [PATCH 05/46] Fix variable name typo --- roles/custom/matrix-synapse/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/custom/matrix-synapse/defaults/main.yml b/roles/custom/matrix-synapse/defaults/main.yml index eecd061b8..1804d243c 100644 --- a/roles/custom/matrix-synapse/defaults/main.yml +++ b/roles/custom/matrix-synapse/defaults/main.yml @@ -297,7 +297,7 @@ matrix_synapse_container_extra_arguments_auto: [] matrix_synapse_container_arguments: "{{ matrix_synapse_container_extra_arguments + matrix_synapse_container_extra_arguments_auto }}" # matrix_synapse_container_master_extra_arguments contains arguments specific to the master process whereas -# matrix_synapse_container_extra_arguments contains arguments the apply to all synapse worker processes. +# matrix_synapse_container_arguments contains arguments the apply to all Synapse containers (master and worker). matrix_synapse_container_master_extra_arguments: [] # List of systemd services that matrix-synapse.service depends on From 185f54a4c7626b550e4003a5c2f1564051eaa48c Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 26 Jan 2024 08:55:53 +0200 Subject: [PATCH 06/46] Upgrade Prometheus (v2.49.1-0 -> v2.49.1-1) --- requirements.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.yml b/requirements.yml index ae08139ea..d5067141f 100644 --- a/requirements.yml +++ b/requirements.yml @@ -46,7 +46,7 @@ version: 7eadc992ca952fc29bf3fab5aa6335fa82ff01e5 name: postgres_backup - src: git+https://github.com/mother-of-all-self-hosting/ansible-role-prometheus.git - version: v2.49.1-0 + version: v2.49.1-1 name: prometheus - src: git+https://github.com/mother-of-all-self-hosting/ansible-role-prometheus-node-exporter.git version: v1.7.0-3 From 5ca4d6ebc5baa402d34508570e3f571f929d39d7 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 26 Jan 2024 12:09:54 +0200 Subject: [PATCH 07/46] Add validation check for matrix_nginx_proxy_container_labels_traefik_proxy_matrix_federation_hostname --- roles/custom/matrix_playbook_migration/tasks/validate_config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/custom/matrix_playbook_migration/tasks/validate_config.yml b/roles/custom/matrix_playbook_migration/tasks/validate_config.yml index bc428b386..51e0440d1 100644 --- a/roles/custom/matrix_playbook_migration/tasks/validate_config.yml +++ b/roles/custom/matrix_playbook_migration/tasks/validate_config.yml @@ -301,6 +301,7 @@ - {'old': 'matrix_nginx_proxy_proxy_synapse_additional_server_configuration_blocks', 'new': ''} - {'old': 'matrix_nginx_proxy_proxy_matrix_additional_server_configuration_blocks', 'new': ''} - {'old': 'matrix_nginx_proxy_access_log_enabled', 'new': 'devture_traefik_config_accessLog_enabled'} + - {'old': 'matrix_nginx_proxy_container_labels_traefik_proxy_matrix_federation_hostname', 'new': ''} - {'old': 'etherpad_nginx_proxy_dimension_integration_path_prefix', 'new': ''} - {'old': 'matrix_prometheus_services_proxy_connect_prometheus_node_exporter_metrics_proxying_enabled', 'new': ''} From b48b06d2f82f2d8481ca791b3c4f193b87993eae Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 26 Jan 2024 12:10:34 +0200 Subject: [PATCH 08/46] Add missing bracket --- .../custom/matrix_playbook_migration/tasks/validate_config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/custom/matrix_playbook_migration/tasks/validate_config.yml b/roles/custom/matrix_playbook_migration/tasks/validate_config.yml index 51e0440d1..2100905db 100644 --- a/roles/custom/matrix_playbook_migration/tasks/validate_config.yml +++ b/roles/custom/matrix_playbook_migration/tasks/validate_config.yml @@ -301,7 +301,7 @@ - {'old': 'matrix_nginx_proxy_proxy_synapse_additional_server_configuration_blocks', 'new': ''} - {'old': 'matrix_nginx_proxy_proxy_matrix_additional_server_configuration_blocks', 'new': ''} - {'old': 'matrix_nginx_proxy_access_log_enabled', 'new': 'devture_traefik_config_accessLog_enabled'} - - {'old': 'matrix_nginx_proxy_container_labels_traefik_proxy_matrix_federation_hostname', 'new': ''} - {'old': 'etherpad_nginx_proxy_dimension_integration_enabled', 'new': ''} - {'old': 'etherpad_nginx_proxy_dimension_integration_path_prefix', 'new': ''} - {'old': 'matrix_prometheus_services_proxy_connect_prometheus_node_exporter_metrics_proxying_enabled', 'new': ''} From dafeee92f4e2dc6016cb41a8669b37e38af1ee17 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 26 Jan 2024 12:17:49 +0200 Subject: [PATCH 09/46] Adjust matrix_nginx_proxy_container_labels_traefik_proxy_matrix_federation_hostname validation check message to mention matrix_static_files_file_matrix_server_property_m_server --- .../custom/matrix_playbook_migration/tasks/validate_config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/custom/matrix_playbook_migration/tasks/validate_config.yml b/roles/custom/matrix_playbook_migration/tasks/validate_config.yml index 2100905db..c46fb7fc4 100644 --- a/roles/custom/matrix_playbook_migration/tasks/validate_config.yml +++ b/roles/custom/matrix_playbook_migration/tasks/validate_config.yml @@ -301,7 +301,7 @@ - {'old': 'matrix_nginx_proxy_proxy_synapse_additional_server_configuration_blocks', 'new': ''} - {'old': 'matrix_nginx_proxy_proxy_matrix_additional_server_configuration_blocks', 'new': ''} - {'old': 'matrix_nginx_proxy_access_log_enabled', 'new': 'devture_traefik_config_accessLog_enabled'} - - {'old': 'matrix_nginx_proxy_container_labels_traefik_proxy_matrix_federation_hostname', 'new': ''} + - {'old': 'matrix_nginx_proxy_container_labels_traefik_proxy_matrix_federation_hostname', 'new': ''} - {'old': 'etherpad_nginx_proxy_dimension_integration_enabled', 'new': ''} - {'old': 'etherpad_nginx_proxy_dimension_integration_path_prefix', 'new': ''} - {'old': 'matrix_prometheus_services_proxy_connect_prometheus_node_exporter_metrics_proxying_enabled', 'new': ''} From a1179289a1dc8c8bb8c72e923beff0f407260310 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 26 Jan 2024 12:55:01 +0200 Subject: [PATCH 10/46] Split some homeserver _additional_networks variables into _auto and _custom --- group_vars/matrix_servers | 6 +++--- roles/custom/matrix-dendrite/defaults/main.yml | 4 +++- .../defaults/main.yml | 4 +++- roles/custom/matrix-synapse/defaults/main.yml | 4 +++- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index df5b12ebb..d5034917c 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -3893,7 +3893,7 @@ matrix_synapse_container_manhole_api_host_bind_port: "{{ (matrix_playbook_servic matrix_synapse_container_network: "{{ matrix_homeserver_container_network }}" -matrix_synapse_container_additional_networks: | +matrix_synapse_container_additional_networks_auto: | {{ ( ([matrix_playbook_reverse_proxyable_services_additional_network] if matrix_synapse_container_labels_traefik_enabled and matrix_playbook_reverse_proxyable_services_additional_network else []) @@ -4052,7 +4052,7 @@ matrix_synapse_reverse_proxy_companion_enabled: "{{ matrix_synapse_enabled and m matrix_synapse_reverse_proxy_companion_container_network: "{{ matrix_synapse_container_network }}" -matrix_synapse_reverse_proxy_companion_container_additional_networks: | +matrix_synapse_reverse_proxy_companion_container_additional_networks_auto: | {{ ( ([matrix_playbook_reverse_proxyable_services_additional_network] if matrix_synapse_reverse_proxy_companion_container_labels_traefik_enabled and matrix_playbook_reverse_proxyable_services_additional_network else []) @@ -4585,7 +4585,7 @@ matrix_dendrite_federation_enabled: "{{ matrix_homeserver_federation_enabled }}" matrix_dendrite_container_network: "{{ matrix_homeserver_container_network }}" -matrix_dendrite_container_additional_networks: | +matrix_dendrite_container_additional_networks_auto: | {{ ( ([matrix_playbook_reverse_proxyable_services_additional_network] if matrix_dendrite_container_labels_traefik_enabled and matrix_playbook_reverse_proxyable_services_additional_network else []) diff --git a/roles/custom/matrix-dendrite/defaults/main.yml b/roles/custom/matrix-dendrite/defaults/main.yml index d602a860e..a3854315f 100644 --- a/roles/custom/matrix-dendrite/defaults/main.yml +++ b/roles/custom/matrix-dendrite/defaults/main.yml @@ -48,7 +48,9 @@ matrix_dendrite_container_network: "" # A list of additional container networks that the container would be connected to. # The role does not create these networks, so make sure they already exist. # Use this to expose this container to another reverse proxy, which runs in a different container network. -matrix_dendrite_container_additional_networks: [] +matrix_dendrite_container_additional_networks: "{{ matrix_dendrite_container_additional_networks_auto + matrix_dendrite_container_additional_networks_custom }}" +matrix_dendrite_container_additional_networks_auto: [] +matrix_dendrite_container_additional_networks_custom: [] # Controls whether the matrix-dendrite container exposes the HTTP port (tcp/{{ matrix_dendrite_http_bind_port }} in the container). # diff --git a/roles/custom/matrix-synapse-reverse-proxy-companion/defaults/main.yml b/roles/custom/matrix-synapse-reverse-proxy-companion/defaults/main.yml index d0638a244..b4ab73b0d 100644 --- a/roles/custom/matrix-synapse-reverse-proxy-companion/defaults/main.yml +++ b/roles/custom/matrix-synapse-reverse-proxy-companion/defaults/main.yml @@ -35,7 +35,9 @@ matrix_synapse_reverse_proxy_companion_container_network: "" # A list of additional container networks that matrix-synapse-reverse-proxy-companion would be connected to. # The playbook does not create these networks, so make sure they already exist. -matrix_synapse_reverse_proxy_companion_container_additional_networks: [] +matrix_synapse_reverse_proxy_companion_container_additional_networks: "{{ matrix_synapse_reverse_proxy_companion_container_additional_networks_auto + matrix_synapse_reverse_proxy_companion_container_additional_networks_custom }}" +matrix_synapse_reverse_proxy_companion_container_additional_networks_auto: [] +matrix_synapse_reverse_proxy_companion_container_additional_networks_custom: [] # Controls whether the matrix-synapse-reverse-proxy-companion container exposes its HTTP Client-Server API port (tcp/8008 in the container). # diff --git a/roles/custom/matrix-synapse/defaults/main.yml b/roles/custom/matrix-synapse/defaults/main.yml index 1804d243c..d35fe2e5a 100644 --- a/roles/custom/matrix-synapse/defaults/main.yml +++ b/roles/custom/matrix-synapse/defaults/main.yml @@ -132,7 +132,9 @@ matrix_synapse_container_network: '' # A list of additional container networks that the container would be connected to. # The role does not create these networks, so make sure they already exist. # Use this to expose this container to another reverse proxy, which runs in a different container network. -matrix_synapse_container_additional_networks: [] +matrix_synapse_container_additional_networks: "{{ matrix_synapse_container_additional_networks_auto + matrix_synapse_container_additional_networks_custom }}" +matrix_synapse_container_additional_networks_auto: [] +matrix_synapse_container_additional_networks_custom: [] # Controls whether the matrix-synapse container exposes the Client/Server API port (tcp/{{ matrix_synapse_container_client_api_port }} in the container). # From a9eba7ab329f3c61e19df08a5d37217f60d4c97f Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 26 Jan 2024 13:07:09 +0200 Subject: [PATCH 11/46] Fix turn: fallback URIs missing due to Jinja operator priorities --- group_vars/matrix_servers | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index d5034917c..d32f73cf1 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -3960,17 +3960,15 @@ matrix_synapse_email_notif_from: "Matrix <{{ exim_relay_sender_address }}>" matrix_synapse_turn_uris: | {{ - [] - + - [ + ([ 'turns:' + matrix_server_fqn_matrix + '?transport=udp', 'turns:' + matrix_server_fqn_matrix + '?transport=tcp', - ] if matrix_coturn_enabled and matrix_coturn_tls_enabled else [] + ] if matrix_coturn_enabled and matrix_coturn_tls_enabled else []) + - [ + ([ 'turn:' + matrix_server_fqn_matrix + '?transport=udp', 'turn:' + matrix_server_fqn_matrix + '?transport=tcp', - ] if matrix_coturn_enabled else [] + ] if matrix_coturn_enabled else []) }} matrix_synapse_turn_shared_secret: "{{ matrix_coturn_turn_static_auth_secret if matrix_coturn_enabled else '' }}" @@ -4705,17 +4703,15 @@ matrix_conduit_container_labels_internal_client_api_traefik_entrypoints: "{{ mat matrix_conduit_turn_uris: | {{ - [] - + - [ + ([ 'turns:' + matrix_server_fqn_matrix + '?transport=udp', 'turns:' + matrix_server_fqn_matrix + '?transport=tcp', - ] if matrix_coturn_enabled and matrix_coturn_tls_enabled else [] + ] if matrix_coturn_enabled and matrix_coturn_tls_enabled else []) + - [ + ([ 'turn:' + matrix_server_fqn_matrix + '?transport=udp', 'turn:' + matrix_server_fqn_matrix + '?transport=tcp', - ] if matrix_coturn_enabled else [] + ] if matrix_coturn_enabled else []) }} matrix_conduit_turn_secret: "{{ matrix_coturn_turn_static_auth_secret if matrix_coturn_enabled else '' }}" From 1468c080656b0fa26dc3d6f070a65f3c8224f718 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 26 Jan 2024 16:04:55 +0200 Subject: [PATCH 12/46] Wire matrix_server_fqn_matrix_federation to matrix_SERVICE_*_public_federation_api_traefik_hostname for ease of use --- group_vars/matrix_servers | 3 +++ .../custom/matrix_playbook_migration/tasks/validate_config.yml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index d32f73cf1..4075308d1 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -3921,6 +3921,7 @@ matrix_synapse_container_labels_public_client_root_redirection_url: "{{ (('https matrix_synapse_container_labels_public_client_synapse_admin_api_enabled: "{{ matrix_synapse_admin_enabled }}" +matrix_synapse_container_labels_public_federation_api_traefik_hostname: "{{ matrix_server_fqn_matrix_federation }}" matrix_synapse_container_labels_public_federation_api_traefik_entrypoints: "{{ matrix_federation_traefik_entrypoint_name }}" matrix_synapse_container_labels_public_federation_api_traefik_tls: "{{ matrix_federation_traefik_entrypoint_tls }}" matrix_synapse_container_labels_public_federation_api_traefik_tls_certResolver: "{{ matrix_synapse_container_labels_traefik_tls_certResolver }}" # noqa var-naming @@ -4605,6 +4606,7 @@ matrix_dendrite_container_labels_public_client_synapse_admin_api_enabled: "{{ ma matrix_dendrite_container_labels_public_client_root_redirection_enabled: "{{ matrix_dendrite_container_labels_public_client_root_redirection_url != '' }}" matrix_dendrite_container_labels_public_client_root_redirection_url: "{{ (('https://' if matrix_playbook_ssl_enabled else 'http://') + matrix_server_fqn_element) if matrix_client_element_enabled else '' }}" +matrix_dendrite_container_labels_public_federation_api_traefik_hostname: "{{ matrix_server_fqn_matrix_federation }}" matrix_dendrite_container_labels_public_federation_api_traefik_entrypoints: "{{ matrix_federation_traefik_entrypoint_name }}" matrix_dendrite_container_labels_public_federation_api_traefik_tls: "{{ matrix_federation_traefik_entrypoint_tls }}" @@ -4695,6 +4697,7 @@ matrix_conduit_container_labels_traefik_tls_certResolver: "{{ devture_traefik_ce matrix_conduit_container_labels_public_client_root_redirection_enabled: "{{ matrix_conduit_container_labels_public_client_root_redirection_url != '' }}" matrix_conduit_container_labels_public_client_root_redirection_url: "{{ (('https://' if matrix_playbook_ssl_enabled else 'http://') + matrix_server_fqn_element) if matrix_client_element_enabled else '' }}" +matrix_conduit_container_labels_public_federation_api_traefik_hostname: "{{ matrix_server_fqn_matrix_federation }}" matrix_conduit_container_labels_public_federation_api_traefik_entrypoints: "{{ matrix_federation_traefik_entrypoint_name }}" matrix_conduit_container_labels_public_federation_api_traefik_tls: "{{ matrix_federation_traefik_entrypoint_tls }}" diff --git a/roles/custom/matrix_playbook_migration/tasks/validate_config.yml b/roles/custom/matrix_playbook_migration/tasks/validate_config.yml index c46fb7fc4..d8958fdd1 100644 --- a/roles/custom/matrix_playbook_migration/tasks/validate_config.yml +++ b/roles/custom/matrix_playbook_migration/tasks/validate_config.yml @@ -301,7 +301,7 @@ - {'old': 'matrix_nginx_proxy_proxy_synapse_additional_server_configuration_blocks', 'new': ''} - {'old': 'matrix_nginx_proxy_proxy_matrix_additional_server_configuration_blocks', 'new': ''} - {'old': 'matrix_nginx_proxy_access_log_enabled', 'new': 'devture_traefik_config_accessLog_enabled'} - - {'old': 'matrix_nginx_proxy_container_labels_traefik_proxy_matrix_federation_hostname', 'new': ''} + - {'old': 'matrix_nginx_proxy_container_labels_traefik_proxy_matrix_federation_hostname', 'new': 'matrix_server_fqn_matrix_federation'} - {'old': 'etherpad_nginx_proxy_dimension_integration_enabled', 'new': ''} - {'old': 'etherpad_nginx_proxy_dimension_integration_path_prefix', 'new': ''} - {'old': 'matrix_prometheus_services_proxy_connect_prometheus_node_exporter_metrics_proxying_enabled', 'new': ''} From 4a2ad1583e5b8d4026f1256adc9ddf00dedad7a1 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 26 Jan 2024 14:05:29 +0000 Subject: [PATCH 13/46] Update matrixdotorg/dendrite-monolith Docker tag to v0.13.6 --- roles/custom/matrix-dendrite/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/custom/matrix-dendrite/defaults/main.yml b/roles/custom/matrix-dendrite/defaults/main.yml index a3854315f..0e2c7223b 100644 --- a/roles/custom/matrix-dendrite/defaults/main.yml +++ b/roles/custom/matrix-dendrite/defaults/main.yml @@ -13,7 +13,7 @@ matrix_dendrite_docker_image_path: "matrixdotorg/dendrite-monolith" matrix_dendrite_docker_image: "{{ matrix_dendrite_docker_image_name_prefix }}{{ matrix_dendrite_docker_image_path }}:{{ matrix_dendrite_docker_image_tag }}" matrix_dendrite_docker_image_name_prefix: "{{ 'localhost/' if matrix_dendrite_container_image_self_build else matrix_container_global_registry_prefix }}" # renovate: datasource=docker depName=matrixdotorg/dendrite-monolith -matrix_dendrite_docker_image_tag: "v0.13.5" +matrix_dendrite_docker_image_tag: "v0.13.6" matrix_dendrite_docker_image_force_pull: "{{ matrix_dendrite_docker_image.endswith(':latest') }}" matrix_dendrite_base_path: "{{ matrix_base_data_path }}/dendrite" From 5ca527066d1b8ff34d2d84b3dfbc9d48c5afad9a Mon Sep 17 00:00:00 2001 From: Tiago Carrondo <2323546+tcarrondo@users.noreply.github.com> Date: Sat, 27 Jan 2024 16:03:37 +0000 Subject: [PATCH 14/46] Fix s3-storage migrate and shell (#3136) * Fix s3-storage migrate and shell: container needs attachment to postgres network also * Connect to s3-storage-provider migrate to multiple networks in multiple steps Multiple `--network` calls lead to: > docker: Error response from daemon: Container cannot be connected to network endpoints: NETWORK_1 NETWORK_2. * Connect to s3-storage-provider shell to multiple networks in multiple steps --------- Co-authored-by: Slavi Pantaleev --- .../ext/s3-storage-provider/bin/migrate.j2 | 14 ++++++++++++-- .../synapse/ext/s3-storage-provider/bin/shell.j2 | 15 +++++++++++++-- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/bin/migrate.j2 b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/bin/migrate.j2 index 5c3a534b5..8d8c4d0d7 100644 --- a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/bin/migrate.j2 +++ b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/bin/migrate.j2 @@ -1,7 +1,9 @@ #jinja2: lstrip_blocks: "True" #!/bin/bash +set -euo pipefail -{{ devture_systemd_docker_base_host_command_docker }} run \ +container_id=$(\ + {{ devture_systemd_docker_base_host_command_docker }} create \ --rm \ --env-file={{ matrix_synapse_ext_s3_storage_provider_base_path }}/env \ --mount type=bind,src={{ matrix_synapse_storage_path }},dst=/matrix-media-store-parent,bind-propagation=slave \ @@ -10,4 +12,12 @@ --network={{ matrix_synapse_container_network }} \ --entrypoint=/bin/bash \ {{ matrix_synapse_docker_image_final }} \ - -c 's3_media_upload update-db $UPDATE_DB_DURATION && s3_media_upload --no-progress check-deleted $MEDIA_PATH && s3_media_upload --no-progress upload $MEDIA_PATH $BUCKET --delete --storage-class $STORAGE_CLASS --endpoint-url $ENDPOINT {% if matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_enabled %}--sse-customer-algo $SSE_CUSTOMER_ALGO --sse-customer-key $SSE_CUSTOMER_KEY{% endif %}' + -c 's3_media_upload update-db $UPDATE_DB_DURATION && s3_media_upload --no-progress check-deleted $MEDIA_PATH && s3_media_upload --no-progress upload $MEDIA_PATH $BUCKET --delete --storage-class $STORAGE_CLASS --endpoint-url $ENDPOINT {% if matrix_synapse_ext_synapse_s3_storage_provider_config_sse_customer_enabled %}--sse-customer-algo $SSE_CUSTOMER_ALGO --sse-customer-key $SSE_CUSTOMER_KEY{% endif %}' \ +) + +{# We need to connect to the Postgres network, which should be in this list. #} +{% for network in matrix_synapse_container_additional_networks %} +{{ devture_systemd_docker_base_host_command_docker }} network connect {{ network }} $container_id +{% endfor %} + +{{ devture_systemd_docker_base_host_command_docker }} start --attach $container_id diff --git a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/bin/shell.j2 b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/bin/shell.j2 index 6f3804cc4..239554a1c 100644 --- a/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/bin/shell.j2 +++ b/roles/custom/matrix-synapse/templates/synapse/ext/s3-storage-provider/bin/shell.j2 @@ -1,13 +1,24 @@ #jinja2: lstrip_blocks: "True" #!/bin/bash +set -euo pipefail -{{ devture_systemd_docker_base_host_command_docker }} run \ +container_id=$(\ + {{ devture_systemd_docker_base_host_command_docker }} create \ -it \ --rm \ + --name=matrix-synapse-s3-storage-provider-shell \ --env-file={{ matrix_synapse_ext_s3_storage_provider_base_path }}/env \ --mount type=bind,src={{ matrix_synapse_storage_path }},dst=/matrix-media-store-parent,bind-propagation=slave \ --mount type=bind,src={{ matrix_synapse_ext_s3_storage_provider_data_path }},dst=/data \ --workdir=/data \ --network={{ matrix_synapse_container_network }} \ --entrypoint=/bin/bash \ - {{ matrix_synapse_docker_image_final }} + {{ matrix_synapse_docker_image_final }} \ +) + +{# We need to connect to the Postgres network, which should be in this list. #} +{% for network in matrix_synapse_container_additional_networks %} +{{ devture_systemd_docker_base_host_command_docker }} network connect {{ network }} $container_id +{% endfor %} + +{{ devture_systemd_docker_base_host_command_docker }} start --attach -i $container_id From 45e46f82bbcd24c221044d2cf655f11e919f87ce Mon Sep 17 00:00:00 2001 From: Wave <86343914+Wave6677@users.noreply.github.com> Date: Sun, 28 Jan 2024 04:56:43 +0000 Subject: [PATCH 15/46] Fix typo in configuring-playbook-bot-matrix-registration-bot.md (#3137) * Fix typo in configuring-playbook-bot-matrix-registration-bot.md changed "loook like" to "Tokens look like" * Minor rewording --------- Co-authored-by: Slavi Pantaleev --- docs/configuring-playbook-bot-matrix-registration-bot.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/configuring-playbook-bot-matrix-registration-bot.md b/docs/configuring-playbook-bot-matrix-registration-bot.md index 30a9a1a78..703714836 100644 --- a/docs/configuring-playbook-bot-matrix-registration-bot.md +++ b/docs/configuring-playbook-bot-matrix-registration-bot.md @@ -3,8 +3,7 @@ The playbook can install and configure [matrix-registration-bot](https://github.com/moan0s/matrix-registration-bot) for you. The bot allows you to easily **create and manage registration tokens** aka. invitation codes. -It can be used for an invitation-based server, -where you invite someone by sending them a registration token (loook like this: `rbalQ0zkaDSRQCOp`). They can register as normal but have to provide a valid registration token in a final step of the registration. +It can be used for an invitation-based server, where you invite someone by sending them a registration token (tokens look like this: `rbalQ0zkaDSRQCOp`). They can register as per normal but have to provide a valid registration token in the final step of the registration process. See the project's [documentation](https://github.com/moan0s/matrix-registration-bot#supported-commands) to learn what it does and why it might be useful to you. From 4bf4fc4f621cf4b99a3570838212e1b48f41b839 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 30 Jan 2024 20:31:24 +0200 Subject: [PATCH 16/46] Upgrade Prometheus (v2.49.1-1 -> v2.49.1-2) --- requirements.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.yml b/requirements.yml index d5067141f..da1d49f8c 100644 --- a/requirements.yml +++ b/requirements.yml @@ -46,7 +46,7 @@ version: 7eadc992ca952fc29bf3fab5aa6335fa82ff01e5 name: postgres_backup - src: git+https://github.com/mother-of-all-self-hosting/ansible-role-prometheus.git - version: v2.49.1-1 + version: v2.49.1-2 name: prometheus - src: git+https://github.com/mother-of-all-self-hosting/ansible-role-prometheus-node-exporter.git version: v1.7.0-3 From 2ba4b94b998d0c49137f77a011c7dd9ad4c9923b Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 30 Jan 2024 20:31:47 +0200 Subject: [PATCH 17/46] Use prometheus_container_additional_networks_auto, instead of prometheus_container_additional_networks --- group_vars/matrix_servers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 4075308d1..115aea49d 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -4279,7 +4279,7 @@ prometheus_gid: "{{ matrix_user_gid }}" prometheus_container_network: "{{ matrix_monitoring_container_network }}" -prometheus_container_additional_networks: | +prometheus_container_additional_networks_auto: | {{ ( ([prometheus_node_exporter_container_network] if prometheus_self_node_scraper_enabled and prometheus_node_exporter_container_network != prometheus_container_network else []) From b167f483965a3789fab5e72b5208a7913f0db89c Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 30 Jan 2024 18:32:14 +0000 Subject: [PATCH 18/46] Update ghcr.io/element-hq/synapse Docker tag to v1.100.0 --- roles/custom/matrix-synapse/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/custom/matrix-synapse/defaults/main.yml b/roles/custom/matrix-synapse/defaults/main.yml index d35fe2e5a..8e3169b7b 100644 --- a/roles/custom/matrix-synapse/defaults/main.yml +++ b/roles/custom/matrix-synapse/defaults/main.yml @@ -16,7 +16,7 @@ matrix_synapse_enabled: true matrix_synapse_github_org_and_repo: element-hq/synapse # renovate: datasource=docker depName=ghcr.io/element-hq/synapse -matrix_synapse_version: v1.99.0 +matrix_synapse_version: v1.100.0 matrix_synapse_username: '' matrix_synapse_uid: '' From a91f14ee0d871b5bdc01702f64ac7bc38afdec7b Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 30 Jan 2024 21:08:51 +0200 Subject: [PATCH 19/46] Upgrade Grafana (v10.3.1-0 -> v10.3.1-1) --- requirements.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.yml b/requirements.yml index da1d49f8c..87c0533c0 100644 --- a/requirements.yml +++ b/requirements.yml @@ -22,7 +22,7 @@ version: v4.97-r0-0-1 name: exim_relay - src: git+https://gitlab.com/etke.cc/roles/grafana.git - version: v10.3.1-0 + version: v10.3.1-1 name: grafana - src: git+https://github.com/mother-of-all-self-hosting/ansible-role-jitsi.git version: v9111-1 From 674658039e69ba64358660d48530d4dad7de752e Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 30 Jan 2024 21:09:33 +0200 Subject: [PATCH 20/46] Switch from grafana_container_additional_networks to grafana_container_additional_networks_auto --- group_vars/matrix_servers | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 115aea49d..c594ee26e 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -4381,10 +4381,10 @@ grafana_base_path: "{{ matrix_base_data_path }}/grafana" grafana_container_network: "{{ matrix_monitoring_container_network }}" -grafana_container_additional_networks: | +grafana_container_additional_networks_auto: | {{ ( - ([matrix_playbook_reverse_proxyable_services_additional_network] if matrix_playbook_reverse_proxyable_services_additional_network else []) + ([matrix_playbook_reverse_proxyable_services_additional_network] if matrix_playbook_reverse_proxyable_services_additional_network and grafana_container_labels_traefik_enabled else []) + ([prometheus_container_network] if prometheus_enabled and prometheus_container_network != grafana_container_network else []) ) | unique From 672b42848fc2059af4f70317b8094a07084b4d0d Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 30 Jan 2024 21:18:31 +0200 Subject: [PATCH 21/46] Upgrade Grafana (v10.3.1-1 -> v10.3.1-2) --- requirements.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.yml b/requirements.yml index 87c0533c0..649ff5463 100644 --- a/requirements.yml +++ b/requirements.yml @@ -22,7 +22,7 @@ version: v4.97-r0-0-1 name: exim_relay - src: git+https://gitlab.com/etke.cc/roles/grafana.git - version: v10.3.1-1 + version: v10.3.1-2 name: grafana - src: git+https://github.com/mother-of-all-self-hosting/ansible-role-jitsi.git version: v9111-1 From 8c69ff8d03ab1db45765745e6cffd77110daa28f Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 30 Jan 2024 21:37:18 +0200 Subject: [PATCH 22/46] Upgrade Postgres (v16.1-5 -> v16.1-6) --- requirements.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.yml b/requirements.yml index 649ff5463..6fffac1f4 100644 --- a/requirements.yml +++ b/requirements.yml @@ -40,7 +40,7 @@ version: ff2fd42e1c1a9e28e3312bbd725395f9c2fc7f16 name: playbook_state_preserver - src: git+https://github.com/devture/com.devture.ansible.role.postgres.git - version: v16.1-5 + version: v16.1-6 name: postgres - src: git+https://github.com/devture/com.devture.ansible.role.postgres_backup.git version: 7eadc992ca952fc29bf3fab5aa6335fa82ff01e5 From 578d00a54a75267e44098725c284217ecf577b7b Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 31 Jan 2024 12:13:20 +0200 Subject: [PATCH 23/46] Default to root-path-redirection on the base domain if index.html creation is disabled This is a break in backward-compatibility for people disabling `index.html` creation via the playbook but are managing their static website files in another way (AUX role, etc). --- group_vars/matrix_servers | 4 ++++ roles/custom/matrix-static-files/defaults/main.yml | 6 ++++++ .../matrix-static-files/tasks/validate_config.yml | 2 ++ roles/custom/matrix-static-files/templates/labels.j2 | 12 ++++++++++++ 4 files changed, 24 insertions(+) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index c594ee26e..f735cd1da 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -4848,6 +4848,10 @@ matrix_static_files_container_labels_well_known_matrix_endpoint_traefik_hostname # but we pass the hostname, so that enabling it is easy. matrix_static_files_container_labels_base_domain_traefik_hostname: "{{ matrix_domain }}" +# If we're not serving a static webpage, serve a redirect instead of a 404. +matrix_static_files_container_labels_base_domain_root_path_redirection_enabled: "{{ not matrix_static_files_file_index_html_enabled }}" +matrix_static_files_container_labels_base_domain_root_path_redirection_url: "https://{{ matrix_server_fqn_matrix }}" + matrix_static_files_file_matrix_client_property_io_element_jitsi_preferred_domain: "{{ matrix_server_fqn_jitsi if jitsi_enabled else '' }}" matrix_static_files_file_matrix_client_property_org_matrix_msc3575_proxy_url: "{{ matrix_homeserver_sliding_sync_url }}" diff --git a/roles/custom/matrix-static-files/defaults/main.yml b/roles/custom/matrix-static-files/defaults/main.yml index 16f0c7d58..045004f46 100644 --- a/roles/custom/matrix-static-files/defaults/main.yml +++ b/roles/custom/matrix-static-files/defaults/main.yml @@ -81,6 +81,9 @@ matrix_static_files_container_labels_base_domain_traefik_priority: 0 matrix_static_files_container_labels_base_domain_traefik_entrypoints: "{{ matrix_static_files_container_labels_traefik_entrypoints }}" matrix_static_files_container_labels_base_domain_traefik_tls: "{{ matrix_static_files_container_labels_base_domain_traefik_entrypoints != 'web' }}" matrix_static_files_container_labels_base_domain_traefik_tls_certResolver: "{{ matrix_static_files_container_labels_traefik_tls_certResolver }}" # noqa var-naming +# Controls whether the root path (/) at the base domain would respond with a redirect to some URL. +matrix_static_files_container_labels_base_domain_root_path_redirection_enabled: false +matrix_static_files_container_labels_base_domain_root_path_redirection_url: "" # matrix_static_files_container_labels_additional_labels contains a multiline string with additional labels to add to the container label file. # See `../templates/labels.j2` for details. @@ -343,6 +346,9 @@ matrix_static_files_file_matrix_support_configuration: "{{ matrix_static_files_f # # You can also use the auxiliary role (https://github.com/mother-of-all-self-hosting/ansible-role-aux) to create files in # the public directory (matrix_static_files_public_path) by yourself. +# If you're disabling this but are preparing some HTML files by yourself, +# consider explicitly disabling `matrix_static_files_container_labels_base_domain_root_path_redirection_enabled` as well. +# The default behavior when this is disabled is to enable root-path-redirection. # # Because you may wish to manage these static files yourself, disabling this variable will intentionally not delete an already existing `index.html` file. matrix_static_files_file_index_html_enabled: "{{ matrix_static_files_container_labels_base_domain_enabled }}" diff --git a/roles/custom/matrix-static-files/tasks/validate_config.yml b/roles/custom/matrix-static-files/tasks/validate_config.yml index 6d4cc7978..ca079d029 100644 --- a/roles/custom/matrix-static-files/tasks/validate_config.yml +++ b/roles/custom/matrix-static-files/tasks/validate_config.yml @@ -11,3 +11,5 @@ - {'name': 'matrix_static_files_container_labels_base_domain_traefik_hostname', when: "{{ matrix_static_files_container_labels_base_domain_enabled }}"} - {'name': 'matrix_static_files_container_labels_base_domain_traefik_path_prefix', when: "{{ matrix_static_files_container_labels_base_domain_enabled }}"} + + - {'name': 'matrix_static_files_container_labels_base_domain_root_path_redirection_url', when: "{{ matrix_static_files_container_labels_base_domain_enabled and matrix_static_files_container_labels_base_domain_root_path_redirection_enabled }}"} diff --git a/roles/custom/matrix-static-files/templates/labels.j2 b/roles/custom/matrix-static-files/templates/labels.j2 index e7776cb82..46fc49490 100644 --- a/roles/custom/matrix-static-files/templates/labels.j2 +++ b/roles/custom/matrix-static-files/templates/labels.j2 @@ -56,12 +56,24 @@ traefik.http.routers.{{ matrix_static_files_identifier }}-well-known.tls.certRes # # ############################################################ +{% set middlewares = [] %} + +{% if matrix_static_files_container_labels_base_domain_root_path_redirection_enabled %} +traefik.http.middlewares.{{ matrix_static_files_identifier }}-root-path-redirect.redirectregex.regex=^https://{{ matrix_static_files_container_labels_base_domain_traefik_hostname }}{{ matrix_static_files_container_labels_base_domain_traefik_path_prefix }}$ +traefik.http.middlewares.{{ matrix_static_files_identifier }}-root-path-redirect.redirectregex.replacement={{ matrix_static_files_container_labels_base_domain_root_path_redirection_url }} +{% set middlewares = middlewares + [matrix_static_files_identifier + '-root-path-redirect'] %} +{% endif %} + traefik.http.routers.{{ matrix_static_files_identifier }}-base-domain.rule={{ matrix_static_files_container_labels_base_domain_traefik_rule }} {% if matrix_static_files_container_labels_base_domain_traefik_priority | int > 0 %} traefik.http.routers.{{ matrix_static_files_identifier }}-base-domain.priority={{ matrix_static_files_container_labels_base_domain_traefik_priority }} {% endif %} +{% if middlewares | length > 0 %} +traefik.http.routers.{{ matrix_static_files_identifier }}-base-domain.middlewares={{ middlewares | join(',') }} +{% endif %} + traefik.http.routers.{{ matrix_static_files_identifier }}-base-domain.service={{ matrix_static_files_identifier }} traefik.http.routers.{{ matrix_static_files_identifier }}-base-domain.entrypoints={{ matrix_static_files_container_labels_base_domain_traefik_entrypoints }} traefik.http.routers.{{ matrix_static_files_identifier }}-base-domain.tls={{ matrix_static_files_container_labels_base_domain_traefik_tls | to_json }} From 502db3583133cf2de80c634ba74a8e0553df2243 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 31 Jan 2024 20:11:11 +0000 Subject: [PATCH 24/46] Update vectorim/element-web Docker tag to v1.11.57 --- roles/custom/matrix-client-element/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/custom/matrix-client-element/defaults/main.yml b/roles/custom/matrix-client-element/defaults/main.yml index 305deae19..934bf885a 100644 --- a/roles/custom/matrix-client-element/defaults/main.yml +++ b/roles/custom/matrix-client-element/defaults/main.yml @@ -11,7 +11,7 @@ matrix_client_element_container_image_self_build_repo: "https://github.com/eleme matrix_client_element_container_image_self_build_low_memory_system_patch_enabled: "{{ ansible_memtotal_mb < 4096 }}" # renovate: datasource=docker depName=vectorim/element-web -matrix_client_element_version: v1.11.55 +matrix_client_element_version: v1.11.57 matrix_client_element_docker_image: "{{ matrix_client_element_docker_image_name_prefix }}vectorim/element-web:{{ matrix_client_element_version }}" matrix_client_element_docker_image_name_prefix: "{{ 'localhost/' if matrix_client_element_container_image_self_build else matrix_container_global_registry_prefix }}" From 2e08d65e7afd6cae0da8ccb9dd608635520200a6 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 1 Feb 2024 15:56:20 +0200 Subject: [PATCH 25/46] Upgrade Jitsi (v9111-1 -> v9220-0) --- requirements.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.yml b/requirements.yml index 6fffac1f4..5e8fd6783 100644 --- a/requirements.yml +++ b/requirements.yml @@ -25,7 +25,7 @@ version: v10.3.1-2 name: grafana - src: git+https://github.com/mother-of-all-self-hosting/ansible-role-jitsi.git - version: v9111-1 + version: v9220-0 name: jitsi - src: git+https://gitlab.com/etke.cc/roles/ntfy.git version: v2.8.0-1 From 533f42fe08c67fff7ad2396b9b29d34f887420e1 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 1 Feb 2024 17:11:26 +0000 Subject: [PATCH 26/46] Update dependency docker to v7.1.0 --- requirements.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.yml b/requirements.yml index 5e8fd6783..3c51ab86e 100644 --- a/requirements.yml +++ b/requirements.yml @@ -10,7 +10,7 @@ version: v0.1.1-3 name: container_socket_proxy - src: git+https://github.com/geerlingguy/ansible-role-docker - version: 7.0.2 + version: 7.1.0 name: docker - src: git+https://github.com/devture/com.devture.ansible.role.docker_sdk_for_python.git version: 129c8590e106b83e6f4c259649a613c6279e937a From 2c06aa1d04256b9764e4f0f45f625f82867889ff Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 1 Feb 2024 20:11:51 +0000 Subject: [PATCH 27/46] Update gnuxie/draupnir Docker tag to v1.86.1 --- roles/custom/matrix-bot-draupnir/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/custom/matrix-bot-draupnir/defaults/main.yml b/roles/custom/matrix-bot-draupnir/defaults/main.yml index e8ab01146..3a4bdc250 100644 --- a/roles/custom/matrix-bot-draupnir/defaults/main.yml +++ b/roles/custom/matrix-bot-draupnir/defaults/main.yml @@ -5,7 +5,7 @@ matrix_bot_draupnir_enabled: true # renovate: datasource=docker depName=gnuxie/draupnir -matrix_bot_draupnir_version: "v1.86.0" +matrix_bot_draupnir_version: "v1.86.1" matrix_bot_draupnir_container_image_self_build: false matrix_bot_draupnir_container_image_self_build_repo: "https://github.com/the-draupnir-project/Draupnir.git" From 6e2bcc79324e02094449154b707a26e543a4ef9c Mon Sep 17 00:00:00 2001 From: Ed Geraghty Date: Fri, 2 Feb 2024 20:09:21 +0000 Subject: [PATCH 28/46] Add upstream `proxy_protocol` instructions to traefik (#3150) * Add upstream `proxy_protocol` instructions to traefik * Fix YAML indentation to use spaces --------- Co-authored-by: Slavi Pantaleev --- docs/configuring-playbook-traefik.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/configuring-playbook-traefik.md b/docs/configuring-playbook-traefik.md index 5f4753891..dc4d88cf6 100644 --- a/docs/configuring-playbook-traefik.md +++ b/docs/configuring-playbook-traefik.md @@ -137,3 +137,25 @@ Changing the `url` to one with an `http://` prefix would allow to connect to the With these changes, all TCP traffic will be reverse-proxied to the target system. **WARNING**: This configuration might lead to problems or need additional steps when a [certbot](https://certbot.eff.org/) behind Traefik also tries to manage [Let's Encrypt](https://letsencrypt.org/) certificates, as Traefik captures all traffic to ```PathPrefix(`/.well-known/acme-challenge/`)```. + + +## Traefik behind a `proxy_protocol` reverse-proxy + +If you run a reverse-proxy which speaks `proxy_protocol`, add the following to your configuration file: + +```yaml +devture_traefik_configuration_extension_yaml: | + entryPoints: + web-secure: + proxyProtocol: + trustedIPs: + - "127.0.0.1/32" + - "/32" + - "/128" + matrix-federation: + proxyProtocol: + trustedIPs: + - "127.0.0.1/32" + - "/32" + - "/128" +``` From 76a265f9a1b9024810f2e67cdff09786a6dfc01a Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sat, 3 Feb 2024 08:05:52 +0200 Subject: [PATCH 29/46] Document new base-domain root-path redirection behavior --- CHANGELOG.md | 11 +++++++++++ docs/configuring-playbook-base-domain-serving.md | 4 ++++ 2 files changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e8ece22b..7a9473760 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +# 2024-01-31 + +## (Backward-compatibility break) Minor changes necessary for some people serving a static website at the base domain + +This only affects people who are [Serving a static website at the base domain](./docs/configuring-playbook-base-domain-serving.md#serving-a-static-website-at-the-base-domain), but not managing its `index.html` through the playbook. + +That is, for people who have `matrix_static_files_file_index_html_enabled: false` in their `vars.yml` configuration, the playbook has a new default behavior. Since the playbook is not managing the `index.html` file, it will default to a more sensible way of handling the base domain - redirecting `https://DOMAIN/` to `https://matrix.DOMAIN/`, instead of serving a 404 page. + +If you are managing your static website by yourself (by dropping files into `/matrix/static-files/public` somehow), then you probably don't wish for such redirection to happen. You can disable it by adding `matrix_static_files_container_labels_base_domain_enabled: false` to your `vars.yml` configuration file. + + # 2024-01-20 ## Support for more efficient (specialized) Synapse workers diff --git a/docs/configuring-playbook-base-domain-serving.md b/docs/configuring-playbook-base-domain-serving.md index f1cf992d1..d070abc59 100644 --- a/docs/configuring-playbook-base-domain-serving.md +++ b/docs/configuring-playbook-base-domain-serving.md @@ -42,6 +42,10 @@ matrix_static_files_container_labels_base_domain_enabled: true # Prevent the default index.html file from being installed matrix_static_files_file_index_html_enabled: false + +# Disable the automatic redirectin of `https://DOMAIN/` to `https://matrix.DOMAIN/`. +# This gets automatically enabled when you disable `matrix_static_files_file_index_html_enabled`, as we're doing above. +matrix_static_files_container_labels_base_domain_enabled: false ``` With this configuration, Ansible will no longer mess around with the `/matrix/static-files/public/index.html` file. From 1160e3212621d2bba1c67ef32780a8c0f6252236 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sat, 3 Feb 2024 18:48:24 +0200 Subject: [PATCH 30/46] Fix incorrect variable name for base-domain root-path redirection Fixes a typo in 76a265f9a1b902481 Fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/3153 --- CHANGELOG.md | 2 +- docs/configuring-playbook-base-domain-serving.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a9473760..d96326a17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ This only affects people who are [Serving a static website at the base domain](. That is, for people who have `matrix_static_files_file_index_html_enabled: false` in their `vars.yml` configuration, the playbook has a new default behavior. Since the playbook is not managing the `index.html` file, it will default to a more sensible way of handling the base domain - redirecting `https://DOMAIN/` to `https://matrix.DOMAIN/`, instead of serving a 404 page. -If you are managing your static website by yourself (by dropping files into `/matrix/static-files/public` somehow), then you probably don't wish for such redirection to happen. You can disable it by adding `matrix_static_files_container_labels_base_domain_enabled: false` to your `vars.yml` configuration file. +If you are managing your static website by yourself (by dropping files into `/matrix/static-files/public` somehow), then you probably don't wish for such redirection to happen. You can disable it by adding `matrix_static_files_container_labels_base_domain_root_path_redirection_enabled: false` to your `vars.yml` configuration file. # 2024-01-20 diff --git a/docs/configuring-playbook-base-domain-serving.md b/docs/configuring-playbook-base-domain-serving.md index d070abc59..95f7547a3 100644 --- a/docs/configuring-playbook-base-domain-serving.md +++ b/docs/configuring-playbook-base-domain-serving.md @@ -45,7 +45,7 @@ matrix_static_files_file_index_html_enabled: false # Disable the automatic redirectin of `https://DOMAIN/` to `https://matrix.DOMAIN/`. # This gets automatically enabled when you disable `matrix_static_files_file_index_html_enabled`, as we're doing above. -matrix_static_files_container_labels_base_domain_enabled: false +matrix_static_files_container_labels_base_domain_root_path_redirection_enabled: false ``` With this configuration, Ansible will no longer mess around with the `/matrix/static-files/public/index.html` file. From 929aee302240e56e0ea37c8f94cf76d8447362d2 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sat, 3 Feb 2024 18:52:26 +0200 Subject: [PATCH 31/46] Fix incorrect prefix for Synapse worker metrics Fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/3154 --- roles/custom/matrix-synapse/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/custom/matrix-synapse/defaults/main.yml b/roles/custom/matrix-synapse/defaults/main.yml index 8e3169b7b..cbb8d2196 100644 --- a/roles/custom/matrix-synapse/defaults/main.yml +++ b/roles/custom/matrix-synapse/defaults/main.yml @@ -752,7 +752,7 @@ matrix_synapse_worker_container_labels_traefik_hostname: "{{ matrix_synapse_cont # Controls whether labels will be added that expose metrics (see `matrix_synapse_metrics_proxying_enabled`) matrix_synapse_worker_container_labels_public_metrics_enabled: "{{ matrix_synapse_metrics_enabled and matrix_synapse_metrics_proxying_enabled }}" # The `__WORKER_ID__` placeholder will be replaced with the actual worker id during label-file generation (see `../templates/worker-labels.j2`). -matrix_synapse_worker_container_labels_public_metrics_traefik_path: "{{ matrix_synapse_metrics_proxying_path_prefix }}/__WORKER_ID__" +matrix_synapse_worker_container_labels_public_metrics_traefik_path: "{{ matrix_synapse_metrics_proxying_path_prefix }}/worker/__WORKER_ID__" matrix_synapse_worker_container_labels_public_metrics_traefik_rule: "Host(`{{ matrix_synapse_metrics_proxying_hostname }}`) && Path(`{{ matrix_synapse_worker_container_labels_public_metrics_traefik_path }}`)" matrix_synapse_worker_container_labels_public_metrics_traefik_priority: 0 matrix_synapse_worker_container_labels_public_metrics_traefik_entrypoints: "{{ matrix_synapse_container_labels_traefik_entrypoints }}" From e01aa667e7dd0a34f1f5af1572d73db7b602a9fd Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sat, 3 Feb 2024 18:53:14 +0200 Subject: [PATCH 32/46] Fix some comments in worker-labels for Synapse Related to 929aee302240e56e0e and https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/3154 --- .../custom/matrix-synapse/templates/synapse/worker-labels.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/custom/matrix-synapse/templates/synapse/worker-labels.j2 b/roles/custom/matrix-synapse/templates/synapse/worker-labels.j2 index 35ef12da1..9cf4feaed 100644 --- a/roles/custom/matrix-synapse/templates/synapse/worker-labels.j2 +++ b/roles/custom/matrix-synapse/templates/synapse/worker-labels.j2 @@ -8,7 +8,7 @@ traefik.docker.network={{ matrix_synapse_worker_container_labels_traefik_docker_ traefik.http.services.{{ matrix_synapse_worker_container_name }}-metrics.loadbalancer.server.port={{ matrix_synapse_worker_details.metrics_port }} {# - Metrics (e.g. /metrics/synapse/__WORKER_ID__) + Metrics (e.g. /metrics/synapse/worker/__WORKER_ID__) #} {% if matrix_synapse_worker_container_labels_public_metrics_enabled %} @@ -42,7 +42,7 @@ traefik.http.routers.{{ matrix_synapse_worker_container_name }}-metrics.tls.cert {% endif %} {# - /Metrics (e.g. /metrics/synapse/__WORKER_ID__) + /Metrics (e.g. /metrics/synapse/worker/__WORKER_ID__) #} From c68e9dc2eb5d5a66b5a43615c2787b4c69c834ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingy=20d=C3=B6t=20Net?= Date: Sun, 4 Feb 2024 09:31:32 -0800 Subject: [PATCH 33/46] Update configuring-playbook.md `mkdir` with multiple subdirs needs `-p` --- docs/configuring-playbook.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuring-playbook.md b/docs/configuring-playbook.md index 1d6227856..413db7b8d 100644 --- a/docs/configuring-playbook.md +++ b/docs/configuring-playbook.md @@ -8,7 +8,7 @@ To configure the playbook, you need to have done the following things: You can then follow these steps inside the playbook directory: -1. create a directory to hold your configuration (`mkdir inventory/host_vars/matrix.`) +1. create a directory to hold your configuration (`mkdir -p inventory/host_vars/matrix.`) 1. copy the sample configuration file (`cp examples/vars.yml inventory/host_vars/matrix./vars.yml`) From 8b027efb65e6f0febee39cadf87054ef508934ed Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 5 Feb 2024 18:39:36 +0200 Subject: [PATCH 34/46] Upgrade mautrix-signal (de8c8d97c23 -> 103666990f3) --- roles/custom/matrix-bridge-mautrix-signal/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/custom/matrix-bridge-mautrix-signal/defaults/main.yml b/roles/custom/matrix-bridge-mautrix-signal/defaults/main.yml index dd255070c..3a30d9d1c 100644 --- a/roles/custom/matrix-bridge-mautrix-signal/defaults/main.yml +++ b/roles/custom/matrix-bridge-mautrix-signal/defaults/main.yml @@ -9,7 +9,7 @@ matrix_mautrix_signal_container_image_self_build_repo: "https://mau.dev/mautrix/ matrix_mautrix_signal_container_image_self_build_branch: "{{ 'main' if matrix_mautrix_signal_version == 'latest' else matrix_mautrix_signal_version }}" # renovate: datasource=docker depName=dock.mau.dev/mautrix/signal -matrix_mautrix_signal_version: de8c8d97c23dc1982686a698162b45f1f112155b +matrix_mautrix_signal_version: 103666990f30a692c63dd84a499b0dd390cef8a4 # See: https://mau.dev/mautrix/signal/container_registry matrix_mautrix_signal_docker_image: "{{ matrix_mautrix_signal_docker_image_name_prefix }}mautrix/signal:{{ matrix_mautrix_signal_docker_image_tag }}" From 7f337fc9a67b47bd10cfe2122e552a48ddccd81c Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 5 Feb 2024 19:07:51 +0200 Subject: [PATCH 35/46] Upgrade synapse-admin (0.8.7 -> 0.9.0) --- roles/custom/matrix-synapse-admin/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/custom/matrix-synapse-admin/defaults/main.yml b/roles/custom/matrix-synapse-admin/defaults/main.yml index 1dee1d756..a23416134 100644 --- a/roles/custom/matrix-synapse-admin/defaults/main.yml +++ b/roles/custom/matrix-synapse-admin/defaults/main.yml @@ -12,7 +12,7 @@ matrix_synapse_admin_container_image_self_build: false matrix_synapse_admin_container_image_self_build_repo: "https://github.com/Awesome-Technologies/synapse-admin.git" # renovate: datasource=docker depName=awesometechnologies/synapse-admin -matrix_synapse_admin_version: 0.8.7 +matrix_synapse_admin_version: 0.9.0 matrix_synapse_admin_docker_image: "{{ matrix_synapse_admin_docker_image_name_prefix }}awesometechnologies/synapse-admin:{{ matrix_synapse_admin_version }}" matrix_synapse_admin_docker_image_name_prefix: "{{ 'localhost/' if matrix_synapse_admin_container_image_self_build else matrix_container_global_registry_prefix }}" matrix_synapse_admin_docker_image_force_pull: "{{ matrix_synapse_admin_docker_image.endswith(':latest') }}" From 2baea7ce7b8c896bf75e450a0459f36ff90fe356 Mon Sep 17 00:00:00 2001 From: Aine Date: Mon, 5 Feb 2024 22:07:45 +0200 Subject: [PATCH 36/46] buscarron v1.4.0 --- .../matrix-bot-buscarron/defaults/main.yml | 17 +---------------- .../matrix-bot-buscarron/templates/env.j2 | 4 +++- 2 files changed, 4 insertions(+), 17 deletions(-) diff --git a/roles/custom/matrix-bot-buscarron/defaults/main.yml b/roles/custom/matrix-bot-buscarron/defaults/main.yml index f06320ed4..116155cef 100644 --- a/roles/custom/matrix-bot-buscarron/defaults/main.yml +++ b/roles/custom/matrix-bot-buscarron/defaults/main.yml @@ -6,7 +6,7 @@ matrix_bot_buscarron_enabled: true # renovate: datasource=docker depName=registry.gitlab.com/etke.cc/buscarron -matrix_bot_buscarron_version: v1.3.1 +matrix_bot_buscarron_version: v1.4.0 # The hostname at which Buscarron is served. matrix_bot_buscarron_hostname: '' @@ -145,9 +145,6 @@ matrix_bot_buscarron_homeserver: "" # forms configuration matrix_bot_buscarron_forms: [] -# Disable encryption -matrix_bot_buscarron_noencryption: false - # Sentry DSN matrix_bot_buscarron_sentry: '' @@ -157,18 +154,6 @@ matrix_bot_buscarron_loglevel: INFO # list of spammers with wildcards support, eg: *@spam.com spam@*, spam@spam.com matrix_bot_buscarron_spamlist: [] -# spam hosts/domains. -# deprecated, use matrix_bot_buscarron_spamlist -matrix_bot_buscarron_spam_hosts: [] - -# spam email addresses -# deprecated, use matrix_bot_buscarron_spamlist -matrix_bot_buscarron_spam_emails: [] - -# spam email localparts -# deprecated, use matrix_bot_buscarron_spamlist -matrix_bot_buscarron_spam_localparts: [] - # Banlist size matrix_bot_buscarron_ban_size: 10000 diff --git a/roles/custom/matrix-bot-buscarron/templates/env.j2 b/roles/custom/matrix-bot-buscarron/templates/env.j2 index 1f71802a5..ea3f8c21f 100644 --- a/roles/custom/matrix-bot-buscarron/templates/env.j2 +++ b/roles/custom/matrix-bot-buscarron/templates/env.j2 @@ -16,17 +16,19 @@ BUSCARRON_PM_FROM={{ matrix_bot_buscarron_pm_from }} BUSCARRON_PM_REPLYTO={{ matrix_bot_buscarron_pm_replyto }} BUSCARRON_SMTP_FROM={{ matrix_bot_buscarron_smtp_from }} BUSCARRON_SMTP_VALIDATION={{ matrix_bot_buscarron_smtp_validation }} -BUSCARRON_NOENCRYPTION={{ matrix_bot_buscarron_noencryption }} BUSCARRON_METRICS_LOGIN={{ matrix_bot_buscarron_metrics_login }} BUSCARRON_METRICS_PASSWORD={{ matrix_bot_buscarron_metrics_password }} BUSCARRON_METRICS_IPS={{ matrix_bot_buscarron_metrics_ips|default([])|join(" ") }} {% set forms = [] %} {% for form in matrix_bot_buscarron_forms -%}{{- forms.append(form.name) -}} BUSCARRON_{{ form.name|upper }}_ROOM={{ form.room|default('') }} +BUSCARRON_{{ form.name|upper }}_TEXT={{ form.text|default('') }} BUSCARRON_{{ form.name|upper }}_REDIRECT={{ form.redirect|default('') }} +BUSCARRON_{{ form.name|upper }}_REDIRECT_REJECT={{ form.redirect_reject|default('') }} BUSCARRON_{{ form.name|upper }}_HASDOMAIN={{ form.hasdomain|default('') }} BUSCARRON_{{ form.name|upper }}_HASEMAIL={{ form.hasemail|default('') }} BUSCARRON_{{ form.name|upper }}_RATELIMIT={{ form.ratelimit|default('') }} +BUSCARRON_{{ form.name|upper }}_RATELIMIT_SHARED={{ form.ratelimit_shared|default(false) }} BUSCARRON_{{ form.name|upper }}_EXTENSIONS={{ form.extensions|default('')|join(' ') }} BUSCARRON_{{ form.name|upper }}_CONFIRMATION_SUBJECT={{ form.confirmation_subject|default('') }} BUSCARRON_{{ form.name|upper }}_CONFIRMATION_BODY={{ form.confirmation_body|default('') }} From a07345a42ee9081eaf2ff71cb0b3b7c7e4d73bdb Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Mon, 5 Feb 2024 22:12:24 +0200 Subject: [PATCH 37/46] update honoroit (v0.9.19 -> v0.9.20) **Warning**: [CI pipeline is in progress](https://gitlab.com/etke.cc/honoroit/-/pipelines/1165360868) changelog: * safer reaction forwarding * fix duplicated prefix and suffix on completed requests * add missing `!ho help` entries * add new `!ho count` command * count requests by homeserver and by MXID * add new `!ho config` command set - configure honoroit directly from the chat * mautrix-go 0.15.x+ migration * shared secret auth support * account data encyption support removed env vars (automatic migration): * HONOROIT_TEXT_* * HONOROIT_ALLOWEDUSERS * HONOROIT_IGNOREDROOMS * HONOROIT_IGNORENOTHREAD * HONOROIT_NOENCRYPTION --- roles/custom/matrix-bot-honoroit/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/custom/matrix-bot-honoroit/defaults/main.yml b/roles/custom/matrix-bot-honoroit/defaults/main.yml index a3430ab24..76ab590f6 100644 --- a/roles/custom/matrix-bot-honoroit/defaults/main.yml +++ b/roles/custom/matrix-bot-honoroit/defaults/main.yml @@ -21,7 +21,7 @@ matrix_bot_honoroit_docker_repo_version: "{{ matrix_bot_honoroit_version }}" matrix_bot_honoroit_docker_src_files_path: "{{ matrix_base_data_path }}/honoroit/docker-src" # renovate: datasource=docker depName=registry.gitlab.com/etke.cc/honoroit -matrix_bot_honoroit_version: v0.9.19 +matrix_bot_honoroit_version: v0.9.20 matrix_bot_honoroit_docker_image: "{{ matrix_bot_honoroit_docker_image_name_prefix }}etke.cc/honoroit:{{ matrix_bot_honoroit_version }}" matrix_bot_honoroit_docker_image_name_prefix: "{{ 'localhost/' if matrix_bot_honoroit_container_image_self_build else 'registry.gitlab.com/' }}" matrix_bot_honoroit_docker_image_force_pull: "{{ matrix_bot_honoroit_docker_image.endswith(':latest') }}" From 928b21acf4c0e356409d72f819acde8ccd90dd4d Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 6 Feb 2024 07:23:56 +0200 Subject: [PATCH 38/46] Add variable-deprecation task for Buscarron Related to https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/3157 --- .../matrix-bot-buscarron/tasks/validate_config.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/roles/custom/matrix-bot-buscarron/tasks/validate_config.yml b/roles/custom/matrix-bot-buscarron/tasks/validate_config.yml index 0694a6578..0e968e55a 100644 --- a/roles/custom/matrix-bot-buscarron/tasks/validate_config.yml +++ b/roles/custom/matrix-bot-buscarron/tasks/validate_config.yml @@ -1,4 +1,15 @@ --- +- name: (Deprecation) Catch and report renamed Buscarron settings + ansible.builtin.fail: + msg: >- + Your configuration contains a variable, which now has a different name. + Please change your configuration to rename the variable (`{{ item.old }}` -> `{{ item.new }}`). + when: "item.old in vars" + with_items: + - {'old': 'matrix_bot_buscarron_noencryption', 'new': ''} + - {'old': 'matrix_bot_buscarron_spam_hosts', 'new': ''} + - {'old': 'matrix_bot_buscarron_spam_emails', 'new': ''} + - {'old': 'matrix_bot_buscarron_spam_localparts', 'new': ''} - name: Fail if required Buscarron settings not defined ansible.builtin.fail: From 066790783275dbf5a521652337c056c263066256 Mon Sep 17 00:00:00 2001 From: needo37 Date: Thu, 8 Feb 2024 03:44:38 +0000 Subject: [PATCH 39/46] Update configuring-playbook-bridge-mautrix-whatsapp.md Backfilling is now supported. Updating documentation. --- ...figuring-playbook-bridge-mautrix-whatsapp.md | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/docs/configuring-playbook-bridge-mautrix-whatsapp.md b/docs/configuring-playbook-bridge-mautrix-whatsapp.md index 1794afbd4..caa7edee3 100644 --- a/docs/configuring-playbook-bridge-mautrix-whatsapp.md +++ b/docs/configuring-playbook-bridge-mautrix-whatsapp.md @@ -24,23 +24,6 @@ matrix_mautrix_whatsapp_bridge_relay_admin_only: false If you want to activate the relay bot in a room, use `!wa set-relay`. Use `!wa unset-relay` to deactivate. -## Enable backfilling history -This requires a server with MSC2716 support, which is currently an experimental feature in synapse. -Note that as of Synapse 1.46, there are still some bugs with the implementation, especially if using event persistence workers. -Use the following playbook configuration: - -```yaml -matrix_synapse_configuration_extension_yaml: | - experimental_features: - msc2716_enabled: true -``` -```yaml -matrix_mautrix_whatsapp_configuration_extension_yaml: - bridge: - history_sync: - backfill: true -``` - ## Set up Double Puppeting If you'd like to use [Double Puppeting](https://docs.mau.fi/bridges/general/double-puppeting.html) (hint: you most likely do), you have 2 ways of going about it. From 518615a97933db64ebbd562a198e4e1370fd9117 Mon Sep 17 00:00:00 2001 From: needo37 Date: Thu, 8 Feb 2024 06:13:07 +0000 Subject: [PATCH 40/46] Update signal config.yaml.j2 merging upstream changes (#3164) * Update signal config.yaml.j2 merging upstream changes * Add raw/endraw around displayname_template for mautrix-signal --------- Co-authored-by: Slavi Pantaleev --- .../matrix-bridge-mautrix-signal/templates/config.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/custom/matrix-bridge-mautrix-signal/templates/config.yaml.j2 b/roles/custom/matrix-bridge-mautrix-signal/templates/config.yaml.j2 index 181637c03..182e43eb1 100644 --- a/roles/custom/matrix-bridge-mautrix-signal/templates/config.yaml.j2 +++ b/roles/custom/matrix-bridge-mautrix-signal/templates/config.yaml.j2 @@ -101,7 +101,7 @@ bridge: # {{.UUID}} - The UUID of the Signal user. # {{.AboutEmoji}} - The emoji set by the user in their profile. # {% endraw %} - displayname_template: "{{ '{{.ProfileName}} (Signal)' }}" + displayname_template: {% raw %}'{{or .ProfileName .PhoneNumber "Unknown user"}}'{% endraw %} # Whether to explicitly set the avatar and room name for private chat portal rooms. # If set to `default`, this will be enabled in encrypted rooms and disabled in unencrypted rooms. # If set to `always`, all DM rooms will have explicit names and avatars set. From 8a9a700cfca59ee1c9be528960d64c3d222b2cfc Mon Sep 17 00:00:00 2001 From: needo37 Date: Thu, 8 Feb 2024 06:15:17 +0000 Subject: [PATCH 41/46] Bring config.yaml.j2 in line with upstream (#3163) * Bring config.yaml.j2 in line with upstream * Update config.yaml.j2 --- .../templates/config.yaml.j2 | 116 +++++++++++------- 1 file changed, 73 insertions(+), 43 deletions(-) diff --git a/roles/custom/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2 b/roles/custom/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2 index ad381f6aa..80e0f683a 100644 --- a/roles/custom/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2 +++ b/roles/custom/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2 @@ -54,16 +54,17 @@ appservice: # Whether or not to receive ephemeral events via appservice transactions. # Requires MSC2409 support (i.e. Synapse 1.22+). - # You should disable bridge -> sync_with_custom_puppets when this is enabled. - ephemeral_events: false + ephemeral_events: true + + # Should incoming events be handled asynchronously? + # This may be necessary for large public instances with lots of messages going through. + # However, messages will not be guaranteed to be bridged in the same order they were sent in. + async_transactions: false # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify. as_token: "{{ matrix_mautrix_whatsapp_appservice_token }}" hs_token: "{{ matrix_mautrix_whatsapp_homeserver_token }}" -# Segment API key to track some events, like provisioning API login and encryption errors. -segment_key: null - # Prometheus config. metrics: # Enable prometheus metrics? @@ -92,7 +93,7 @@ bridge: # The following variables are also available, but will cause problems on multi-user instances: # {{ '{{.FullName}}' }} - full name from contact list # {{ '{{.FirstName}}' }} - first name from contact list - displayname_template: "{{ '{{if .BusinessName}}{{.BusinessName}}{{else if .PushName}}{{.PushName}}{{else}}{{.JID}}{{end}} (WA)' }}" + displayname_template: "{{ '{{or .BusinessName .PushName .JID}} (WhatsApp)' }}" # Should the bridge create a space for each logged-in user and add bridged rooms to it? # Users who logged in before turning this on should run `!wa sync space` to create and fill the space for the first time. personal_filtering_spaces: {{ matrix_mautrix_whatsapp_bridge_personal_filtering_spaces | to_json }} @@ -109,23 +110,38 @@ bridge: portal_message_buffer: 128 # Settings for handling history sync payloads. history_sync: - # Should the bridge create portals for chats in the history sync payload? - create_portals: true - # Enable backfilling history sync payloads from WhatsApp using batch sending? - # This requires a server with MSC2716 support, which is currently an experimental feature in synapse. - # It can be enabled by setting experimental_features -> msc2716_enabled to true in homeserver.yaml. - # Note that prior to Synapse 1.49, there were some bugs with the implementation, especially if using event persistence workers. - # There are also still some issues in Synapse's federation implementation. - backfill: false - # Use double puppets for backfilling? - # In order to use this, the double puppets must be in the appservice's user ID namespace - # (because the bridge can't use the double puppet access token with batch sending). - # This only affects double puppets on the local server, double puppets on other servers will never be used. - # Doesn't work out of box with this playbook - double_puppet_backfill: false + # Enable backfilling history sync payloads from WhatsApp? + backfill: true + # The maximum number of initial conversations that should be synced. + # Other conversations will be backfilled on demand when receiving a message or when initiating a direct chat. + max_initial_conversations: -1 + # Maximum number of messages to backfill in each conversation. + # Set to -1 to disable limit. + message_count: 50 # Should the bridge request a full sync from the phone when logging in? # This bumps the size of history syncs from 3 months to 1 year. request_full_sync: false + # Configuration parameters that are sent to the phone along with the request full sync flag. + # By default (when the values are null or 0), the config isn't sent at all. + full_sync_config: + # Number of days of history to request. + # The limit seems to be around 3 years, but using higher values doesn't break. + days_limit: null + # This is presumably the maximum size of the transferred history sync blob, which may affect what the phone includes in the blob. + size_mb_limit: null + # This is presumably the local storage quota, which may affect what the phone includes in the history sync blob. + storage_quota_mb: null + # If this value is greater than 0, then if the conversation's last message was more than + # this number of hours ago, then the conversation will automatically be marked it as read. + # Conversations that have a last message that is less than this number of hours ago will + # have their unread status synced from WhatsApp. + unread_hours_threshold: 0 + + ############################################################################### + # The settings below are only applicable for backfilling using batch sending, # + # which is no longer supported in Synapse. # + ############################################################################### + # Settings for media requests. If the media expired, then it will not # be on the WA servers. # Media can always be requested by reacting with the ♻️ (recycle) emoji. @@ -142,17 +158,6 @@ bridge: # If request_method is "local_time", what time should the requests # be sent (in minutes after midnight)? request_local_time: 120 - # The maximum number of initial conversations that should be synced. - # Other conversations will be backfilled on demand when the start PM - # provisioning endpoint is used or when a message comes in from that - # chat. - max_initial_conversations: -1 - # If this value is greater than 0, then if the conversation's last - # message was more than this number of hours ago, then the conversation - # will automatically be marked it as read. - # Conversations that have a last message that is less than this number - # of hours ago will have their unread status synced from WhatsApp. - unread_hours_threshold: 0 # Settings for immediate backfills. These backfills should generally be # small and their main purpose is to populate each of the initial chats # (as configured by max_initial_conversations) with a few messages so @@ -192,12 +197,11 @@ bridge: - start_days_ago: -1 max_batch_events: 500 batch_delay: 10 + # Should puppet avatars be fetched from the server even if an avatar is already set? user_avatar_sync: true # Should Matrix users leaving groups be bridged to WhatsApp? bridge_matrix_leave: true - # Should the bridge sync with double puppeting to receive EDUs that aren't normally sent to appservices. - sync_with_custom_puppets: true # Should the bridge update the m.direct account data event when double puppeting is enabled. # Note that updating the m.direct event is not atomic (except with mautrix-asmux) # and is therefore prone to race conditions. @@ -210,7 +214,6 @@ bridge: # When double puppeting is enabled, users can use `!wa toggle` to change whether # presence and read receipts are bridged. These settings set the default values. # Existing users won't be affected when these are changed. - default_bridge_receipts: true default_bridge_presence: true # Send the presence as "available" to whatsapp when users start typing on a portal. # This works as a workaround for homeservers that do not support presence, and allows @@ -235,7 +238,7 @@ bridge: login_shared_secret_map: {{ matrix_mautrix_whatsapp_bridge_login_shared_secret_map|to_json }} # Should the bridge explicitly set the avatar and room name for private chat portal rooms? # This is implicitly enabled in encrypted rooms. - private_chat_portal_meta: false + private_chat_portal_meta: default # Should group members be synced in parallel? This makes member sync faster parallel_member_sync: false # Should Matrix m.notice-type messages be bridged? @@ -273,10 +276,6 @@ bridge: # Whether or not created rooms should have federation enabled. # If false, created portal rooms will never be federated. federate_rooms: {{ matrix_mautrix_whatsapp_federate_rooms|to_json }} - # Whether to enable disappearing messages in groups. If enabled, then the expiration time of - # the messages will be determined by the first user to read the message, rather than individually. - # If the bridge only has a single user, this can be turned on safely. - disappearing_messages_in_groups: false # Should the bridge never send alerts to the bridge management room? # These are mostly things like the user being logged out. disable_bridge_alerts: false @@ -290,11 +289,15 @@ bridge: # Send captions in the same message as images. This will send data compatible with both MSC2530 and MSC3552. # This is currently not supported in most clients. caption_in_message: false + # Send galleries as a single event? This is not an MSC (yet). + beeper_galleries: false # Should polls be sent using MSC3381 event types? extev_polls: false - # Should Matrix edits be bridged to WhatsApp edits? - # Official WhatsApp clients don't render edits yet, but once they do, the bridge should work with them right away. - send_whatsapp_edits: false + # Should cross-chat replies from WhatsApp be bridged? Most servers and clients don't support this. + cross_room_replies: false + # Disable generating reply fallbacks? Some extremely bad clients still rely on them, + # but they're being phased out and will be completely removed in the future. + disable_reply_fallbacks: false # Maximum time for handling Matrix events. Duration strings formatted for https://pkg.go.dev/time#ParseDuration # Null means there's no enforced timeout. message_handling_timeout: @@ -338,7 +341,30 @@ bridge: # Enable key sharing? If enabled, key requests for rooms where users are in will be fulfilled. # You must use a client that supports requesting keys from other users to use this feature. allow_key_sharing: {{ matrix_mautrix_whatsapp_bridge_encryption_key_sharing_allow|to_json }} - # What level of device verification should be required from users? + # Should users mentions be in the event wire content to enable the server to send push notifications? + plaintext_mentions: false + # Options for deleting megolm sessions from the bridge. + delete_keys: + # Beeper-specific: delete outbound sessions when hungryserv confirms + # that the user has uploaded the key to key backup. + delete_outbound_on_ack: false + # Don't store outbound sessions in the inbound table. + dont_store_outbound: false + # Ratchet megolm sessions forward after decrypting messages. + ratchet_on_decrypt: false + # Delete fully used keys (index >= max_messages) after decrypting messages. + delete_fully_used_on_decrypt: false + # Delete previous megolm sessions from same device when receiving a new one. + delete_prev_on_new_session: false + # Delete megolm sessions received from a device when the device is deleted. + delete_on_device_delete: false + # Periodically delete megolm sessions when 2x max_age has passed since receiving the session. + periodically_delete_expired: false + # Delete inbound megolm sessions that don't have the received_at field used for + # automatic ratcheting and expired session deletion. This is meant as a migration + # to delete old keys prior to the bridge update. + delete_outdated_inbound: false + # What level of device verification should be required from users? # # Valid levels: # unverified - Send keys to all device in the room. @@ -373,6 +399,10 @@ bridge: # default. messages: 100 + # Disable rotating keys when a user's devices change? + # You should not enable this option unless you understand all the implications. + disable_device_change_key_rotation: false + # Settings for provisioning API provisioning: # Prefix for the provisioning API paths. From 193d20013fe20596b40ed1117d8ac851162f44fa Mon Sep 17 00:00:00 2001 From: needo37 Date: Thu, 8 Feb 2024 09:16:29 +0000 Subject: [PATCH 42/46] Update Signal config.yaml.j2 Not sure why but the endraw is not working. --- .../matrix-bridge-mautrix-signal/templates/config.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/custom/matrix-bridge-mautrix-signal/templates/config.yaml.j2 b/roles/custom/matrix-bridge-mautrix-signal/templates/config.yaml.j2 index 182e43eb1..0dd496aa8 100644 --- a/roles/custom/matrix-bridge-mautrix-signal/templates/config.yaml.j2 +++ b/roles/custom/matrix-bridge-mautrix-signal/templates/config.yaml.j2 @@ -101,7 +101,7 @@ bridge: # {{.UUID}} - The UUID of the Signal user. # {{.AboutEmoji}} - The emoji set by the user in their profile. # {% endraw %} - displayname_template: {% raw %}'{{or .ProfileName .PhoneNumber "Unknown user"}}'{% endraw %} + displayname_template: "{{ '{{or .ProfileName .PhoneNumber "Unknown user"}} (Signal)' }}" # Whether to explicitly set the avatar and room name for private chat portal rooms. # If set to `default`, this will be enabled in encrypted rooms and disabled in unencrypted rooms. # If set to `always`, all DM rooms will have explicit names and avatars set. From f3c69562fae30e67fdbc690a02c52e38bdff09e1 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 8 Feb 2024 11:46:59 +0200 Subject: [PATCH 43/46] Use devture_postgres_container_network for all rust-synapse-compress-state tasks Using `matrix_synapse_container_network` for this task may have worked before, when everything was in the same `matrix` network, but not anymore. Fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/3165 --- .../tasks/rust-synapse-compress-state/compress_room.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/custom/matrix-synapse/tasks/rust-synapse-compress-state/compress_room.yml b/roles/custom/matrix-synapse/tasks/rust-synapse-compress-state/compress_room.yml index 7f5e05666..07fdbd1b1 100644 --- a/roles/custom/matrix-synapse/tasks/rust-synapse-compress-state/compress_room.yml +++ b/roles/custom/matrix-synapse/tasks/rust-synapse-compress-state/compress_room.yml @@ -9,7 +9,7 @@ {{ devture_systemd_docker_base_host_command_docker }} run --rm --name matrix-rust-synapse-compress-state-compress-room --user={{ matrix_synapse_uid }}:{{ matrix_synapse_gid }} --cap-drop=ALL - --network={{ matrix_synapse_container_network }} + --network={{ devture_postgres_container_network }} --mount type=bind,src={{ matrix_synapse_rust_synapse_compress_state_base_path }},dst=/work {{ matrix_synapse_rust_synapse_compress_state_docker_image }} {{ matrix_synapse_rust_synapse_compress_state_synapse_compress_state_in_container_path }} -t -o /work/state-compressor.sql From e9a2b91da6ce9523089ab835b95f1806f27311da Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 8 Feb 2024 12:42:59 +0200 Subject: [PATCH 44/46] Enable federation API labels if the federation port is enabled `matrix_synapse_federation_port_enabled` is defined like this: ``` matrix_synapse_federation_port_enabled: "{{ matrix_synapse_federation_enabled or matrix_synapse_federation_port_openid_resource_required }}" ``` Previously, people that disabled federation, but needed the `openid` listener were running without these federation-related labels. In this patch, we're also dropping the `not matrix_synapse_workers_enabled` condition, because.. none of the Matrix-related labels would be applied anyway when workers are enabled, thanks to `matrix_synapse_container_labels_matrix_related_labels_enabled`. Fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/3127 --- roles/custom/matrix-synapse/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/custom/matrix-synapse/defaults/main.yml b/roles/custom/matrix-synapse/defaults/main.yml index cbb8d2196..5067a1432 100644 --- a/roles/custom/matrix-synapse/defaults/main.yml +++ b/roles/custom/matrix-synapse/defaults/main.yml @@ -253,7 +253,7 @@ matrix_synapse_container_labels_public_client_synapse_admin_api_traefik_tls_cert # Controls whether labels will be added that expose the Server-Server API (Federation API). # Regardless of whether this is enabled, it may or may not take effect due to the value of other variables. # See `matrix_synapse_container_labels_traefik_enabled` or `matrix_synapse_container_labels_matrix_related_labels_enabled` -matrix_synapse_container_labels_public_federation_api_enabled: "{{ matrix_synapse_federation_enabled and matrix_synapse_federation_port_enabled and not matrix_synapse_workers_enabled }}" +matrix_synapse_container_labels_public_federation_api_enabled: "{{ matrix_synapse_federation_port_enabled }}" matrix_synapse_container_labels_public_federation_api_traefik_hostname: "{{ matrix_synapse_container_labels_traefik_hostname }}" matrix_synapse_container_labels_public_federation_api_traefik_path_prefix: /_matrix matrix_synapse_container_labels_public_federation_api_traefik_rule: "Host(`{{ matrix_synapse_container_labels_public_federation_api_traefik_hostname }}`) && PathPrefix(`{{ matrix_synapse_container_labels_public_federation_api_traefik_path_prefix }}`)" From 41ca1a1d964e4dd5889b847003611ea95a2151b8 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 8 Feb 2024 16:39:29 +0200 Subject: [PATCH 45/46] Upgrade synapse-admin (0.9.0 -> 0.9.1) --- roles/custom/matrix-synapse-admin/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/custom/matrix-synapse-admin/defaults/main.yml b/roles/custom/matrix-synapse-admin/defaults/main.yml index a23416134..c69e66be2 100644 --- a/roles/custom/matrix-synapse-admin/defaults/main.yml +++ b/roles/custom/matrix-synapse-admin/defaults/main.yml @@ -12,7 +12,7 @@ matrix_synapse_admin_container_image_self_build: false matrix_synapse_admin_container_image_self_build_repo: "https://github.com/Awesome-Technologies/synapse-admin.git" # renovate: datasource=docker depName=awesometechnologies/synapse-admin -matrix_synapse_admin_version: 0.9.0 +matrix_synapse_admin_version: 0.9.1 matrix_synapse_admin_docker_image: "{{ matrix_synapse_admin_docker_image_name_prefix }}awesometechnologies/synapse-admin:{{ matrix_synapse_admin_version }}" matrix_synapse_admin_docker_image_name_prefix: "{{ 'localhost/' if matrix_synapse_admin_container_image_self_build else matrix_container_global_registry_prefix }}" matrix_synapse_admin_docker_image_force_pull: "{{ matrix_synapse_admin_docker_image.endswith(':latest') }}" From 2096d13bbddf7b9da29827bc0e127e93603c3a3f Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Thu, 8 Feb 2024 21:17:12 +0200 Subject: [PATCH 46/46] fix buscarron old vars --- roles/custom/matrix-bot-buscarron/templates/env.j2 | 3 --- 1 file changed, 3 deletions(-) diff --git a/roles/custom/matrix-bot-buscarron/templates/env.j2 b/roles/custom/matrix-bot-buscarron/templates/env.j2 index ea3f8c21f..acfe99eba 100644 --- a/roles/custom/matrix-bot-buscarron/templates/env.j2 +++ b/roles/custom/matrix-bot-buscarron/templates/env.j2 @@ -4,9 +4,6 @@ BUSCARRON_HOMESERVER={{ matrix_bot_buscarron_homeserver }} BUSCARRON_DB_DSN={{ matrix_bot_buscarron_database_connection_string }} BUSCARRON_DB_DIALECT={{ matrix_bot_buscarron_database_dialect }} BUSCARRON_SPAMLIST={{ matrix_bot_buscarron_spamlist|join(" ") }} -BUSCARRON_SPAM_HOSTS={{ matrix_bot_buscarron_spam_hosts|join(" ") }} -BUSCARRON_SPAM_EMAILS={{ matrix_bot_buscarron_spam_emails|join(" ") }} -BUSCARRON_SPAM_LOCALPARTS={{ matrix_bot_buscarron_spam_localparts|join(" ") }} BUSCARRON_SENTRY={{ matrix_bot_buscarron_sentry }} BUSCARRON_LOGLEVEL={{ matrix_bot_buscarron_loglevel }} BUSCARRON_BAN_SIZE={{ matrix_bot_buscarron_ban_size }}