From c0f89ec2520e68f67488f43cf8b16b0abd55e6ba Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 6 Jul 2023 11:14:22 +0300 Subject: [PATCH 01/43] Upgrade Grafana (v10.0.1-0 -> v10.0.1-1) --- requirements.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.yml b/requirements.yml index 5a20ffe83..da29a3791 100644 --- a/requirements.yml +++ b/requirements.yml @@ -35,7 +35,7 @@ version: 6.1.0 name: geerlingguy.docker - src: git+https://gitlab.com/etke.cc/roles/grafana.git - version: v10.0.1-0 + version: v10.0.1-1 - src: git+https://github.com/mother-of-all-self-hosting/ansible-role-jitsi.git version: v8615-0 name: jitsi From 9c77486e59641fd6df51c164b53768a665894aaa Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 6 Jul 2023 11:38:07 +0300 Subject: [PATCH 02/43] Upgrade Dendrite (v0.13.0 -> v0.13.1) --- roles/custom/matrix-dendrite/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/custom/matrix-dendrite/defaults/main.yml b/roles/custom/matrix-dendrite/defaults/main.yml index 1a83c7039..5cfbfe15e 100644 --- a/roles/custom/matrix-dendrite/defaults/main.yml +++ b/roles/custom/matrix-dendrite/defaults/main.yml @@ -10,7 +10,7 @@ matrix_dendrite_container_image_self_build_repo: "https://github.com/matrix-org/ matrix_dendrite_docker_image_path: "matrixdotorg/dendrite-monolith" matrix_dendrite_docker_image: "{{ matrix_dendrite_docker_image_name_prefix }}{{ matrix_dendrite_docker_image_path }}:{{ matrix_dendrite_docker_image_tag }}" matrix_dendrite_docker_image_name_prefix: "{{ 'localhost/' if matrix_dendrite_container_image_self_build else matrix_container_global_registry_prefix }}" -matrix_dendrite_docker_image_tag: "v0.13.0" +matrix_dendrite_docker_image_tag: "v0.13.1" matrix_dendrite_docker_image_force_pull: "{{ matrix_dendrite_docker_image.endswith(':latest') }}" matrix_dendrite_base_path: "{{ matrix_base_data_path }}/dendrite" From d02f6b8b93b5e70039ede01f593b88fad36ebecf Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 6 Jul 2023 20:18:18 +0300 Subject: [PATCH 03/43] Force-build customized container image for Synapse when its Dockerfile changes --- roles/custom/matrix-synapse/tasks/synapse/setup_install.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/custom/matrix-synapse/tasks/synapse/setup_install.yml b/roles/custom/matrix-synapse/tasks/synapse/setup_install.yml index 169be60bf..7bdf5ddc8 100644 --- a/roles/custom/matrix-synapse/tasks/synapse/setup_install.yml +++ b/roles/custom/matrix-synapse/tasks/synapse/setup_install.yml @@ -72,12 +72,13 @@ owner: "{{ matrix_synapse_uid }}" group: "{{ matrix_synapse_gid }}" mode: 0640 + register: matrix_synapse_container_image_customizations_dockerfile_result - name: Ensure customized Docker image for Synapse is built community.docker.docker_image: name: "{{ matrix_synapse_docker_image_customized }}" source: build - force_source: "{{ matrix_synapse_docker_image_customized_force_source }}" + force_source: "{{ matrix_synapse_container_image_customizations_dockerfile_result.changed or matrix_synapse_docker_image_customized_force_source }}" build: dockerfile: Dockerfile path: "{{ matrix_synapse_customized_docker_src_files_path }}" From 3fc217ded2a588bb667fde19a778cf51f8d9e1d2 Mon Sep 17 00:00:00 2001 From: David Mehren Date: Sun, 9 Jul 2023 15:45:42 +0200 Subject: [PATCH 04/43] Upgrade sliding-sync (v0.99.1 -> v0.99.3) --- roles/custom/matrix-sliding-sync/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/custom/matrix-sliding-sync/defaults/main.yml b/roles/custom/matrix-sliding-sync/defaults/main.yml index fa6e65a46..1f28d3053 100644 --- a/roles/custom/matrix-sliding-sync/defaults/main.yml +++ b/roles/custom/matrix-sliding-sync/defaults/main.yml @@ -4,7 +4,7 @@ matrix_sliding_sync_enabled: true -matrix_sliding_sync_version: v0.99.1 +matrix_sliding_sync_version: v0.99.3 matrix_sliding_sync_scheme: https From db47c6f1e3a5607f731289ffde4863ba0ae68d5e Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 11 Jul 2023 14:43:44 +0300 Subject: [PATCH 05/43] Upgrade Grafana (v10.0.1-1 -> v10.0.2-0) --- requirements.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.yml b/requirements.yml index da29a3791..05e1120ce 100644 --- a/requirements.yml +++ b/requirements.yml @@ -35,7 +35,7 @@ version: 6.1.0 name: geerlingguy.docker - src: git+https://gitlab.com/etke.cc/roles/grafana.git - version: v10.0.1-1 + version: v10.0.2-0 - src: git+https://github.com/mother-of-all-self-hosting/ansible-role-jitsi.git version: v8615-0 name: jitsi From c09c1265e8f2ac1da04a38a4269a767e99d4d4d8 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 11 Jul 2023 17:20:21 +0300 Subject: [PATCH 06/43] Stop using deprecated worker settings (worker_replication_host, worker_replication_http_port) Related to: - https://github.com/matrix-org/synapse/commit/e4f545c452df817daa2f22dfda906f3451d98351 - https://github.com/matrix-org/synapse/commit/2481b7dfa41c1c890346136f04344a4e1660ef32 We've prepared for this by adding the `main` process to the `instance_map` a long time ago, in 49cb8b7b11a72b. --- roles/custom/matrix-synapse/templates/synapse/worker.yaml.j2 | 5 ----- 1 file changed, 5 deletions(-) diff --git a/roles/custom/matrix-synapse/templates/synapse/worker.yaml.j2 b/roles/custom/matrix-synapse/templates/synapse/worker.yaml.j2 index f0e6fe90a..18b96a55f 100644 --- a/roles/custom/matrix-synapse/templates/synapse/worker.yaml.j2 +++ b/roles/custom/matrix-synapse/templates/synapse/worker.yaml.j2 @@ -5,11 +5,6 @@ worker_name: {{ matrix_synapse_worker_details.name }} worker_daemonize: false worker_log_config: /data/{{ matrix_server_fqn_matrix }}.log.config -{% if matrix_synapse_replication_listener_enabled %} -worker_replication_host: matrix-synapse -worker_replication_http_port: {{ matrix_synapse_replication_http_port }} -{% endif %} - {% set http_resources = [] %} {% if matrix_synapse_worker_details.type == 'user_dir' %} From 3037bf3a562fb50e91add0962a3ab7467ebb4a38 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Jul 2023 16:16:59 +0000 Subject: [PATCH 07/43] Bump ansible-community/ansible-lint-action from 6.16.0 to 6.17.0 Bumps [ansible-community/ansible-lint-action](https://github.com/ansible-community/ansible-lint-action) from 6.16.0 to 6.17.0. - [Release notes](https://github.com/ansible-community/ansible-lint-action/releases) - [Commits](https://github.com/ansible-community/ansible-lint-action/compare/v6.16.0...v6.17.0) --- updated-dependencies: - dependency-name: ansible-community/ansible-lint-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/matrix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/matrix.yml b/.github/workflows/matrix.yml index 579ab7192..f38ae3525 100644 --- a/.github/workflows/matrix.yml +++ b/.github/workflows/matrix.yml @@ -21,6 +21,6 @@ jobs: - name: Check out uses: actions/checkout@v3 - name: Run ansible-lint - uses: ansible-community/ansible-lint-action@v6.16.0 + uses: ansible-community/ansible-lint-action@v6.17.0 with: path: roles/custom From 68c9652947d0c32cb6a752e7fe5f73c4fc0dd1ac Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 12 Jul 2023 08:05:40 +0300 Subject: [PATCH 08/43] Upgrade Grafana (v10.0.2-0 -> v10.0.2-1) --- requirements.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.yml b/requirements.yml index 05e1120ce..3cad12d4a 100644 --- a/requirements.yml +++ b/requirements.yml @@ -35,7 +35,7 @@ version: 6.1.0 name: geerlingguy.docker - src: git+https://gitlab.com/etke.cc/roles/grafana.git - version: v10.0.2-0 + version: v10.0.2-1 - src: git+https://github.com/mother-of-all-self-hosting/ansible-role-jitsi.git version: v8615-0 name: jitsi From 9d76e7391381e8cfdbcac58e525537770ec88c1e Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 12 Jul 2023 08:05:45 +0300 Subject: [PATCH 09/43] Upgrade Jitsi (v8615-0 -> v8615-2) --- requirements.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.yml b/requirements.yml index 3cad12d4a..a8c398215 100644 --- a/requirements.yml +++ b/requirements.yml @@ -37,7 +37,7 @@ - src: git+https://gitlab.com/etke.cc/roles/grafana.git version: v10.0.2-1 - src: git+https://github.com/mother-of-all-self-hosting/ansible-role-jitsi.git - version: v8615-0 + version: v8615-2 name: jitsi - src: git+https://gitlab.com/etke.cc/roles/ntfy.git version: v2.6.2-0 From 54412c361d2bc36f89daaadf4819522ed5ca21bf Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 12 Jul 2023 08:15:58 +0300 Subject: [PATCH 10/43] Disable var-naming[no-role-prefix] ansible-lint rule --- .config/ansible-lint.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.config/ansible-lint.yml b/.config/ansible-lint.yml index 00d62f207..0ff5748cb 100644 --- a/.config/ansible-lint.yml +++ b/.config/ansible-lint.yml @@ -9,6 +9,7 @@ skip_list: - schema - command-instead-of-shell - role-name + - var-naming[no-role-prefix] # We frequently load configuration from a template (into a variable), then merge that with another variable (configuration extension) # before finally dumping it to a file. - template-instead-of-copy From 78bd1dbd1bc8b760e7a1f4fd9d6fa76a91d0655d Mon Sep 17 00:00:00 2001 From: Michael Hollister Date: Wed, 12 Jul 2023 01:09:27 -0500 Subject: [PATCH 11/43] Added matrix-media-repo role --- README.md | 1 + .../configuring-playbook-matrix-media-repo.md | 106 ++ ...configuring-playbook-prometheus-grafana.md | 1 + docs/configuring-playbook-s3.md | 3 +- group_vars/matrix_servers | 67 +- .../matrix-media-repo/defaults/main.yml | 681 ++++++++++++ roles/custom/matrix-media-repo/tasks/main.yml | 17 + .../matrix-media-repo/tasks/setup_install.yml | 74 ++ .../tasks/setup_uninstall.yml | 19 + .../templates/grafana/media-repo.json | 991 ++++++++++++++++++ .../templates/media-repo/media-repo.yaml.j2 | 359 +++++++ .../systemd/matrix-media-repo.service.j2 | 52 + .../matrix-nginx-proxy/defaults/main.yml | 5 + .../nginx/conf.d/matrix-domain.conf.j2 | 90 ++ .../defaults/main.yml | 20 + roles/custom/matrix-synapse/defaults/main.yml | 6 +- setup.yml | 1 + 17 files changed, 2486 insertions(+), 7 deletions(-) create mode 100644 docs/configuring-playbook-matrix-media-repo.md create mode 100644 roles/custom/matrix-media-repo/defaults/main.yml create mode 100644 roles/custom/matrix-media-repo/tasks/main.yml create mode 100644 roles/custom/matrix-media-repo/tasks/setup_install.yml create mode 100644 roles/custom/matrix-media-repo/tasks/setup_uninstall.yml create mode 100644 roles/custom/matrix-media-repo/templates/grafana/media-repo.json create mode 100644 roles/custom/matrix-media-repo/templates/media-repo/media-repo.yaml.j2 create mode 100644 roles/custom/matrix-media-repo/templates/media-repo/systemd/matrix-media-repo.service.j2 diff --git a/README.md b/README.md index 8b0265092..912febc04 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,7 @@ Use alternative file storage to the default `media_store` folder. | ---- | -------- | ----------- | ------------- | | [Goofys](https://github.com/kahing/goofys) | x | [Amazon S3](https://aws.amazon.com/s3/) (or other S3-compatible object store) storage for Synapse's content repository (`media_store`) files | [Link](docs/configuring-playbook-s3-goofys.md) | | [synapse-s3-storage-provider](https://github.com/matrix-org/synapse-s3-storage-provider) | x | [Amazon S3](https://aws.amazon.com/s3/) (or other S3-compatible object store) storage for Synapse's content repository (`media_store`) files | [Link](docs/configuring-playbook-s3.md) | +| [matrix-media-repo](https://github.com/turt2live/matrix-media-repo) | x | matrix-media-repo is a highly customizable multi-domain media repository for Matrix. Intended for medium to large deployments, this media repo de-duplicates media while being fully compliant with the specification. | [Link](docs/configuring-playbook-media-repo.md) | ### Bridges diff --git a/docs/configuring-playbook-matrix-media-repo.md b/docs/configuring-playbook-matrix-media-repo.md new file mode 100644 index 000000000..e011e1e66 --- /dev/null +++ b/docs/configuring-playbook-matrix-media-repo.md @@ -0,0 +1,106 @@ +# Setting up matrix-media-repo (optional) + +matrix-media-repo is a highly customizable multi-domain media repository for Matrix. Intended for medium to large environments consisting of several homeservers, this media repo de-duplicates media (including remote media) while being fully compliant with the specification. + +Smaller/individual homeservers can still make use of this project's features, though it may be difficult to set up or have higher than expected resource consumption - please do your research before deploying this as this project may not be useful for your environment. + +More documentation about the project can be found at: https://docs.t2bot.io/matrix-media-repo/ + +## Quickstart + +Add the following configuration to your `inventory/host_vars/matrix.DOMAIN/vars.yml` file: + +```yaml +matrix_media_repo_enabled: true + +# (optional) Turned off by default +# matrix_media_repo_metrics_enabled: true +``` + +The repo is pre-configured for integrating with the postgres database, NGINX proxy and prometheus/grafana (if metrics enabled) from this playbook for all the available homeserver roles. When the media repo is enabled, other media store roles should be disabled (if using Synapse with other media store roles). + +By default, the media-repo will use the local filesystem for data storage. Additional options include `s3` and `IPFS` (experimental). Access token caching is also enabled by default since the logout endpoints are proxied through the media repo. + +## Configuring the media-repo + +Additional common configuration options: +```yaml + +# The postgres database pooling options + +# The maximum number of connects to hold open. More of these allow for more concurrent +# processes to happen. +matrix_media_repo_database_max_connections: 25 + +# The maximum number of connects to leave idle. More of these reduces the time it takes +# to serve requests in low-traffic scenarios. +matrix_media_repo_database_max_idle_connections: 5 + +# These users have full access to the administrative functions of the media repository. +# See https://github.com/turt2live/matrix-media-repo/blob/release-v1.2.8/docs/admin.md for information on what these people can do. They must belong to one of the +# configured homeservers above. +matrix_media_repo_admins: + admins: [] +# admins: +# - "@your_username:example.org" + +# Datastores are places where media should be persisted. This isn't dedicated for just uploads: +# thumbnails and other misc data is also stored in these places. The media repo, when looking +# for a datastore to use, will always use the smallest datastore first. +matrix_media_repo_datastores: + datastores: + - type: file + enabled: true # Enable this to set up data storage. + # Datastores can be split into many areas when handling uploads. Media is still de-duplicated + # across all datastores (local content which duplicates remote content will re-use the remote + # content's location). This option is useful if your datastore is becoming very large, or if + # you want faster storage for a particular kind of media. + # + # The kinds available are: + # thumbnails - Used to store thumbnails of media (local and remote). + # remote_media - Original copies of remote media (servers not configured by this repo). + # local_media - Original uploads for local media. + # archives - Archives of content (GDPR and similar requests). + forKinds: ["thumbnails", "remote_media", "local_media", "archives"] + opts: + path: /data/media + + - type: s3 + enabled: false # Enable this to set up s3 uploads + forKinds: ["thumbnails", "remote_media", "local_media", "archives"] + opts: + # The s3 uploader needs a temporary location to buffer files to reduce memory usage on + # small file uploads. If the file size is unknown, the file is written to this location + # before being uploaded to s3 (then the file is deleted). If you aren't concerned about + # memory usage, set this to an empty string. + tempPath: "/tmp/mediarepo_s3_upload" + endpoint: sfo2.digitaloceanspaces.com + accessKeyId: "" + accessSecret: "" + ssl: true + bucketName: "your-media-bucket" + # An optional region for where this S3 endpoint is located. Typically not needed, though + # some providers will need this (like Scaleway). Uncomment to use. + #region: "sfo2" + # An optional storage class for tuning how the media is stored at s3. + # See https://aws.amazon.com/s3/storage-classes/ for details; uncomment to use. + #storageClass: STANDARD + + # The media repo does support an IPFS datastore, but only if the IPFS feature is enabled. If + # the feature is not enabled, this will not work. Note that IPFS support is experimental at + # the moment and not recommended for general use. + # + # NOTE: Everything you upload to IPFS will be publicly accessible, even when the media repo + # puts authentication on the download endpoints. Only use this option for cases where you + # expect your media to be publicly accessible. + - type: ipfs + enabled: false # Enable this to use IPFS support + forKinds: ["local_media"] + # The IPFS datastore currently has no options. It will use the daemon or HTTP API configured + # in the IPFS section of your main config. + opts: {} + +``` + +Full list of configuration options with documentation can be found in `roles/custom/matrix-media-repo/templates/defaults/main.yml` + diff --git a/docs/configuring-playbook-prometheus-grafana.md b/docs/configuring-playbook-prometheus-grafana.md index 7e4764c20..49a47f1a5 100644 --- a/docs/configuring-playbook-prometheus-grafana.md +++ b/docs/configuring-playbook-prometheus-grafana.md @@ -83,6 +83,7 @@ Name | Description `matrix_bridge_hookshot_metrics_proxying_enabled`|Set this to `true` to expose the [Hookshot](configuring-playbook-bridge-hookshot.md) metrics on `https://matrix.DOMAIN/metrics/hookshot` (only takes effect if `matrix_nginx_proxy_proxy_matrix_metrics_enabled: true`) `matrix_SERVICE_metrics_proxying_enabled`|Various other services/roles may provide similar `_metrics_enabled` and `_metrics_proxying_enabled` variables for exposing their metrics. Refer to each role for details. Only takes effect if `matrix_nginx_proxy_proxy_matrix_metrics_enabled: true` `matrix_nginx_proxy_proxy_matrix_metrics_additional_user_location_configuration_blocks`|Add nginx `location` blocks to this list if you'd like to expose additional exporters manually (see below) +`matrix_media_repo_metrics_enabled`|Set this to `true` to make media-repo expose metrics (locally, on the container network) Example for how to make use of `matrix_nginx_proxy_proxy_matrix_metrics_additional_user_location_configuration_blocks` for exposing additional metrics locations: ```nginx diff --git a/docs/configuring-playbook-s3.md b/docs/configuring-playbook-s3.md index 539f96d32..941ae0db2 100644 --- a/docs/configuring-playbook-s3.md +++ b/docs/configuring-playbook-s3.md @@ -9,7 +9,7 @@ First, [choose an Object Storage provider](#choosing-an-object-storage-provider) Then, [create the S3 bucket](#bucket-creation-and-security-configuration). -Finally, [set up S3 storage for Synapse](#setting-up) (with [Goofys](configuring-playbook-s3-goofys.md) or [synapse-s3-storage-provider](configuring-playbook-synapse-s3-storage-provider.md)). +Finally, [set up S3 storage for Synapse](#setting-up) (with [Goofys](configuring-playbook-s3-goofys.md), [synapse-s3-storage-provider](configuring-playbook-synapse-s3-storage-provider.md), or use s3 datastore with the [matrix-media-repo](https://docs.t2bot.io/matrix-media-repo/configuration/s3-datastore.html)). ## Choosing an Object Storage provider @@ -105,3 +105,4 @@ To set up Synapse to store files in S3, follow the instructions for the method o - using [synapse-s3-storage-provider](configuring-playbook-synapse-s3-storage-provider.md) (recommended) - using [Goofys to mount the S3 store to the local filesystem](configuring-playbook-s3-goofys.md) +- using [matrix-media-repo](https://docs.t2bot.io/matrix-media-repo/configuration/s3-datastore.html) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index d225cc302..cc60b9758 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -326,6 +326,8 @@ devture_systemd_service_manager_services_list_auto: | + ([{'name': 'matrix-ma1sd.service', 'priority': 2000, 'groups': ['matrix', 'ma1sd']}] if matrix_ma1sd_enabled else []) + + ([{'name': 'matrix-media-repo.service', 'priority': 4000, 'groups': ['matrix', 'media_store']}] if matrix_media_repo_enabled else []) + + ([{'name': 'matrix-mailer.service', 'priority': 2000, 'groups': ['matrix', 'mailer']}] if matrix_mailer_enabled else []) + ([{'name': 'matrix-nginx-proxy.service', 'priority': 3000, 'groups': ['matrix', 'nginx', 'reverse-proxies']}] if matrix_nginx_proxy_enabled else []) @@ -395,7 +397,6 @@ devture_systemd_service_manager_services_list_auto: | ######################################################################## - ###################################################################### # # com.devture.ansible.role.playbook_state_preserver @@ -418,7 +419,6 @@ devture_playbook_state_preserver_commit_hash_preservation_dst: "{{ matrix_base_d ###################################################################### - ###################################################################### # # matrix-base @@ -2557,6 +2557,38 @@ matrix_ma1sd_database_password: "{{ '%s' | format(matrix_homeserver_generic_secr # ###################################################################### +###################################################################### +# +# matrix-media-repo +# +###################################################################### + +matrix_media_repo_enabled: false +matrix_media_repo_identifier: matrix-media-repo +matrix_media_repo_container_network: "{{ matrix_docker_network }}" + +matrix_media_repo_container_labels_traefik_enabled: false +matrix_media_repo_container_labels_traefik_docker_network: "{{ matrix_playbook_reverse_proxyable_services_additional_network }}" +matrix_media_repo_container_labels_traefik_entrypoints: "{{ devture_traefik_entrypoint_primary }}" +matrix_media_repo_container_labels_traefik_tls_certResolver: "{{ devture_traefik_certResolver_primary }}" + +matrix_media_repo_database_hostname: "{{ devture_postgres_connection_hostname if devture_postgres_enabled else '' }}" +matrix_media_repo_database_username: matrix_media_repo +matrix_media_repo_database_password: "{{ '%s' | format(matrix_homeserver_generic_secret_key) | password_hash('sha512', 'mediarepo.db', rounds=655555) | to_uuid }}" +matrix_media_repo_database_name: matrix_media_repo + +matrix_media_repo_systemd_required_services_list: | + {{ + (['docker.service']) + + + ([devture_postgres_identifier ~ '.service'] if devture_postgres_enabled else []) + }} + +###################################################################### +# +# /matrix-media-repo +# +###################################################################### ###################################################################### # @@ -2638,6 +2670,10 @@ matrix_nginx_proxy_proxy_matrix_identity_api_enabled: "{{ matrix_ma1sd_enabled } matrix_nginx_proxy_proxy_matrix_identity_api_addr_with_container: "matrix-ma1sd:{{ matrix_ma1sd_container_port }}" matrix_nginx_proxy_proxy_matrix_identity_api_addr_sans_container: "127.0.0.1:{{ matrix_ma1sd_container_port }}" +matrix_nginx_proxy_proxy_media_repo_enabled: "{{ matrix_media_repo_enabled }}" +matrix_nginx_proxy_proxy_media_repo_addr_with_container: "matrix-media-repo:{{ matrix_media_repo_port }}" +matrix_nginx_proxy_proxy_media_repo_addr_sans_container: "127.0.0.1:{{ matrix_media_repo_port }}" + # By default, we do TLS termination for the Matrix Federation API (port 8448) at matrix-nginx-proxy. # Unless this is handled there OR Synapse's federation listener port is disabled, we'll reverse-proxy. matrix_nginx_proxy_proxy_matrix_federation_api_enabled: |- @@ -2696,6 +2732,8 @@ matrix_nginx_proxy_systemd_wanted_services_list: | + (['matrix-ma1sd.service'] if matrix_ma1sd_enabled else []) + + (['matrix-media-repo.service'] if matrix_media_repo_enabled else []) + + (['matrix-client-cinny.service'] if matrix_client_cinny_enabled and matrix_playbook_reverse_proxy_type in ['playbook-managed-nginx', 'other-nginx-non-container'] else []) + (['matrix-bot-buscarron.service'] if matrix_bot_buscarron_enabled and matrix_playbook_reverse_proxy_type in ['playbook-managed-nginx', 'other-nginx-non-container'] else []) @@ -3050,6 +3088,12 @@ devture_postgres_managed_databases_auto: | 'username': prometheus_postgres_exporter_database_username, 'password': prometheus_postgres_exporter_database_password, }] if (prometheus_postgres_exporter_enabled and prometheus_postgres_exporter_database_hostname == devture_postgres_connection_hostname) else []) + + + ([{ + 'name': matrix_media_repo_database_name, + 'username': matrix_media_repo_database_username, + 'password': matrix_media_repo_database_password, + }] if (matrix_media_repo_enabled and matrix_media_repo_database_hostname == devture_postgres_connection_hostname) else []) }} @@ -3424,6 +3468,9 @@ matrix_synapse_redis_password: "{{ redis_connection_password if redis_enabled el matrix_synapse_container_extra_arguments_auto: "{{ matrix_homeserver_container_extra_arguments_auto }}" matrix_synapse_app_service_config_files_auto: "{{ matrix_homeserver_app_service_config_files_auto }}" +# Disable creation of media repository Synapse worker when using media-repo +matrix_synapse_ext_media_repo_enabled: "{{ matrix_media_repo_enabled }}" + ###################################################################### # # /matrix-synapse @@ -3653,6 +3700,8 @@ prometheus_container_additional_networks: | ([matrix_hookshot_container_network] if matrix_prometheus_services_connect_scraper_hookshot_enabled and matrix_hookshot_container_network != prometheus_container_network else []) + ([matrix_prometheus_nginxlog_exporter_container_network] if matrix_prometheus_services_connect_scraper_nginxlog_enabled and matrix_prometheus_nginxlog_exporter_container_network != prometheus_container_network else []) + + + ([matrix_media_repo_container_network] if matrix_prometheus_services_connect_scraper_media_repo_enabled and matrix_media_repo_container_network != prometheus_container_network else []) ) | unique }} @@ -3678,6 +3727,8 @@ prometheus_config_scrape_configs_auto: | (matrix_prometheus_services_connect_scraper_hookshot_scrape_configs if matrix_prometheus_services_connect_scraper_hookshot_enabled else []) + (matrix_prometheus_services_connect_scraper_nginxlog_scrape_configs if matrix_prometheus_services_connect_scraper_nginxlog_enabled else []) + + + (matrix_prometheus_services_connect_scraper_media_repo_scrape_configs if matrix_prometheus_services_connect_scraper_media_repo_enabled else []) }} ###################################################################### @@ -3713,6 +3764,9 @@ matrix_prometheus_services_connect_scraper_hookshot_static_configs_target: "{{ m matrix_prometheus_services_connect_scraper_nginxlog_enabled: "{{ matrix_prometheus_nginxlog_exporter_enabled }}" matrix_prometheus_services_connect_scraper_nginxlog_static_configs_target: "{{ matrix_prometheus_nginxlog_exporter_container_hostname }}:{{ matrix_prometheus_nginxlog_exporter_container_metrics_port | string }}" +matrix_prometheus_services_connect_scraper_media_repo_enabled: "{{ matrix_media_repo_enabled and matrix_media_repo_metrics_enabled }}" +matrix_prometheus_services_connect_scraper_media_repo_static_configs_target: "{{ matrix_media_repo_identifier }}:{{ matrix_media_repo_metrics_port }}" + ###################################################################### # # /matrix-prometheus-services-connect @@ -3777,6 +3831,8 @@ grafana_dashboard_download_urls: | (prometheus_postgres_exporter_dashboard_urls if prometheus_postgres_exporter_enabled else []) + (matrix_prometheus_nginxlog_exporter_dashboard_urls if matrix_prometheus_nginxlog_exporter_enabled else []) + + + (matrix_media_repo_dashboard_urls if matrix_media_repo_metrics_enabled else []) }} grafana_provisioning_dashboard_template_files: | @@ -3785,6 +3841,11 @@ grafana_provisioning_dashboard_template_files: | 'path': 'roles/custom/matrix-prometheus-nginxlog-exporter/templates/grafana/nginx-proxy.json', 'name': 'nginx-proxy.json', }] if matrix_prometheus_nginxlog_exporter_enabled else []) + + + ([{ + 'path': 'roles/custom/matrix-media-repo/templates/grafana/media-repo.json', + 'name': 'media-repo.json', + }] if matrix_media_repo_metrics_enabled else []) }} grafana_default_home_dashboard_path: |- @@ -3803,7 +3864,6 @@ grafana_default_home_dashboard_path: |- ###################################################################### - ###################################################################### # # matrix-registration @@ -3853,7 +3913,6 @@ matrix_registration_database_password: "{{ '%s' | format(matrix_homeserver_gener ###################################################################### - ###################################################################### # # matrix-sliding-sync diff --git a/roles/custom/matrix-media-repo/defaults/main.yml b/roles/custom/matrix-media-repo/defaults/main.yml new file mode 100644 index 000000000..87ffcbe84 --- /dev/null +++ b/roles/custom/matrix-media-repo/defaults/main.yml @@ -0,0 +1,681 @@ +--- +# matrix-media-repo is a highly customizable multi-domain media repository for Matrix. +# Intended for medium to large environments consisting of several homeservers, this +# media repo de-duplicates media (including remote media) while being fully compliant +# with the specification. +# See: https://github.com/turt2live/matrix-media-repo + +matrix_media_repo_enabled: true + +matrix_media_repo_container_image_self_build: false +matrix_media_repo_container_image_self_build_repo: "https://github.com/turt2live/matrix-media-repo.git" + +matrix_media_repo_docker_image_path: "turt2live/matrix-media-repo" +matrix_media_repo_docker_image: "{{ matrix_media_repo_docker_image_name_prefix }}{{ matrix_media_repo_docker_image_path }}:{{ matrix_media_repo_docker_image_tag }}" +matrix_media_repo_docker_image_name_prefix: "{{ 'localhost/' if matrix_media_repo_container_image_self_build else matrix_container_global_registry_prefix }}" +matrix_media_repo_docker_image_tag: "v1.2.13" +matrix_media_repo_docker_image_force_pull: "{{ matrix_media_repo_docker_image.endswith(':latest') }}" + +matrix_media_repo_base_path: "{{ matrix_base_data_path }}/media-repo" +matrix_media_repo_docker_src_files_path: "{{ matrix_media_repo_base_path }}/docker-src" + +# List of systemd services that matrix-conduit.service depends on +matrix_media_repo_systemd_required_services_list: ["docker.service"] + +# List of systemd services that matrix-conduit.service wants +matrix_media_repo_systemd_wanted_services_list: [] + +# The base container network. It will be auto-created by this role if it doesn't exist already. +matrix_media_repo_container_network: "{{ matrix_docker_network }}" + +# A list of additional container networks that the container would be connected to. +# The role does not create these networks, so make sure they already exist. +# Use this to expose this container to another reverse proxy, which runs in a different container network. +matrix_media_repo_container_additional_networks: [] + +# Extra arguments for the Docker container +matrix_media_repo_container_extra_arguments: [] + +# matrix_media_repo_dashboard_urls contains a list of URLs with Grafana dashboard definitions. +# If the Grafana role is enabled, these dashboards will be downloaded. +matrix_media_repo_dashboard_urls: + - https://raw.githubusercontent.com/spantaleev/matrix-docker-ansible-deploy/master/roles/custom/matrix-media-repo/templates/grafana/media-repo.json + +# ***************************************************************************** +# Configuration File Settings +# ***************************************************************************** + +# General repo configuration +matrix_media_repo_bind_address: '0.0.0.0' +matrix_media_repo_port: 8000 + +# Where to store the logs, relative to where the repo is started from. Logs will be automatically +# rotated every day and held for 14 days. To disable the repo logging to files, set this to +# "-" (including quotation marks). +# +# Note: to change the log directory you'll have to restart the repository. This setting cannot be +# live reloaded. +matrix_media_repo_log_directory: logs + +# Set to true to enable color coding in your logs. Note that this may cause escape sequences to +# appear in logs which render them unreadable, which is why colors are disabled by default. +matrix_media_repo_log_colors: false + +# Set to true to enable JSON logging for consumption by things like logstash. Note that this is +# incompatible with the log color option and will always render without colors. +matrix_media_repo_json_logs: false + +# The log level to log at. Note that this will need to be at least "info" to receive support. +# +# Values (in increasing spam): panic | fatal | error | warn | info | debug | trace +matrix_media_repo_log_level: "info" + +# If true, the media repo will accept any X-Forwarded-For header without validation. In most cases +# this option should be left as "false". Note that the media repo already expects an X-Forwarded-For +# header, but validates it to ensure the IP being given makes sense. +matrix_media_repo_trust_any_forwarded_address: false + +# If false, the media repo will not use the X-Forwarded-Host header commonly added by reverse proxies. +# Typically this should remain as true, though in some circumstances it may need to be disabled. +# See https://github.com/turt2live/matrix-media-repo/issues/202 for more information. +matrix_media_repo_use_forwarded_host: true + +# Options for dealing with federation + +# On a per-host basis, the number of consecutive failures in calling the host before the +# media repo will back off. This defaults to 20 if not given. Note that 404 errors from +# the remote server do not count towards this. +matrix_media_repo_federation_backoff_at: 20 + +# The database configuration for the media repository +# Do NOT put your homeserver's existing database credentials here. Create a new database and +# user instead. Using the same server is fine, just not the same username and database. +matrix_media_repo_database_username: "matrix_media_repo" +matrix_media_repo_database_password: "your_password" +matrix_media_repo_database_hostname: "matrix-postgres" +matrix_media_repo_database_port: 5432 +matrix_media_repo_database_name: "matrix_media_repo" + +# Currently only "postgres" is supported. +matrix_media_repo_database_postgres: "postgres://{{ matrix_media_repo_database_username }}:{{ matrix_media_repo_database_password }}@{{ matrix_media_repo_database_hostname }}:{{ matrix_media_repo_database_port }}/{{ matrix_media_repo_database_name }}?sslmode=disable" + +# The database pooling options + +# The maximum number of connects to hold open. More of these allow for more concurrent +# processes to happen. +matrix_media_repo_database_max_connections: 25 + +# The maximum number of connects to leave idle. More of these reduces the time it takes +# to serve requests in low-traffic scenarios. +matrix_media_repo_database_max_idle_connections: 5 + +# The configuration for the homeservers this media repository is known to control. Servers +# not listed here will not be able to upload media. +matrix_media_repo_homeservers: + homeservers: + # This should match the server_name of your homeserver, and the Host header + # provided to the media repo. + - name: "{{ matrix_server_fqn_matrix }}" + + # The base URL to where the homeserver can actually be reached + csApi: "https://{{ matrix_server_fqn_matrix }}/" + + # The number of consecutive failures in calling this homeserver before the + # media repository will start backing off. This defaults to 10 if not given. + backoffAt: 10 + + # The kind of admin API the homeserver supports. If set to "matrix", + # the media repo will use the Synapse-defined endpoints under the + # unstable client-server API. When this is "synapse", the new /_synapse + # endpoints will be used instead. Unknown values are treated as the + # default, "matrix". + adminApiKind: "matrix" + +# Options for controlling how access tokens work with the media repo. It is recommended that if +# you are going to use these options that the `/logout` and `/logout/all` client-server endpoints +# be proxied through this process. They will also be called on the homeserver, and the response +# sent straight through the client - they are simply used to invalidate the cache faster for +# a particular user. Without these, the access tokens might still work for a short period of time +# after the user has already invalidated them. +# +# This will also cache errors from the homeserver. +# +# Note that when this config block is used outside of a per-domain config, all hosts will be +# subject to the same cache. This also means that application services on limited homeservers +# could be authorized on the wrong domain. +# +# *************************************************************************** +# * IT IS HIGHLY RECOMMENDED TO USE PER-DOMAIN CONFIGS WITH THIS FEATURE. * +# *************************************************************************** +matrix_media_repo_access_tokens: + accessTokens: + # The maximum time a cached access token will be considered valid. Set to zero (the default) + # to disable the cache and constantly hit the homeserver. This is recommended to be set to + # 43200 (12 hours) on servers with the logout endpoints proxied through the media repo, and + # zero for servers who do not proxy the endpoints through. + maxCacheTimeSeconds: 43200 + + # Whether or not to use the `appservices` config option below. If disabled (the default), + # the regular access token cache will be used for each user, potentially leading to high + # memory usage. + useLocalAppserviceConfig: false + + # The application services (and their namespaces) registered on the homeserver. Only used + # if `useLocalAppserviceConfig` is enabled (recommended). + # + # Usually the appservice will provide you with these config details - they'll just need + # translating from the appservice registration to here. Note that this does not require + # all options from the registration, and only requires the bare minimum required to run + # the media repo. + # appservices: + # - id: Name_of_appservice_for_your_reference + # asToken: Secret_token_for_appservices_to_use + # senderUserId: "@_example_bridge:yourdomain.com" + # userNamespaces: + # - regex: "@_example_bridge_.+:yourdomain.com" + # # A note about regexes: it is best to suffix *all* namespaces with the homeserver + # # domain users are valid for, as otherwise the appservice can use any user with + # # any domain name it feels like, even if that domain is not configured with the + # # media repo. This will lead to inaccurate reporting in the case of the media + # # repo, and potentially leading to media being considered "remote". + +# These users have full access to the administrative functions of the media repository. +# See docs/admin.md for information on what these people can do. They must belong to one of the +# configured homeservers above. +matrix_media_repo_admins: + admins: [] +# admins: +# - "@your_username:example.org" + +# Shared secret auth is useful for applications building on top of the media repository, such +# as a management interface. The `token` provided here is treated as a repository administrator +# when shared secret auth is enabled: if the `token` is used in place of an access token, the' +# request will be authorized. This is not limited to any particular domain, giving applications +# the ability to use it on any configured hostname. +# Set this to true to enable shared secret auth. +matrix_media_repo_shared_secret_auth_enabled: false + +# Use a secure value here to prevent unauthorized access to the media repository. +matrix_media_repo_shared_secret_auth_token: "PutSomeRandomSecureValueHere" + +# Datastores are places where media should be persisted. This isn't dedicated for just uploads: +# thumbnails and other misc data is also stored in these places. The media repo, when looking +# for a datastore to use, will always use the smallest datastore first. +matrix_media_repo_datastores: + datastores: + - type: file + enabled: true # Enable this to set up data storage. + # Datastores can be split into many areas when handling uploads. Media is still de-duplicated + # across all datastores (local content which duplicates remote content will re-use the remote + # content's location). This option is useful if your datastore is becoming very large, or if + # you want faster storage for a particular kind of media. + # + # The kinds available are: + # thumbnails - Used to store thumbnails of media (local and remote). + # remote_media - Original copies of remote media (servers not configured by this repo). + # local_media - Original uploads for local media. + # archives - Archives of content (GDPR and similar requests). + forKinds: ["thumbnails", "remote_media", "local_media", "archives"] + opts: + path: /data/media + + - type: s3 + enabled: false # Enable this to set up s3 uploads + forKinds: ["thumbnails", "remote_media", "local_media", "archives"] + opts: + # The s3 uploader needs a temporary location to buffer files to reduce memory usage on + # small file uploads. If the file size is unknown, the file is written to this location + # before being uploaded to s3 (then the file is deleted). If you aren't concerned about + # memory usage, set this to an empty string. + tempPath: "/tmp/mediarepo_s3_upload" + endpoint: sfo2.digitaloceanspaces.com + accessKeyId: "" + accessSecret: "" + ssl: true + bucketName: "your-media-bucket" + # An optional region for where this S3 endpoint is located. Typically not needed, though + # some providers will need this (like Scaleway). Uncomment to use. + # region: "sfo2" + # An optional storage class for tuning how the media is stored at s3. + # See https://aws.amazon.com/s3/storage-classes/ for details; uncomment to use. + # storageClass: STANDARD + + # The media repo does support an IPFS datastore, but only if the IPFS feature is enabled. If + # the feature is not enabled, this will not work. Note that IPFS support is experimental at + # the moment and not recommended for general use. + # + # NOTE: Everything you upload to IPFS will be publicly accessible, even when the media repo + # puts authentication on the download endpoints. Only use this option for cases where you + # expect your media to be publicly accessible. + - type: ipfs + enabled: false # Enable this to use IPFS support + forKinds: ["local_media"] + # The IPFS datastore currently has no options. It will use the daemon or HTTP API configured + # in the IPFS section of your main config. + opts: {} + +# Options for controlling archives. Archives are exports of a particular user's content for +# the purpose of GDPR or moving media to a different server. + +# Whether archiving is enabled or not. Default enabled. +matrix_media_repo_archiving_enabled: true +# If true, users can request a copy of their own data. By default, only repository administrators +# can request a copy. +# This includes the ability for homeserver admins to request a copy of their own server's +# data, as known to the repo. +matrix_media_repo_archiving_self_service: false +# The number of bytes to target per archive before breaking up the files. This is independent +# of any file upload limits and will require a similar amount of memory when performing an export. +# The file size is also a target, not a guarantee - it is possible to have files that are smaller +# or larger than the target. This is recommended to be approximately double the size of your +# file upload limit, provided there is enough memory available for the demand of exporting. +matrix_media_repo_archiving_target_bytes_per_part: 209715200 # 200mb default + +# The file upload settings for the media repository +matrix_media_repo_uploads: + uploads: + # The maximum individual file size a user can upload. + maxBytes: 104857600 # 100MB default, 0 to disable + + # The minimum number of bytes to let people upload. This is recommended to be non-zero to + # ensure that the "cost" of running the media repo is worthwhile - small file uploads tend + # to waste more CPU and database resources than small files, thus a default of 100 bytes + # is applied here as an approximate break-even point. + minBytes: 100 # 100 bytes by default + + # The number of bytes to claim as the maximum size for uploads for the limits API. If this + # is not provided then the maxBytes setting will be used instead. This is useful to provide + # if the media repo's settings and the reverse proxy do not match for maximum request size. + # This is purely for informational reasons and does not actually limit any functionality. + # Set this to -1 to indicate that there is no limit. Zero will force the use of maxBytes. + reportedMaxBytes: 0 + + # Options for limiting how much content a user can upload. Quotas are applied to content + # associated with a user regardless of de-duplication. Quotas which affect remote servers + # or users will not take effect. When a user exceeds their quota they will be unable to + # upload any more media. + quotas: + # Whether or not quotas are enabled/enforced. Note that even when disabled the media repo + # will track how much media a user has uploaded. This is disabled by default. + enabled: false + + # The quota rules that affect users. The first rule to match the uploader will take effect. + # An implied rule which matches all users and has no quota is always last in this list, + # meaning that if no rules are supplied then users will be able to upload anything. Similarly, + # if no rules match a user then the implied rule will match, allowing the user to have no + # quota. The quota will let the user upload to 1 media past their quota, meaning that from + # a statistics perspective the user might exceed their quota however only by a small amount. + users: + - glob: "@*:*" # Affect all users. Use asterisks (*) to match any character. + maxBytes: 53687063712 # 50GB default, 0 to disable + +# Settings related to downloading files from the media repository + +# The maximum number of bytes to download from other servers +matrix_media_repo_downloads_max_bytes: 104857600 # 100MB default, 0 to disable + +# The number of workers to use when downloading remote media. Raise this number if remote +# media is downloading slowly or timing out. +# +# Maximum memory usage = numWorkers multiplied by the maximum download size +# Average memory usage is dependent on how many concurrent downloads your users are doing. +matrix_media_repo_downloads_num_workers: 10 + +# How long, in minutes, to cache errors related to downloading remote media. Once this time +# has passed, the media is able to be re-requested. +matrix_media_repo_downloads_failure_cache_minutes: 5 + +# The cache control settings for downloads. This can help speed up downloads for users by +# keeping popular media in the cache. This cache is also used for thumbnails. +matrix_media_repo_downloads_cache_enabled: true + +# The maximum size of cache to have. Higher numbers are better. +matrix_media_repo_downloads_cache_max_size_bytes: 1048576000 # 1GB default + +# The maximum file size to cache. This should normally be the same size as your maximum +# upload size. +matrix_media_repo_downloads_cache_max_file_size_bytes: 104857600 # 100MB default + +# The number of minutes to track how many downloads a file gets +matrix_media_repo_downloads_cache_tracked_minutes: 30 + +# The number of downloads a file must receive in the window above (trackedMinutes) in +# order to be cached. +matrix_media_repo_downloads_cache_min_downloads: 5 + +# The minimum amount of time an item should remain in the cache. This prevents the cache +# from cycling out the file if it needs more room during this time. Note that the media +# repo regularly cleans out media which is past this point from the cache, so this number +# may need increasing depending on your use case. If the maxSizeBytes is reached for the +# media repo, and some cached items are still under this timer, new items will not be able +# to enter the cache. When this happens, consider raising maxSizeBytes or lowering this +# timer. +matrix_media_repo_downloads_cache_min_cache_time_seconds: 300 + +# The minimum amount of time an item should remain outside the cache once it is removed. +matrix_media_repo_downloads_cache_min_evicted_time_seconds: 60 + +# How many days after a piece of remote content is downloaded before it expires. It can be +# re-downloaded on demand, this just helps free up space in your datastore. Set to zero or +# negative to disable. Defaults to disabled. +matrix_media_repo_downloads_expire_after_days: 0 + +# URL Preview settings +matrix_media_repo_url_previews: + urlPreviews: + enabled: true # If enabled, the preview_url routes will be accessible + maxPageSizeBytes: 10485760 # 10MB default, 0 to disable + + # If true, the media repository will try to provide previews for URLs with invalid or unsafe + # certificates. If false (the default), the media repo will fail requests to said URLs. + previewUnsafeCertificates: false + + # Note: URL previews are limited to a given number of words, which are then limited to a number + # of characters, taking off the last word if it needs to. This also applies for the title. + + numWords: 50 # The number of words to include in a preview (maximum) + maxLength: 200 # The maximum number of characters for a description + + numTitleWords: 30 # The maximum number of words to include in a preview's title + maxTitleLength: 150 # The maximum number of characters for a title + + # The mime types to preview when OpenGraph previews cannot be rendered. OpenGraph previews are + # calculated on anything matching "text/*". To have a thumbnail in the preview the URL must be + # an image and the image's type must be allowed by the thumbnailer. + filePreviewTypes: + - "image/*" + + # The number of workers to use when generating url previews. Raise this number if url + # previews are slow or timing out. + # + # Maximum memory usage = numWorkers multiplied by the maximum page size + # Average memory usage is dependent on how many concurrent urls your users are previewing. + numWorkers: 10 + + # Either allowedNetworks or disallowedNetworks must be provided. If both are provided, they + # will be merged. URL previews will be disabled if neither is supplied. Each entry must be + # a CIDR range. + disallowedNetworks: + - "127.0.0.1/8" + - "10.0.0.0/8" + - "172.16.0.0/12" + - "192.168.0.0/16" + - "100.64.0.0/10" + - "169.254.0.0/16" + - '::1/128' + - 'fe80::/64' + - 'fc00::/7' + allowedNetworks: + # "Everything". The blacklist will help limit this. + # This is the default value for this field. + - "0.0.0.0/0" + + # How many days after a preview is generated before it expires and is deleted. The preview + # can be regenerated safely - this just helps free up some space in your database. Set to + # zero or negative to disable. Defaults to disabled. + expireAfterDays: 0 + + # The default Accept-Language header to supply when generating URL previews when one isn't + # supplied by the client. + # Reference: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Language + defaultLanguage: "en-US,en" + + # When true, oEmbed previews will be enabled. Typically these kinds of previews are used for + # sites that do not support OpenGraph or page scraping, such as Twitter. For information on + # specifying providers for oEmbed, including your own, see the following documentation: + # https://docs.t2bot.io/matrix-media-repo/url-previews/oembed.html + # Defaults to disabled. + oEmbed: false + +# The thumbnail configuration for the media repository. +matrix_media_repo_thumbnails: + thumbnails: + # The maximum number of bytes an image can be before the thumbnailer refuses. + maxSourceBytes: 10485760 # 10MB default, 0 to disable + + # The maximum number of pixels an image can have before the thumbnailer refuses. Note that + # this only applies to image types: file types like audio and video are affected solely by + # the maxSourceBytes. + maxPixels: 32000000 # 32M default + + # The number of workers to use when generating thumbnails. Raise this number if thumbnails + # are slow to generate or timing out. + # + # Maximum memory usage = numWorkers multiplied by the maximum image source size + # Average memory usage is dependent on how many thumbnails are being generated by your users + numWorkers: 100 + + # All thumbnails are generated into one of the sizes listed here. The first size is used as + # the default for when no width or height is requested. The media repository will return + # either an exact match or the next largest size of thumbnail. + sizes: + - width: 32 + height: 32 + - width: 96 + height: 96 + - width: 320 + height: 240 + - width: 640 + height: 480 + - width: 768 # This size is primarily used for audio thumbnailing. + height: 240 + - width: 800 + height: 600 + + # To allow for thumbnails to be any size, not just in the sizes specified above, set this to + # true (default false). When enabled, whatever size requested by the client will be generated + # up to a maximum of the largest possible dimensions in the `sizes` list. For best results, + # specify only one size in the `sizes` list when this option is enabled. + dynamicSizing: false + + # The content types to thumbnail when requested. Types that are not supported by the media repo + # will not be thumbnailed (adding application/json here won't work). Clients may still not request + # thumbnails for these types - this won't make clients automatically thumbnail these file types. + types: + - "image/jpeg" + - "image/jpg" + - "image/png" + - "image/apng" + - "image/gif" + - "image/heif" + - "image/webp" + # - "image/svg+xml" # Be sure to have ImageMagick installed to thumbnail SVG files + - "audio/mpeg" + - "audio/ogg" + - "audio/wav" + - "audio/flac" + # - "video/mp4" # Be sure to have ffmpeg installed to thumbnail video files + + # Animated thumbnails can be CPU intensive to generate. To disable the generation of animated + # thumbnails, set this to false. If disabled, regular thumbnails will be returned. + allowAnimated: true + + # Default to animated thumbnails, if available + defaultAnimated: false + + # The maximum file size to thumbnail when a capable animated thumbnail is requested. If the image + # is larger than this, the thumbnail will be generated as a static image. + maxAnimateSizeBytes: 10485760 # 10MB default, 0 to disable + + # On a scale of 0 (start of animation) to 1 (end of animation), where should the thumbnailer try + # and thumbnail animated content? Defaults to 0.5 (middle of animation). + stillFrame: 0.5 + + # How many days after a thumbnail is generated before it expires and is deleted. The thumbnail + # can be regenerated safely - this just helps free up some space in your datastores. Set to + # zero or negative to disable. Defaults to disabled. + expireAfterDays: 0 + +# Controls for the rate limit functionality + +# Set this to false if rate limiting is handled at a higher level or you don't want it enabled. +matrix_media_repo_rate_limit_enabled: true + +# The number of requests per second before an IP will be rate limited. Must be a whole number. +matrix_media_repo_rate_limit_requests_per_second: 1 + +# The number of requests an IP can send at once before the rate limit is actually considered. +matrix_media_repo_rate_limit_burst: 10 + +# Identicons are generated avatars for a given username. Some clients use these to give users a +# default avatar after signing up. Identicons are not part of the official matrix spec, therefore +# this feature is completely optional. +matrix_media_repo_identicons_enabled: true + +# The quarantine media settings. + +# If true, when a thumbnail of quarantined media is requested an image will be returned. If no +# image is given in the thumbnailPath below then a generated image will be provided. This does +# not affect regular downloads of files. +matrix_media_repo_quarantine_replace_thumbnails: true + +# If true, when media which has been quarantined is requested an image will be returned. If +# no image is given in the thumbnailPath below then a generated image will be provided. This +# will replace media which is not an image (ie: quarantining a PDF will replace the PDF with +# an image). +matrix_media_repo_quarantine_replace_downloads: false + +# If provided, the given image will be returned as a thumbnail for media that is quarantined. +matrix_media_repo_quarantine_thumbnail_path: "" + +# If true, administrators of the configured homeservers may quarantine media for their server +# only. Global administrators can quarantine any media (local or remote) regardless of this +# flag. +matrix_media_repo_quarantine_allow_local_admins: true + +# The various timeouts that the media repo will use. + +# The maximum amount of time the media repo should spend trying to fetch a resource that is +# being previewed. +matrix_media_repo_timeouts_url_preview_timeout_seconds: 10 + +# The maximum amount of time the media repo will spend making remote requests to other repos +# or homeservers. This is primarily used to download media. +matrix_media_repo_timeouts_federation_timeout_seconds: 120 + +# The maximum amount of time the media repo will spend talking to your configured homeservers. +# This is usually used to verify a user's identity. +matrix_media_repo_timeouts_client_server_timeout_seconds: 30 + +# Prometheus metrics configuration +# For an example Grafana dashboard, import the following JSON: +# https://github.com/turt2live/matrix-media-repo/blob/master/docs/grafana.json + +# If true, the bindAddress and port below will serve GET /metrics for Prometheus to scrape. +matrix_media_repo_metrics_enabled: false + +# The address to listen on. Typically "127.0.0.1" or "0.0.0.0" for all interfaces. +matrix_media_repo_metrics_bind_address: "0.0.0.0" + +# The port to listen on. Cannot be the same as the general web server port. +matrix_media_repo_metrics_port: 9000 + +# Plugins are optional pieces of the media repo used to extend the functionality offered. +# Currently there are only antispam plugins, but in future there should be more options. +# Plugins are not supported on per-domain paths and are instead repo-wide. For more +# information on writing plugins, please visit #matrix-media-repo:t2bot.io on Matrix. +matrix_media_repo_plugins: + plugins: [] + + # An example OCR plugin to block images with certain text. Note that the Docker image + # for the media repo automatically ships this at /plugins/plugin_antispam_ocr +# - exec: /plugins/plugin_antispam_ocr +# config: +# # The URL to your OCR server (https://github.com/otiai10/ocrserver) +# ocrServer: "http://localhost:8080" +# # The keywords to scan for. The image must contain at least one of the keywords +# # from each list to qualify for spam. +# keywordGroups: +# - - elon +# - musk +# - elonmusk +# - - bitcoin +# # The minimum (and maximum) sizes of images to process. +# minSizeBytes: 20000 +# maxSizeBytes: 200000 +# # The types of files to process +# types: ["image/png", "image/jpeg", "image/jpg"] +# # The user ID regex to check against +# userIds: "@telegram_.*" +# # How much of the image's height, starting from the top, to consider before +# # discarding the rest. Set to 1.0 to consider the whole image. +# percentageOfHeight: 0.35 + +# Options for controlling various MSCs/unstable features of the media repo +# Sections of this config might disappear or be added over time. By default all +# features are disabled in here and must be explicitly enabled to be used. +matrix_media_repo_feature_support: + featureSupport: + # MSC2248 - Blurhash + MSC2448: + # Whether or not this MSC is enabled for use in the media repo + enabled: false + + # Maximum dimensions for converting a blurhash to an image. When no width and + # height options are supplied, the default will be half these values. + maxWidth: 1024 + maxHeight: 1024 + + # Thumbnail size in pixels to use to generate the blurhash string + thumbWidth: 64 + thumbHeight: 64 + + # The X and Y components to use. Higher numbers blur less, lower numbers blur more. + xComponents: 4 + yComponents: 3 + + # The amount of contrast to apply when converting a blurhash to an image. Lower values + # make the effect more subtle, larger values make it stronger. + punch: 1 + + # IPFS Support + # This is currently experimental and might not work at all. + IPFS: + # Whether or not IPFS support is enabled for use in the media repo. + enabled: false + + # Options for the built in IPFS daemon + builtInDaemon: + # Enable this to spawn an in-process IPFS node to use instead of a localhost + # HTTP agent. If this is disabled, the media repo will assume you have an HTTP + # IPFS agent running and accessible. Defaults to using a daemon (true). + enabled: true + + # If the Daemon is enabled, set this to the location where the IPFS files should + # be stored. If you're using Docker, this should be something like "/data/ipfs" + # so it can be mapped to a volume. + repoPath: "./ipfs" + + # Support for redis as a cache mechanism + # + # Note: Enabling Redis support will mean that the existing cache mechanism will do nothing. + # It can be safely disabled once Redis support is enabled. + # + # See docs/redis.md for more information on how this works and how to set it up. + redis: + # Whether or not use Redis instead of in-process caching. + enabled: false + + # The Redis shards that should be used by the media repo in the ring. The names of the + # shards are for your reference and have no bearing on the connection, but must be unique. + shards: + - name: "server1" + addr: ":7000" + - name: "server2" + addr: ":7001" + - name: "server3" + addr: ":7002" + +# Optional sentry (https://sentry.io/) configuration for the media repo + +# Whether or not to set up error reporting. Defaults to off. +matrix_media_repo_sentry_enabled: false + +# Get this value from the setup instructions in Sentry +matrix_media_repo_sentry_dsn: "https://examplePublicKey@ingest.sentry.io/0" + +# Optional environment flag. Defaults to an empty string. +matrix_media_repo_sentry_environment: "" + +# Whether or not to turn on sentry's built in debugging. This will increase log output. +matrix_media_repo_sentry_debug: false diff --git a/roles/custom/matrix-media-repo/tasks/main.yml b/roles/custom/matrix-media-repo/tasks/main.yml new file mode 100644 index 000000000..59f962a28 --- /dev/null +++ b/roles/custom/matrix-media-repo/tasks/main.yml @@ -0,0 +1,17 @@ +--- + +- tags: + - setup-all + - setup-media-repo + - install-all + - install-media-repo + block: + - when: matrix_media_repo_enabled | bool + ansible.builtin.include_tasks: "{{ role_path }}/tasks/setup_install.yml" + +- tags: + - setup-all + - setup-media-repo + block: + - when: not matrix_media_repo_enabled | bool + ansible.builtin.include_tasks: "{{ role_path }}/tasks/setup_uninstall.yml" diff --git a/roles/custom/matrix-media-repo/tasks/setup_install.yml b/roles/custom/matrix-media-repo/tasks/setup_install.yml new file mode 100644 index 000000000..af02d4b09 --- /dev/null +++ b/roles/custom/matrix-media-repo/tasks/setup_install.yml @@ -0,0 +1,74 @@ +--- + +- name: Ensure media-repo paths exist + ansible.builtin.file: + path: "{{ item.path }}" + state: directory + mode: 0750 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + with_items: + - path: "{{ matrix_media_repo_base_path }}" + when: true + - path: "{{ matrix_media_repo_docker_src_files_path }}" + when: "{{ matrix_media_repo_container_image_self_build }}" + when: "item.when | bool" + +- name: Ensure media-repo configuration installed + ansible.builtin.template: + src: "{{ role_path }}/templates/media-repo/media-repo.yaml.j2" + dest: "{{ matrix_media_repo_base_path }}/media-repo.yaml" + mode: 0640 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + +- name: Ensure media-repo Docker image is pulled + community.docker.docker_image: + name: "{{ matrix_media_repo_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_media_repo_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_media_repo_docker_image_force_pull }}" + when: "not matrix_media_repo_container_image_self_build | bool" + register: result + retries: "{{ devture_playbook_help_container_retries_count }}" + delay: "{{ devture_playbook_help_container_retries_delay }}" + until: result is not failed + +- when: "matrix_media_repo_container_image_self_build | bool" + block: + - name: Ensure media-repo repository is present on self-build + ansible.builtin.git: + repo: "{{ matrix_media_repo_container_image_self_build_repo }}" + dest: "{{ matrix_media_repo_docker_src_files_path }}" + version: "{{ matrix_media_repo_docker_image.split(':')[1] }}" + force: "yes" + become: true + become_user: "{{ matrix_user_username }}" + register: matrix_media_repo_git_pull_results + + - name: Check if media-repo Docker image exists + ansible.builtin.command: "{{ devture_systemd_docker_base_host_command_docker }} images --quiet --filter 'reference={{ matrix_media_repo_docker_image }}'" + register: matrix_media_repo_docker_image_check_result + changed_when: false + + # Invoking the `docker build` command here, instead of calling the `docker_image` Ansible module, + # because the latter does not support BuildKit. + # See: https://github.com/ansible-collections/community.general/issues/514 + - name: Ensure media-repo Docker image is built + ansible.builtin.command: + cmd: "{{ devture_systemd_docker_base_host_command_docker }} build -t {{ matrix_media_repo_docker_image }} {{ matrix_media_repo_docker_src_files_path }}" + environment: + DOCKER_BUILDKIT: 1 + changed_when: true + when: "matrix_media_repo_git_pull_results.changed | bool or matrix_media_repo_docker_image_check_result.stdout == ''" + +- name: Ensure media-repo container network is created + community.general.docker_network: + name: "{{ matrix_media_repo_container_network }}" + driver: bridge + +- name: Ensure matrix-media-repo.service installed + ansible.builtin.template: + src: "{{ role_path }}/templates/media-repo/systemd/matrix-media-repo.service.j2" + dest: "{{ devture_systemd_docker_base_systemd_path }}/matrix-media-repo.service" + mode: 0640 diff --git a/roles/custom/matrix-media-repo/tasks/setup_uninstall.yml b/roles/custom/matrix-media-repo/tasks/setup_uninstall.yml new file mode 100644 index 000000000..d5f08994e --- /dev/null +++ b/roles/custom/matrix-media-repo/tasks/setup_uninstall.yml @@ -0,0 +1,19 @@ +--- + +- name: Check existence of matrix-media-repo service + ansible.builtin.stat: + path: "{{ devture_systemd_docker_base_systemd_path }}/matrix-media-repo.service" + register: matrix_media_repo_service_stat + +- when: matrix_media_repo_service_stat.stat.exists | bool + block: + - name: Ensure matrix-media-repo is stopped + ansible.builtin.systemd: + name: matrix-media-repo + state: stopped + daemon_reload: true + + - name: Ensure matrix-media-repo.service doesn't exist + ansible.builtin.file: + path: "{{ devture_systemd_docker_base_systemd_path }}/matrix-media-repo.service" + state: absent diff --git a/roles/custom/matrix-media-repo/templates/grafana/media-repo.json b/roles/custom/matrix-media-repo/templates/grafana/media-repo.json new file mode 100644 index 000000000..1fb68e0a9 --- /dev/null +++ b/roles/custom/matrix-media-repo/templates/grafana/media-repo.json @@ -0,0 +1,991 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.3.1" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 9, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "hertz" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(media_http_requests_total[2m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ '{{host}}: {{method}} {{action}}' }}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(media_invalid_http_requests_total[2m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ 'Invalid Host: {{method}} {{action}}' }}", + "refId": "B" + } + ], + "title": "HTTP Requsts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "hertz" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(media_http_responses_total[2m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ '{{host}}: {{method}} {{action}} {{statusCode}}' }}", + "refId": "A" + } + ], + "title": "HTTP Responses", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 8, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "go_memstats_alloc_bytes", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "memory usage (alloc)", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "go_memstats_sys_bytes", + "interval": "", + "legendFormat": "memory usage (sys)", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "go_memstats_heap_alloc_bytes", + "interval": "", + "legendFormat": "heap usage (alloc)", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "go_memstats_heap_idle_bytes", + "interval": "", + "legendFormat": "heap usage (idle)", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "go_memstats_heap_inuse_bytes", + "interval": "", + "legendFormat": "heap usage (used)", + "refId": "E" + } + ], + "title": "Memory Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 4, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "media_cache_num_bytes_used", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ 'size of cache: {{cache}}' }}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "media_cache_num_live_bytes_used", + "interval": "", + "legendFormat": "{{ 'live size of cache: {{cache}}' }}", + "refId": "C" + } + ], + "title": "Cache Size (Bytes)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 9, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "media_cache_num_items", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ 'items in cache: {{cache}}' }}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "media_cache_num_live_items", + "interval": "", + "legendFormat": "{{ 'live items in cache: {{cache}}' }}", + "refId": "C" + } + ], + "title": "Cache Size (# of items)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "hertz" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 5, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(media_cache_hits_total[2m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ 'hits in {{cache}}' }}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(media_cache_misses_total[2m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ 'misses in {{cache}}' }}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(media_cache_evictions_total[2m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ 'evictions due to {{reason}} in {{cache}}' }}", + "refId": "C" + } + ], + "title": "Cache Operations", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "hertz" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 6, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(media_thumbnails_generated_total[2m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ '{{origin}} {{width}}x{{height}} {{method}} animated={{animated}}' }}", + "refId": "A" + } + ], + "title": "Thumbnail Generation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "hertz" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 7, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(media_downloaded_total[2m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ 'downloads from {{origin}}' }}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(media_url_previews_generated_total[2m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ 'preview with engine: {{type}}' }}", + "refId": "B" + } + ], + "title": "Resource Handling", + "type": "timeseries" + } + ], + "refresh": "1m", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Media Repo Dashboard", + "uid": "xJUZ3xfmk", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/roles/custom/matrix-media-repo/templates/media-repo/media-repo.yaml.j2 b/roles/custom/matrix-media-repo/templates/media-repo/media-repo.yaml.j2 new file mode 100644 index 000000000..0595f576d --- /dev/null +++ b/roles/custom/matrix-media-repo/templates/media-repo/media-repo.yaml.j2 @@ -0,0 +1,359 @@ +# General repo configuration +repo: + bindAddress: {{ matrix_media_repo_bind_address }} + port: {{ matrix_media_repo_port }} + + # Where to store the logs, relative to where the repo is started from. Logs will be automatically + # rotated every day and held for 14 days. To disable the repo logging to files, set this to + # "-" (including quotation marks). + # + # Note: to change the log directory you'll have to restart the repository. This setting cannot be + # live reloaded. + logDirectory: {{ matrix_media_repo_log_directory }} + + # Set to true to enable color coding in your logs. Note that this may cause escape sequences to + # appear in logs which render them unreadable, which is why colors are disabled by default. + logColors: {{ matrix_media_repo_log_colors }} + + # Set to true to enable JSON logging for consumption by things like logstash. Note that this is + # incompatible with the log color option and will always render without colors. + jsonLogs: {{ matrix_media_repo_json_logs }} + + # The log level to log at. Note that this will need to be at least "info" to receive support. + # + # Values (in increasing spam): panic | fatal | error | warn | info | debug | trace + logLevel: {{ matrix_media_repo_log_level }} + + # If true, the media repo will accept any X-Forwarded-For header without validation. In most cases + # this option should be left as "false". Note that the media repo already expects an X-Forwarded-For + # header, but validates it to ensure the IP being given makes sense. + trustAnyForwardedAddress: {{ matrix_media_repo_trust_any_forwarded_address }} + + # If false, the media repo will not use the X-Forwarded-Host header commonly added by reverse proxies. + # Typically this should remain as true, though in some circumstances it may need to be disabled. + # See https://github.com/turt2live/matrix-media-repo/issues/202 for more information. + useForwardedHost: {{ matrix_media_repo_use_forwarded_host }} + +# Options for dealing with federation +federation: + # On a per-host basis, the number of consecutive failures in calling the host before the + # media repo will back off. This defaults to 20 if not given. Note that 404 errors from + # the remote server do not count towards this. + backoffAt: {{ matrix_media_repo_federation_backoff_at }} + +# The database configuration for the media repository +# Do NOT put your homeserver's existing database credentials here. Create a new database and +# user instead. Using the same server is fine, just not the same username and database. +database: + # Currently only "postgres" is supported. + postgres: {{ matrix_media_repo_database_postgres }} + + # The database pooling options + pool: + # The maximum number of connects to hold open. More of these allow for more concurrent + # processes to happen. + maxConnections: {{ matrix_media_repo_database_max_connections }} + + # The maximum number of connects to leave idle. More of these reduces the time it takes + # to serve requests in low-traffic scenarios. + maxIdleConnections: {{ matrix_media_repo_database_max_idle_connections }} + +# The configuration for the homeservers this media repository is known to control. Servers +# not listed here will not be able to upload media. +{# +homeservers: + - name: example.org # This should match the server_name of your homeserver, and the Host header + # provided to the media repo. + csApi: "https://example.org/" # The base URL to where the homeserver can actually be reached + backoffAt: 10 # The number of consecutive failures in calling this homeserver before the + # media repository will start backing off. This defaults to 10 if not given. + adminApiKind: "matrix" # The kind of admin API the homeserver supports. If set to "matrix", + # the media repo will use the Synapse-defined endpoints under the + # unstable client-server API. When this is "synapse", the new /_synapse + # endpoints will be used instead. Unknown values are treated as the + # default, "matrix". +#} +{{ matrix_media_repo_homeservers | to_nice_yaml(indent=2, sort_keys=false) }} + +# Options for controlling how access tokens work with the media repo. It is recommended that if +# you are going to use these options that the `/logout` and `/logout/all` client-server endpoints +# be proxied through this process. They will also be called on the homeserver, and the response +# sent straight through the client - they are simply used to invalidate the cache faster for +# a particular user. Without these, the access tokens might still work for a short period of time +# after the user has already invalidated them. +# +# This will also cache errors from the homeserver. +# +# Note that when this config block is used outside of a per-domain config, all hosts will be +# subject to the same cache. This also means that application services on limited homeservers +# could be authorized on the wrong domain. +# +# *************************************************************************** +# * IT IS HIGHLY RECOMMENDED TO USE PER-DOMAIN CONFIGS WITH THIS FEATURE. * +# *************************************************************************** +{{ matrix_media_repo_access_tokens | to_nice_yaml(indent=2, sort_keys=false) }} + +# These users have full access to the administrative functions of the media repository. +# See docs/admin.md for information on what these people can do. They must belong to one of the +# configured homeservers above. +{{ matrix_media_repo_admins | to_nice_yaml(indent=2, sort_keys=false) }} + +# Shared secret auth is useful for applications building on top of the media repository, such +# as a management interface. The `token` provided here is treated as a repository administrator +# when shared secret auth is enabled: if the `token` is used in place of an access token, the' +# request will be authorized. This is not limited to any particular domain, giving applications +# the ability to use it on any configured hostname. +sharedSecretAuth: + # Set this to true to enable shared secret auth. + enabled: {{ matrix_media_repo_shared_secret_auth_enabled }} + + # Use a secure value here to prevent unauthorized access to the media repository. + token: {{ matrix_media_repo_shared_secret_auth_token }} + +# Datastores are places where media should be persisted. This isn't dedicated for just uploads: +# thumbnails and other misc data is also stored in these places. The media repo, when looking +# for a datastore to use, will always use the smallest datastore first. +{# +datastores: + - type: file + enabled: false # Enable this to set up data storage. + # Datastores can be split into many areas when handling uploads. Media is still de-duplicated + # across all datastores (local content which duplicates remote content will re-use the remote + # content's location). This option is useful if your datastore is becoming very large, or if + # you want faster storage for a particular kind of media. + # + # The kinds available are: + # thumbnails - Used to store thumbnails of media (local and remote). + # remote_media - Original copies of remote media (servers not configured by this repo). + # local_media - Original uploads for local media. + # archives - Archives of content (GDPR and similar requests). + forKinds: ["thumbnails"] + opts: + path: /var/matrix/media + + - type: s3 + enabled: false # Enable this to set up s3 uploads + forKinds: ["thumbnails", "remote_media", "local_media", "archives"] + opts: + # The s3 uploader needs a temporary location to buffer files to reduce memory usage on + # small file uploads. If the file size is unknown, the file is written to this location + # before being uploaded to s3 (then the file is deleted). If you aren't concerned about + # memory usage, set this to an empty string. + tempPath: "/tmp/mediarepo_s3_upload" + endpoint: sfo2.digitaloceanspaces.com + accessKeyId: "" + accessSecret: "" + ssl: true + bucketName: "your-media-bucket" + # An optional region for where this S3 endpoint is located. Typically not needed, though + # some providers will need this (like Scaleway). Uncomment to use. + #region: "sfo2" + + # The media repo does support an IPFS datastore, but only if the IPFS feature is enabled. If + # the feature is not enabled, this will not work. Note that IPFS support is experimental at + # the moment and not recommended for general use. + # + # NOTE: Everything you upload to IPFS will be publicly accessible, even when the media repo + # puts authentication on the download endpoints. Only use this option for cases where you + # expect your media to be publicly accessible. + - type: ipfs + enabled: false # Enable this to use IPFS support + forKinds: ["local_media"] + # The IPFS datastore currently has no options. It will use the daemon or HTTP API configured + # in the IPFS section of your main config. + opts: {} +#} +{{ matrix_media_repo_datastores | to_nice_yaml(indent=2, sort_keys=false) }} + +# Options for controlling archives. Archives are exports of a particular user's content for +# the purpose of GDPR or moving media to a different server. +archiving: + # Whether archiving is enabled or not. Default enabled. + enabled: {{ matrix_media_repo_archiving_enabled }} + # If true, users can request a copy of their own data. By default, only repository administrators + # can request a copy. + # This includes the ability for homeserver admins to request a copy of their own server's + # data, as known to the repo. + selfService: {{ matrix_media_repo_archiving_self_service }} + # The number of bytes to target per archive before breaking up the files. This is independent + # of any file upload limits and will require a similar amount of memory when performing an export. + # The file size is also a target, not a guarantee - it is possible to have files that are smaller + # or larger than the target. This is recommended to be approximately double the size of your + # file upload limit, provided there is enough memory available for the demand of exporting. + targetBytesPerPart: {{ matrix_media_repo_archiving_target_bytes_per_part }} # 200mb default + +# The file upload settings for the media repository +{{ matrix_media_repo_uploads | to_nice_yaml(indent=2, sort_keys=false) }} + +# Settings related to downloading files from the media repository +downloads: + # The maximum number of bytes to download from other servers + maxBytes: {{ matrix_media_repo_downloads_max_bytes }} # 100MB default, 0 to disable + + # The number of workers to use when downloading remote media. Raise this number if remote + # media is downloading slowly or timing out. + # + # Maximum memory usage = numWorkers multiplied by the maximum download size + # Average memory usage is dependent on how many concurrent downloads your users are doing. + numWorkers: {{ matrix_media_repo_downloads_num_workers }} + + # How long, in minutes, to cache errors related to downloading remote media. Once this time + # has passed, the media is able to be re-requested. + failureCacheMinutes: {{ matrix_media_repo_downloads_failure_cache_minutes }} + + # The cache control settings for downloads. This can help speed up downloads for users by + # keeping popular media in the cache. This cache is also used for thumbnails. + cache: + enabled: {{ matrix_media_repo_downloads_cache_enabled }} + + # The maximum size of cache to have. Higher numbers are better. + maxSizeBytes: {{ matrix_media_repo_downloads_cache_max_size_bytes }} # 1GB default + + # The maximum file size to cache. This should normally be the same size as your maximum + # upload size. + maxFileSizeBytes: {{ matrix_media_repo_downloads_cache_max_file_size_bytes }} # 100MB default + + # The number of minutes to track how many downloads a file gets + trackedMinutes: {{ matrix_media_repo_downloads_cache_tracked_minutes }} + + # The number of downloads a file must receive in the window above (trackedMinutes) in + # order to be cached. + minDownloads: {{ matrix_media_repo_downloads_cache_min_downloads }} + + # The minimum amount of time an item should remain in the cache. This prevents the cache + # from cycling out the file if it needs more room during this time. Note that the media + # repo regularly cleans out media which is past this point from the cache, so this number + # may need increasing depending on your use case. If the maxSizeBytes is reached for the + # media repo, and some cached items are still under this timer, new items will not be able + # to enter the cache. When this happens, consider raising maxSizeBytes or lowering this + # timer. + minCacheTimeSeconds: {{ matrix_media_repo_downloads_cache_min_cache_time_seconds }} + + # The minimum amount of time an item should remain outside the cache once it is removed. + minEvictedTimeSeconds: {{ matrix_media_repo_downloads_cache_min_evicted_time_seconds }} + + # How many days after a piece of remote content is downloaded before it expires. It can be + # re-downloaded on demand, this just helps free up space in your datastore. Set to zero or + # negative to disable. Defaults to disabled. + expireAfterDays: {{ matrix_media_repo_downloads_expire_after_days }} + +# URL Preview settings +{{ matrix_media_repo_url_previews | to_nice_yaml(indent=2) }} + +# The thumbnail configuration for the media repository. +{{ matrix_media_repo_thumbnails | to_nice_yaml(indent=2) }} + +# Controls for the rate limit functionality +rateLimit: + # Set this to false if rate limiting is handled at a higher level or you don't want it enabled. + enabled: {{ matrix_media_repo_rate_limit_enabled }} + + # The number of requests per second before an IP will be rate limited. Must be a whole number. + requestsPerSecond: {{ matrix_media_repo_rate_limit_requests_per_second }} + + # The number of requests an IP can send at once before the rate limit is actually considered. + burst: {{ matrix_media_repo_rate_limit_burst }} + +# Identicons are generated avatars for a given username. Some clients use these to give users a +# default avatar after signing up. Identicons are not part of the official matrix spec, therefore +# this feature is completely optional. +identicons: + enabled: {{ matrix_media_repo_identicons_enabled }} + +# The quarantine media settings. +quarantine: + # If true, when a thumbnail of quarantined media is requested an image will be returned. If no + # image is given in the thumbnailPath below then a generated image will be provided. This does + # not affect regular downloads of files. + replaceThumbnails: {{ matrix_media_repo_quarantine_replace_thumbnails }} + + # If true, when media which has been quarantined is requested an image will be returned. If + # no image is given in the thumbnailPath below then a generated image will be provided. This + # will replace media which is not an image (ie: quarantining a PDF will replace the PDF with + # an image). + replaceDownloads: {{ matrix_media_repo_quarantine_replace_downloads }} + + # If provided, the given image will be returned as a thumbnail for media that is quarantined. + #thumbnailPath: "/path/to/thumbnail.png" + thumbnailPath: {{ "" if matrix_media_repo_quarantine_thumbnail_path == "" else matrix_media_repo_quarantine_thumbnail_path }} + + # If true, administrators of the configured homeservers may quarantine media for their server + # only. Global administrators can quarantine any media (local or remote) regardless of this + # flag. + allowLocalAdmins: {{ matrix_media_repo_quarantine_allow_local_admins }} + +# The various timeouts that the media repo will use. +timeouts: + # The maximum amount of time the media repo should spend trying to fetch a resource that is + # being previewed. + urlPreviewTimeoutSeconds: {{ matrix_media_repo_timeouts_url_preview_timeout_seconds }} + + # The maximum amount of time the media repo will spend making remote requests to other repos + # or homeservers. This is primarily used to download media. + federationTimeoutSeconds: {{ matrix_media_repo_timeouts_federation_timeout_seconds }} + + # The maximum amount of time the media repo will spend talking to your configured homeservers. + # This is usually used to verify a user's identity. + clientServerTimeoutSeconds: {{ matrix_media_repo_timeouts_client_server_timeout_seconds }} + +# Prometheus metrics configuration +# For an example Grafana dashboard, import the following JSON: +# https://github.com/turt2live/matrix-media-repo/blob/master/docs/grafana.json +metrics: + # If true, the bindAddress and port below will serve GET /metrics for Prometheus to scrape. + enabled: {{ matrix_media_repo_metrics_enabled }} + + # The address to listen on. Typically "127.0.0.1" or "0.0.0.0" for all interfaces. + bindAddress: {{ matrix_media_repo_metrics_bind_address }} + + # The port to listen on. Cannot be the same as the general web server port. + port: {{ matrix_media_repo_metrics_port }} + +# Plugins are optional pieces of the media repo used to extend the functionality offered. +# Currently there are only antispam plugins, but in future there should be more options. +# Plugins are not supported on per-domain paths and are instead repo-wide. For more +# information on writing plugins, please visit #matrix-media-repo:t2bot.io on Matrix. +{{ matrix_media_repo_plugins | to_nice_yaml(indent=2) }} + + # An example OCR plugin to block images with certain text. Note that the Docker image + # for the media repo automatically ships this at /plugins/plugin_antispam_ocr +# - exec: /plugins/plugin_antispam_ocr +# config: +# # The URL to your OCR server (https://github.com/otiai10/ocrserver) +# ocrServer: "http://localhost:8080" +# # The keywords to scan for. The image must contain at least one of the keywords +# # from each list to qualify for spam. +# keywordGroups: +# - - elon +# - musk +# - elonmusk +# - - bitcoin +# # The minimum (and maximum) sizes of images to process. +# minSizeBytes: 20000 +# maxSizeBytes: 200000 +# # The types of files to process +# types: ["image/png", "image/jpeg", "image/jpg"] +# # The user ID regex to check against +# userIds: "@telegram_.*" +# # How much of the image's height, starting from the top, to consider before +# # discarding the rest. Set to 1.0 to consider the whole image. +# percentageOfHeight: 0.35 + +# Options for controlling various MSCs/unstable features of the media repo +# Sections of this config might disappear or be added over time. By default all +# features are disabled in here and must be explicitly enabled to be used. +{{ matrix_media_repo_feature_support | to_nice_yaml(indent=2) }} + +# Optional sentry (https://sentry.io/) configuration for the media repo +sentry: + # Whether or not to set up error reporting. Defaults to off. + enabled: {{ matrix_media_repo_sentry_enabled }} + + # Get this value from the setup instructions in Sentry + dsn: {{ matrix_media_repo_sentry_dsn }} + + # Optional environment flag. Defaults to an empty string. + environment: {{ "" if matrix_media_repo_sentry_environment == "" else matrix_media_repo_sentry_environment }} + + # Whether or not to turn on sentry's built in debugging. This will increase log output. + debug: {{ matrix_media_repo_sentry_debug }} \ No newline at end of file diff --git a/roles/custom/matrix-media-repo/templates/media-repo/systemd/matrix-media-repo.service.j2 b/roles/custom/matrix-media-repo/templates/media-repo/systemd/matrix-media-repo.service.j2 new file mode 100644 index 000000000..ac33e3fee --- /dev/null +++ b/roles/custom/matrix-media-repo/templates/media-repo/systemd/matrix-media-repo.service.j2 @@ -0,0 +1,52 @@ +#jinja2: lstrip_blocks: "True" +[Unit] +Description=Matrix media-repo +{% for service in matrix_media_repo_systemd_required_services_list %} +Requires={{ service }} +After={{ service }} +{% endfor %} +{% for service in matrix_media_repo_systemd_wanted_services_list %} +Wants={{ service }} +{% endfor %} +DefaultDependencies=no + +[Service] +Type=simple +Environment="HOME={{ devture_systemd_docker_base_systemd_unit_home_path }}" +ExecStartPre=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} kill matrix-media-repo 2>/dev/null || true' +ExecStartPre=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} rm matrix-media-repo 2>/dev/null || true' + +ExecStartPre={{ devture_systemd_docker_base_host_command_docker }} create \ + --rm \ + --name=matrix-media-repo \ + --log-driver=none \ + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ + --cap-drop=ALL \ + --network={{ matrix_docker_network }} \ + -p {{ matrix_media_repo_port }}:{{ matrix_media_repo_port }} \ + {% if matrix_media_repo_metrics_enabled %} + -p {{ matrix_media_repo_metrics_port }}:{{ matrix_media_repo_metrics_port }} \ + {% endif %} + -v {{ matrix_media_repo_base_path }}:/data:z \ + --workdir='/data' \ + --entrypoint='media_repo' \ + {% for arg in matrix_media_repo_container_extra_arguments %} + {{ arg }} \ + {% endfor %} + {{ matrix_media_repo_docker_image }} + +{% for network in matrix_media_repo_container_additional_networks %} +ExecStartPre={{ devture_systemd_docker_base_host_command_docker }} network connect {{ network }} matrix-media-repo +{% endfor %} + +ExecStart={{ devture_systemd_docker_base_host_command_docker }} start --attach matrix-media-repo + +ExecStop=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} kill matrix-media-repo 2>/dev/null || true' +ExecStop=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} rm matrix-media-repo 2>/dev/null || true' +ExecReload={{ devture_systemd_docker_base_host_command_docker }} exec matrix-media-repo /bin/sh -c 'kill -HUP 1' +Restart=always +RestartSec=30 +SyslogIdentifier=matrix-media-repo + +[Install] +WantedBy=multi-user.target diff --git a/roles/custom/matrix-nginx-proxy/defaults/main.yml b/roles/custom/matrix-nginx-proxy/defaults/main.yml index 8ad11b375..d8d73a869 100644 --- a/roles/custom/matrix-nginx-proxy/defaults/main.yml +++ b/roles/custom/matrix-nginx-proxy/defaults/main.yml @@ -333,6 +333,11 @@ matrix_nginx_proxy_proxy_matrix_identity_api_enabled: false matrix_nginx_proxy_proxy_matrix_identity_api_addr_with_container: "matrix-ma1sd:{{ matrix_ma1sd_container_port }}" matrix_nginx_proxy_proxy_matrix_identity_api_addr_sans_container: "127.0.0.1:{{ matrix_ma1sd_container_port }}" +# Controls whether proxying for the media repo (`/_matrix/media`) should be done (on the matrix domain) +matrix_nginx_proxy_proxy_media_repo_enabled: false +matrix_nginx_proxy_proxy_media_repo_addr_with_container: "matrix-media-repo:{{ matrix_media_repo_port }}" +matrix_nginx_proxy_proxy_media_repo_addr_sans_container: "127.0.0.1:{{ matrix_media_repo_port }}" + # The addresses where the Matrix Client API is. # Certain extensions (like matrix-corporal) may override this in order to capture all traffic. matrix_nginx_proxy_proxy_matrix_client_api_addr_with_container: "matrix-nginx-proxy:12080" diff --git a/roles/custom/matrix-nginx-proxy/templates/nginx/conf.d/matrix-domain.conf.j2 b/roles/custom/matrix-nginx-proxy/templates/nginx/conf.d/matrix-domain.conf.j2 index 63d45bc63..8ecaf64e9 100644 --- a/roles/custom/matrix-nginx-proxy/templates/nginx/conf.d/matrix-domain.conf.j2 +++ b/roles/custom/matrix-nginx-proxy/templates/nginx/conf.d/matrix-domain.conf.j2 @@ -94,6 +94,96 @@ } {% endif %} + {% if matrix_nginx_proxy_proxy_media_repo_enabled %} + # Redirect all media endpoints to the media-repo + location ^~ /_matrix/media { + {% if matrix_nginx_proxy_enabled %} + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; + set $backend "{{ matrix_nginx_proxy_proxy_media_repo_addr_with_container }}"; + proxy_pass http://$backend; + {% else %} + {# Generic configuration for use outside of our container setup #} + proxy_pass http://{{ matrix_nginx_proxy_proxy_media_repo_addr_sans_container }}; + {% endif %} + + # Make sure this matches your homeserver in media-repo.yaml + # You may have to manually specify it if using delegation or the + # incoming Host doesn't match. + proxy_set_header Host $host; + + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $remote_addr; + } + + # Redirect other endpoints registered by the media-repo to its container + # /_matrix/client/r0/logout + # /_matrix/client/r0/logout/all + location ^~ /_matrix/client/(r0|v1|v3|unstable)/(logout|logout/all) { + {% if matrix_nginx_proxy_enabled %} + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; + set $backend "{{ matrix_nginx_proxy_proxy_media_repo_addr_with_container }}"; + proxy_pass http://$backend; + {% else %} + {# Generic configuration for use outside of our container setup #} + proxy_pass http://{{ matrix_nginx_proxy_proxy_media_repo_addr_sans_container }}; + {% endif %} + + # Make sure this matches your homeserver in media-repo.yaml + # You may have to manually specify it if using delegation or the + # incoming Host doesn't match. + proxy_set_header Host $host; + + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $remote_addr; + } + + # Redirect other endpoints registered by the media-repo to its container + # /_matrix/client/r0/admin/purge_media_cache + # /_matrix/client/r0/admin/quarantine_media/{roomId:[^/]+} + location ^~ /_matrix/client/(r0|v1|v3|unstable)/admin/(purge_media_cache|quarantine_media/.*) { + {% if matrix_nginx_proxy_enabled %} + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; + set $backend "{{ matrix_nginx_proxy_proxy_media_repo_addr_with_container }}"; + proxy_pass http://$backend; + {% else %} + {# Generic configuration for use outside of our container setup #} + proxy_pass http://{{ matrix_nginx_proxy_proxy_media_repo_addr_sans_container }}; + {% endif %} + + # Make sure this matches your homeserver in media-repo.yaml + # You may have to manually specify it if using delegation or the + # incoming Host doesn't match. + proxy_set_header Host $host; + + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $remote_addr; + } + + # Redirect other endpoints registered by the media-repo to its container + location ^~ /_matrix/client/unstable/io.t2bot.media { + {% if matrix_nginx_proxy_enabled %} + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; + set $backend "{{ matrix_nginx_proxy_proxy_media_repo_addr_with_container }}"; + proxy_pass http://$backend; + {% else %} + {# Generic configuration for use outside of our container setup #} + proxy_pass http://{{ matrix_nginx_proxy_proxy_media_repo_addr_sans_container }}; + {% endif %} + + # Make sure this matches your homeserver in media-repo.yaml + # You may have to manually specify it if using delegation or the + # incoming Host doesn't match. + proxy_set_header Host $host; + + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endif %} + {% if matrix_nginx_proxy_proxy_matrix_user_directory_search_enabled %} location ^~ /_matrix/client/r0/user_directory/search { {% if matrix_nginx_proxy_enabled %} diff --git a/roles/custom/matrix-prometheus-services-connect/defaults/main.yml b/roles/custom/matrix-prometheus-services-connect/defaults/main.yml index 748f88c5c..f06d8da4e 100644 --- a/roles/custom/matrix-prometheus-services-connect/defaults/main.yml +++ b/roles/custom/matrix-prometheus-services-connect/defaults/main.yml @@ -142,3 +142,23 @@ matrix_prometheus_services_connect_scraper_nginxlog_scrape_configs: | 'static_configs': matrix_prometheus_services_connect_scraper_nginxlog_static_configs, }] }} + +# Controls whether media-repo shall be scraped +matrix_prometheus_services_connect_scraper_media_repo_enabled: false +matrix_prometheus_services_connect_scraper_media_repo_job_name: media-repo +matrix_prometheus_services_connect_scraper_media_repo_metrics_path: /metrics +matrix_prometheus_services_connect_scraper_media_repo_scrape_interval: 15s +matrix_prometheus_services_connect_scraper_media_repo_scrape_timeout: 15s +matrix_prometheus_services_connect_scraper_media_repo_static_configs: "{{ [{'targets': [matrix_prometheus_services_connect_scraper_media_repo_static_configs_target]}] }}" +matrix_prometheus_services_connect_scraper_media_repo_static_configs_target: '' +# The final scrape config for the media-repo scraper +matrix_prometheus_services_connect_scraper_media_repo_scrape_configs: | + {{ + [{ + 'job_name': matrix_prometheus_services_connect_scraper_media_repo_job_name, + 'metrics_path': matrix_prometheus_services_connect_scraper_media_repo_metrics_path, + 'scrape_interval': matrix_prometheus_services_connect_scraper_media_repo_scrape_interval, + 'scrape_timeout': matrix_prometheus_services_connect_scraper_media_repo_scrape_timeout, + 'static_configs': matrix_prometheus_services_connect_scraper_media_repo_static_configs, + }] + }} diff --git a/roles/custom/matrix-synapse/defaults/main.yml b/roles/custom/matrix-synapse/defaults/main.yml index 964fde115..96bb42561 100644 --- a/roles/custom/matrix-synapse/defaults/main.yml +++ b/roles/custom/matrix-synapse/defaults/main.yml @@ -633,14 +633,14 @@ matrix_synapse_workers_federation_sender_workers_metrics_range_start: 19400 # Adjusting this value manually is generally not necessary. matrix_synapse_federation_sender_instances: [] -matrix_synapse_workers_media_repository_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['media_repository_workers_count'] }}" +matrix_synapse_workers_media_repository_workers_count: "{{ matrix_synapse_workers_presets[matrix_synapse_workers_preset]['media_repository_workers_count'] if not matrix_synapse_ext_media_repo_enabled else 0 }}" matrix_synapse_workers_media_repository_workers_port_range_start: 18551 matrix_synapse_workers_media_repository_workers_metrics_range_start: 19551 # matrix_synapse_enable_media_repo controls if the main Synapse process should serve media repository endpoints or if it should be left to media_repository workers (see `matrix_synapse_workers_media_repository_workers_count`). # This is enabled if workers are disabled, or if they are enabled, but there are no media repository workers. # Adjusting this value manually is generally not necessary. -matrix_synapse_enable_media_repo: "{{ not matrix_synapse_workers_enabled or (matrix_synapse_workers_enabled_list | selectattr('type', 'equalto', 'media_repository') | list | length == 0) }}" +matrix_synapse_enable_media_repo: "{{ not matrix_synapse_ext_media_repo_enabled and (not matrix_synapse_workers_enabled or (matrix_synapse_workers_enabled_list | selectattr('type', 'equalto', 'media_repository') | list | length == 0)) }}" # matrix_synapse_media_instance_running_background_jobs populates the `media_instance_running_background_jobs` Synapse configuration used when Synapse workers are in use (`matrix_synapse_workers_enabled`). # `media_instance_running_background_jobs` is meant to point to a single media-repository worker, which is dedicated to running background tasks that maintain the media repository. @@ -901,6 +901,8 @@ matrix_synapse_ext_synapse_s3_storage_provider_update_db_day_count: 0 # This is a systemd timer OnCalendar definition. Learn more here: https://man.archlinux.org/man/systemd.time.7#CALENDAR_EVENTS matrix_synapse_ext_synapse_s3_storage_provider_periodic_migration_schedule: '*-*-* 05:00:00' +matrix_synapse_ext_media_repo_enabled: false + matrix_s3_media_store_enabled: false matrix_s3_media_store_custom_endpoint_enabled: false matrix_s3_goofys_docker_image: "{{ matrix_s3_goofys_docker_image_name_prefix }}ewoutp/goofys:latest" diff --git a/setup.yml b/setup.yml index 266f3b9e5..c7a6a3639 100644 --- a/setup.yml +++ b/setup.yml @@ -113,6 +113,7 @@ - galaxy/ntfy - custom/matrix-nginx-proxy - custom/matrix-coturn + - custom/matrix-media-repo - role: galaxy/auxiliary From 036c823a061eb120994a33cef71aa35eecbe8223 Mon Sep 17 00:00:00 2001 From: QEDeD Date: Wed, 12 Jul 2023 18:10:47 +0200 Subject: [PATCH 12/43] Explain the option of manually defining your public IP in the documentation for Coturn I was very surprised by the fact that a rather important configuration for coturn was "hidden" in the Hosts file, but not mentioned here. Therefore my suggestion is to explicitly mention it here, as I believe that is that natural for people to look. --- docs/configuring-playbook-turn.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/configuring-playbook-turn.md b/docs/configuring-playbook-turn.md index df5419bd4..7b2c58bdb 100644 --- a/docs/configuring-playbook-turn.md +++ b/docs/configuring-playbook-turn.md @@ -15,6 +15,13 @@ matrix_coturn_enabled: false In that case, Synapse would not point to any Coturn servers and audio/video call functionality may fail. +## Manually defining your public IP +In the `hosts` file we explicitly ask for your server's external IP address when defining `ansible_host`, because the same value is used for configuring Coturn. +If you'd rather use a local IP for `ansible_host`, make sure to set up `matrix_coturn_turn_external_ip_address` replacing `YOUR_PUBLIC_IP` with the pubic IP used by the server. + +```yaml +matrix_coturn_turn_external_ip_address: "YOUR_PUBLIC_IP" +``` ## Using your own external Coturn server From b7c0690542094fb2f57b499b5f7561e47cdb5431 Mon Sep 17 00:00:00 2001 From: QEDeD Date: Wed, 12 Jul 2023 18:25:20 +0200 Subject: [PATCH 13/43] Explicitly refer users to the relevant roles file Explicitly refer users to the relevant roles file so people know where to find the additional configuration options. --- docs/configuring-playbook-turn.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/configuring-playbook-turn.md b/docs/configuring-playbook-turn.md index 7b2c58bdb..365fea4b1 100644 --- a/docs/configuring-playbook-turn.md +++ b/docs/configuring-playbook-turn.md @@ -47,3 +47,6 @@ jitsi_web_stun_servers: - stun:HOSTNAME_OR_IP:PORT ``` You can put multiple host/port combinations if you like. + +## Further variables and configuration options +To see all the available configuration options, check roles/custom/matrix-coturn/defaults/main.yml From 4ee26fab2f8a80f2d9062b9e36d639e6e1d8100c Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 13 Jul 2023 12:29:08 +0300 Subject: [PATCH 14/43] Upgrade postgres_backup --- requirements.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.yml b/requirements.yml index a8c398215..b1c4751fc 100644 --- a/requirements.yml +++ b/requirements.yml @@ -18,7 +18,7 @@ - src: git+https://github.com/devture/com.devture.ansible.role.postgres.git version: v15.3-0 - src: git+https://github.com/devture/com.devture.ansible.role.postgres_backup.git - version: 8e9ec48a09284c84704d7a2dce17da35f181574d + version: a0cc7c1c696872ba8880d9c5e5a54098de825030 - src: git+https://github.com/devture/com.devture.ansible.role.systemd_docker_base.git version: v1.0.0-0 - src: git+https://github.com/devture/com.devture.ansible.role.systemd_service_manager.git From 3f2b14f63060fcb5c3f465e565e0dd39079e3e50 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 13 Jul 2023 12:32:51 +0300 Subject: [PATCH 15/43] Upgrade backup-borg (v1.2.4-1.7.15-0 -> v1.2.4-1.7.15-1) --- requirements.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.yml b/requirements.yml index b1c4751fc..bac5fea32 100644 --- a/requirements.yml +++ b/requirements.yml @@ -4,7 +4,7 @@ version: v1.0.0-1 name: auxiliary - src: git+https://gitlab.com/etke.cc/roles/backup_borg.git - version: v1.2.4-1.7.15-0 + version: v1.2.4-1.7.15-1 - src: git+https://github.com/devture/com.devture.ansible.role.container_socket_proxy.git version: v0.1.1-2 - src: git+https://github.com/devture/com.devture.ansible.role.docker_sdk_for_python.git From facaeb5abe33017ebe114000b1c638d4bd0f9c87 Mon Sep 17 00:00:00 2001 From: Michael Hollister Date: Thu, 13 Jul 2023 11:03:36 -0500 Subject: [PATCH 16/43] Document purpose of media_repo variable Co-authored-by: Slavi Pantaleev --- roles/custom/matrix-synapse/defaults/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/custom/matrix-synapse/defaults/main.yml b/roles/custom/matrix-synapse/defaults/main.yml index 96bb42561..2cd119629 100644 --- a/roles/custom/matrix-synapse/defaults/main.yml +++ b/roles/custom/matrix-synapse/defaults/main.yml @@ -901,6 +901,8 @@ matrix_synapse_ext_synapse_s3_storage_provider_update_db_day_count: 0 # This is a systemd timer OnCalendar definition. Learn more here: https://man.archlinux.org/man/systemd.time.7#CALENDAR_EVENTS matrix_synapse_ext_synapse_s3_storage_provider_periodic_migration_schedule: '*-*-* 05:00:00' +# Specifies whether an external media repository is enabled. +# If it is, the Synapse media repo and media-repo workers will be disabled automatically. matrix_synapse_ext_media_repo_enabled: false matrix_s3_media_store_enabled: false From d565c1607b989d241b2eef09e520046a34497830 Mon Sep 17 00:00:00 2001 From: Michael Hollister Date: Thu, 13 Jul 2023 11:06:59 -0500 Subject: [PATCH 17/43] Replaced hard coded string with identifier variable Co-authored-by: Slavi Pantaleev --- group_vars/matrix_servers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index cc60b9758..d5184e233 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -2671,7 +2671,7 @@ matrix_nginx_proxy_proxy_matrix_identity_api_addr_with_container: "matrix-ma1sd: matrix_nginx_proxy_proxy_matrix_identity_api_addr_sans_container: "127.0.0.1:{{ matrix_ma1sd_container_port }}" matrix_nginx_proxy_proxy_media_repo_enabled: "{{ matrix_media_repo_enabled }}" -matrix_nginx_proxy_proxy_media_repo_addr_with_container: "matrix-media-repo:{{ matrix_media_repo_port }}" +matrix_nginx_proxy_proxy_media_repo_addr_with_container: "{{ matrix_media_repo_identifier }}:{{ matrix_media_repo_port }}" matrix_nginx_proxy_proxy_media_repo_addr_sans_container: "127.0.0.1:{{ matrix_media_repo_port }}" # By default, we do TLS termination for the Matrix Federation API (port 8448) at matrix-nginx-proxy. From 6365118067c69a136300cfe763a9aec87fc99ff4 Mon Sep 17 00:00:00 2001 From: Michael Hollister Date: Thu, 13 Jul 2023 12:23:48 -0500 Subject: [PATCH 18/43] Removed additional logging to filesystem by default --- roles/custom/matrix-media-repo/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/custom/matrix-media-repo/defaults/main.yml b/roles/custom/matrix-media-repo/defaults/main.yml index 87ffcbe84..707a9e9a5 100644 --- a/roles/custom/matrix-media-repo/defaults/main.yml +++ b/roles/custom/matrix-media-repo/defaults/main.yml @@ -55,7 +55,7 @@ matrix_media_repo_port: 8000 # # Note: to change the log directory you'll have to restart the repository. This setting cannot be # live reloaded. -matrix_media_repo_log_directory: logs +matrix_media_repo_log_directory: "{{ '\"-\"' }}" # Set to true to enable color coding in your logs. Note that this may cause escape sequences to # appear in logs which render them unreadable, which is why colors are disabled by default. From b9f5aa034428d600cacd1b936d9ddbbdfcb1b3a3 Mon Sep 17 00:00:00 2001 From: Michael Hollister Date: Thu, 13 Jul 2023 13:23:34 -0500 Subject: [PATCH 19/43] Removed exposing ports by default --- roles/custom/matrix-media-repo/defaults/main.yml | 10 ++++++++++ .../media-repo/systemd/matrix-media-repo.service.j2 | 8 +++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/roles/custom/matrix-media-repo/defaults/main.yml b/roles/custom/matrix-media-repo/defaults/main.yml index 707a9e9a5..d1790a8b2 100644 --- a/roles/custom/matrix-media-repo/defaults/main.yml +++ b/roles/custom/matrix-media-repo/defaults/main.yml @@ -33,6 +33,16 @@ matrix_media_repo_container_network: "{{ matrix_docker_network }}" # Use this to expose this container to another reverse proxy, which runs in a different container network. matrix_media_repo_container_additional_networks: [] +# Controls whether the matrix-media-repo container exposes its HTTP port (tcp/8000 in the container). +# +# Takes an ":" or "" value (e.g. "127.0.0.1:8000"), or empty string to not expose. +matrix_media_repo_container_http_host_bind_port: "" + +# Controls whether the matrix-media-repo container exposes its metrics port (tcp/9000 in the container). +# +# Takes an ":" or "" value (e.g. "127.0.0.1:9000"), or empty string to not expose. +matrix_media_repo_container_metrics_host_bind_port: "" + # Extra arguments for the Docker container matrix_media_repo_container_extra_arguments: [] diff --git a/roles/custom/matrix-media-repo/templates/media-repo/systemd/matrix-media-repo.service.j2 b/roles/custom/matrix-media-repo/templates/media-repo/systemd/matrix-media-repo.service.j2 index ac33e3fee..091337962 100644 --- a/roles/custom/matrix-media-repo/templates/media-repo/systemd/matrix-media-repo.service.j2 +++ b/roles/custom/matrix-media-repo/templates/media-repo/systemd/matrix-media-repo.service.j2 @@ -23,9 +23,11 @@ ExecStartPre={{ devture_systemd_docker_base_host_command_docker }} create \ --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ --cap-drop=ALL \ --network={{ matrix_docker_network }} \ - -p {{ matrix_media_repo_port }}:{{ matrix_media_repo_port }} \ - {% if matrix_media_repo_metrics_enabled %} - -p {{ matrix_media_repo_metrics_port }}:{{ matrix_media_repo_metrics_port }} \ + {% if matrix_media_repo_container_http_host_bind_port %} + -p {{ matrix_media_repo_container_http_host_bind_port }}:{{ matrix_media_repo_port }} \ + {% endif %} + {% if matrix_media_repo_metrics_enabled and matrix_media_repo_container_metrics_host_bind_port %} + -p {{ matrix_media_repo_container_metrics_host_bind_port }}:{{ matrix_media_repo_metrics_port }} \ {% endif %} -v {{ matrix_media_repo_base_path }}:/data:z \ --workdir='/data' \ From ecb24dcaabafcf4ceccd175bb4b437475c154740 Mon Sep 17 00:00:00 2001 From: Michael Hollister Date: Thu, 13 Jul 2023 15:02:21 -0500 Subject: [PATCH 20/43] Changed volume mounting and directory structure --- roles/custom/matrix-media-repo/defaults/main.yml | 2 ++ .../matrix-media-repo/tasks/setup_install.yml | 16 +++++++++++++++- .../templates/media-repo/env.j2 | 1 + .../systemd/matrix-media-repo.service.j2 | 4 +++- 4 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 roles/custom/matrix-media-repo/templates/media-repo/env.j2 diff --git a/roles/custom/matrix-media-repo/defaults/main.yml b/roles/custom/matrix-media-repo/defaults/main.yml index d1790a8b2..5be31c4b3 100644 --- a/roles/custom/matrix-media-repo/defaults/main.yml +++ b/roles/custom/matrix-media-repo/defaults/main.yml @@ -17,6 +17,8 @@ matrix_media_repo_docker_image_tag: "v1.2.13" matrix_media_repo_docker_image_force_pull: "{{ matrix_media_repo_docker_image.endswith(':latest') }}" matrix_media_repo_base_path: "{{ matrix_base_data_path }}/media-repo" +matrix_media_repo_config_path: "{{ matrix_media_repo_base_path }}/config" +matrix_media_repo_data_path: "{{ matrix_media_repo_base_path }}/data" matrix_media_repo_docker_src_files_path: "{{ matrix_media_repo_base_path }}/docker-src" # List of systemd services that matrix-conduit.service depends on diff --git a/roles/custom/matrix-media-repo/tasks/setup_install.yml b/roles/custom/matrix-media-repo/tasks/setup_install.yml index af02d4b09..5e7399059 100644 --- a/roles/custom/matrix-media-repo/tasks/setup_install.yml +++ b/roles/custom/matrix-media-repo/tasks/setup_install.yml @@ -10,14 +10,28 @@ with_items: - path: "{{ matrix_media_repo_base_path }}" when: true + - path: "{{ matrix_media_repo_config_path }}" + when: true + - path: "{{ matrix_media_repo_data_path }}" + when: true - path: "{{ matrix_media_repo_docker_src_files_path }}" when: "{{ matrix_media_repo_container_image_self_build }}" when: "item.when | bool" +- name: Ensure media-repo support files installed + ansible.builtin.template: + src: "{{ role_path }}/templates/media-repo/{{ item }}.j2" + dest: "{{ matrix_media_repo_base_path }}/{{ item }}" + mode: 0640 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + with_items: + - env + - name: Ensure media-repo configuration installed ansible.builtin.template: src: "{{ role_path }}/templates/media-repo/media-repo.yaml.j2" - dest: "{{ matrix_media_repo_base_path }}/media-repo.yaml" + dest: "{{ matrix_media_repo_config_path }}/media-repo.yaml" mode: 0640 owner: "{{ matrix_user_username }}" group: "{{ matrix_user_groupname }}" diff --git a/roles/custom/matrix-media-repo/templates/media-repo/env.j2 b/roles/custom/matrix-media-repo/templates/media-repo/env.j2 new file mode 100644 index 000000000..8b26f77d7 --- /dev/null +++ b/roles/custom/matrix-media-repo/templates/media-repo/env.j2 @@ -0,0 +1 @@ +REPO_CONFIG=/config/media-repo.yaml diff --git a/roles/custom/matrix-media-repo/templates/media-repo/systemd/matrix-media-repo.service.j2 b/roles/custom/matrix-media-repo/templates/media-repo/systemd/matrix-media-repo.service.j2 index 091337962..33b5199a6 100644 --- a/roles/custom/matrix-media-repo/templates/media-repo/systemd/matrix-media-repo.service.j2 +++ b/roles/custom/matrix-media-repo/templates/media-repo/systemd/matrix-media-repo.service.j2 @@ -23,13 +23,15 @@ ExecStartPre={{ devture_systemd_docker_base_host_command_docker }} create \ --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ --cap-drop=ALL \ --network={{ matrix_docker_network }} \ + --env-file={{ matrix_media_repo_base_path }}/env \ {% if matrix_media_repo_container_http_host_bind_port %} -p {{ matrix_media_repo_container_http_host_bind_port }}:{{ matrix_media_repo_port }} \ {% endif %} {% if matrix_media_repo_metrics_enabled and matrix_media_repo_container_metrics_host_bind_port %} -p {{ matrix_media_repo_container_metrics_host_bind_port }}:{{ matrix_media_repo_metrics_port }} \ {% endif %} - -v {{ matrix_media_repo_base_path }}:/data:z \ + --mount type=bind,src={{ matrix_media_repo_config_path }},dst=/config,ro \ + --mount type=bind,src={{ matrix_media_repo_data_path }},dst=/data \ --workdir='/data' \ --entrypoint='media_repo' \ {% for arg in matrix_media_repo_container_extra_arguments %} From c043463ee18fac2a080c6af0c8f41eef88936856 Mon Sep 17 00:00:00 2001 From: Michael Hollister Date: Thu, 13 Jul 2023 15:19:12 -0500 Subject: [PATCH 21/43] Fixed conflicting default values for enabling media-repo --- roles/custom/matrix-media-repo/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/custom/matrix-media-repo/defaults/main.yml b/roles/custom/matrix-media-repo/defaults/main.yml index 5be31c4b3..a89ea8a58 100644 --- a/roles/custom/matrix-media-repo/defaults/main.yml +++ b/roles/custom/matrix-media-repo/defaults/main.yml @@ -5,7 +5,7 @@ # with the specification. # See: https://github.com/turt2live/matrix-media-repo -matrix_media_repo_enabled: true +matrix_media_repo_enabled: false matrix_media_repo_container_image_self_build: false matrix_media_repo_container_image_self_build_repo: "https://github.com/turt2live/matrix-media-repo.git" From d120b0c153e3313171017baf89d1c97a5e5b51e1 Mon Sep 17 00:00:00 2001 From: Michael Hollister Date: Thu, 13 Jul 2023 21:16:07 -0500 Subject: [PATCH 22/43] Updated code to use identifier variable --- .../custom/matrix-media-repo/defaults/main.yml | 9 +++++++-- .../matrix-media-repo/tasks/setup_install.yml | 4 ++-- .../tasks/setup_uninstall.yml | 12 ++++++------ .../systemd/matrix-media-repo.service.j2 | 18 +++++++++--------- 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/roles/custom/matrix-media-repo/defaults/main.yml b/roles/custom/matrix-media-repo/defaults/main.yml index a89ea8a58..a08ad0418 100644 --- a/roles/custom/matrix-media-repo/defaults/main.yml +++ b/roles/custom/matrix-media-repo/defaults/main.yml @@ -7,6 +7,11 @@ matrix_media_repo_enabled: false +# matrix_media_repo_identifier controls the identifier of this media-repo instance, which influences: +# - the default storage path +# - the names of systemd services +matrix_media_repo_identifier: matrix-media-repo + matrix_media_repo_container_image_self_build: false matrix_media_repo_container_image_self_build_repo: "https://github.com/turt2live/matrix-media-repo.git" @@ -16,7 +21,7 @@ matrix_media_repo_docker_image_name_prefix: "{{ 'localhost/' if matrix_media_rep matrix_media_repo_docker_image_tag: "v1.2.13" matrix_media_repo_docker_image_force_pull: "{{ matrix_media_repo_docker_image.endswith(':latest') }}" -matrix_media_repo_base_path: "{{ matrix_base_data_path }}/media-repo" +matrix_media_repo_base_path: "{{ matrix_base_data_path }}/{{ matrix_media_repo_identifier }}" matrix_media_repo_config_path: "{{ matrix_media_repo_base_path }}/config" matrix_media_repo_data_path: "{{ matrix_media_repo_base_path }}/data" matrix_media_repo_docker_src_files_path: "{{ matrix_media_repo_base_path }}/docker-src" @@ -104,7 +109,7 @@ matrix_media_repo_federation_backoff_at: 20 # user instead. Using the same server is fine, just not the same username and database. matrix_media_repo_database_username: "matrix_media_repo" matrix_media_repo_database_password: "your_password" -matrix_media_repo_database_hostname: "matrix-postgres" +matrix_media_repo_database_hostname: "{{ devture_postgres_identifier }}" matrix_media_repo_database_port: 5432 matrix_media_repo_database_name: "matrix_media_repo" diff --git a/roles/custom/matrix-media-repo/tasks/setup_install.yml b/roles/custom/matrix-media-repo/tasks/setup_install.yml index 5e7399059..3bcbed966 100644 --- a/roles/custom/matrix-media-repo/tasks/setup_install.yml +++ b/roles/custom/matrix-media-repo/tasks/setup_install.yml @@ -81,8 +81,8 @@ name: "{{ matrix_media_repo_container_network }}" driver: bridge -- name: Ensure matrix-media-repo.service installed +- name: Ensure media-repo service installed ansible.builtin.template: src: "{{ role_path }}/templates/media-repo/systemd/matrix-media-repo.service.j2" - dest: "{{ devture_systemd_docker_base_systemd_path }}/matrix-media-repo.service" + dest: "{{ devture_systemd_docker_base_systemd_path }}/{{ matrix_media_repo_identifier }}.service" mode: 0640 diff --git a/roles/custom/matrix-media-repo/tasks/setup_uninstall.yml b/roles/custom/matrix-media-repo/tasks/setup_uninstall.yml index d5f08994e..449cd48b8 100644 --- a/roles/custom/matrix-media-repo/tasks/setup_uninstall.yml +++ b/roles/custom/matrix-media-repo/tasks/setup_uninstall.yml @@ -1,19 +1,19 @@ --- -- name: Check existence of matrix-media-repo service +- name: Check existence of media-repo service ansible.builtin.stat: - path: "{{ devture_systemd_docker_base_systemd_path }}/matrix-media-repo.service" + path: "{{ devture_systemd_docker_base_systemd_path }}/{{ matrix_media_repo_identifier }}.service" register: matrix_media_repo_service_stat - when: matrix_media_repo_service_stat.stat.exists | bool block: - - name: Ensure matrix-media-repo is stopped + - name: Ensure media-repo is stopped ansible.builtin.systemd: - name: matrix-media-repo + name: "{{ matrix_media_repo_identifier }}" state: stopped daemon_reload: true - - name: Ensure matrix-media-repo.service doesn't exist + - name: Ensure media-repo service doesn't exist ansible.builtin.file: - path: "{{ devture_systemd_docker_base_systemd_path }}/matrix-media-repo.service" + path: "{{ devture_systemd_docker_base_systemd_path }}/{{ matrix_media_repo_identifier }}.service" state: absent diff --git a/roles/custom/matrix-media-repo/templates/media-repo/systemd/matrix-media-repo.service.j2 b/roles/custom/matrix-media-repo/templates/media-repo/systemd/matrix-media-repo.service.j2 index 33b5199a6..0e73cb6ce 100644 --- a/roles/custom/matrix-media-repo/templates/media-repo/systemd/matrix-media-repo.service.j2 +++ b/roles/custom/matrix-media-repo/templates/media-repo/systemd/matrix-media-repo.service.j2 @@ -13,12 +13,12 @@ DefaultDependencies=no [Service] Type=simple Environment="HOME={{ devture_systemd_docker_base_systemd_unit_home_path }}" -ExecStartPre=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} kill matrix-media-repo 2>/dev/null || true' -ExecStartPre=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} rm matrix-media-repo 2>/dev/null || true' +ExecStartPre=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} kill {{ matrix_media_repo_identifier }} 2>/dev/null || true' +ExecStartPre=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} rm {{ matrix_media_repo_identifier }} 2>/dev/null || true' ExecStartPre={{ devture_systemd_docker_base_host_command_docker }} create \ --rm \ - --name=matrix-media-repo \ + --name={{ matrix_media_repo_identifier }} \ --log-driver=none \ --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ --cap-drop=ALL \ @@ -40,17 +40,17 @@ ExecStartPre={{ devture_systemd_docker_base_host_command_docker }} create \ {{ matrix_media_repo_docker_image }} {% for network in matrix_media_repo_container_additional_networks %} -ExecStartPre={{ devture_systemd_docker_base_host_command_docker }} network connect {{ network }} matrix-media-repo +ExecStartPre={{ devture_systemd_docker_base_host_command_docker }} network connect {{ network }} {{ matrix_media_repo_identifier }} {% endfor %} -ExecStart={{ devture_systemd_docker_base_host_command_docker }} start --attach matrix-media-repo +ExecStart={{ devture_systemd_docker_base_host_command_docker }} start --attach {{ matrix_media_repo_identifier }} -ExecStop=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} kill matrix-media-repo 2>/dev/null || true' -ExecStop=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} rm matrix-media-repo 2>/dev/null || true' -ExecReload={{ devture_systemd_docker_base_host_command_docker }} exec matrix-media-repo /bin/sh -c 'kill -HUP 1' +ExecStop=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} kill {{ matrix_media_repo_identifier }} 2>/dev/null || true' +ExecStop=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} rm {{ matrix_media_repo_identifier }} 2>/dev/null || true' +ExecReload={{ devture_systemd_docker_base_host_command_docker }} exec {{ matrix_media_repo_identifier }} /bin/sh -c 'kill -HUP 1' Restart=always RestartSec=30 -SyslogIdentifier=matrix-media-repo +SyslogIdentifier={{ matrix_media_repo_identifier }} [Install] WantedBy=multi-user.target From 28fa644c30d29b05deb82812027a985ad077cff0 Mon Sep 17 00:00:00 2001 From: Michael Hollister Date: Thu, 13 Jul 2023 21:19:07 -0500 Subject: [PATCH 23/43] Removed redeclration of matrix_media_repo_identifier --- group_vars/matrix_servers | 1 - 1 file changed, 1 deletion(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index d5184e233..aa709f86f 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -2564,7 +2564,6 @@ matrix_ma1sd_database_password: "{{ '%s' | format(matrix_homeserver_generic_secr ###################################################################### matrix_media_repo_enabled: false -matrix_media_repo_identifier: matrix-media-repo matrix_media_repo_container_network: "{{ matrix_docker_network }}" matrix_media_repo_container_labels_traefik_enabled: false From 73edde39920eba64abbd58891def7f3253e45128 Mon Sep 17 00:00:00 2001 From: Michael Hollister Date: Thu, 13 Jul 2023 23:12:24 -0500 Subject: [PATCH 24/43] Replaced additional hardcoded service names with identifer variable --- group_vars/matrix_servers | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index aa709f86f..b008d694c 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -326,7 +326,7 @@ devture_systemd_service_manager_services_list_auto: | + ([{'name': 'matrix-ma1sd.service', 'priority': 2000, 'groups': ['matrix', 'ma1sd']}] if matrix_ma1sd_enabled else []) + - ([{'name': 'matrix-media-repo.service', 'priority': 4000, 'groups': ['matrix', 'media_store']}] if matrix_media_repo_enabled else []) + ([{'name': (matrix_media_repo_identifier + '.service'), 'priority': 4000, 'groups': ['matrix', 'media_store']}] if matrix_media_repo_enabled else []) + ([{'name': 'matrix-mailer.service', 'priority': 2000, 'groups': ['matrix', 'mailer']}] if matrix_mailer_enabled else []) + @@ -2731,7 +2731,7 @@ matrix_nginx_proxy_systemd_wanted_services_list: | + (['matrix-ma1sd.service'] if matrix_ma1sd_enabled else []) + - (['matrix-media-repo.service'] if matrix_media_repo_enabled else []) + ([(matrix_media_repo_identifier + '.service')] if matrix_media_repo_enabled else []) + (['matrix-client-cinny.service'] if matrix_client_cinny_enabled and matrix_playbook_reverse_proxy_type in ['playbook-managed-nginx', 'other-nginx-non-container'] else []) + From c8b6b6e0347a92f7e92c0ceb673611d4cf290897 Mon Sep 17 00:00:00 2001 From: Michael Hollister Date: Fri, 14 Jul 2023 00:20:07 -0500 Subject: [PATCH 25/43] Added additioanl documentation to rendered config and added to_json filters --- .../matrix-media-repo/defaults/main.yml | 4 +- .../templates/media-repo/media-repo.yaml.j2 | 500 +++++++++++++----- 2 files changed, 382 insertions(+), 122 deletions(-) diff --git a/roles/custom/matrix-media-repo/defaults/main.yml b/roles/custom/matrix-media-repo/defaults/main.yml index a08ad0418..e2304d42e 100644 --- a/roles/custom/matrix-media-repo/defaults/main.yml +++ b/roles/custom/matrix-media-repo/defaults/main.yml @@ -72,7 +72,7 @@ matrix_media_repo_port: 8000 # # Note: to change the log directory you'll have to restart the repository. This setting cannot be # live reloaded. -matrix_media_repo_log_directory: "{{ '\"-\"' }}" +matrix_media_repo_log_directory: "-" # Set to true to enable color coding in your logs. Note that this may cause escape sequences to # appear in logs which render them unreadable, which is why colors are disabled by default. @@ -146,7 +146,7 @@ matrix_media_repo_homeservers: # unstable client-server API. When this is "synapse", the new /_synapse # endpoints will be used instead. Unknown values are treated as the # default, "matrix". - adminApiKind: "matrix" + adminApiKind: "{{ 'synapse' if matrix_homeserver_implementation == 'synapse' else 'matrix' }}" # Options for controlling how access tokens work with the media repo. It is recommended that if # you are going to use these options that the `/logout` and `/logout/all` client-server endpoints diff --git a/roles/custom/matrix-media-repo/templates/media-repo/media-repo.yaml.j2 b/roles/custom/matrix-media-repo/templates/media-repo/media-repo.yaml.j2 index 0595f576d..c304c1c2c 100644 --- a/roles/custom/matrix-media-repo/templates/media-repo/media-repo.yaml.j2 +++ b/roles/custom/matrix-media-repo/templates/media-repo/media-repo.yaml.j2 @@ -1,7 +1,7 @@ # General repo configuration repo: - bindAddress: {{ matrix_media_repo_bind_address }} - port: {{ matrix_media_repo_port }} + bindAddress: {{ matrix_media_repo_bind_address | to_json }} + port: {{ matrix_media_repo_port | to_json }} # Where to store the logs, relative to where the repo is started from. Logs will be automatically # rotated every day and held for 14 days. To disable the repo logging to files, set this to @@ -9,71 +9,69 @@ repo: # # Note: to change the log directory you'll have to restart the repository. This setting cannot be # live reloaded. - logDirectory: {{ matrix_media_repo_log_directory }} + logDirectory: {{ matrix_media_repo_log_directory | to_json }} # Set to true to enable color coding in your logs. Note that this may cause escape sequences to # appear in logs which render them unreadable, which is why colors are disabled by default. - logColors: {{ matrix_media_repo_log_colors }} + logColors: {{ matrix_media_repo_log_colors | to_json }} # Set to true to enable JSON logging for consumption by things like logstash. Note that this is # incompatible with the log color option and will always render without colors. - jsonLogs: {{ matrix_media_repo_json_logs }} + jsonLogs: {{ matrix_media_repo_json_logs | to_json }} # The log level to log at. Note that this will need to be at least "info" to receive support. # # Values (in increasing spam): panic | fatal | error | warn | info | debug | trace - logLevel: {{ matrix_media_repo_log_level }} + logLevel: {{ matrix_media_repo_log_level | to_json }} # If true, the media repo will accept any X-Forwarded-For header without validation. In most cases # this option should be left as "false". Note that the media repo already expects an X-Forwarded-For # header, but validates it to ensure the IP being given makes sense. - trustAnyForwardedAddress: {{ matrix_media_repo_trust_any_forwarded_address }} + trustAnyForwardedAddress: {{ matrix_media_repo_trust_any_forwarded_address | to_json }} # If false, the media repo will not use the X-Forwarded-Host header commonly added by reverse proxies. # Typically this should remain as true, though in some circumstances it may need to be disabled. # See https://github.com/turt2live/matrix-media-repo/issues/202 for more information. - useForwardedHost: {{ matrix_media_repo_use_forwarded_host }} + useForwardedHost: {{ matrix_media_repo_use_forwarded_host | to_json }} # Options for dealing with federation federation: # On a per-host basis, the number of consecutive failures in calling the host before the # media repo will back off. This defaults to 20 if not given. Note that 404 errors from # the remote server do not count towards this. - backoffAt: {{ matrix_media_repo_federation_backoff_at }} + backoffAt: {{ matrix_media_repo_federation_backoff_at | to_json }} # The database configuration for the media repository # Do NOT put your homeserver's existing database credentials here. Create a new database and # user instead. Using the same server is fine, just not the same username and database. database: # Currently only "postgres" is supported. - postgres: {{ matrix_media_repo_database_postgres }} + postgres: {{ matrix_media_repo_database_postgres | to_json }} # The database pooling options pool: # The maximum number of connects to hold open. More of these allow for more concurrent # processes to happen. - maxConnections: {{ matrix_media_repo_database_max_connections }} + maxConnections: {{ matrix_media_repo_database_max_connections | to_json }} # The maximum number of connects to leave idle. More of these reduces the time it takes # to serve requests in low-traffic scenarios. - maxIdleConnections: {{ matrix_media_repo_database_max_idle_connections }} + maxIdleConnections: {{ matrix_media_repo_database_max_idle_connections | to_json }} # The configuration for the homeservers this media repository is known to control. Servers # not listed here will not be able to upload media. -{# -homeservers: - - name: example.org # This should match the server_name of your homeserver, and the Host header - # provided to the media repo. - csApi: "https://example.org/" # The base URL to where the homeserver can actually be reached - backoffAt: 10 # The number of consecutive failures in calling this homeserver before the - # media repository will start backing off. This defaults to 10 if not given. - adminApiKind: "matrix" # The kind of admin API the homeserver supports. If set to "matrix", - # the media repo will use the Synapse-defined endpoints under the - # unstable client-server API. When this is "synapse", the new /_synapse - # endpoints will be used instead. Unknown values are treated as the - # default, "matrix". -#} -{{ matrix_media_repo_homeservers | to_nice_yaml(indent=2, sort_keys=false) }} +#homeservers: +# - name: example.org # This should match the server_name of your homeserver, and the Host header +# # provided to the media repo. +# csApi: "https://example.org/" # The base URL to where the homeserver can actually be reached +# backoffAt: 10 # The number of consecutive failures in calling this homeserver before the +# # media repository will start backing off. This defaults to 10 if not given. +# adminApiKind: "matrix" # The kind of admin API the homeserver supports. If set to "matrix", +# # the media repo will use the Synapse-defined endpoints under the +# # unstable client-server API. When this is "synapse", the new /_synapse +# # endpoints will be used instead. Unknown values are treated as the +# # default, "matrix". +{{ matrix_media_repo_homeservers | to_json | from_json | to_nice_yaml(indent=2, width=999999, sort_keys=false) }} # Options for controlling how access tokens work with the media repo. It is recommended that if # you are going to use these options that the `/logout` and `/logout/all` client-server endpoints @@ -91,12 +89,42 @@ homeservers: # *************************************************************************** # * IT IS HIGHLY RECOMMENDED TO USE PER-DOMAIN CONFIGS WITH THIS FEATURE. * # *************************************************************************** -{{ matrix_media_repo_access_tokens | to_nice_yaml(indent=2, sort_keys=false) }} +# accessTokens: +# # The maximum time a cached access token will be considered valid. Set to zero (the default) +# # to disable the cache and constantly hit the homeserver. This is recommended to be set to +# # 43200 (12 hours) on servers with the logout endpoints proxied through the media repo, and +# # zero for servers who do not proxy the endpoints through. +# maxCacheTimeSeconds: 0 +# +# # Whether or not to use the `appservices` config option below. If disabled (the default), +# # the regular access token cache will be used for each user, potentially leading to high +# # memory usage. +# useLocalAppserviceConfig: false +# +# # The application services (and their namespaces) registered on the homeserver. Only used +# # if `useLocalAppserviceConfig` is enabled (recommended). +# # +# # Usually the appservice will provide you with these config details - they'll just need +# # translating from the appservice registration to here. Note that this does not require +# # all options from the registration, and only requires the bare minimum required to run +# # the media repo. +# appservices: +# - id: Name_of_appservice_for_your_reference +# asToken: Secret_token_for_appservices_to_use +# senderUserId: "@_example_bridge:yourdomain.com" +# userNamespaces: +# - regex: "@_example_bridge_.+:yourdomain.com" +# # A note about regexes: it is best to suffix *all* namespaces with the homeserver +# # domain users are valid for, as otherwise the appservice can use any user with +# # any domain name it feels like, even if that domain is not configured with the +# # media repo. This will lead to inaccurate reporting in the case of the media +# # repo, and potentially leading to media being considered "remote". +{{ matrix_media_repo_access_tokens | to_json | from_json | to_nice_yaml(indent=2, width=999999, sort_keys=false) }} # These users have full access to the administrative functions of the media repository. # See docs/admin.md for information on what these people can do. They must belong to one of the # configured homeservers above. -{{ matrix_media_repo_admins | to_nice_yaml(indent=2, sort_keys=false) }} +{{ matrix_media_repo_admins | to_json | from_json | to_nice_yaml(indent=2, width=999999, sort_keys=false) }} # Shared secret auth is useful for applications building on top of the media repository, such # as a management interface. The `token` provided here is treated as a repository administrator @@ -105,120 +133,153 @@ homeservers: # the ability to use it on any configured hostname. sharedSecretAuth: # Set this to true to enable shared secret auth. - enabled: {{ matrix_media_repo_shared_secret_auth_enabled }} + enabled: {{ matrix_media_repo_shared_secret_auth_enabled | to_json }} # Use a secure value here to prevent unauthorized access to the media repository. - token: {{ matrix_media_repo_shared_secret_auth_token }} + token: {{ matrix_media_repo_shared_secret_auth_token | to_json }} # Datastores are places where media should be persisted. This isn't dedicated for just uploads: # thumbnails and other misc data is also stored in these places. The media repo, when looking # for a datastore to use, will always use the smallest datastore first. -{# -datastores: - - type: file - enabled: false # Enable this to set up data storage. - # Datastores can be split into many areas when handling uploads. Media is still de-duplicated - # across all datastores (local content which duplicates remote content will re-use the remote - # content's location). This option is useful if your datastore is becoming very large, or if - # you want faster storage for a particular kind of media. - # - # The kinds available are: - # thumbnails - Used to store thumbnails of media (local and remote). - # remote_media - Original copies of remote media (servers not configured by this repo). - # local_media - Original uploads for local media. - # archives - Archives of content (GDPR and similar requests). - forKinds: ["thumbnails"] - opts: - path: /var/matrix/media - - - type: s3 - enabled: false # Enable this to set up s3 uploads - forKinds: ["thumbnails", "remote_media", "local_media", "archives"] - opts: - # The s3 uploader needs a temporary location to buffer files to reduce memory usage on - # small file uploads. If the file size is unknown, the file is written to this location - # before being uploaded to s3 (then the file is deleted). If you aren't concerned about - # memory usage, set this to an empty string. - tempPath: "/tmp/mediarepo_s3_upload" - endpoint: sfo2.digitaloceanspaces.com - accessKeyId: "" - accessSecret: "" - ssl: true - bucketName: "your-media-bucket" - # An optional region for where this S3 endpoint is located. Typically not needed, though - # some providers will need this (like Scaleway). Uncomment to use. - #region: "sfo2" - - # The media repo does support an IPFS datastore, but only if the IPFS feature is enabled. If - # the feature is not enabled, this will not work. Note that IPFS support is experimental at - # the moment and not recommended for general use. - # - # NOTE: Everything you upload to IPFS will be publicly accessible, even when the media repo - # puts authentication on the download endpoints. Only use this option for cases where you - # expect your media to be publicly accessible. - - type: ipfs - enabled: false # Enable this to use IPFS support - forKinds: ["local_media"] - # The IPFS datastore currently has no options. It will use the daemon or HTTP API configured - # in the IPFS section of your main config. - opts: {} -#} -{{ matrix_media_repo_datastores | to_nice_yaml(indent=2, sort_keys=false) }} +# datastores: +# - type: file +# enabled: false # Enable this to set up data storage. +# # Datastores can be split into many areas when handling uploads. Media is still de-duplicated +# # across all datastores (local content which duplicates remote content will re-use the remote +# # content's location). This option is useful if your datastore is becoming very large, or if +# # you want faster storage for a particular kind of media. +# # +# # The kinds available are: +# # thumbnails - Used to store thumbnails of media (local and remote). +# # remote_media - Original copies of remote media (servers not configured by this repo). +# # local_media - Original uploads for local media. +# # archives - Archives of content (GDPR and similar requests). +# forKinds: ["thumbnails"] +# opts: +# path: /var/matrix/media +# +# - type: s3 +# enabled: false # Enable this to set up s3 uploads +# forKinds: ["thumbnails", "remote_media", "local_media", "archives"] +# opts: +# # The s3 uploader needs a temporary location to buffer files to reduce memory usage on +# # small file uploads. If the file size is unknown, the file is written to this location +# # before being uploaded to s3 (then the file is deleted). If you aren't concerned about +# # memory usage, set this to an empty string. +# tempPath: "/tmp/mediarepo_s3_upload" +# endpoint: sfo2.digitaloceanspaces.com +# accessKeyId: "" +# accessSecret: "" +# ssl: true +# bucketName: "your-media-bucket" +# # An optional region for where this S3 endpoint is located. Typically not needed, though +# # some providers will need this (like Scaleway). Uncomment to use. +# #region: "sfo2" +# +# # The media repo does support an IPFS datastore, but only if the IPFS feature is enabled. If +# # the feature is not enabled, this will not work. Note that IPFS support is experimental at +# # the moment and not recommended for general use. +# # +# # NOTE: Everything you upload to IPFS will be publicly accessible, even when the media repo +# # puts authentication on the download endpoints. Only use this option for cases where you +# # expect your media to be publicly accessible. +# - type: ipfs +# enabled: false # Enable this to use IPFS support +# forKinds: ["local_media"] +# # The IPFS datastore currently has no options. It will use the daemon or HTTP API configured +# # in the IPFS section of your main config. +# opts: {} +{{ matrix_media_repo_datastores | to_json | from_json | to_nice_yaml(indent=2, width=999999, sort_keys=false) }} # Options for controlling archives. Archives are exports of a particular user's content for # the purpose of GDPR or moving media to a different server. archiving: # Whether archiving is enabled or not. Default enabled. - enabled: {{ matrix_media_repo_archiving_enabled }} + enabled: {{ matrix_media_repo_archiving_enabled | to_json }} # If true, users can request a copy of their own data. By default, only repository administrators # can request a copy. # This includes the ability for homeserver admins to request a copy of their own server's # data, as known to the repo. - selfService: {{ matrix_media_repo_archiving_self_service }} + selfService: {{ matrix_media_repo_archiving_self_service | to_json }} # The number of bytes to target per archive before breaking up the files. This is independent # of any file upload limits and will require a similar amount of memory when performing an export. # The file size is also a target, not a guarantee - it is possible to have files that are smaller # or larger than the target. This is recommended to be approximately double the size of your # file upload limit, provided there is enough memory available for the demand of exporting. - targetBytesPerPart: {{ matrix_media_repo_archiving_target_bytes_per_part }} # 200mb default + targetBytesPerPart: {{ matrix_media_repo_archiving_target_bytes_per_part | to_json }} # 200mb default # The file upload settings for the media repository -{{ matrix_media_repo_uploads | to_nice_yaml(indent=2, sort_keys=false) }} +# uploads: +# # The maximum individual file size a user can upload. +# maxBytes: 104857600 # 100MB default, 0 to disable +# +# # The minimum number of bytes to let people upload. This is recommended to be non-zero to +# # ensure that the "cost" of running the media repo is worthwhile - small file uploads tend +# # to waste more CPU and database resources than small files, thus a default of 100 bytes +# # is applied here as an approximate break-even point. +# minBytes: 100 # 100 bytes by default +# +# # The number of bytes to claim as the maximum size for uploads for the limits API. If this +# # is not provided then the maxBytes setting will be used instead. This is useful to provide +# # if the media repo's settings and the reverse proxy do not match for maximum request size. +# # This is purely for informational reasons and does not actually limit any functionality. +# # Set this to -1 to indicate that there is no limit. Zero will force the use of maxBytes. +# #reportedMaxBytes: 104857600 +# +# # Options for limiting how much content a user can upload. Quotas are applied to content +# # associated with a user regardless of de-duplication. Quotas which affect remote servers +# # or users will not take effect. When a user exceeds their quota they will be unable to +# # upload any more media. +# quotas: +# # Whether or not quotas are enabled/enforced. Note that even when disabled the media repo +# # will track how much media a user has uploaded. This is disabled by default. +# enabled: false +# +# # The quota rules that affect users. The first rule to match the uploader will take effect. +# # An implied rule which matches all users and has no quota is always last in this list, +# # meaning that if no rules are supplied then users will be able to upload anything. Similarly, +# # if no rules match a user then the implied rule will match, allowing the user to have no +# # quota. The quota will let the user upload to 1 media past their quota, meaning that from +# # a statistics perspective the user might exceed their quota however only by a small amount. +# users: +# - glob: "@*:*" # Affect all users. Use asterisks (*) to match any character. +# maxBytes: 53687063712 # 50GB default, 0 to disable +{{ matrix_media_repo_uploads | to_json | from_json | to_nice_yaml(indent=2, width=999999, sort_keys=false) }} # Settings related to downloading files from the media repository downloads: # The maximum number of bytes to download from other servers - maxBytes: {{ matrix_media_repo_downloads_max_bytes }} # 100MB default, 0 to disable + maxBytes: {{ matrix_media_repo_downloads_max_bytes | to_json }} # 100MB default, 0 to disable # The number of workers to use when downloading remote media. Raise this number if remote # media is downloading slowly or timing out. # # Maximum memory usage = numWorkers multiplied by the maximum download size # Average memory usage is dependent on how many concurrent downloads your users are doing. - numWorkers: {{ matrix_media_repo_downloads_num_workers }} + numWorkers: {{ matrix_media_repo_downloads_num_workers | to_json }} # How long, in minutes, to cache errors related to downloading remote media. Once this time # has passed, the media is able to be re-requested. - failureCacheMinutes: {{ matrix_media_repo_downloads_failure_cache_minutes }} + failureCacheMinutes: {{ matrix_media_repo_downloads_failure_cache_minutes | to_json }} # The cache control settings for downloads. This can help speed up downloads for users by # keeping popular media in the cache. This cache is also used for thumbnails. cache: - enabled: {{ matrix_media_repo_downloads_cache_enabled }} + enabled: {{ matrix_media_repo_downloads_cache_enabled | to_json }} # The maximum size of cache to have. Higher numbers are better. - maxSizeBytes: {{ matrix_media_repo_downloads_cache_max_size_bytes }} # 1GB default + maxSizeBytes: {{ matrix_media_repo_downloads_cache_max_size_bytes | to_json }} # 1GB default # The maximum file size to cache. This should normally be the same size as your maximum # upload size. - maxFileSizeBytes: {{ matrix_media_repo_downloads_cache_max_file_size_bytes }} # 100MB default + maxFileSizeBytes: {{ matrix_media_repo_downloads_cache_max_file_size_bytes | to_json }} # 100MB default # The number of minutes to track how many downloads a file gets - trackedMinutes: {{ matrix_media_repo_downloads_cache_tracked_minutes }} + trackedMinutes: {{ matrix_media_repo_downloads_cache_tracked_minutes | to_json }} # The number of downloads a file must receive in the window above (trackedMinutes) in # order to be cached. - minDownloads: {{ matrix_media_repo_downloads_cache_min_downloads }} + minDownloads: {{ matrix_media_repo_downloads_cache_min_downloads | to_json }} # The minimum amount of time an item should remain in the cache. This prevents the cache # from cycling out the file if it needs more room during this time. Note that the media @@ -227,96 +288,234 @@ downloads: # media repo, and some cached items are still under this timer, new items will not be able # to enter the cache. When this happens, consider raising maxSizeBytes or lowering this # timer. - minCacheTimeSeconds: {{ matrix_media_repo_downloads_cache_min_cache_time_seconds }} + minCacheTimeSeconds: {{ matrix_media_repo_downloads_cache_min_cache_time_seconds | to_json }} # The minimum amount of time an item should remain outside the cache once it is removed. - minEvictedTimeSeconds: {{ matrix_media_repo_downloads_cache_min_evicted_time_seconds }} + minEvictedTimeSeconds: {{ matrix_media_repo_downloads_cache_min_evicted_time_seconds | to_json }} # How many days after a piece of remote content is downloaded before it expires. It can be # re-downloaded on demand, this just helps free up space in your datastore. Set to zero or # negative to disable. Defaults to disabled. - expireAfterDays: {{ matrix_media_repo_downloads_expire_after_days }} + expireAfterDays: {{ matrix_media_repo_downloads_expire_after_days | to_json }} # URL Preview settings -{{ matrix_media_repo_url_previews | to_nice_yaml(indent=2) }} +# urlPreviews: +# enabled: true # If enabled, the preview_url routes will be accessible +# maxPageSizeBytes: 10485760 # 10MB default, 0 to disable +# +# # If true, the media repository will try to provide previews for URLs with invalid or unsafe +# # certificates. If false (the default), the media repo will fail requests to said URLs. +# previewUnsafeCertificates: false +# +# # Note: URL previews are limited to a given number of words, which are then limited to a number +# # of characters, taking off the last word if it needs to. This also applies for the title. +# +# numWords: 50 # The number of words to include in a preview (maximum) +# maxLength: 200 # The maximum number of characters for a description +# +# numTitleWords: 30 # The maximum number of words to include in a preview's title +# maxTitleLength: 150 # The maximum number of characters for a title +# +# # The mime types to preview when OpenGraph previews cannot be rendered. OpenGraph previews are +# # calculated on anything matching "text/*". To have a thumbnail in the preview the URL must be +# # an image and the image's type must be allowed by the thumbnailer. +# filePreviewTypes: +# - "image/*" +# +# # The number of workers to use when generating url previews. Raise this number if url +# # previews are slow or timing out. +# # +# # Maximum memory usage = numWorkers multiplied by the maximum page size +# # Average memory usage is dependent on how many concurrent urls your users are previewing. +# numWorkers: 10 +# +# # Either allowedNetworks or disallowedNetworks must be provided. If both are provided, they +# # will be merged. URL previews will be disabled if neither is supplied. Each entry must be +# # a CIDR range. +# disallowedNetworks: +# - "127.0.0.1/8" +# - "10.0.0.0/8" +# - "172.16.0.0/12" +# - "192.168.0.0/16" +# - "100.64.0.0/10" +# - "169.254.0.0/16" +# - '::1/128' +# - 'fe80::/64' +# - 'fc00::/7' +# allowedNetworks: +# - "0.0.0.0/0" # "Everything". The blacklist will help limit this. +# # This is the default value for this field. +# +# # How many days after a preview is generated before it expires and is deleted. The preview +# # can be regenerated safely - this just helps free up some space in your database. Set to +# # zero or negative to disable. Defaults to disabled. +# expireAfterDays: 0 +# +# # The default Accept-Language header to supply when generating URL previews when one isn't +# # supplied by the client. +# # Reference: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Language +# defaultLanguage: "en-US,en" +# +# # When true, oEmbed previews will be enabled. Typically these kinds of previews are used for +# # sites that do not support OpenGraph or page scraping, such as Twitter. For information on +# # specifying providers for oEmbed, including your own, see the following documentation: +# # https://docs.t2bot.io/matrix-media-repo/url-previews/oembed.html +# # Defaults to disabled. +# oEmbed: false +{{ matrix_media_repo_url_previews | to_json | from_json | to_nice_yaml(indent=2, width=999999, sort_keys=false)}} # The thumbnail configuration for the media repository. -{{ matrix_media_repo_thumbnails | to_nice_yaml(indent=2) }} +# thumbnails: +# # The maximum number of bytes an image can be before the thumbnailer refuses. +# maxSourceBytes: 10485760 # 10MB default, 0 to disable +# +# # The maximum number of pixels an image can have before the thumbnailer refuses. Note that +# # this only applies to image types: file types like audio and video are affected solely by +# # the maxSourceBytes. +# maxPixels: 32000000 # 32M default +# +# # The number of workers to use when generating thumbnails. Raise this number if thumbnails +# # are slow to generate or timing out. +# # +# # Maximum memory usage = numWorkers multiplied by the maximum image source size +# # Average memory usage is dependent on how many thumbnails are being generated by your users +# numWorkers: 100 +# +# # All thumbnails are generated into one of the sizes listed here. The first size is used as +# # the default for when no width or height is requested. The media repository will return +# # either an exact match or the next largest size of thumbnail. +# sizes: +# - width: 32 +# height: 32 +# - width: 96 +# height: 96 +# - width: 320 +# height: 240 +# - width: 640 +# height: 480 +# - width: 768 # This size is primarily used for audio thumbnailing. +# height: 240 +# - width: 800 +# height: 600 +# +# # To allow for thumbnails to be any size, not just in the sizes specified above, set this to +# # true (default false). When enabled, whatever size requested by the client will be generated +# # up to a maximum of the largest possible dimensions in the `sizes` list. For best results, +# # specify only one size in the `sizes` list when this option is enabled. +# dynamicSizing: false +# +# # The content types to thumbnail when requested. Types that are not supported by the media repo +# # will not be thumbnailed (adding application/json here won't work). Clients may still not request +# # thumbnails for these types - this won't make clients automatically thumbnail these file types. +# types: +# - "image/jpeg" +# - "image/jpg" +# - "image/png" +# - "image/apng" +# - "image/gif" +# - "image/heif" +# - "image/webp" +# #- "image/svg+xml" # Be sure to have ImageMagick installed to thumbnail SVG files +# - "audio/mpeg" +# - "audio/ogg" +# - "audio/wav" +# - "audio/flac" +# #- "video/mp4" # Be sure to have ffmpeg installed to thumbnail video files +# +# # Animated thumbnails can be CPU intensive to generate. To disable the generation of animated +# # thumbnails, set this to false. If disabled, regular thumbnails will be returned. +# allowAnimated: true +# +# # Default to animated thumbnails, if available +# defaultAnimated: false +# +# # The maximum file size to thumbnail when a capable animated thumbnail is requested. If the image +# # is larger than this, the thumbnail will be generated as a static image. +# maxAnimateSizeBytes: 10485760 # 10MB default, 0 to disable +# +# # On a scale of 0 (start of animation) to 1 (end of animation), where should the thumbnailer try +# # and thumbnail animated content? Defaults to 0.5 (middle of animation). +# stillFrame: 0.5 +# +# # How many days after a thumbnail is generated before it expires and is deleted. The thumbnail +# # can be regenerated safely - this just helps free up some space in your datastores. Set to +# # zero or negative to disable. Defaults to disabled. +# expireAfterDays: 0 +{{ matrix_media_repo_thumbnails | to_json | from_json | to_nice_yaml(indent=2, width=999999, sort_keys=false) }} # Controls for the rate limit functionality rateLimit: # Set this to false if rate limiting is handled at a higher level or you don't want it enabled. - enabled: {{ matrix_media_repo_rate_limit_enabled }} + enabled: {{ matrix_media_repo_rate_limit_enabled | to_json }} # The number of requests per second before an IP will be rate limited. Must be a whole number. - requestsPerSecond: {{ matrix_media_repo_rate_limit_requests_per_second }} + requestsPerSecond: {{ matrix_media_repo_rate_limit_requests_per_second | to_json }} # The number of requests an IP can send at once before the rate limit is actually considered. - burst: {{ matrix_media_repo_rate_limit_burst }} + burst: {{ matrix_media_repo_rate_limit_burst | to_json }} # Identicons are generated avatars for a given username. Some clients use these to give users a # default avatar after signing up. Identicons are not part of the official matrix spec, therefore # this feature is completely optional. identicons: - enabled: {{ matrix_media_repo_identicons_enabled }} + enabled: {{ matrix_media_repo_identicons_enabled | to_json }} # The quarantine media settings. quarantine: # If true, when a thumbnail of quarantined media is requested an image will be returned. If no # image is given in the thumbnailPath below then a generated image will be provided. This does # not affect regular downloads of files. - replaceThumbnails: {{ matrix_media_repo_quarantine_replace_thumbnails }} + replaceThumbnails: {{ matrix_media_repo_quarantine_replace_thumbnails | to_json }} # If true, when media which has been quarantined is requested an image will be returned. If # no image is given in the thumbnailPath below then a generated image will be provided. This # will replace media which is not an image (ie: quarantining a PDF will replace the PDF with # an image). - replaceDownloads: {{ matrix_media_repo_quarantine_replace_downloads }} + replaceDownloads: {{ matrix_media_repo_quarantine_replace_downloads | to_json }} # If provided, the given image will be returned as a thumbnail for media that is quarantined. #thumbnailPath: "/path/to/thumbnail.png" - thumbnailPath: {{ "" if matrix_media_repo_quarantine_thumbnail_path == "" else matrix_media_repo_quarantine_thumbnail_path }} + thumbnailPath: {{ "" if matrix_media_repo_quarantine_thumbnail_path == "" else matrix_media_repo_quarantine_thumbnail_path | to_json }} # If true, administrators of the configured homeservers may quarantine media for their server # only. Global administrators can quarantine any media (local or remote) regardless of this # flag. - allowLocalAdmins: {{ matrix_media_repo_quarantine_allow_local_admins }} + allowLocalAdmins: {{ matrix_media_repo_quarantine_allow_local_admins | to_json }} # The various timeouts that the media repo will use. timeouts: # The maximum amount of time the media repo should spend trying to fetch a resource that is # being previewed. - urlPreviewTimeoutSeconds: {{ matrix_media_repo_timeouts_url_preview_timeout_seconds }} + urlPreviewTimeoutSeconds: {{ matrix_media_repo_timeouts_url_preview_timeout_seconds | to_json }} # The maximum amount of time the media repo will spend making remote requests to other repos # or homeservers. This is primarily used to download media. - federationTimeoutSeconds: {{ matrix_media_repo_timeouts_federation_timeout_seconds }} + federationTimeoutSeconds: {{ matrix_media_repo_timeouts_federation_timeout_seconds | to_json }} # The maximum amount of time the media repo will spend talking to your configured homeservers. # This is usually used to verify a user's identity. - clientServerTimeoutSeconds: {{ matrix_media_repo_timeouts_client_server_timeout_seconds }} + clientServerTimeoutSeconds: {{ matrix_media_repo_timeouts_client_server_timeout_seconds | to_json }} # Prometheus metrics configuration # For an example Grafana dashboard, import the following JSON: # https://github.com/turt2live/matrix-media-repo/blob/master/docs/grafana.json metrics: # If true, the bindAddress and port below will serve GET /metrics for Prometheus to scrape. - enabled: {{ matrix_media_repo_metrics_enabled }} + enabled: {{ matrix_media_repo_metrics_enabled | to_json }} # The address to listen on. Typically "127.0.0.1" or "0.0.0.0" for all interfaces. - bindAddress: {{ matrix_media_repo_metrics_bind_address }} + bindAddress: {{ matrix_media_repo_metrics_bind_address | to_json }} # The port to listen on. Cannot be the same as the general web server port. - port: {{ matrix_media_repo_metrics_port }} + port: {{ matrix_media_repo_metrics_port | to_json }} # Plugins are optional pieces of the media repo used to extend the functionality offered. # Currently there are only antispam plugins, but in future there should be more options. # Plugins are not supported on per-domain paths and are instead repo-wide. For more # information on writing plugins, please visit #matrix-media-repo:t2bot.io on Matrix. -{{ matrix_media_repo_plugins | to_nice_yaml(indent=2) }} - # An example OCR plugin to block images with certain text. Note that the Docker image - # for the media repo automatically ships this at /plugins/plugin_antispam_ocr +# An example OCR plugin to block images with certain text. Note that the Docker image +# for the media repo automatically ships this at /plugins/plugin_antispam_ocr # - exec: /plugins/plugin_antispam_ocr # config: # # The URL to your OCR server (https://github.com/otiai10/ocrserver) @@ -338,22 +537,83 @@ metrics: # # How much of the image's height, starting from the top, to consider before # # discarding the rest. Set to 1.0 to consider the whole image. # percentageOfHeight: 0.35 +{{ matrix_media_repo_plugins | to_json | from_json | to_nice_yaml(indent=2, width=999999, sort_keys=false) }} # Options for controlling various MSCs/unstable features of the media repo # Sections of this config might disappear or be added over time. By default all # features are disabled in here and must be explicitly enabled to be used. -{{ matrix_media_repo_feature_support | to_nice_yaml(indent=2) }} +# featureSupport: +# # MSC2248 - Blurhash +# MSC2448: +# # Whether or not this MSC is enabled for use in the media repo +# enabled: false +# +# # Maximum dimensions for converting a blurhash to an image. When no width and +# # height options are supplied, the default will be half these values. +# maxWidth: 1024 +# maxHeight: 1024 +# +# # Thumbnail size in pixels to use to generate the blurhash string +# thumbWidth: 64 +# thumbHeight: 64 +# +# # The X and Y components to use. Higher numbers blur less, lower numbers blur more. +# xComponents: 4 +# yComponents: 3 +# +# # The amount of contrast to apply when converting a blurhash to an image. Lower values +# # make the effect more subtle, larger values make it stronger. +# punch: 1 +# +# # IPFS Support +# # This is currently experimental and might not work at all. +# IPFS: +# # Whether or not IPFS support is enabled for use in the media repo. +# enabled: false +# +# # Options for the built in IPFS daemon +# builtInDaemon: +# # Enable this to spawn an in-process IPFS node to use instead of a localhost +# # HTTP agent. If this is disabled, the media repo will assume you have an HTTP +# # IPFS agent running and accessible. Defaults to using a daemon (true). +# enabled: true +# +# # If the Daemon is enabled, set this to the location where the IPFS files should +# # be stored. If you're using Docker, this should be something like "/data/ipfs" +# # so it can be mapped to a volume. +# repoPath: "./ipfs" +# +# # Support for redis as a cache mechanism +# # +# # Note: Enabling Redis support will mean that the existing cache mechanism will do nothing. +# # It can be safely disabled once Redis support is enabled. +# # +# # See docs/redis.md for more information on how this works and how to set it up. +# redis: +# # Whether or not use Redis instead of in-process caching. +# enabled: false +# +# # The Redis shards that should be used by the media repo in the ring. The names of the +# # shards are for your reference and have no bearing on the connection, but must be unique. +# shards: +# - name: "server1" +# addr: ":7000" +# - name: "server2" +# addr: ":7001" +# - name: "server3" +# addr: ":7002" +{{ matrix_media_repo_feature_support | to_json | from_json | to_nice_yaml(indent=2, width=999999, sort_keys=false) }} # Optional sentry (https://sentry.io/) configuration for the media repo sentry: # Whether or not to set up error reporting. Defaults to off. - enabled: {{ matrix_media_repo_sentry_enabled }} + enabled: {{ matrix_media_repo_sentry_enabled | to_json }} # Get this value from the setup instructions in Sentry - dsn: {{ matrix_media_repo_sentry_dsn }} + dsn: {{ matrix_media_repo_sentry_dsn | to_json }} # Optional environment flag. Defaults to an empty string. - environment: {{ "" if matrix_media_repo_sentry_environment == "" else matrix_media_repo_sentry_environment }} + environment: {{ "" if matrix_media_repo_sentry_environment == "" else matrix_media_repo_sentry_environment | to_json }} # Whether or not to turn on sentry's built in debugging. This will increase log output. - debug: {{ matrix_media_repo_sentry_debug }} \ No newline at end of file + debug: {{ matrix_media_repo_sentry_debug | to_json }} \ No newline at end of file From 63ee8e5bb9ac624f6041098cc9f3d9f42090d090 Mon Sep 17 00:00:00 2001 From: Michael Hollister Date: Fri, 14 Jul 2023 01:31:35 -0500 Subject: [PATCH 26/43] Replaced csApi config with proxy container url --- roles/custom/matrix-media-repo/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/custom/matrix-media-repo/defaults/main.yml b/roles/custom/matrix-media-repo/defaults/main.yml index e2304d42e..312e02588 100644 --- a/roles/custom/matrix-media-repo/defaults/main.yml +++ b/roles/custom/matrix-media-repo/defaults/main.yml @@ -135,7 +135,7 @@ matrix_media_repo_homeservers: - name: "{{ matrix_server_fqn_matrix }}" # The base URL to where the homeserver can actually be reached - csApi: "https://{{ matrix_server_fqn_matrix }}/" + csApi: "http://{{ matrix_nginx_proxy_proxy_matrix_client_api_addr_with_container }}" # The number of consecutive failures in calling this homeserver before the # media repository will start backing off. This defaults to 10 if not given. From 3e93613a3f280e1ca9253bab7f496aa4fbef0f49 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 14 Jul 2023 09:49:30 +0300 Subject: [PATCH 27/43] Upgrade Heisenbridge (1.14.2 -> 1.14.3) --- roles/custom/matrix-bridge-heisenbridge/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/custom/matrix-bridge-heisenbridge/defaults/main.yml b/roles/custom/matrix-bridge-heisenbridge/defaults/main.yml index ba5471cce..5dfe38105 100644 --- a/roles/custom/matrix-bridge-heisenbridge/defaults/main.yml +++ b/roles/custom/matrix-bridge-heisenbridge/defaults/main.yml @@ -4,7 +4,7 @@ matrix_heisenbridge_enabled: true -matrix_heisenbridge_version: 1.14.2 +matrix_heisenbridge_version: 1.14.3 matrix_heisenbridge_docker_image: "{{ matrix_container_global_registry_prefix }}hif1/heisenbridge:{{ matrix_heisenbridge_version }}" matrix_heisenbridge_docker_image_force_pull: "{{ matrix_heisenbridge_docker_image.endswith(':latest') }}" From 4e4f107daaa165541c384407fb47ed90246dc09d Mon Sep 17 00:00:00 2001 From: David Mehren Date: Sun, 16 Jul 2023 09:21:30 +0200 Subject: [PATCH 28/43] Upgrade sliding-sync (v0.99.3 -> v0.99.4) --- roles/custom/matrix-sliding-sync/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/custom/matrix-sliding-sync/defaults/main.yml b/roles/custom/matrix-sliding-sync/defaults/main.yml index 1f28d3053..cfc55ecd2 100644 --- a/roles/custom/matrix-sliding-sync/defaults/main.yml +++ b/roles/custom/matrix-sliding-sync/defaults/main.yml @@ -4,7 +4,7 @@ matrix_sliding_sync_enabled: true -matrix_sliding_sync_version: v0.99.3 +matrix_sliding_sync_version: v0.99.4 matrix_sliding_sync_scheme: https From ccc99d6c73a2693c0f9c889da7c251d7e0a25eb7 Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Sun, 16 Jul 2023 14:12:11 +0300 Subject: [PATCH 29/43] Update mautrix-discord 0.5.0 -> 0.6.0 --- roles/custom/matrix-bridge-mautrix-discord/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/custom/matrix-bridge-mautrix-discord/defaults/main.yml b/roles/custom/matrix-bridge-mautrix-discord/defaults/main.yml index dbb73aa45..f9b853bc6 100644 --- a/roles/custom/matrix-bridge-mautrix-discord/defaults/main.yml +++ b/roles/custom/matrix-bridge-mautrix-discord/defaults/main.yml @@ -8,7 +8,7 @@ matrix_mautrix_discord_container_image_self_build: false matrix_mautrix_discord_container_image_self_build_repo: "https://mau.dev/mautrix/discord.git" matrix_mautrix_discord_container_image_self_build_branch: "{{ 'main' if matrix_mautrix_discord_version == 'latest' else matrix_mautrix_discord_version }}" -matrix_mautrix_discord_version: v0.5.0 +matrix_mautrix_discord_version: v0.6.0 # See: https://mau.dev/mautrix/discord/container_registry matrix_mautrix_discord_docker_image: "{{ matrix_mautrix_discord_docker_image_name_prefix }}mautrix/discord:{{ matrix_mautrix_discord_version }}" matrix_mautrix_discord_docker_image_name_prefix: "{{ 'localhost/' if matrix_mautrix_discord_container_image_self_build else 'dock.mau.dev/' }}" From bc0b73dd70128fbc11162e8c705038a5f11ed515 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 17 Jul 2023 08:07:49 +0300 Subject: [PATCH 30/43] Improve if condition for including Postgres in matrix_media_repo_systemd_required_services_list --- group_vars/matrix_servers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index b008d694c..0a7c9543f 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -2580,7 +2580,7 @@ matrix_media_repo_systemd_required_services_list: | {{ (['docker.service']) + - ([devture_postgres_identifier ~ '.service'] if devture_postgres_enabled else []) + ([devture_postgres_identifier ~ '.service'] if devture_postgres_enabled and matrix_media_repo_database_hostname == devture_postgres_connection_hostname else []) }} ###################################################################### From 5fb1020adaeedea57f232ef48187109b3cac1475 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 17 Jul 2023 08:09:31 +0300 Subject: [PATCH 31/43] Use full name of matrix-media-repo for Ansible tags (*-media-repo -> *-matrix-media-repo) --- roles/custom/matrix-media-repo/tasks/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/custom/matrix-media-repo/tasks/main.yml b/roles/custom/matrix-media-repo/tasks/main.yml index 59f962a28..03c26ec5b 100644 --- a/roles/custom/matrix-media-repo/tasks/main.yml +++ b/roles/custom/matrix-media-repo/tasks/main.yml @@ -2,16 +2,16 @@ - tags: - setup-all - - setup-media-repo + - setup-matrix-media-repo - install-all - - install-media-repo + - install-matrix-media-repo block: - when: matrix_media_repo_enabled | bool ansible.builtin.include_tasks: "{{ role_path }}/tasks/setup_install.yml" - tags: - setup-all - - setup-media-repo + - setup-matrix-media-repo block: - when: not matrix_media_repo_enabled | bool ansible.builtin.include_tasks: "{{ role_path }}/tasks/setup_uninstall.yml" From 95bfa4e87e25c8c53687bd3bc055bb2fa27c6138 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 17 Jul 2023 08:11:23 +0300 Subject: [PATCH 32/43] Put matrix-media-repo.service in the matrix-media-repo group Making the group match the Ansible task tags allows people to do `just install-service matrix-media-repo` and have that trigger both `--tags=matrix-media-repo` and also restart just that single group (`matrix-media-repo`). --- group_vars/matrix_servers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 0a7c9543f..f932dcc35 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -326,7 +326,7 @@ devture_systemd_service_manager_services_list_auto: | + ([{'name': 'matrix-ma1sd.service', 'priority': 2000, 'groups': ['matrix', 'ma1sd']}] if matrix_ma1sd_enabled else []) + - ([{'name': (matrix_media_repo_identifier + '.service'), 'priority': 4000, 'groups': ['matrix', 'media_store']}] if matrix_media_repo_enabled else []) + ([{'name': (matrix_media_repo_identifier + '.service'), 'priority': 4000, 'groups': ['matrix', 'matrix-media-repo']}] if matrix_media_repo_enabled else []) + ([{'name': 'matrix-mailer.service', 'priority': 2000, 'groups': ['matrix', 'mailer']}] if matrix_mailer_enabled else []) + From 1df533d351213792252d83c8759ef5cf4f8ebbe9 Mon Sep 17 00:00:00 2001 From: Shreyas Ajjarapu Date: Mon, 17 Jul 2023 11:20:11 -0500 Subject: [PATCH 33/43] Update main.yml --- roles/custom/matrix-bridge-mautrix-whatsapp/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/custom/matrix-bridge-mautrix-whatsapp/defaults/main.yml b/roles/custom/matrix-bridge-mautrix-whatsapp/defaults/main.yml index 9ee461b5c..ab2921e31 100644 --- a/roles/custom/matrix-bridge-mautrix-whatsapp/defaults/main.yml +++ b/roles/custom/matrix-bridge-mautrix-whatsapp/defaults/main.yml @@ -8,7 +8,7 @@ matrix_mautrix_whatsapp_container_image_self_build: false matrix_mautrix_whatsapp_container_image_self_build_repo: "https://mau.dev/mautrix/whatsapp.git" matrix_mautrix_whatsapp_container_image_self_build_branch: "{{ 'master' if matrix_mautrix_whatsapp_version == 'latest' else matrix_mautrix_whatsapp_version }}" -matrix_mautrix_whatsapp_version: v0.8.6 +matrix_mautrix_whatsapp_version: v0.9.0 # See: https://mau.dev/mautrix/whatsapp/container_registry matrix_mautrix_whatsapp_docker_image: "{{ matrix_mautrix_whatsapp_docker_image_name_prefix }}mautrix/whatsapp:{{ matrix_mautrix_whatsapp_version }}" matrix_mautrix_whatsapp_docker_image_name_prefix: "{{ 'localhost/' if matrix_mautrix_whatsapp_container_image_self_build else 'dock.mau.dev/' }}" From 68556400ba74030a78ad09ee3d6411fb5861b1f4 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 17 Jul 2023 22:37:53 +0300 Subject: [PATCH 34/43] Announce matrix-media-repo Related to https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/2783 --- CHANGELOG.md | 9 +++++++++ docs/configuring-playbook-matrix-media-repo.md | 8 ++++---- docs/configuring-playbook-s3.md | 2 ++ docs/configuring-playbook.md | 2 ++ 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70b17b000..c1870a273 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +# 2023-07-17 + +## matrix-media-repo support + +Thanks to [Michael Hollister](https://github.com/Michael-Hollister) from [FUTO](https://www.futo.org/), the creators of the [Circles app](https://circu.li/), the playbook can now set up [matrix-media-repo](https://github.com/turt2live/matrix-media-repo) - an alternative way to store homeserver media files, powered by a homeserver-independent implementation which supports S3 storage, IPFS, deduplication and other advanced features. + +To learn more see our [Storing Matrix media files using matrix-media-repo](docs/configuring-playbook-matrix-media-repo.md) documentation page. + + # 2023-05-25 ## Enabling `forget_rooms_on_leave` by default for Synapse diff --git a/docs/configuring-playbook-matrix-media-repo.md b/docs/configuring-playbook-matrix-media-repo.md index e011e1e66..d5d6eda37 100644 --- a/docs/configuring-playbook-matrix-media-repo.md +++ b/docs/configuring-playbook-matrix-media-repo.md @@ -1,10 +1,10 @@ # Setting up matrix-media-repo (optional) -matrix-media-repo is a highly customizable multi-domain media repository for Matrix. Intended for medium to large environments consisting of several homeservers, this media repo de-duplicates media (including remote media) while being fully compliant with the specification. +[matrix-media-repo](https://docs.t2bot.io/matrix-media-repo/) is a highly customizable multi-domain media repository for Matrix. Intended for medium to large environments consisting of several homeservers, this media repo de-duplicates media (including remote media) while being fully compliant with the specification. -Smaller/individual homeservers can still make use of this project's features, though it may be difficult to set up or have higher than expected resource consumption - please do your research before deploying this as this project may not be useful for your environment. +Smaller/individual homeservers can still make use of this project's features, though it may be difficult to set up or have higher than expected resource consumption. Please do your research before deploying this as this project may not be useful for your environment. -More documentation about the project can be found at: https://docs.t2bot.io/matrix-media-repo/ +For a simpler alternative (which allows you to offload your media repository storage to S3, etc.), you can [configure S3 storage](configuring-playbook-s3.md) instead of setting up matrix-media-repo. ## Quickstart @@ -17,7 +17,7 @@ matrix_media_repo_enabled: true # matrix_media_repo_metrics_enabled: true ``` -The repo is pre-configured for integrating with the postgres database, NGINX proxy and prometheus/grafana (if metrics enabled) from this playbook for all the available homeserver roles. When the media repo is enabled, other media store roles should be disabled (if using Synapse with other media store roles). +The repo is pre-configured for integrating with the Postgres database, NGINX proxy and [Prometheus/Grafana](configuring-playbook-prometheus-grafana.md) (if metrics enabled) from this playbook for all the available homeserver roles. When the media repo is enabled, other media store roles should be disabled (if using Synapse with other media store roles). By default, the media-repo will use the local filesystem for data storage. Additional options include `s3` and `IPFS` (experimental). Access token caching is also enabled by default since the logout endpoints are proxied through the media repo. diff --git a/docs/configuring-playbook-s3.md b/docs/configuring-playbook-s3.md index 941ae0db2..f5a182840 100644 --- a/docs/configuring-playbook-s3.md +++ b/docs/configuring-playbook-s3.md @@ -5,6 +5,8 @@ If that's alright, you can skip this. As an alternative to storing media files on the local filesystem, you can store them on [Amazon S3](https://aws.amazon.com/s3/) or another S3-compatible object store. +You can do this either by sticking to Synapse's media repository and making that use S3 (read below for this method), or by switching to an external media storage implementation like [matrix-media-repo](configuring-playbook-matrix-media-repo.md). + First, [choose an Object Storage provider](#choosing-an-object-storage-provider). Then, [create the S3 bucket](#bucket-creation-and-security-configuration). diff --git a/docs/configuring-playbook.md b/docs/configuring-playbook.md index 31eea8953..8f598f6bd 100644 --- a/docs/configuring-playbook.md +++ b/docs/configuring-playbook.md @@ -51,6 +51,8 @@ When you're done with all the configuration you'd like to do, continue with [Ins - [Configuring Element](configuring-playbook-client-element.md) (optional) +- [Storing Matrix media files using matrix-media-repo](configuring-playbook-matrix-media-repo.md) (optional) + - [Storing Matrix media files on Amazon S3](configuring-playbook-s3.md) (optional) - [Using an external PostgreSQL server](configuring-playbook-external-postgres.md) (optional) From 60c34d701af574926971d2c965c12590e955da6d Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 18 Jul 2023 18:12:47 +0300 Subject: [PATCH 35/43] Use prebuilt container images for matrix-sliding-sync on ARM64 As mentioned in https://github.com/matrix-org/sliding-sync/issues/31#issuecomment-1640321110 images are available for arm64 already. --- group_vars/matrix_servers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index f932dcc35..8d7197f15 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -3927,7 +3927,7 @@ matrix_sliding_sync_hostname: "{{ matrix_server_fqn_matrix }}" matrix_sliding_sync_path_prefix: /sliding-sync -matrix_sliding_sync_container_image_self_build: "{{ matrix_architecture not in ['amd64'] }}" +matrix_sliding_sync_container_image_self_build: "{{ matrix_architecture not in ['amd64', 'arm64'] }}" matrix_sliding_sync_container_additional_networks: | {{ From 2fad66a3719ad09347e70e012ea4ca3e2175e40d Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 18 Jul 2023 20:04:51 +0300 Subject: [PATCH 36/43] Upgrade Synapse (v1.87.0 -> v1.88.0) --- roles/custom/matrix-synapse/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/custom/matrix-synapse/defaults/main.yml b/roles/custom/matrix-synapse/defaults/main.yml index 6548f3561..9cc5fffcf 100644 --- a/roles/custom/matrix-synapse/defaults/main.yml +++ b/roles/custom/matrix-synapse/defaults/main.yml @@ -4,7 +4,7 @@ matrix_synapse_enabled: true -matrix_synapse_version: v1.87.0 +matrix_synapse_version: v1.88.0 matrix_synapse_username: '' matrix_synapse_uid: '' From 64396419ba8c70fdde5679324a7f7c2fbdb1eaa5 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 18 Jul 2023 20:40:44 +0300 Subject: [PATCH 37/43] Upgrade synapse-s3-storage-provider (1.2.0 -> 1.2.1) --- roles/custom/matrix-synapse/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/custom/matrix-synapse/defaults/main.yml b/roles/custom/matrix-synapse/defaults/main.yml index 9cc5fffcf..48d8e8d98 100644 --- a/roles/custom/matrix-synapse/defaults/main.yml +++ b/roles/custom/matrix-synapse/defaults/main.yml @@ -870,7 +870,7 @@ matrix_synapse_ext_encryption_config_yaml: | # Installing it requires building a customized Docker image for Synapse (see `matrix_synapse_container_image_customizations_enabled`). # Enabling this will enable customizations and inject the appropriate Dockerfile clauses for installing synapse-s3-storage-provider. matrix_synapse_ext_synapse_s3_storage_provider_enabled: false -matrix_synapse_ext_synapse_s3_storage_provider_version: 1.2.0 +matrix_synapse_ext_synapse_s3_storage_provider_version: 1.2.1 # Controls whether media from this (local) server is stored in s3-storage-provider matrix_synapse_ext_synapse_s3_storage_provider_store_local: true # Controls whether media from remote servers is stored in s3-storage-provider From 330dfd4eaf804bcefdb44ab9efe635d49c663df0 Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Tue, 18 Jul 2023 22:38:37 +0300 Subject: [PATCH 38/43] Update element 1.11.35 -> 1.11.36 --- roles/custom/matrix-client-element/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/custom/matrix-client-element/defaults/main.yml b/roles/custom/matrix-client-element/defaults/main.yml index 368639cb0..9199a1d5a 100644 --- a/roles/custom/matrix-client-element/defaults/main.yml +++ b/roles/custom/matrix-client-element/defaults/main.yml @@ -10,7 +10,7 @@ matrix_client_element_container_image_self_build_repo: "https://github.com/vecto # - https://github.com/vector-im/element-web/issues/19544 matrix_client_element_container_image_self_build_low_memory_system_patch_enabled: "{{ ansible_memtotal_mb < 4096 }}" -matrix_client_element_version: v1.11.35 +matrix_client_element_version: v1.11.36 matrix_client_element_docker_image: "{{ matrix_client_element_docker_image_name_prefix }}vectorim/element-web:{{ matrix_client_element_version }}" matrix_client_element_docker_image_name_prefix: "{{ 'localhost/' if matrix_client_element_container_image_self_build else matrix_container_global_registry_prefix }}" matrix_client_element_docker_image_force_pull: "{{ matrix_client_element_docker_image.endswith(':latest') }}" From a5cb6272d8891532735aee79238059f1d9ea5ef6 Mon Sep 17 00:00:00 2001 From: Aine Date: Fri, 21 Jul 2023 13:22:05 +0300 Subject: [PATCH 39/43] add global bridges relay mode switch --- docs/configuring-playbook-mautrix-bridges.md | 22 +++++++++++++++++++ roles/custom/matrix-base/defaults/main.yml | 3 +++ .../defaults/main.yml | 2 ++ .../templates/config.yaml.j2 | 2 +- .../defaults/main.yml | 3 +++ .../templates/config.yaml.j2 | 2 +- .../defaults/main.yml | 3 +++ .../templates/config.yaml.j2 | 17 ++++++++++++++ .../defaults/main.yml | 2 +- .../defaults/main.yml | 2 +- 10 files changed, 54 insertions(+), 4 deletions(-) diff --git a/docs/configuring-playbook-mautrix-bridges.md b/docs/configuring-playbook-mautrix-bridges.md index 392be47df..c6e78d02d 100644 --- a/docs/configuring-playbook-mautrix-bridges.md +++ b/docs/configuring-playbook-mautrix-bridges.md @@ -32,14 +32,18 @@ matrix_mautrix_SERVICENAME_configuration_extension_yaml: | '@YOUR_USERNAME:{{ matrix_domain }}': admin ``` +## encryption + Encryption support is off by default. If you would like to enable encryption, add the following to your `vars.yml` file: **for all bridges with encryption support**: + ```yaml matrix_bridges_encryption_enabled: true ``` **Alternatively**, for a specific bridge: + ```yaml matrix_mautrix_SERVICENAME_configuration_extension_yaml: | bridge: @@ -48,6 +52,24 @@ matrix_mautrix_SERVICENAME_configuration_extension_yaml: | default: true ``` +## relay mode + +Relay mode is off by default. If you would like to enable relay mode, add the following to your `vars.yml` file: + +**for all bridges with relay mode support**: + +```yaml +matrix_bridges_relay_enabled: true +``` + +**Alternatively**, for a specific bridge: + +```yaml +matrix_mautrix_SERVICENAME_configuration_extension_yaml: | + bridge: + relay: + enabled: true +``` You can only have one `matrix_mautrix_SERVICENAME_configuration_extension_yaml` definition in `vars.yml` per bridge, so if you need multiple pieces of configuration there, just merge them like this: diff --git a/roles/custom/matrix-base/defaults/main.yml b/roles/custom/matrix-base/defaults/main.yml index 35974bba9..637ce3bb9 100644 --- a/roles/custom/matrix-base/defaults/main.yml +++ b/roles/custom/matrix-base/defaults/main.yml @@ -16,6 +16,9 @@ matrix_admin: '' # Global var to enable/disable encryption across all bridges with encryption support matrix_bridges_encryption_enabled: false +# Global var to enable/disable relay mode across all bridges with relay mode support +matrix_bridges_relay_enabled: false + # matrix_homeserver_enabled controls whether to enable the homeserver systemd service, etc. # # Unless you're wrapping this playbook in another one diff --git a/roles/custom/matrix-bridge-appservice-kakaotalk/defaults/main.yml b/roles/custom/matrix-bridge-appservice-kakaotalk/defaults/main.yml index 86c3366c3..9f86be5fb 100644 --- a/roles/custom/matrix-bridge-appservice-kakaotalk/defaults/main.yml +++ b/roles/custom/matrix-bridge-appservice-kakaotalk/defaults/main.yml @@ -110,6 +110,8 @@ matrix_appservice_kakaotalk_login_shared_secret: '' matrix_appservice_kakaotalk_bridge_login_shared_secret_map: "{{ {matrix_appservice_kakaotalk_homeserver_domain: matrix_appservice_kakaotalk_login_shared_secret} if matrix_appservice_kakaotalk_login_shared_secret else {} }}" +matrix_appservice_kakaotalk_bridge_relay_enabled: "{{ matrix_bridges_relay_enabled }}" + matrix_appservice_kakaotalk_bridge_permissions: | {{ {matrix_appservice_kakaotalk_homeserver_domain: 'user'} diff --git a/roles/custom/matrix-bridge-appservice-kakaotalk/templates/config.yaml.j2 b/roles/custom/matrix-bridge-appservice-kakaotalk/templates/config.yaml.j2 index 803d443fd..cf3644a9b 100644 --- a/roles/custom/matrix-bridge-appservice-kakaotalk/templates/config.yaml.j2 +++ b/roles/custom/matrix-bridge-appservice-kakaotalk/templates/config.yaml.j2 @@ -220,7 +220,7 @@ bridge: relay: # Whether relay mode should be allowed. If allowed, `!kt set-relay` can be used to turn any # authenticated user into a relaybot for that chat. - enabled: false + enabled: {{ matrix_appservice_kakaotalk_bridge_relay_enabled }} # The formats to use when sending messages to KakaoTalk via a relay user. # # Available variables: diff --git a/roles/custom/matrix-bridge-mautrix-facebook/defaults/main.yml b/roles/custom/matrix-bridge-mautrix-facebook/defaults/main.yml index dcd9fdfa0..0b48aaad5 100644 --- a/roles/custom/matrix-bridge-mautrix-facebook/defaults/main.yml +++ b/roles/custom/matrix-bridge-mautrix-facebook/defaults/main.yml @@ -103,6 +103,9 @@ matrix_mautrix_facebook_login_shared_secret: '' matrix_mautrix_facebook_bridge_login_shared_secret_map: "{{ {matrix_mautrix_facebook_homeserver_domain: matrix_mautrix_facebook_login_shared_secret} if matrix_mautrix_facebook_login_shared_secret else {} }}" +# Enable bridge relay bot functionality +matrix_mautrix_facebook_relay_enabled: "{{ matrix_bridges_relay_enabled }}" + matrix_mautrix_facebook_appservice_bot_username: facebookbot matrix_mautrix_facebook_bridge_presence: true diff --git a/roles/custom/matrix-bridge-mautrix-facebook/templates/config.yaml.j2 b/roles/custom/matrix-bridge-mautrix-facebook/templates/config.yaml.j2 index 636b442b9..7ec9342a2 100644 --- a/roles/custom/matrix-bridge-mautrix-facebook/templates/config.yaml.j2 +++ b/roles/custom/matrix-bridge-mautrix-facebook/templates/config.yaml.j2 @@ -206,7 +206,7 @@ bridge: relay: # Whether relay mode should be allowed. If allowed, `!fb set-relay` can be used to turn any # authenticated user into a relaybot for that chat. - enabled: false + enabled: {{ matrix_mautrix_facebook_relay_enabled }} # The formats to use when sending messages to Messenger via a relay user. # # Available variables: diff --git a/roles/custom/matrix-bridge-mautrix-instagram/defaults/main.yml b/roles/custom/matrix-bridge-mautrix-instagram/defaults/main.yml index 2288fb776..cef492889 100644 --- a/roles/custom/matrix-bridge-mautrix-instagram/defaults/main.yml +++ b/roles/custom/matrix-bridge-mautrix-instagram/defaults/main.yml @@ -83,6 +83,9 @@ matrix_mautrix_instagram_login_shared_secret: '' matrix_mautrix_instagram_bridge_login_shared_secret_map: "{{ {matrix_mautrix_instagram_homeserver_domain: matrix_mautrix_instagram_login_shared_secret} if matrix_mautrix_instagram_login_shared_secret else {} }}" +# Enable bridge relay bot functionality +matrix_mautrix_instagram_relay_enabled: "{{ matrix_bridges_relay_enabled }}" + matrix_mautrix_instagram_appservice_bot_username: instagrambot matrix_mautrix_instagram_bridge_presence: true diff --git a/roles/custom/matrix-bridge-mautrix-instagram/templates/config.yaml.j2 b/roles/custom/matrix-bridge-mautrix-instagram/templates/config.yaml.j2 index 1949a2532..e3d4be521 100644 --- a/roles/custom/matrix-bridge-mautrix-instagram/templates/config.yaml.j2 +++ b/roles/custom/matrix-bridge-mautrix-instagram/templates/config.yaml.j2 @@ -196,6 +196,23 @@ bridge: # The shared secret to authorize users of the API. # Set to "generate" to generate and save a new token. shared_secret: generate + relay: + # Whether relay mode should be allowed. If allowed, `!ig set-relay` can be used to turn any + # authenticated user into a relaybot for that chat. + enabled: {{ matrix_mautrix_instagram_relay_enabled }} + # The formats to use when sending messages to Instagram via a relay user. + # + # Available variables: + # $sender_displayname - The display name of the sender (e.g. Example User) + # $sender_username - The username (Matrix ID localpart) of the sender (e.g. exampleuser) + # $sender_mxid - The Matrix ID of the sender (e.g. @exampleuser:example.com) + # $message - The message content + # + # Note that Instagram doesn't support captions for images, so images won't include any indication of being relayed. + message_formats: + m.text: '$sender_displayname: $message' + m.notice: '$sender_displayname: $message' + m.emote: '* $sender_displayname $message' # Python logging configuration. # diff --git a/roles/custom/matrix-bridge-mautrix-signal/defaults/main.yml b/roles/custom/matrix-bridge-mautrix-signal/defaults/main.yml index ca9bab54a..7675e7b17 100644 --- a/roles/custom/matrix-bridge-mautrix-signal/defaults/main.yml +++ b/roles/custom/matrix-bridge-mautrix-signal/defaults/main.yml @@ -104,7 +104,7 @@ matrix_mautrix_signal_appservice_database: "{{ matrix_mautrix_signal_login_shared_secret: '' # Enable bridge relay bot functionality -matrix_mautrix_signal_relaybot_enabled: false +matrix_mautrix_signal_relaybot_enabled: "{{ matrix_bridges_relay_enabled }}" # Permissions for using the bridge. # Permitted values: diff --git a/roles/custom/matrix-bridge-mautrix-whatsapp/defaults/main.yml b/roles/custom/matrix-bridge-mautrix-whatsapp/defaults/main.yml index ab2921e31..a308811a7 100644 --- a/roles/custom/matrix-bridge-mautrix-whatsapp/defaults/main.yml +++ b/roles/custom/matrix-bridge-mautrix-whatsapp/defaults/main.yml @@ -113,7 +113,7 @@ matrix_mautrix_whatsapp_bridge_permissions: | }} # Enable bridge relay functionality -matrix_mautrix_whatsapp_bridge_relay_enabled: false +matrix_mautrix_whatsapp_bridge_relay_enabled: "{{ matrix_bridges_relay_enabled }}" # Only allow admins on this home server to set themselves as a relay user matrix_mautrix_whatsapp_bridge_relay_admin_only: true From 5ea6aa3e50d20a4a2e50e84810c20411a1f58448 Mon Sep 17 00:00:00 2001 From: Shreyas Ajjarapu Date: Fri, 21 Jul 2023 06:33:52 -0500 Subject: [PATCH 40/43] Added Google Messager Bridge (#2794) * intial commit * changed * Reorderd * merge old changes * added changes to matrix_servers * Remove duplicate discord * Update main.yml * added google message to configuring-playbook.md * Changed docs to add new changes * Changed bug? * Removed problem j2 values * Rename a service files * change how password hash string * Changed port number * Change how the local part works * Revert "Merge pull request #8 from shreyasajj/wsproxy" This reverts commit bb1b8fc67ca39f63ca77e70077be99cb2b32c4de, reversing changes made to cce6ba5f9d74f89172488afc8b1ef124031de8c1. --------- Co-authored-by: Shreyas Ajjarapu --- README.md | 1 + ...uring-playbook-bridge-mautrix-gmessages.md | 38 +++ docs/configuring-playbook.md | 2 + docs/container-images.md | 2 + docs/self-building.md | 1 + group_vars/matrix_servers | 63 +++- .../defaults/main.yml | 151 +++++++++ .../tasks/inject_into_nginx_proxy.yml | 35 ++ .../tasks/main.yml | 29 ++ .../tasks/setup_install.yml | 140 ++++++++ .../tasks/setup_uninstall.yml | 20 ++ .../tasks/validate_config.yml | 20 ++ .../templates/config.yaml.j2 | 298 ++++++++++++++++++ .../matrix-mautrix-gmessages.service.j2 | 43 +++ setup.yml | 1 + 15 files changed, 841 insertions(+), 3 deletions(-) create mode 100644 docs/configuring-playbook-bridge-mautrix-gmessages.md create mode 100644 roles/custom/matrix-bridge-mautrix-gmessages/defaults/main.yml create mode 100644 roles/custom/matrix-bridge-mautrix-gmessages/tasks/inject_into_nginx_proxy.yml create mode 100644 roles/custom/matrix-bridge-mautrix-gmessages/tasks/main.yml create mode 100644 roles/custom/matrix-bridge-mautrix-gmessages/tasks/setup_install.yml create mode 100644 roles/custom/matrix-bridge-mautrix-gmessages/tasks/setup_uninstall.yml create mode 100644 roles/custom/matrix-bridge-mautrix-gmessages/tasks/validate_config.yml create mode 100644 roles/custom/matrix-bridge-mautrix-gmessages/templates/config.yaml.j2 create mode 100644 roles/custom/matrix-bridge-mautrix-gmessages/templates/systemd/matrix-mautrix-gmessages.service.j2 diff --git a/README.md b/README.md index 912febc04..71d46f822 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,7 @@ Bridges can be used to connect your matrix installation with third-party communi | [mautrix-discord](https://github.com/mautrix/discord) | x | Bridge for bridging your Matrix server to [Discord](https://discord.com/) | [Link](docs/configuring-playbook-bridge-mautrix-discord.md) | | [mautrix-slack](https://github.com/mautrix/slack) | x | Bridge for bridging your Matrix server to [Slack](https://slack.com/) | [Link](docs/configuring-playbook-bridge-mautrix-slack.md) | | [mautrix-telegram](https://github.com/mautrix/telegram) | x | Bridge for bridging your Matrix server to [Telegram](https://telegram.org/) | [Link](docs/configuring-playbook-bridge-mautrix-telegram.md) | +| [mautrix-gmessages](https://github.com/mautrix/gmessages) | x | Bridge for bridging your Matrix server to [Google Messages](https://messages.google.com/) | [Link](docs/configuring-playbook-bridge-mautrix-gmessages.md) | | [mautrix-whatsapp](https://github.com/mautrix/whatsapp) | x | Bridge for bridging your Matrix server to [WhatsApp](https://www.whatsapp.com/) | [Link](docs/configuring-playbook-bridge-mautrix-whatsapp.md) | | [mautrix-facebook](https://github.com/mautrix/facebook) | x | Bridge for bridging your Matrix server to [Facebook](https://facebook.com/) | [Link](docs/configuring-playbook-bridge-mautrix-facebook.md) | | [mautrix-twitter](https://github.com/mautrix/twitter) | x | Bridge for bridging your Matrix server to [Twitter](https://twitter.com/) | [Link](docs/configuring-playbook-bridge-mautrix-twitter.md) | diff --git a/docs/configuring-playbook-bridge-mautrix-gmessages.md b/docs/configuring-playbook-bridge-mautrix-gmessages.md new file mode 100644 index 000000000..6b2b18e61 --- /dev/null +++ b/docs/configuring-playbook-bridge-mautrix-gmessages.md @@ -0,0 +1,38 @@ +# Setting up Mautrix gmessages (optional) + +The playbook can install and configure [mautrix-gmessages](https://github.com/mautrix/gmessages) for you. + +See the project's [documentation](https://docs.mau.fi/bridges/go/gmessages/index.html) to learn what it does and why it might be useful to you. + +Use the following playbook configuration: + +```yaml +matrix_mautrix_gmessages_enabled: true +``` + +## Set up Double Puppeting + +If you'd like to use [Double Puppeting](https://docs.mau.fi/bridges/general/double-puppeting.html) (hint: you most likely do), you have 2 ways of going about it. + +### Method 1: automatically, by enabling Shared Secret Auth + +The bridge will automatically perform Double Puppeting if you enable [Shared Secret Auth](configuring-playbook-shared-secret-auth.md) for this playbook. + +This is the recommended way of setting up Double Puppeting, as it's easier to accomplish, works for all your users automatically, and has less of a chance of breaking in the future. + +### Method 2: manually, by asking each user to provide a working access token + +**Note**: This method for enabling Double Puppeting can be configured only after you've already set up bridging (see [Usage](#usage)). + +When using this method, **each user** that wishes to enable Double Puppeting needs to follow the following steps: + +- retrieve a Matrix access token for yourself. Refer to the documentation on [how to do that](obtaining-access-tokens.md). + +- send the access token to the bot. Example: `login-matrix MATRIX_ACCESS_TOKEN_HERE` + +- make sure you don't log out the `Mautrix-gmessages` device some time in the future, as that would break the Double Puppeting feature + + +## Usage + +You then need to start a chat with `@gmessagesbot:YOUR_DOMAIN` (where `YOUR_DOMAIN` is your base domain, not the `matrix.` domain). diff --git a/docs/configuring-playbook.md b/docs/configuring-playbook.md index 8f598f6bd..a4746a62c 100644 --- a/docs/configuring-playbook.md +++ b/docs/configuring-playbook.md @@ -114,6 +114,8 @@ When you're done with all the configuration you'd like to do, continue with [Ins - [Setting up Mautrix Slack bridging](configuring-playbook-bridge-mautrix-slack.md) (optional) +- [Setting up Mautrix Google Messages bridging](configuring-playbook-bridge-mautrix-gmessages.md) (optional) + - [Setting up Mautrix Whatsapp bridging](configuring-playbook-bridge-mautrix-whatsapp.md) (optional) - [Setting up Mautrix Facebook bridging](configuring-playbook-bridge-mautrix-facebook.md) (optional) diff --git a/docs/container-images.md b/docs/container-images.md index 737a44578..e89161f7e 100644 --- a/docs/container-images.md +++ b/docs/container-images.md @@ -46,6 +46,8 @@ These services are not part of our default installation, but can be enabled by [ - [mautrix/telegram](https://mau.dev/mautrix/telegram/container_registry) - the [mautrix-telegram](https://github.com/mautrix/telegram) bridge to [Telegram](https://telegram.org/) (optional) +- [mautrix/gmessages](https://mau.dev/mautrix/gmessages/container_registry) - the [mautrix-gmessages](https://github.com/mautrix/gmessages) bridge to [Google Messages](https://messages.google.com/) (optional) + - [mautrix/whatsapp](https://mau.dev/mautrix/whatsapp/container_registry) - the [mautrix-whatsapp](https://github.com/mautrix/whatsapp) bridge to [Whatsapp](https://www.whatsapp.com/) (optional) - [mautrix/facebook](https://mau.dev/mautrix/facebook/container_registry) - the [mautrix-facebook](https://github.com/mautrix/facebook) bridge to [Facebook](https://facebook.com/) (optional) diff --git a/docs/self-building.md b/docs/self-building.md index ad29fc2da..3fe826da1 100644 --- a/docs/self-building.md +++ b/docs/self-building.md @@ -32,6 +32,7 @@ Possibly outdated list of roles where self-building the Docker image is currentl - `matrix-bridge-mautrix-googlechat` - `matrix-bridge-mautrix-telegram` - `matrix-bridge-mautrix-signal` +- `matrix-bridge-mautrix-gmessages` - `matrix-bridge-mautrix-whatsapp` - `matrix-bridge-mx-puppet-steam` - `matrix-bot-mjolnir` diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 8d7197f15..5fc5c25d8 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -101,6 +101,8 @@ matrix_homeserver_container_extra_arguments_auto: | + (['--mount type=bind,src=' + matrix_mautrix_twitter_config_path + '/registration.yaml,dst=/matrix-mautrix-twitter-registration.yaml,ro'] if matrix_mautrix_twitter_enabled else []) + + (['--mount type=bind,src=' + matrix_mautrix_gmessages_config_path + '/registration.yaml,dst=/matrix-mautrix-gmessages-registration.yaml,ro'] if matrix_mautrix_gmessages_enabled else []) + + (['--mount type=bind,src=' + matrix_mautrix_whatsapp_config_path + '/registration.yaml,dst=/matrix-mautrix-whatsapp-registration.yaml,ro'] if matrix_mautrix_whatsapp_enabled else []) + (['--mount type=bind,src=' + matrix_mx_puppet_discord_config_path + '/registration.yaml,dst=/matrix-mx-puppet-discord-registration.yaml,ro'] if matrix_mx_puppet_discord_enabled else []) @@ -158,6 +160,8 @@ matrix_homeserver_app_service_config_files_auto: | + (['/matrix-mautrix-twitter-registration.yaml'] if matrix_mautrix_twitter_enabled else []) + + (['/matrix-mautrix-gmessages-registration.yaml'] if matrix_mautrix_gmessages_enabled else []) + + (['/matrix-mautrix-whatsapp-registration.yaml'] if matrix_mautrix_whatsapp_enabled else []) + (['/matrix-mx-puppet-discord-registration.yaml'] if matrix_mx_puppet_discord_enabled else []) @@ -270,6 +274,8 @@ devture_systemd_service_manager_services_list_auto: | + ([{'name': 'matrix-mautrix-twitter.service', 'priority': 2000, 'groups': ['matrix', 'bridges', 'mautrix-twitter']}] if matrix_mautrix_twitter_enabled else []) + + ([{'name': 'matrix-mautrix-gmessages.service', 'priority': 2000, 'groups': ['matrix', 'bridges', 'mautrix-gmessages']}] if matrix_mautrix_gmessages_enabled else []) + + ([{'name': 'matrix-mautrix-whatsapp.service', 'priority': 2000, 'groups': ['matrix', 'bridges', 'mautrix-whatsapp']}] if matrix_mautrix_whatsapp_enabled else []) + ([{'name': 'matrix-mx-puppet-discord.service', 'priority': 2000, 'groups': ['matrix', 'bridges', 'mx-puppet-discord']}] if matrix_mx_puppet_discord_enabled else []) @@ -1153,6 +1159,51 @@ matrix_mautrix_twitter_database_password: "{{ '%s' | format(matrix_homeserver_ge # ###################################################################### +###################################################################### +# +# matrix-bridge-mautrix-gmessages +# +###################################################################### + +# We don't enable bridges by default. +matrix_mautrix_gmessages_enabled: false + +matrix_mautrix_gmessages_container_image_self_build: "{{ matrix_architecture not in ['arm64', 'amd64'] }}" + +matrix_mautrix_gmessages_systemd_required_services_list: | + {{ + ['docker.service'] + + + ['matrix-' + matrix_homeserver_implementation + '.service'] + + + ([devture_postgres_identifier ~ '.service'] if devture_postgres_enabled else []) + + + (['matrix-nginx-proxy.service'] if matrix_nginx_proxy_enabled else []) + }} + +matrix_mautrix_gmessages_appservice_token: "{{ '%s' | format(matrix_homeserver_generic_secret_key) | password_hash('sha512', 'gmessage.as.token', rounds=655555) | to_uuid }}" + +matrix_mautrix_gmessages_homeserver_token: "{{ '%s' | format(matrix_homeserver_generic_secret_key) | password_hash('sha512', 'gmessage.hs.token', rounds=655555) | to_uuid }}" + +matrix_mautrix_gmessages_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}" + +# People using an external Prometheus server will need to toggle all of these to be able to consume metrics remotely: +# - `matrix_mautrix_gmessages_metrics_enabled` +# - `matrix_mautrix_gmessages_proxying_metrics_enabled` +# - `matrix_nginx_proxy_proxy_matrix_metrics_enabled` +matrix_mautrix_gmessages_metrics_enabled: "{{ prometheus_enabled }}" + +# Postgres is the default, except if not using internal Postgres server +matrix_mautrix_gmessages_database_engine: "{{ 'postgres' if devture_postgres_enabled else 'sqlite' }}" +matrix_mautrix_gmessages_database_hostname: "{{ devture_postgres_connection_hostname if devture_postgres_enabled else '' }}" +matrix_mautrix_gmessages_database_password: "{{ '%s' | format(matrix_homeserver_generic_secret_key) | password_hash('sha512', 'maugmessages.db', rounds=655555) | to_uuid }}" + +###################################################################### +# +# /matrix-bridge-mautrix-gmessages +# +###################################################################### + ###################################################################### # # matrix-bridge-mautrix-whatsapp @@ -1182,10 +1233,10 @@ matrix_mautrix_whatsapp_homeserver_token: "{{ '%s' | format(matrix_homeserver_ge matrix_mautrix_whatsapp_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}" # People using an external Prometheus server will need to toggle all of these to be able to consume metrics remotely: -# - `matrix_mautrix_twitter_metrics_enabled` -# - `matrix_mautrix_twitter_proxying_metrics_enabled` +# - `matrix_mautrix_whatsapp_metrics_enabled` +# - `matrix_mautrix_whatsapp_proxying_metrics_enabled` # - `matrix_nginx_proxy_proxy_matrix_metrics_enabled` -matrix_mautrix_twitter_metrics_enabled: "{{ prometheus_enabled }}" +matrix_mautrix_whatsapp_metrics_enabled: "{{ prometheus_enabled }}" # Postgres is the default, except if not using internal Postgres server matrix_mautrix_whatsapp_database_engine: "{{ 'postgres' if devture_postgres_enabled else 'sqlite' }}" @@ -3016,6 +3067,12 @@ devture_postgres_managed_databases_auto: | 'password': matrix_mautrix_twitter_database_password, }] if (matrix_mautrix_twitter_enabled and matrix_mautrix_twitter_database_engine == 'postgres' and matrix_mautrix_twitter_database_hostname == devture_postgres_connection_hostname) else []) + + ([{ + 'name': matrix_mautrix_gmessages_database_name, + 'username': matrix_mautrix_gmessages_database_username, + 'password': matrix_mautrix_gmessages_database_password, + }] if (matrix_mautrix_gmessages_enabled and matrix_mautrix_gmessages_database_engine == 'postgres' and matrix_mautrix_gmessages_database_hostname == devture_postgres_connection_hostname) else []) + + ([{ 'name': matrix_mautrix_whatsapp_database_name, 'username': matrix_mautrix_whatsapp_database_username, diff --git a/roles/custom/matrix-bridge-mautrix-gmessages/defaults/main.yml b/roles/custom/matrix-bridge-mautrix-gmessages/defaults/main.yml new file mode 100644 index 000000000..6a6c14b0a --- /dev/null +++ b/roles/custom/matrix-bridge-mautrix-gmessages/defaults/main.yml @@ -0,0 +1,151 @@ +--- +# mautrix-gmessages is a Matrix <-> gmessages bridge +# Project source code URL: https://github.com/mautrix/gmessages + +matrix_mautrix_gmessages_enabled: true + +matrix_mautrix_gmessages_container_image_self_build: false +matrix_mautrix_gmessages_container_image_self_build_repo: "https://github.com/mautrix/gmessages.git" +matrix_mautrix_gmessages_container_image_self_build_branch: "{{ 'master' if matrix_mautrix_gmessages_version == 'latest' else matrix_mautrix_gmessages_version }}" + +matrix_mautrix_gmessages_version: latest +# See: https://mau.dev/mautrix/gmessages/container_registry +matrix_mautrix_gmessages_docker_image: "{{ matrix_mautrix_gmessages_docker_image_name_prefix }}mautrix/gmessages:{{ matrix_mautrix_gmessages_version }}" +matrix_mautrix_gmessages_docker_image_name_prefix: "{{ 'localhost/' if matrix_mautrix_gmessages_container_image_self_build else 'dock.mau.dev/' }}" +matrix_mautrix_gmessages_docker_image_force_pull: "{{ matrix_mautrix_gmessages_docker_image.endswith(':latest') }}" + +matrix_mautrix_gmessages_base_path: "{{ matrix_base_data_path }}/mautrix-gmessages" +matrix_mautrix_gmessages_config_path: "{{ matrix_mautrix_gmessages_base_path }}/config" +matrix_mautrix_gmessages_data_path: "{{ matrix_mautrix_gmessages_base_path }}/data" +matrix_mautrix_gmessages_docker_src_files_path: "{{ matrix_mautrix_gmessages_base_path }}/docker-src" + +matrix_mautrix_gmessages_homeserver_address: "{{ matrix_homeserver_container_url }}" +matrix_mautrix_gmessages_homeserver_domain: "{{ matrix_domain }}" +matrix_mautrix_gmessages_appservice_address: "http://matrix-mautrix-gmessages:8080" + +matrix_mautrix_gmessages_command_prefix: "!gm" + +# A list of extra arguments to pass to the container +matrix_mautrix_gmessages_container_extra_arguments: [] + +# List of systemd services that matrix-mautrix-gmessages.service depends on. +matrix_mautrix_gmessages_systemd_required_services_list: ['docker.service'] + +# List of systemd services that matrix-mautrix-gmessages.service wants +matrix_mautrix_gmessages_systemd_wanted_services_list: [] + +matrix_mautrix_gmessages_appservice_token: '' +matrix_mautrix_gmessages_homeserver_token: '' + +matrix_mautrix_gmessages_appservice_bot_username: gmessagesbot + +# Minimum severity of journal log messages. +# Options: debug, info, warn, error, fatal +matrix_mautrix_gmessages_logging_level: 'warn' + +# Whether or not created rooms should have federation enabled. +# If false, created portal rooms will never be federated. +matrix_mautrix_gmessages_federate_rooms: true + +# Whether or not metrics endpoint should be enabled. +# Enabling them is usually enough for a local (in-container) Prometheus to consume them. +# If metrics need to be consumed by another (external) Prometheus server, consider exposing them via `matrix_mautrix_gmessages_metrics_proxying_enabled`. +matrix_mautrix_gmessages_metrics_enabled: false + +# Controls whether metrics should be proxied (exposed) on `matrix.DOMAIN/metrics/mautrix-gmessages`. +# This will only work take effect if `matrix_nginx_proxy_proxy_matrix_metrics_enabled: true`. +# See the `matrix-nginx-proxy` role for details about enabling `matrix_nginx_proxy_proxy_matrix_metrics_enabled`. +matrix_mautrix_gmessages_metrics_proxying_enabled: false + +# Database-related configuration fields. +# +# To use SQLite, stick to these defaults. +# +# To use Postgres: +# - change the engine (`matrix_mautrix_gmessages_database_engine: 'postgres'`) +# - adjust your database credentials via the `matrix_mautrix_gmessages_database_*` variables +matrix_mautrix_gmessages_database_engine: 'sqlite' + +matrix_mautrix_gmessages_sqlite_database_path_local: "{{ matrix_mautrix_gmessages_data_path }}/mautrix-gmessages.db" +matrix_mautrix_gmessages_sqlite_database_path_in_container: "/data/mautrix-gmessages.db" + +matrix_mautrix_gmessages_database_username: 'matrix_mautrix_gmessages' +matrix_mautrix_gmessages_database_password: 'some-password' +matrix_mautrix_gmessages_database_hostname: '' +matrix_mautrix_gmessages_database_port: 5432 +matrix_mautrix_gmessages_database_name: 'matrix_mautrix_gmessages' + +matrix_mautrix_gmessages_database_connection_string: 'postgresql://{{ matrix_mautrix_gmessages_database_username }}:{{ matrix_mautrix_gmessages_database_password }}@{{ matrix_mautrix_gmessages_database_hostname }}:{{ matrix_mautrix_gmessages_database_port }}/{{ matrix_mautrix_gmessages_database_name }}?sslmode=disable' + +matrix_mautrix_gmessages_appservice_database_type: "{{ + { + 'sqlite': 'sqlite3', + 'postgres':'postgres', + }[matrix_mautrix_gmessages_database_engine] +}}" + +matrix_mautrix_gmessages_appservice_database_uri: "{{ + { + 'sqlite': matrix_mautrix_gmessages_sqlite_database_path_in_container, + 'postgres': matrix_mautrix_gmessages_database_connection_string, + }[matrix_mautrix_gmessages_database_engine] +}}" + +# Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth). +matrix_mautrix_gmessages_login_shared_secret: '' +matrix_mautrix_gmessages_bridge_login_shared_secret_map: + "{{ {matrix_mautrix_gmessages_homeserver_domain: matrix_mautrix_gmessages_login_shared_secret} if matrix_mautrix_gmessages_login_shared_secret else {} }}" + +# Enable End-to-bridge encryption +matrix_mautrix_gmessages_bridge_encryption_allow: "{{ matrix_bridges_encryption_enabled }}" +matrix_mautrix_gmessages_bridge_encryption_default: "{{ matrix_mautrix_gmessages_bridge_encryption_allow }}" +matrix_mautrix_gmessages_bridge_encryption_key_sharing_allow: "{{ matrix_mautrix_gmessages_bridge_encryption_allow }}" + +matrix_mautrix_gmessages_bridge_personal_filtering_spaces: true +matrix_mautrix_gmessages_bridge_mute_bridging: true + +matrix_mautrix_gmessages_bridge_permissions: | + {{ + {matrix_mautrix_gmessages_homeserver_domain: 'user'} + | combine({matrix_admin: 'admin'} if matrix_admin else {}) + }} + +# Default mautrix-gmessages configuration template which covers the generic use case. +# You can customize it by controlling the various variables inside it. +# +# For a more advanced customization, you can extend the default (see `matrix_mautrix_gmessages_configuration_extension_yaml`) +# or completely replace this variable with your own template. +matrix_mautrix_gmessages_configuration_yaml: "{{ lookup('template', 'templates/config.yaml.j2') }}" + +matrix_mautrix_gmessages_configuration_extension_yaml: | + # Your custom YAML configuration goes here. + # This configuration extends the default starting configuration (`matrix_mautrix_gmessages_configuration_yaml`). + # + # You can override individual variables from the default configuration, or introduce new ones. + # + # If you need something more special, you can take full control by + # completely redefining `matrix_mautrix_gmessages_configuration_yaml`. + +matrix_mautrix_gmessages_configuration_extension: "{{ matrix_mautrix_gmessages_configuration_extension_yaml | from_yaml if matrix_mautrix_gmessages_configuration_extension_yaml | from_yaml is mapping else {} }}" + +# Holds the final configuration (a combination of the default and its extension). +# You most likely don't need to touch this variable. Instead, see `matrix_mautrix_gmessages_configuration_yaml`. +matrix_mautrix_gmessages_configuration: "{{ matrix_mautrix_gmessages_configuration_yaml | from_yaml | combine(matrix_mautrix_gmessages_configuration_extension, recursive=True) }}" + +matrix_mautrix_gmessages_registration_yaml: | + id: gmessages + url: {{ matrix_mautrix_gmessages_appservice_address }} + as_token: "{{ matrix_mautrix_gmessages_appservice_token }}" + hs_token: "{{ matrix_mautrix_gmessages_homeserver_token }}" + # See https://github.com/mautrix/signal/issues/43 + sender_localpart: _bot_{{ matrix_mautrix_gmessages_appservice_bot_username }} + rate_limited: false + namespaces: + users: + - regex: '^@gmessages_.+:{{ matrix_mautrix_gmessages_homeserver_domain | regex_escape }}$' + exclusive: true + - exclusive: true + regex: '^@{{ matrix_mautrix_gmessages_appservice_bot_username | regex_escape }}:{{ matrix_mautrix_gmessages_homeserver_domain | regex_escape }}$' + de.sorunome.msc2409.push_ephemeral: true + +matrix_mautrix_gmessages_registration: "{{ matrix_mautrix_gmessages_registration_yaml | from_yaml }}" diff --git a/roles/custom/matrix-bridge-mautrix-gmessages/tasks/inject_into_nginx_proxy.yml b/roles/custom/matrix-bridge-mautrix-gmessages/tasks/inject_into_nginx_proxy.yml new file mode 100644 index 000000000..c5cb1ba8a --- /dev/null +++ b/roles/custom/matrix-bridge-mautrix-gmessages/tasks/inject_into_nginx_proxy.yml @@ -0,0 +1,35 @@ +--- + +- name: Fail if matrix-nginx-proxy role already executed + ansible.builtin.fail: + msg: >- + Trying to append mautrix-gmessages-metrics's reverse-proxying configuration to matrix-nginx-proxy, + but it's pointless since the matrix-nginx-proxy role had already executed. + To fix this, please change the order of roles in your playbook, + so that the matrix-nginx-proxy role would run after the matrix-bridge-mautrix-gmessages role. + when: matrix_nginx_proxy_role_executed | default(False) | bool + +- when: matrix_mautrix_gmessages_metrics_proxying_enabled | bool + block: + - name: Generate mautrix-gmessages metrics proxying configuration for matrix-nginx-proxy (matrix.DOMAIN/metrics/mautrix-gmessages) + ansible.builtin.set_fact: + matrix_mautrix_gmessages_nginx_metrics_configuration_block: | + location /metrics/mautrix-gmessages { + {% if matrix_nginx_proxy_enabled | default(False) %} + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver 127.0.0.11 valid=5s; + set $backend "matrix-mautrix-gmessages:8001"; + proxy_pass http://$backend/metrics; + {% else %} + return 404 "matrix-nginx-proxy is disabled and no host port was bound to the container, so metrics are unavailable"; + {% endif %} + } + + - name: Register mautrix-gmessages metrics proxying configuration with matrix-nginx-proxy (matrix.DOMAIN/metrics/mautrix-gmessages) + ansible.builtin.set_fact: + matrix_nginx_proxy_proxy_matrix_metrics_additional_system_location_configuration_blocks: | + {{ + matrix_nginx_proxy_proxy_matrix_metrics_additional_system_location_configuration_blocks | default([]) + + + [matrix_mautrix_gmessages_nginx_metrics_configuration_block] + }} diff --git a/roles/custom/matrix-bridge-mautrix-gmessages/tasks/main.yml b/roles/custom/matrix-bridge-mautrix-gmessages/tasks/main.yml new file mode 100644 index 000000000..45da31da8 --- /dev/null +++ b/roles/custom/matrix-bridge-mautrix-gmessages/tasks/main.yml @@ -0,0 +1,29 @@ +--- + +- tags: + - setup-all + - setup-nginx-proxy + - install-all + - install-nginx-proxy + block: + - when: matrix_mautrix_gmessages_enabled | bool and matrix_mautrix_gmessages_metrics_enabled | bool + ansible.builtin.include_tasks: "{{ role_path }}/tasks/inject_into_nginx_proxy.yml" + +- tags: + - setup-all + - setup-mautrix-gmessages + - install-all + - install-mautrix-gmessages + block: + - when: matrix_mautrix_gmessages_enabled | bool + ansible.builtin.include_tasks: "{{ role_path }}/tasks/validate_config.yml" + + - when: matrix_mautrix_gmessages_enabled | bool + ansible.builtin.include_tasks: "{{ role_path }}/tasks/setup_install.yml" + +- tags: + - setup-all + - setup-mautrix-gmessages + block: + - when: not matrix_mautrix_gmessages_enabled | bool + ansible.builtin.include_tasks: "{{ role_path }}/tasks/setup_uninstall.yml" diff --git a/roles/custom/matrix-bridge-mautrix-gmessages/tasks/setup_install.yml b/roles/custom/matrix-bridge-mautrix-gmessages/tasks/setup_install.yml new file mode 100644 index 000000000..73038c4c1 --- /dev/null +++ b/roles/custom/matrix-bridge-mautrix-gmessages/tasks/setup_install.yml @@ -0,0 +1,140 @@ +--- + +- ansible.builtin.set_fact: + matrix_mautrix_gmessages_requires_restart: false + +- when: "matrix_mautrix_gmessages_database_engine == 'postgres'" + block: + - name: Check if an SQLite database already exists + ansible.builtin.stat: + path: "{{ matrix_mautrix_gmessages_sqlite_database_path_local }}" + register: matrix_mautrix_gmessages_sqlite_database_path_local_stat_result + + - when: "matrix_mautrix_gmessages_sqlite_database_path_local_stat_result.stat.exists | bool" + block: + - ansible.builtin.include_role: + name: galaxy/com.devture.ansible.role.postgres + tasks_from: migrate_db_to_postgres + vars: + devture_postgres_db_migration_request: + src: "{{ matrix_mautrix_gmessages_sqlite_database_path_local }}" + dst: "{{ matrix_mautrix_gmessages_database_connection_string }}" + caller: "{{ role_path | basename }}" + engine_variable_name: 'matrix_mautrix_gmessages_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-mautrix-gmessages.service'] + pgloader_options: ['--with "quote identifiers"'] + + - ansible.builtin.set_fact: + matrix_mautrix_gmessages_requires_restart: true + +- name: Ensure Mautrix gmessages paths exists + ansible.builtin.file: + path: "{{ item.path }}" + state: directory + mode: 0750 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + with_items: + - {path: "{{ matrix_mautrix_gmessages_base_path }}", when: true} + - {path: "{{ matrix_mautrix_gmessages_config_path }}", when: true} + - {path: "{{ matrix_mautrix_gmessages_data_path }}", when: true} + - {path: "{{ matrix_mautrix_gmessages_docker_src_files_path }}", when: "{{ matrix_mautrix_gmessages_container_image_self_build }}"} + when: item.when | bool + +- name: Ensure Mautrix gmessages image is pulled + community.docker.docker_image: + name: "{{ matrix_mautrix_gmessages_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_mautrix_gmessages_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mautrix_gmessages_docker_image_force_pull }}" + when: not matrix_mautrix_gmessages_container_image_self_build + register: result + retries: "{{ devture_playbook_help_container_retries_count }}" + delay: "{{ devture_playbook_help_container_retries_delay }}" + until: result is not failed + +- name: Ensure Mautrix gmessages repository is present on self-build + ansible.builtin.git: + repo: "{{ matrix_mautrix_gmessages_container_image_self_build_repo }}" + dest: "{{ matrix_mautrix_gmessages_docker_src_files_path }}" + version: "{{ matrix_mautrix_gmessages_container_image_self_build_branch }}" + force: "yes" + become: true + become_user: "{{ matrix_user_username }}" + register: matrix_mautrix_gmessages_git_pull_results + when: "matrix_mautrix_gmessages_container_image_self_build | bool" + +- name: Ensure Mautrix gmessages Docker image is built + community.docker.docker_image: + name: "{{ matrix_mautrix_gmessages_docker_image }}" + source: build + force_source: "{{ matrix_mautrix_gmessages_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mautrix_gmessages_git_pull_results.changed }}" + build: + dockerfile: Dockerfile + path: "{{ matrix_mautrix_gmessages_docker_src_files_path }}" + pull: true + when: "matrix_mautrix_gmessages_container_image_self_build | bool" + +- name: Check if an old database file exists + ansible.builtin.stat: + path: "{{ matrix_mautrix_gmessages_base_path }}/mautrix-gmessages.db" + register: matrix_mautrix_gmessages_stat_database + +- name: Check if an old matrix state file exists + ansible.builtin.stat: + path: "{{ matrix_mautrix_gmessages_base_path }}/mx-state.json" + register: matrix_mautrix_gmessages_stat_mx_state + +- name: (Data relocation) Ensure matrix-mautrix-gmessages.service is stopped + ansible.builtin.service: + name: matrix-mautrix-gmessages + state: stopped + enabled: false + daemon_reload: true + failed_when: false + when: "matrix_mautrix_gmessages_stat_database.stat.exists" + +- name: (Data relocation) Move mautrix-gmessages database file to ./data directory + ansible.builtin.command: + cmd: "mv {{ matrix_mautrix_gmessages_base_path }}/mautrix-gmessages.db {{ matrix_mautrix_gmessages_data_path }}/mautrix-gmessages.db" + creates: "{{ matrix_mautrix_gmessages_data_path }}/mautrix-gmessages.db" + removes: "{{ matrix_mautrix_gmessages_base_path }}/mautrix-gmessages.db" + when: "matrix_mautrix_gmessages_stat_database.stat.exists" + +- name: (Data relocation) Move mautrix-gmessages mx-state file to ./data directory + ansible.builtin.command: + cmd: "mv {{ matrix_mautrix_gmessages_base_path }}/mx-state.json {{ matrix_mautrix_gmessages_data_path }}/mx-state.json" + creates: "{{ matrix_mautrix_gmessages_data_path }}/mx-state.json" + removes: "{{ matrix_mautrix_gmessages_base_path }}/mx-state.json" + when: "matrix_mautrix_gmessages_stat_mx_state.stat.exists" + +- name: Ensure mautrix-gmessages config.yaml installed + ansible.builtin.copy: + content: "{{ matrix_mautrix_gmessages_configuration | to_nice_yaml(indent=2, width=999999) }}" + dest: "{{ matrix_mautrix_gmessages_config_path }}/config.yaml" + mode: 0644 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + +- name: Ensure mautrix-gmessages registration.yaml installed + ansible.builtin.copy: + content: "{{ matrix_mautrix_gmessages_registration | to_nice_yaml(indent=2, width=999999) }}" + dest: "{{ matrix_mautrix_gmessages_config_path }}/registration.yaml" + mode: 0644 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + +- name: Ensure matrix-mautrix-gmessages.service installed + ansible.builtin.template: + src: "{{ role_path }}/templates/systemd/matrix-mautrix-gmessages.service.j2" + dest: "{{ devture_systemd_docker_base_systemd_path }}/matrix-mautrix-gmessages.service" + mode: 0644 + +- name: Ensure matrix-mautrix-gmessages.service restarted, if necessary + ansible.builtin.service: + name: "matrix-mautrix-gmessages.service" + state: restarted + daemon_reload: true + when: "matrix_mautrix_gmessages_requires_restart | bool" diff --git a/roles/custom/matrix-bridge-mautrix-gmessages/tasks/setup_uninstall.yml b/roles/custom/matrix-bridge-mautrix-gmessages/tasks/setup_uninstall.yml new file mode 100644 index 000000000..e324a523e --- /dev/null +++ b/roles/custom/matrix-bridge-mautrix-gmessages/tasks/setup_uninstall.yml @@ -0,0 +1,20 @@ +--- + +- name: Check existence of matrix-mautrix-gmessages service + ansible.builtin.stat: + path: "{{ devture_systemd_docker_base_systemd_path }}/matrix-mautrix-gmessages.service" + register: matrix_mautrix_gmessages_service_stat + +- when: matrix_mautrix_gmessages_service_stat.stat.exists | bool + block: + - name: Ensure matrix-mautrix-gmessages is stopped + ansible.builtin.service: + name: matrix-mautrix-gmessages + state: stopped + enabled: false + daemon_reload: true + + - name: Ensure matrix-mautrix-gmessages.service doesn't exist + ansible.builtin.file: + path: "{{ devture_systemd_docker_base_systemd_path }}/matrix-mautrix-gmessages.service" + state: absent diff --git a/roles/custom/matrix-bridge-mautrix-gmessages/tasks/validate_config.yml b/roles/custom/matrix-bridge-mautrix-gmessages/tasks/validate_config.yml new file mode 100644 index 000000000..acfffa75e --- /dev/null +++ b/roles/custom/matrix-bridge-mautrix-gmessages/tasks/validate_config.yml @@ -0,0 +1,20 @@ +--- + +- name: Fail if required mautrix-gmessages settings not defined + ansible.builtin.fail: + msg: >- + You need to define a required configuration setting (`{{ item.name }}`). + when: "item.when | bool and vars[item.name] == ''" + with_items: + - {'name': 'matrix_mautrix_gmessages_appservice_token', when: true} + - {'name': 'matrix_mautrix_gmessages_homeserver_token', when: true} + - {'name': 'matrix_mautrix_gmessages_database_hostname', when: "{{ matrix_mautrix_gmessages_database_engine == 'postgres' }}"} + +- name: (Deprecation) Catch and report renamed settings + ansible.builtin.fail: + msg: >- + Your configuration contains a variable, which now has a different name. + Please change your configuration to rename the variable (`{{ item.old }}` -> `{{ item.new }}`). + when: "item.old in vars" + with_items: + - {'old': 'matrix_mautrix_gmessages_log_level', 'new': 'matrix_mautrix_gmessages_logging_level'} diff --git a/roles/custom/matrix-bridge-mautrix-gmessages/templates/config.yaml.j2 b/roles/custom/matrix-bridge-mautrix-gmessages/templates/config.yaml.j2 new file mode 100644 index 000000000..a17f6cd1e --- /dev/null +++ b/roles/custom/matrix-bridge-mautrix-gmessages/templates/config.yaml.j2 @@ -0,0 +1,298 @@ +#jinja2: lstrip_blocks: "True" +# Homeserver details. +homeserver: + # The address that this appservice can use to connect to the homeserver. + address: {{ matrix_mautrix_gmessages_homeserver_address }} + # The domain of the homeserver (also known as server_name, used for MXIDs, etc). + domain: {{ matrix_mautrix_gmessages_homeserver_domain }} + + # What software is the homeserver running? + # Standard Matrix homeservers like Synapse, Dendrite and Conduit should just use "standard" here. + software: standard + # The URL to push real-time bridge status to. + # If set, the bridge will make POST requests to this URL whenever a user's google messages connection state changes. + # The bridge will use the appservice as_token to authorize requests. + status_endpoint: null + # Endpoint for reporting per-message status. + message_send_checkpoint_endpoint: null + # Does the homeserver support https://github.com/matrix-org/matrix-spec-proposals/pull/2246? + async_media: false + + # Should the bridge use a websocket for connecting to the homeserver? + # The server side is currently not documented anywhere and is only implemented by mautrix-wsproxy, + # mautrix-asmux (deprecated), and hungryserv (proprietary). + websocket: false + # How often should the websocket be pinged? Pinging will be disabled if this is zero. + ping_interval_seconds: 0 + +# Application service host/registration related details. +# Changing these values requires regeneration of the registration. +appservice: + # The address that the homeserver can use to connect to this appservice. + address: {{ matrix_mautrix_gmessages_appservice_address }} + + # The hostname and port where this appservice should listen. + hostname: 0.0.0.0 + port: 8080 + + # Database config. + database: + # The database type. "sqlite3-fk-wal" and "postgres" are supported. + type: postgres + # The database URI. + # SQLite: A raw file path is supported, but `file:?_txlock=immediate` is recommended. + # https://github.com/mattn/go-sqlite3#connection-string + # Postgres: Connection string. For example, postgres://user:password@host/database?sslmode=disable + # To connect via Unix socket, use something like postgres:///dbname?host=/var/run/postgresql + uri: {{ matrix_mautrix_gmessages_appservice_database_uri|to_json }} + # Maximum number of connections. Mostly relevant for Postgres. + max_open_conns: 20 + max_idle_conns: 2 + # Maximum connection idle time and lifetime before they're closed. Disabled if null. + # Parsed with https://pkg.go.dev/time#ParseDuration + max_conn_idle_time: null + max_conn_lifetime: null + + # The unique ID of this appservice. + id: gmessages + # Appservice bot details. + bot: + # Username of the appservice bot. + username: {{ matrix_mautrix_gmessages_appservice_bot_username|to_json }} + # Display name and avatar for bot. Set to "remove" to remove display name/avatar, leave empty + # to leave display name/avatar as-is. + displayname: Google Messages bridge bot + avatar: mxc://maunium.net/yGOdcrJcwqARZqdzbfuxfhzb + + # Whether or not to receive ephemeral events via appservice transactions. + # Requires MSC2409 support (i.e. Synapse 1.22+). + ephemeral_events: true + + # Should incoming events be handled asynchronously? + # This may be necessary for large public instances with lots of messages going through. + # However, messages will not be guaranteed to be bridged in the same order they were sent in. + async_transactions: false + + # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify. + as_token: "{{ matrix_mautrix_gmessages_appservice_token }}" + hs_token: "{{ matrix_mautrix_gmessages_homeserver_token }}" + +# Segment API key to track some events, like provisioning API login and encryption errors. +segment_key: null +# Optional user_id to use when sending Segment events. If null, defaults to using mxID. +segment_user_id: null + +# Prometheus config. +metrics: + # Enable prometheus metrics? + enabled: {{ matrix_mautrix_gmessages_metrics_enabled | to_json }} + # IP and port where the metrics listener should be. The path is always /metrics + listen: 127.0.0.1:8001 + +google_messages: + # OS name to tell the phone. This is the name that shows up in the paired devices list. + os: mautrix-gmessages + # Browser type to tell the phone. This decides which icon is shown. + # Valid types: OTHER, CHROME, FIREFOX, SAFARI, OPERA, IE, EDGE + browser: OTHER + + # Should the bridge aggressively set itself as the active device if the user opens Google Messages in a browser? + # If this is disabled, the user must manually use the `reconnect` command to reactivate the bridge. + aggressive_reconnect: false + +# Bridge config +bridge: + # Localpart template of MXIDs for SMS users. + # {{ '{{.}}' }} is replaced with an identifier of the recipient. + username_template: "{{ 'gmessages_{{.}}' }}" + # Displayname template for SMS users. + # {{ '{{.FullName}}' }} - Full name provided by the phone + # {{ '{{.FirstName}}' }} - First name provided by the phone + # {{ '{{.PhoneNumber}}' }} - Formatted phone number provided by the phone + displayname_template: "{{ '{{or .FullName .PhoneNumber}}' }}" + # Should the bridge create a space for each logged-in user and add bridged rooms to it? + personal_filtering_spaces: {{ matrix_mautrix_gmessages_bridge_personal_filtering_spaces | to_json }} + # Should the bridge send a read receipt from the bridge bot when a message has been sent to the phone? + delivery_receipts: false + # Whether the bridge should send the message status as a custom com.beeper.message_send_status event. + message_status_events: false + # Whether the bridge should send error notices via m.notice events when a message fails to bridge. + message_error_notices: true + + portal_message_buffer: 128 + + # Should the bridge update the m.direct account data event when double puppeting is enabled. + # Note that updating the m.direct event is not atomic (except with mautrix-asmux) + # and is therefore prone to race conditions. + sync_direct_chat_list: false + # Number of chats to sync when connecting to Google Messages. + initial_chat_sync_count: 25 + # Backfill settings + backfill: + # Number of messages to backfill in new chats. + initial_limit: 50 + # Number of messages to backfill on startup if the last message ID in the chat sync doesn't match the last bridged message. + missed_limit: 100 + + # Servers to always allow double puppeting from + double_puppet_server_map: + "{{ matrix_mautrix_gmessages_homeserver_domain }}": {{ matrix_mautrix_gmessages_homeserver_address }} + # Allow using double puppeting from any server with a valid client .well-known file. + double_puppet_allow_discovery: false + # Shared secrets for https://github.com/devture/matrix-synapse-shared-secret-auth + # + # If set, double puppeting will be enabled automatically for local users + # instead of users having to find an access token and run `login-matrix` + # manually. + login_shared_secret_map: {{ matrix_mautrix_gmessages_bridge_login_shared_secret_map|to_json }} + + # Whether to explicitly set the avatar and room name for private chat portal rooms. + # If set to `default`, this will be enabled in encrypted rooms and disabled in unencrypted rooms. + # If set to `always`, all DM rooms will have explicit names and avatars set. + # If set to `never`, DM rooms will never have names and avatars set. + private_chat_portal_meta: default + # Should Matrix m.notice-type messages be bridged? + bridge_notices: true + # Set this to true to tell the bridge to re-send m.bridge events to all rooms on the next run. + # This field will automatically be changed back to false after it, except if the config file is not writable. + resend_bridge_info: false + # When using double puppeting, should muted chats be muted in Matrix? + mute_bridging: {{ matrix_mautrix_gmessages_bridge_mute_bridging | to_json }} + # When using double puppeting, should archived chats be moved to a specific tag in Matrix? + # This can be set to a tag (e.g. m.lowpriority), or null to disable. + archive_tag: null + # Same as above, but for pinned chats. The favorite tag is called m.favourite + pinned_tag: null + # Should mute status and tags only be bridged when the portal room is created? + tag_only_on_create: true + # Whether or not created rooms should have federation enabled. + # If false, created portal rooms will never be federated. + federate_rooms: {{ matrix_mautrix_gmessages_federate_rooms|to_json }} + # Should the bridge never send alerts to the bridge management room? + # These are mostly things like the user being logged out. + disable_bridge_alerts: false + # Send captions in the same message as images. This will send data compatible with both MSC2530 and MSC3552. + # This is currently not supported in most clients. + caption_in_message: false + + # The prefix for commands. Only required in non-management rooms. + command_prefix: "!gm" + + # Messages sent upon joining a management room. + # Markdown is supported. The defaults are listed below. + management_room_text: + # Sent when joining a room. + welcome: "Hello, I'm a Google Messages bridge bot." + # Sent when joining a management room and the user is already logged in. + welcome_connected: "Use `help` for help." + # Sent when joining a management room and the user is not logged in. + welcome_unconnected: "Use `help` for help or `login` to log in." + # Optional extra text sent when joining a management room. + additional_help: "" + + # End-to-bridge encryption support options. + # + # See https://docs.mau.fi/bridges/general/end-to-bridge-encryption.html for more info. + encryption: + # Allow encryption, work in group chat rooms with e2ee enabled + allow: {{ matrix_mautrix_gmessages_bridge_encryption_allow|to_json }} + # Default to encryption, force-enable encryption in all portals the bridge creates + # This will cause the bridge bot to be in private chats for the encryption to work properly. + default: {{ matrix_mautrix_gmessages_bridge_encryption_default|to_json }} + # Whether to use MSC2409/MSC3202 instead of /sync long polling for receiving encryption-related data. + appservice: false + # Require encryption, drop any unencrypted messages. + require: false + # Enable key sharing? If enabled, key requests for rooms where users are in will be fulfilled. + # You must use a client that supports requesting keys from other users to use this feature. + allow_key_sharing: {{ matrix_mautrix_gmessages_bridge_encryption_key_sharing_allow|to_json }} + # Options for deleting megolm sessions from the bridge. + delete_keys: + # Beeper-specific: delete outbound sessions when hungryserv confirms + # that the user has uploaded the key to key backup. + delete_outbound_on_ack: false + # Don't store outbound sessions in the inbound table. + dont_store_outbound: false + # Ratchet megolm sessions forward after decrypting messages. + ratchet_on_decrypt: false + # Delete fully used keys (index >= max_messages) after decrypting messages. + delete_fully_used_on_decrypt: false + # Delete previous megolm sessions from same device when receiving a new one. + delete_prev_on_new_session: false + # Delete megolm sessions received from a device when the device is deleted. + delete_on_device_delete: false + # Periodically delete megolm sessions when 2x max_age has passed since receiving the session. + periodically_delete_expired: false + # Delete inbound megolm sessions that don't have the received_at field used for + # automatic ratcheting and expired session deletion. This is meant as a migration + # to delete old keys prior to the bridge update. + delete_outdated_inbound: false + # What level of device verification should be required from users? + # + # Valid levels: + # unverified - Send keys to all device in the room. + # cross-signed-untrusted - Require valid cross-signing, but trust all cross-signing keys. + # cross-signed-tofu - Require valid cross-signing, trust cross-signing keys on first use (and reject changes). + # cross-signed-verified - Require valid cross-signing, plus a valid user signature from the bridge bot. + # Note that creating user signatures from the bridge bot is not currently possible. + # verified - Require manual per-device verification + # (currently only possible by modifying the `trust` column in the `crypto_device` database table). + verification_levels: + # Minimum level for which the bridge should send keys to when bridging messages from SMS to Matrix. + receive: unverified + # Minimum level that the bridge should accept for incoming Matrix messages. + send: unverified + # Minimum level that the bridge should require for accepting key requests. + share: cross-signed-tofu + # Options for Megolm room key rotation. These options allow you to + # configure the m.room.encryption event content. See: + # https://spec.matrix.org/v1.3/client-server-api/#mroomencryption for + # more information about that event. + rotation: + # Enable custom Megolm room key rotation settings. Note that these + # settings will only apply to rooms created after this option is + # set. + enable_custom: false + # The maximum number of milliseconds a session should be used + # before changing it. The Matrix spec recommends 604800000 (a week) + # as the default. + milliseconds: 604800000 + # The maximum number of messages that should be sent with a given a + # session before changing it. The Matrix spec recommends 100 as the + # default. + messages: 100 + + # Disable rotating keys when a user's devices change? + # You should not enable this option unless you understand all the implications. + disable_device_change_key_rotation: false + + # Settings for provisioning API + provisioning: + # Prefix for the provisioning API paths. + prefix: /_matrix/provision + # Shared secret for authentication. If set to "generate", a random secret will be generated, + # or if set to "disable", the provisioning API will be disabled. + shared_secret: generate + + # Permissions for using the bridge. + # Permitted values: + # user - Access to use the bridge to link their own Google Messages on android. + # admin - User level and some additional administration tools + # Permitted keys: + # * - All Matrix users + # domain - All users on that homeserver + # mxid - Specific user + permissions: {{ matrix_mautrix_gmessages_bridge_permissions|to_json }} + +# Logging config. See https://github.com/tulir/zeroconfig for details. +logging: + min_level: debug + writers: + - type: stdout + format: pretty-colored + - type: file + format: json + filename: ./logs/mautrix-gmessages.log + max_size: 100 + max_backups: 10 + compress: true diff --git a/roles/custom/matrix-bridge-mautrix-gmessages/templates/systemd/matrix-mautrix-gmessages.service.j2 b/roles/custom/matrix-bridge-mautrix-gmessages/templates/systemd/matrix-mautrix-gmessages.service.j2 new file mode 100644 index 000000000..fb34e95b9 --- /dev/null +++ b/roles/custom/matrix-bridge-mautrix-gmessages/templates/systemd/matrix-mautrix-gmessages.service.j2 @@ -0,0 +1,43 @@ +#jinja2: lstrip_blocks: "True" +[Unit] +Description=Matrix Mautrix gmessages bridge +{% for service in matrix_mautrix_gmessages_systemd_required_services_list %} +Requires={{ service }} +After={{ service }} +{% endfor %} +{% for service in matrix_mautrix_gmessages_systemd_wanted_services_list %} +Wants={{ service }} +{% endfor %} +DefaultDependencies=no + +[Service] +Type=simple +Environment="HOME={{ devture_systemd_docker_base_systemd_unit_home_path }}" +ExecStartPre=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} kill matrix-mautrix-gmessages 2>/dev/null || true' +ExecStartPre=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} rm matrix-mautrix-gmessages 2>/dev/null || true' + +# Intentional delay, so that the homeserver (we likely depend on) can manage to start. +ExecStartPre={{ matrix_host_command_sleep }} 5 + +ExecStart={{ devture_systemd_docker_base_host_command_docker }} run --rm --name matrix-mautrix-gmessages \ + --log-driver=none \ + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ + --cap-drop=ALL \ + --network={{ matrix_docker_network }} \ + -v {{ matrix_mautrix_gmessages_config_path }}:/config:z \ + -v {{ matrix_mautrix_gmessages_data_path }}:/data:z \ + --workdir=/data \ + {% for arg in matrix_mautrix_gmessages_container_extra_arguments %} + {{ arg }} \ + {% endfor %} + {{ matrix_mautrix_gmessages_docker_image }} \ + /usr/bin/mautrix-gmessages -c /config/config.yaml -r /config/registration.yaml + +ExecStop=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} kill matrix-mautrix-gmessages 2>/dev/null || true' +ExecStop=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} rm matrix-mautrix-gmessages 2>/dev/null || true' +Restart=always +RestartSec=30 +SyslogIdentifier=matrix-mautrix-gmessages + +[Install] +WantedBy=multi-user.target diff --git a/setup.yml b/setup.yml index c7a6a3639..4da64ff4b 100644 --- a/setup.yml +++ b/setup.yml @@ -59,6 +59,7 @@ - custom/matrix-bridge-mautrix-instagram - custom/matrix-bridge-mautrix-signal - custom/matrix-bridge-mautrix-telegram + - custom/matrix-bridge-mautrix-gmessages - custom/matrix-bridge-mautrix-whatsapp - custom/matrix-bridge-mautrix-discord - custom/matrix-bridge-mautrix-slack From 0a81083e7dc5932146cc7c75a51e01d6eb49c13b Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 21 Jul 2023 14:37:05 +0300 Subject: [PATCH 41/43] Announce mautrix-gmessages support Related to https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/2794 --- CHANGELOG.md | 6 ++++++ docs/configuring-playbook-bridge-mautrix-gmessages.md | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1870a273..950e38e4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# 2023-07-21 + +## mautrix-gmessages support + +Thanks to [Shreyas Ajjarapu](https://github.com/shreyasajj)'s efforts, the playbook now supports bridging to [Google Messages](https://messages.google.com/) via the [mautrix-gmessages](https://github.com/mautrix/gmessages) bridge. See our [Setting up Mautrix Google Messages bridging](docs/configuring-playbook-bridge-mautrix-gmessages.md) documentation page for getting started. + # 2023-07-17 ## matrix-media-repo support diff --git a/docs/configuring-playbook-bridge-mautrix-gmessages.md b/docs/configuring-playbook-bridge-mautrix-gmessages.md index 6b2b18e61..10981bf16 100644 --- a/docs/configuring-playbook-bridge-mautrix-gmessages.md +++ b/docs/configuring-playbook-bridge-mautrix-gmessages.md @@ -1,6 +1,6 @@ # Setting up Mautrix gmessages (optional) -The playbook can install and configure [mautrix-gmessages](https://github.com/mautrix/gmessages) for you. +The playbook can install and configure [mautrix-gmessages](https://github.com/mautrix/gmessages) for you, for bridging to [Google Messages](https://messages.google.com/). See the project's [documentation](https://docs.mau.fi/bridges/go/gmessages/index.html) to learn what it does and why it might be useful to you. @@ -8,7 +8,7 @@ Use the following playbook configuration: ```yaml matrix_mautrix_gmessages_enabled: true -``` +``` ## Set up Double Puppeting From eea143e6eb8c87cfbd9e785081baad815969f40b Mon Sep 17 00:00:00 2001 From: ThellraAK Date: Fri, 21 Jul 2023 05:47:51 -0800 Subject: [PATCH 42/43] Shortened gmessages salt The salts need to be shorter than 16 chars --- group_vars/matrix_servers | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 5fc5c25d8..f1fbb6c37 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -1176,14 +1176,14 @@ matrix_mautrix_gmessages_systemd_required_services_list: | + ['matrix-' + matrix_homeserver_implementation + '.service'] + - ([devture_postgres_identifier ~ '.service'] if devture_postgres_enabled else []) + ([devture_postgres_identifier ~ '.service'] if devture_postgres_enabled else [])gmessage.as.token + (['matrix-nginx-proxy.service'] if matrix_nginx_proxy_enabled else []) }} -matrix_mautrix_gmessages_appservice_token: "{{ '%s' | format(matrix_homeserver_generic_secret_key) | password_hash('sha512', 'gmessage.as.token', rounds=655555) | to_uuid }}" +matrix_mautrix_gmessages_appservice_token: "{{ '%s' | format(matrix_homeserver_generic_secret_key) | password_hash('sha512', 'gmessa.as.token', rounds=655555) | to_uuid }}" -matrix_mautrix_gmessages_homeserver_token: "{{ '%s' | format(matrix_homeserver_generic_secret_key) | password_hash('sha512', 'gmessage.hs.token', rounds=655555) | to_uuid }}" +matrix_mautrix_gmessages_homeserver_token: "{{ '%s' | format(matrix_homeserver_generic_secret_key) | password_hash('sha512', 'gmessa.hs.token', rounds=655555) | to_uuid }}" matrix_mautrix_gmessages_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}" From 2153c310f76eedc9d715cd2c2fdec3a175ea7fa6 Mon Sep 17 00:00:00 2001 From: ThellraAK Date: Fri, 21 Jul 2023 05:49:52 -0800 Subject: [PATCH 43/43] Update matrix_servers --- group_vars/matrix_servers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index f1fbb6c37..0a30f217b 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -1176,7 +1176,7 @@ matrix_mautrix_gmessages_systemd_required_services_list: | + ['matrix-' + matrix_homeserver_implementation + '.service'] + - ([devture_postgres_identifier ~ '.service'] if devture_postgres_enabled else [])gmessage.as.token + ([devture_postgres_identifier ~ '.service'] if devture_postgres_enabled else []) + (['matrix-nginx-proxy.service'] if matrix_nginx_proxy_enabled else []) }}