From 60aa40845f9e054974a0178b4f00fc0c3622a610 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9la=20Becker?= Date: Tue, 16 Mar 2021 18:51:05 +0100 Subject: [PATCH 01/28] Upgrade Etherpad (1.8.7 -> 1.8.12) --- roles/matrix-etherpad/defaults/main.yml | 2 +- roles/matrix-etherpad/templates/settings.json.j2 | 3 +++ .../templates/systemd/matrix-etherpad.service.j2 | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/roles/matrix-etherpad/defaults/main.yml b/roles/matrix-etherpad/defaults/main.yml index 27589b2a3..908833109 100644 --- a/roles/matrix-etherpad/defaults/main.yml +++ b/roles/matrix-etherpad/defaults/main.yml @@ -2,7 +2,7 @@ matrix_etherpad_enabled: false matrix_etherpad_base_path: "{{ matrix_base_data_path }}/etherpad" -matrix_etherpad_version: 1.8.7 +matrix_etherpad_version: 1.8.12 matrix_etherpad_docker_image: "docker.io/etherpad/etherpad:{{ matrix_etherpad_version }}" matrix_etherpad_docker_image_force_pull: "{{ matrix_etherpad_docker_image.endswith(':latest') }}" diff --git a/roles/matrix-etherpad/templates/settings.json.j2 b/roles/matrix-etherpad/templates/settings.json.j2 index b3b87f43e..377bad988 100644 --- a/roles/matrix-etherpad/templates/settings.json.j2 +++ b/roles/matrix-etherpad/templates/settings.json.j2 @@ -42,6 +42,9 @@ "percentageToScrollWhenUserPressesArrowUp": 0 }, "socketTransportProtocols" : ["xhr-polling", "jsonp-polling", "htmlfile"], + "socketIo": { + "maxHttpBufferSize": 10000 + }, "loadTest": false, "importExportRateLimiting": { "windowMs": 90000, diff --git a/roles/matrix-etherpad/templates/systemd/matrix-etherpad.service.j2 b/roles/matrix-etherpad/templates/systemd/matrix-etherpad.service.j2 index b8a266640..b579036be 100644 --- a/roles/matrix-etherpad/templates/systemd/matrix-etherpad.service.j2 +++ b/roles/matrix-etherpad/templates/systemd/matrix-etherpad.service.j2 @@ -29,7 +29,7 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-etherpad \ {{ arg }} \ {% endfor %} {{ matrix_etherpad_docker_image }} \ - node --experimental-worker /opt/etherpad-lite/node_modules/ep_etherpad-lite/node/server.js \ + node --experimental-worker src/node/server.js \ --settings /data/settings.json --credentials /data/credentials.json \ --sessionkey /data/sessionkey.json --apikey /data/apijey.json From 466827139a32efe3fdb8261bb1e56a225cff15b0 Mon Sep 17 00:00:00 2001 From: Aaron Raimist Date: Wed, 17 Mar 2021 00:54:00 -0500 Subject: [PATCH 02/28] Also check if matrix_ssl_lets_encrypt_support_email is blank --- roles/matrix-nginx-proxy/tasks/validate_config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/matrix-nginx-proxy/tasks/validate_config.yml b/roles/matrix-nginx-proxy/tasks/validate_config.yml index 3197bd978..9661ae5e9 100644 --- a/roles/matrix-nginx-proxy/tasks/validate_config.yml +++ b/roles/matrix-nginx-proxy/tasks/validate_config.yml @@ -40,8 +40,8 @@ - name: Fail if required variables are undefined fail: - msg: "Detected an undefined required variable" + msg: "The `{{ item }}` variable must be defined and have a non-null value" with_items: - "matrix_ssl_lets_encrypt_support_email" - when: "vars[item] is none" + when: "vars[item] == '' or vars[item] is none" when: "matrix_ssl_retrieval_method == 'lets-encrypt'" From 2d7e7680e565eadc226dbb6375ed869cca141290 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9la=20Becker?= Date: Wed, 17 Mar 2021 12:36:45 +0100 Subject: [PATCH 03/28] matrix.{{ matrix_domain }} -> {{ matrix_server_fqn_matrix }} --- roles/matrix-awx/surveys/configure_synapse_admin.json.j2 | 2 +- roles/matrix-bridge-appservice-slack/defaults/main.yml | 2 +- roles/matrix-bridge-appservice-webhooks/defaults/main.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/matrix-awx/surveys/configure_synapse_admin.json.j2 b/roles/matrix-awx/surveys/configure_synapse_admin.json.j2 index f5a740aab..e1f6dc12e 100644 --- a/roles/matrix-awx/surveys/configure_synapse_admin.json.j2 +++ b/roles/matrix-awx/surveys/configure_synapse_admin.json.j2 @@ -4,7 +4,7 @@ "spec": [ { "question_name": "Enable Synapse Admin", - "question_description": "Set if Synapse Admin is enabled or not. If enabled you can access it at https://matrix.{{ matrix_domain }}/synapse-admin.", + "question_description": "Set if Synapse Admin is enabled or not. If enabled you can access it at https://{{ matrix_server_fqn_matrix }}/synapse-admin.", "required": false, "min": null, "max": null, diff --git a/roles/matrix-bridge-appservice-slack/defaults/main.yml b/roles/matrix-bridge-appservice-slack/defaults/main.yml index 1169de4fa..a0032a0dd 100644 --- a/roles/matrix-bridge-appservice-slack/defaults/main.yml +++ b/roles/matrix-bridge-appservice-slack/defaults/main.yml @@ -32,7 +32,7 @@ matrix_appservice_slack_slack_port: 9003 # Takes an ":" or "" value (e.g. "127.0.0.1:9999"), or empty string to not expose. matrix_appservice_slack_container_http_host_bind_port: '' -matrix_appservice_slack_homeserver_media_url: "matrix.{{ matrix_domain }}" +matrix_appservice_slack_homeserver_media_url: "{{ matrix_server_fqn_matrix }}" matrix_appservice_slack_homeserver_url: "http://matrix-synapse:8008" matrix_appservice_slack_homeserver_domain: "{{ matrix_domain }}" matrix_appservice_slack_appservice_url: 'http://matrix-appservice-slack' diff --git a/roles/matrix-bridge-appservice-webhooks/defaults/main.yml b/roles/matrix-bridge-appservice-webhooks/defaults/main.yml index 215afc05f..82e0925c3 100644 --- a/roles/matrix-bridge-appservice-webhooks/defaults/main.yml +++ b/roles/matrix-bridge-appservice-webhooks/defaults/main.yml @@ -28,7 +28,7 @@ matrix_appservice_webhooks_matrix_port: 6789 # Takes an ":" or "" value (e.g. "127.0.0.1:9999"), or empty string to not expose. matrix_appservice_webhooks_container_http_host_bind_port: '' -matrix_appservice_webhooks_homeserver_media_url: "matrix.{{ matrix_domain }}" +matrix_appservice_webhooks_homeserver_media_url: "{{ matrix_server_fqn_matrix }}" matrix_appservice_webhooks_homeserver_url: "http://matrix-synapse:8008" matrix_appservice_webhooks_homeserver_domain: "{{ matrix_domain }}" matrix_appservice_webhooks_appservice_url: 'http://matrix-appservice-webhooks' From 07fb05a4ab9c1dabca2f8c1f76e6625e2841809d Mon Sep 17 00:00:00 2001 From: chagai95 <31655082+chagai95@users.noreply.github.com> Date: Wed, 17 Mar 2021 20:29:17 +0100 Subject: [PATCH 04/28] fix typo ot -> to --- docs/maintenance-migrating.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/maintenance-migrating.md b/docs/maintenance-migrating.md index 71a38e583..d6380b992 100644 --- a/docs/maintenance-migrating.md +++ b/docs/maintenance-migrating.md @@ -1,6 +1,6 @@ # Migrating to new server -1. Prepare by lowering DNS TTL for your domains (`matrix.DOMAIN`, etc.), so that DNS record changes (step 4 below) would happen faster, leading ot less downtime +1. Prepare by lowering DNS TTL for your domains (`matrix.DOMAIN`, etc.), so that DNS record changes (step 4 below) would happen faster, leading to less downtime 2. Stop all services on the old server and make sure they won't be starting again. Execute this on the old server: `systemctl disable --now matrix*` 3. Copy directory `/matrix` from the old server to the new server. Make sure to preserve ownership and permissions (use `cp -p` or `rsync -ar`)! 4. Make sure your DNS records are adjusted to point to the new server's IP address From 32b3650c1245ec5854fbdb52965cda19a8ab0ace Mon Sep 17 00:00:00 2001 From: Aaron Raimist Date: Wed, 17 Mar 2021 18:51:10 -0500 Subject: [PATCH 05/28] Set X-Forwarded-Proto on federation requests --- .../templates/nginx/conf.d/matrix-domain.conf.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-domain.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-domain.conf.j2 index 9d5826189..1d2470a97 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-domain.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-domain.conf.j2 @@ -241,6 +241,7 @@ server { proxy_set_header Host $host; proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; client_body_buffer_size 25M; client_max_body_size {{ matrix_nginx_proxy_proxy_matrix_federation_api_client_max_body_size_mb }}M; From a5ee39266c29c6784175e76b796702fd3fc9a2e8 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 19 Mar 2021 08:16:59 +0200 Subject: [PATCH 06/28] Go through start.py when launching Synapse This allows us to benefit from helpful things it does for us, like enabling jemalloc: https://github.com/matrix-org/synapse/pull/8553 We weren't going through `start.py` before, because it was causing some conflict with our `docker run --user=...` stuff, but it doesn't seem to be a problem anymore. Having done this, we won't need to do things like https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/941 anymore. --- .../templates/synapse/systemd/matrix-synapse-worker.service.j2 | 3 +-- .../templates/synapse/systemd/matrix-synapse.service.j2 | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker.service.j2 index ee858183f..43dc42d1f 100644 --- a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker.service.j2 +++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker.service.j2 @@ -18,7 +18,6 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name {{ matrix_synapse_wor --log-driver=none \ --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ --cap-drop=ALL \ - --entrypoint=python \ --read-only \ --tmpfs=/tmp:rw,noexec,nosuid,size={{ matrix_synapse_tmp_directory_size_mb }}m \ --network={{ matrix_docker_network }} \ @@ -44,7 +43,7 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name {{ matrix_synapse_wor {{ arg }} \ {% endfor %} {{ matrix_synapse_docker_image }} \ - -m synapse.app.{{ matrix_synapse_worker_details.type }} -c /data/homeserver.yaml -c /data/{{ matrix_synapse_worker_config_file_name }} + run -m synapse.app.{{ matrix_synapse_worker_details.type }} -c /data/homeserver.yaml -c /data/{{ matrix_synapse_worker_config_file_name }} ExecStop=-{{ matrix_host_command_docker }} kill {{ matrix_synapse_worker_container_name }} diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 index 5f69e3151..cd3a2f3df 100644 --- a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 +++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 @@ -34,7 +34,6 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-synapse \ --log-driver=none \ --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ --cap-drop=ALL \ - --entrypoint=python \ --read-only \ --tmpfs=/tmp:rw,noexec,nosuid,size={{ matrix_synapse_tmp_directory_size_mb }}m \ --network={{ matrix_docker_network }} \ @@ -62,7 +61,7 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-synapse \ {{ arg }} \ {% endfor %} {{ matrix_synapse_docker_image }} \ - -m synapse.app.homeserver -c /data/homeserver.yaml + run -m synapse.app.homeserver -c /data/homeserver.yaml ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-synapse 2>/dev/null' ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-synapse 2>/dev/null' From f99dcd611fadce7d20a3a804b4aac49d2d3781e2 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 19 Mar 2021 16:44:10 +0200 Subject: [PATCH 07/28] Pass proper UID/GID to Synapse Fixes a regression caused by a5ee39266c29c6. If the user id and group id were different than 991:991 (which used to be a hardcoded default for us long ago), there was a mismatch between what Synapse was trying to use (991:991) and what it was actually started with (in `--user=..`). It was then trying to change ownership, which was failing. This was mostly affecting newer installations which were not using the 991:991 defaults we had long ago (since a1c5a197a93d410). --- .../templates/synapse/systemd/matrix-synapse-worker.service.j2 | 2 ++ .../templates/synapse/systemd/matrix-synapse.service.j2 | 2 ++ 2 files changed, 4 insertions(+) diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker.service.j2 index 43dc42d1f..6c90c9a3e 100644 --- a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker.service.j2 +++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker.service.j2 @@ -17,6 +17,8 @@ ExecStartPre={{ matrix_host_command_sleep }} 5 ExecStart={{ matrix_host_command_docker }} run --rm --name {{ matrix_synapse_worker_container_name }} \ --log-driver=none \ --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ + -e UID={{ matrix_user_uid }} \ + -e GID={{ matrix_user_gid }} \ --cap-drop=ALL \ --read-only \ --tmpfs=/tmp:rw,noexec,nosuid,size={{ matrix_synapse_tmp_directory_size_mb }}m \ diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 index cd3a2f3df..2fbaac7b5 100644 --- a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 +++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 @@ -33,6 +33,8 @@ ExecStartPre={{ matrix_host_command_sleep }} 3 ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-synapse \ --log-driver=none \ --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ + --env=UID={{ matrix_user_uid }} \ + --env=GID={{ matrix_user_gid }} \ --cap-drop=ALL \ --read-only \ --tmpfs=/tmp:rw,noexec,nosuid,size={{ matrix_synapse_tmp_directory_size_mb }}m \ From 1e54b1d1a5fca731ef7fcf1271c06bc2b2b069ea Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 20 Mar 2021 17:21:02 +0800 Subject: [PATCH 08/28] merge upstream --- .../surveys/configure_synapse.json.j2 | 2 +- .../tasks/load_hosting_and_org_variables.yml | 18 ++--- .../tasks/load_matrix_variables.yml | 10 +-- roles/matrix-awx/tasks/main.yml | 65 +++++++++++++++---- .../tasks/set_variables_synapse.yml | 10 +++ 5 files changed, 79 insertions(+), 26 deletions(-) diff --git a/roles/matrix-awx/surveys/configure_synapse.json.j2 b/roles/matrix-awx/surveys/configure_synapse.json.j2 index 32d54646b..8d3b5daf3 100755 --- a/roles/matrix-awx/surveys/configure_synapse.json.j2 +++ b/roles/matrix-awx/surveys/configure_synapse.json.j2 @@ -119,7 +119,7 @@ "default": "{{ matrix_synapse_max_upload_size_mb }}", "choices": "", "new_question": true, - "variable": "matrix_synapse_max_upload_size_mb", + "variable": "matrix_synapse_max_upload_size_mb_raw", "type": "text" }, { diff --git a/roles/matrix-awx/tasks/load_hosting_and_org_variables.yml b/roles/matrix-awx/tasks/load_hosting_and_org_variables.yml index 6789431a7..529a65fcf 100644 --- a/roles/matrix-awx/tasks/load_hosting_and_org_variables.yml +++ b/roles/matrix-awx/tasks/load_hosting_and_org_variables.yml @@ -1,10 +1,12 @@ -- name: Include vars in organisation.yml - include_vars: - file: '/var/lib/awx/projects/clients/{{ member_id }}/organisation.yml' - no_log: True +- tags: always + block: + - name: Include vars in organisation.yml + include_vars: + file: '/var/lib/awx/projects/clients/{{ member_id }}/organisation.yml' + no_log: True -- name: Include vars in hosting_vars.yml - include_vars: - file: '/var/lib/awx/projects/hosting/hosting_vars.yml' - no_log: True + - name: Include vars in hosting_vars.yml + include_vars: + file: '/var/lib/awx/projects/hosting/hosting_vars.yml' + no_log: True diff --git a/roles/matrix-awx/tasks/load_matrix_variables.yml b/roles/matrix-awx/tasks/load_matrix_variables.yml index ff0986d72..ba7de3e9b 100755 --- a/roles/matrix-awx/tasks/load_matrix_variables.yml +++ b/roles/matrix-awx/tasks/load_matrix_variables.yml @@ -1,5 +1,7 @@ -- name: Include vars in matrix_vars.yml - include_vars: - file: '/var/lib/awx/projects/clients/{{ member_id }}/{{ subscription_id }}/matrix_vars.yml' - no_log: True +- tags: always + block: + - name: Include vars in matrix_vars.yml + include_vars: + file: '/var/lib/awx/projects/clients/{{ member_id }}/{{ subscription_id }}/matrix_vars.yml' + no_log: True diff --git a/roles/matrix-awx/tasks/main.yml b/roles/matrix-awx/tasks/main.yml index 20ce2d8e2..4965ae39d 100755 --- a/roles/matrix-awx/tasks/main.yml +++ b/roles/matrix-awx/tasks/main.yml @@ -1,78 +1,117 @@ # Load initial hosting and organisation variables from AWX volume -- include_tasks: "{{ role_path }}/tasks/load_hosting_and_org_variables.yml" +- include_tasks: + file: "load_hosting_and_org_variables.yml" + apply: + tags: always when: run_setup|bool and matrix_awx_enabled|bool tags: - always # Perform a backup of the server -- include_tasks: "{{ role_path }}/tasks/backup_server.yml" +- include_tasks: + file: "backup_server.yml" + apply: + tags: backup-server when: run_setup|bool and matrix_awx_enabled|bool tags: - backup-server # Create a user account if called -- include_tasks: "{{ role_path }}/tasks/create_user.yml" +- include_tasks: + file: "create_user.yml" + apply: + tags: create-user when: run_setup|bool and matrix_awx_enabled|bool tags: - create-user # Perform extra self-check functions -- include_tasks: "{{ role_path }}/tasks/self_check.yml" +- include_tasks: + file: "self_check.yml" + apply: + tags: self-check when: run_setup|bool and matrix_awx_enabled|bool tags: - self-check # Import configs, media repo from /chroot/backup import -- include_tasks: "{{ role_path }}/tasks/import_awx.yml" +- include_tasks: + file: "import_awx.yml" + apply: + tags: import-awx when: run_setup|bool and matrix_awx_enabled|bool tags: - import-awx # Configure SFTP so user can upload a static website or access the servers export -- include_tasks: "{{ role_path }}/tasks/customise_website_access_export.yml" +- include_tasks: + file: "customise_website_access_export.yml" + apply: + tags: setup-nginx-proxy when: run_setup|bool and matrix_awx_enabled|bool tags: - setup-nginx-proxy # Additional playbook to set the variable file during Element configuration -- include_tasks: "{{ role_path }}/tasks/set_variables_element.yml" +- include_tasks: + file: "set_variables_element.yml" + apply: + tags: setup-client-element when: run_setup|bool and matrix_awx_enabled|bool tags: - setup-client-element # Additional playbook to set the variable file during Synapse configuration -- include_tasks: "{{ role_path }}/tasks/set_variables_synapse.yml" +- include_tasks: + file: "set_variables_synapse.yml" + apply: + tags: setup-synapse when: run_setup|bool and matrix_awx_enabled|bool tags: - setup-synapse # Additional playbook to set the variable file during Jitsi configuration -- include_tasks: "{{ role_path }}/tasks/set_variables_jitsi.yml" +- include_tasks: + file: "set_variables_jitsi.yml" + apply: + tags: setup-jitsi when: run_setup|bool and matrix_awx_enabled|bool tags: - setup-jitsi # Additional playbook to set the variable file during Ma1sd configuration -- include_tasks: "{{ role_path }}/tasks/set_variables_ma1sd.yml" +- include_tasks: + file: "set_variables_ma1sd.yml" + apply: + tags: setup-ma1sd when: run_setup|bool and matrix_awx_enabled|bool tags: - setup-ma1sd # Additional playbook to set the variable file during Corporal configuration -- include_tasks: "{{ role_path }}/tasks/set_variables_corporal.yml" +- include_tasks: + file: "set_variables_corporal.yml" + apply: + tags: setup-corporal when: run_setup|bool and matrix_awx_enabled|bool tags: - setup-corporal # Additional playbook to set the variable file during Synapse Admin configuration -- include_tasks: "{{ role_path }}/tasks/set_variables_synapse_admin.yml" +- include_tasks: + file: "set_variables_synapse_admin.yml" + apply: + tags: setup-synapse-admin when: run_setup|bool and matrix_awx_enabled|bool tags: - setup-synapse-admin # Load newly formed matrix variables from tower volume -- include_tasks: "{{ role_path }}/tasks/load_matrix_variables.yml" +- include_tasks: + file: "load_matrix_variables.yml" + apply: + tags: always when: run_setup|bool and matrix_awx_enabled|bool tags: - always diff --git a/roles/matrix-awx/tasks/set_variables_synapse.yml b/roles/matrix-awx/tasks/set_variables_synapse.yml index a8cc418ba..ef7c70ef1 100755 --- a/roles/matrix-awx/tasks/set_variables_synapse.yml +++ b/roles/matrix-awx/tasks/set_variables_synapse.yml @@ -1,4 +1,14 @@ +- name: Limit max upload size to 100MB part 1 + set_fact: + matrix_synapse_max_upload_size_mb: "100" + when: matrix_synapse_max_upload_size_mb_raw|int >= 100 + +- name: Limit max upload size to 100MB part 2 + set_fact: + matrix_synapse_max_upload_size_mb: "{{ matrix_synapse_max_upload_size_mb_raw }}" + when: matrix_synapse_max_upload_size_mb_raw|int < 100 + - name: Record Synapse variables locally on AWX delegate_to: 127.0.0.1 lineinfile: From af240aef3727496c77ce9106639ee545995c077f Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 20 Mar 2021 17:35:30 +0800 Subject: [PATCH 09/28] remove sections from task list that arent needed --- .../tasks/load_hosting_and_org_variables.yml | 18 ++++++++---------- .../matrix-awx/tasks/load_matrix_variables.yml | 10 ++++------ 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/roles/matrix-awx/tasks/load_hosting_and_org_variables.yml b/roles/matrix-awx/tasks/load_hosting_and_org_variables.yml index 529a65fcf..6789431a7 100644 --- a/roles/matrix-awx/tasks/load_hosting_and_org_variables.yml +++ b/roles/matrix-awx/tasks/load_hosting_and_org_variables.yml @@ -1,12 +1,10 @@ -- tags: always - block: - - name: Include vars in organisation.yml - include_vars: - file: '/var/lib/awx/projects/clients/{{ member_id }}/organisation.yml' - no_log: True +- name: Include vars in organisation.yml + include_vars: + file: '/var/lib/awx/projects/clients/{{ member_id }}/organisation.yml' + no_log: True - - name: Include vars in hosting_vars.yml - include_vars: - file: '/var/lib/awx/projects/hosting/hosting_vars.yml' - no_log: True +- name: Include vars in hosting_vars.yml + include_vars: + file: '/var/lib/awx/projects/hosting/hosting_vars.yml' + no_log: True diff --git a/roles/matrix-awx/tasks/load_matrix_variables.yml b/roles/matrix-awx/tasks/load_matrix_variables.yml index ba7de3e9b..ff0986d72 100755 --- a/roles/matrix-awx/tasks/load_matrix_variables.yml +++ b/roles/matrix-awx/tasks/load_matrix_variables.yml @@ -1,7 +1,5 @@ -- tags: always - block: - - name: Include vars in matrix_vars.yml - include_vars: - file: '/var/lib/awx/projects/clients/{{ member_id }}/{{ subscription_id }}/matrix_vars.yml' - no_log: True +- name: Include vars in matrix_vars.yml + include_vars: + file: '/var/lib/awx/projects/clients/{{ member_id }}/{{ subscription_id }}/matrix_vars.yml' + no_log: True From 9a0222fa4742054338778037909f8fa5dc35e10f Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sat, 20 Mar 2021 13:31:34 +0200 Subject: [PATCH 10/28] Add Sygnal support Fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/683 --- CHANGELOG.md | 12 + README.md | 2 + docs/configuring-dns.md | 3 + docs/configuring-playbook-sygnal.md | 71 +++++ docs/configuring-playbook.md | 5 + docs/container-images.md | 2 + group_vars/matrix_servers | 35 +++ roles/matrix-base/defaults/main.yml | 3 + roles/matrix-nginx-proxy/defaults/main.yml | 7 + .../tasks/setup_nginx_proxy.yml | 13 + .../nginx/conf.d/matrix-sygnal.conf.j2 | 79 +++++ roles/matrix-sygnal/defaults/main.yml | 95 ++++++ roles/matrix-sygnal/tasks/init.yml | 3 + roles/matrix-sygnal/tasks/main.yml | 21 ++ roles/matrix-sygnal/tasks/setup_install.yml | 73 +++++ roles/matrix-sygnal/tasks/setup_uninstall.yml | 35 +++ roles/matrix-sygnal/tasks/validate_config.yml | 7 + roles/matrix-sygnal/templates/sygnal.yaml.j2 | 288 ++++++++++++++++++ .../systemd/matrix-sygnal.service.j2 | 42 +++ setup.yml | 1 + 20 files changed, 797 insertions(+) create mode 100644 docs/configuring-playbook-sygnal.md create mode 100644 roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-sygnal.conf.j2 create mode 100644 roles/matrix-sygnal/defaults/main.yml create mode 100644 roles/matrix-sygnal/tasks/init.yml create mode 100644 roles/matrix-sygnal/tasks/main.yml create mode 100644 roles/matrix-sygnal/tasks/setup_install.yml create mode 100644 roles/matrix-sygnal/tasks/setup_uninstall.yml create mode 100644 roles/matrix-sygnal/tasks/validate_config.yml create mode 100644 roles/matrix-sygnal/templates/sygnal.yaml.j2 create mode 100644 roles/matrix-sygnal/templates/systemd/matrix-sygnal.service.j2 diff --git a/CHANGELOG.md b/CHANGELOG.md index aeff7b0ca..36790dce8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ + +# 2021-03-20 + +## Sygnal push gateway support + +The playbook can now install the [Sygnal](https://github.com/matrix-org/sygnal) push gateway for you. + +This is only useful to people who develop/build their own Matrix client applications. + +Additional details are available in our [Setting up Sygnal](docs/configuring-playbook-sygnal.md) docs. + + # 2021-03-16 ## Go-NEB support diff --git a/README.md b/README.md index dde8d924a..9a2726871 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,8 @@ Using this playbook, you can get the following services configured on your serve - (optional) the [Prometheus](https://prometheus.io) time-series database server, the Prometheus [node-exporter](https://prometheus.io/docs/guides/node-exporter/) host metrics exporter, and the [Grafana](https://grafana.com/) web UI - see [Enabling metrics and graphs (Prometheus, Grafana) for your Matrix server](docs/configuring-playbook-prometheus-grafana.md) for setup documentation +- (optional) the [Sygnal](https://github.com/matrix-org/sygnal) push gateway - see [Setting up the Sygnal push gateway](docs/configuring-playbook-sygnal.md) for setup documentation + Basically, this playbook aims to get you up-and-running with all the basic necessities around Matrix, without you having to do anything else. **Note**: the list above is exhaustive. It includes optional or even some advanced components that you will most likely not need. diff --git a/docs/configuring-dns.md b/docs/configuring-dns.md index a3815953b..82bef8a2d 100644 --- a/docs/configuring-dns.md +++ b/docs/configuring-dns.md @@ -35,6 +35,7 @@ If you are using Cloudflare DNS, make sure to disable the proxy and set all reco | CNAME | `jitsi` (*) | - | - | - | `matrix.` | | CNAME | `stats` (*) | - | - | - | `matrix.` | | CNAME | `goneb` (*) | - | - | - | `matrix.` | +| CNAME | `sygnal` (*) | - | - | - | `matrix.` | ## Subdomains setup @@ -51,6 +52,8 @@ The `stats.` subdomain may be necessary, because this playbook coul The `goneb.` subdomain may be necessary, because this playbook could install the [Go-NEB](https://github.com/matrix-org/go-neb) bot. The installation of Go-NEB is disabled by default, it is not a core required component. To learn how to install it, see our [configuring Go-NEB guide](configuring-playbook-bot-go-neb.md). If you do not wish to set up Go-NEB, feel free to skip the `goneb.` DNS record. +The `sygnal.` subdomain may be necessary, because this playbook could install the [Sygnal](https://github.com/matrix-org/sygnal) push gateway. The installation of Sygnal is disabled by default, it is not a core required component. To learn how to install it, see our [configuring Sygnal guide](configuring-playbook-sygnal.md). If you do not wish to set up Sygnal (you probably don't, unless you're also developing/building your own Matrix apps), feel free to skip the `sygnal.` DNS record. + ## `_matrix-identity._tcp` SRV record setup diff --git a/docs/configuring-playbook-sygnal.md b/docs/configuring-playbook-sygnal.md new file mode 100644 index 000000000..b8dc11f0d --- /dev/null +++ b/docs/configuring-playbook-sygnal.md @@ -0,0 +1,71 @@ +# Setting up Sygnal (optional) + +The playbook can install and configure the [Sygnal](https://github.com/matrix-org/sygnal) push gateway for you. + +See the project's [documentation](https://github.com/matrix-org/sygnal) to learn what it does and why it might be useful to you. + +**Note**: most people don't need to install their own gateway. As Sygnal's [Notes for application developers](https://github.com/matrix-org/sygnal/blob/master/docs/applications.md) documentation says: + +> It is not feasible to allow end-users to configure their own Sygnal instance, because the Sygnal instance needs the appropriate FCM or APNs secrets that belong to the application. + +This optional playbook component is only useful to people who develop/build their own Matrix client applications themselves. + + +## Adjusting the playbook configuration + +Add the following configuration to your `inventory/host_vars/matrix.DOMAIN/vars.yml` file (adapt to your needs): + +```yaml +matrix_sygnal_enabled: true + +# You need at least 1 app defined. +# The configuration below is incomplete. Read more below. +matrix_sygnal_apps: + com.example.myapp.ios: + type: apns + keyfile: /data/my_key.p8 + # .. more configuration .. + com.example.myapp.android: + type: gcm + api_key: your_api_key_for_gcm + # .. more configuration .. + +matrix_aux_file_definitions: + - dest: "{{ matrix_sygnal_data_path }}/my_key.p8" + content: | + some + content + here + mode: '0600' + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" +``` + +For a more complete example of available fields and values they can take, see `roles/matrix-sygnal/templates/sygnal.yaml.j2` (or the [upstream `sygnal.yaml.sample` configuration file](https://github.com/matrix-org/sygnal/blob/master/sygnal.yaml.sample)). + +Configuring [GCM/FCM](https://firebase.google.com/docs/cloud-messaging/) is easier, as it only requires that you provide some config values. + +To configure [APNS](https://developer.apple.com/notifications/) (Apple Push Notification Service), you'd need to provide one or more certificate files. +To do that, the above example configuration: + +- makes use of the `matrix-aux` role (and its `matrix_aux_file_definitions` variable) to make the playbook install files into `/matrix/sygnal/data` (the `matrix_sygnal_data_path` variable). See `roles/matrix-aux/defaults/main.yml` for usage examples. It also makes sure the files are owned by `matrix:matrix`, so that Sygnal can read them. Of course, you can also install these files manually yourself, if you'd rather not use `matrix-aux`. + +- references these files in the Sygnal configuration (`matrix_sygnal_apps`) using a path like `/data/..` (the `/matrix/sygnal/data` directory on the host system is mounted into the `/data` directory inside the container) + + +## Installing + +Don't forget to add `sygnal.` to DNS as described in [Configuring DNS](configuring-dns.md) before running the playbook. + +After configuring the playbook, run the [installation](installing.md) command again: + +``` +ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start +``` + + +## Usage + +To make use of your Sygnal installation, you'd need to build your own Matrix client application, which uses the same API keys (for [GCM/FCM](https://firebase.google.com/docs/cloud-messaging/)) and certificates (for [APNS](https://developer.apple.com/notifications/)) and is also pointed to `https://sygnal.DOMAIN` as the configured push server. + +Refer to Sygnal's [Notes for application developers](https://github.com/matrix-org/sygnal/blob/master/docs/applications.md) document. diff --git a/docs/configuring-playbook.md b/docs/configuring-playbook.md index a25f85854..721735d53 100644 --- a/docs/configuring-playbook.md +++ b/docs/configuring-playbook.md @@ -132,3 +132,8 @@ When you're done with all the configuration you'd like to do, continue with [Ins - [Setting up matrix-reminder-bot](configuring-playbook-bot-matrix-reminder-bot.md) (optional) - [Setting up Go-NEB](configuring-playbook-bot-go-neb.md) (optional) + + +### Other specialized services + +- [Setting up the Sygnal push gateway](configuring-playbook-sygnal.md) (optional) diff --git a/docs/container-images.md b/docs/container-images.md index 5bccc2d49..ee9cbd3ca 100644 --- a/docs/container-images.md +++ b/docs/container-images.md @@ -97,3 +97,5 @@ These services are not part of our default installation, but can be enabled by [ - [prom/node-exporter](https://hub.docker.com/r/prom/node-exporter/) - [Prometheus Node Exporter](https://github.com/prometheus/node_exporter/) is an addon for Prometheus that gathers standard system metrics - [grafana/grafana](https://hub.docker.com/r/grafana/grafana/) - [Grafana](https://github.com/grafana/grafana/) is a graphing tool that works well with the above two images. Our playbook also adds two dashboards for [Synapse](https://github.com/matrix-org/synapse/tree/master/contrib/grafana) and [Node Exporter](https://github.com/rfrail3/grafana-dashboards) + +- [matrixdotorg/sygnal](https://hub.docker.com/r/matrixdotorg/sygnal/) - [Sygnal](https://github.com/matrix-org/sygnal) is a reference Push Gateway for Matrix diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index c9cc6ad0e..a6c433cb1 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -1095,6 +1095,7 @@ matrix_nginx_proxy_proxy_dimension_enabled: "{{ matrix_dimension_enabled }}" matrix_nginx_proxy_proxy_bot_go_neb_enabled: "{{ matrix_bot_go_neb_enabled }}" matrix_nginx_proxy_proxy_jitsi_enabled: "{{ matrix_jitsi_enabled }}" matrix_nginx_proxy_proxy_grafana_enabled: "{{ matrix_grafana_enabled }}" +matrix_nginx_proxy_proxy_sygnal_enabled: "{{ matrix_sygnal_enabled }}" matrix_nginx_proxy_proxy_matrix_corporal_api_enabled: "{{ matrix_corporal_enabled and matrix_corporal_http_api_enabled }}" matrix_nginx_proxy_proxy_matrix_corporal_api_addr_with_container: "matrix-corporal:41081" @@ -1169,6 +1170,8 @@ matrix_ssl_domains_to_obtain_certificates_for: | + ([matrix_server_fqn_grafana] if matrix_grafana_enabled else []) + + ([matrix_server_fqn_sygnal] if matrix_sygnal_enabled else []) + + ([matrix_domain] if matrix_nginx_proxy_base_domain_serving_enabled else []) + matrix_ssl_additional_domains_to_obtain_certificates_for @@ -1341,6 +1344,12 @@ matrix_postgres_additional_databases: | 'username': matrix_etherpad_database_username, 'password': matrix_etherpad_database_password, }] if (matrix_etherpad_enabled and matrix_etherpad_database_engine == 'postgres' and matrix_etherpad_database_hostname == 'matrix-postgres') else []) + + + ([{ + 'name': matrix_sygnal_database_name, + 'username': matrix_sygnal_database_username, + 'password': matrix_sygnal_database_password, + }] if (matrix_sygnal_enabled and matrix_sygnal_database_engine == 'postgres' and matrix_sygnal_database_hostname == 'matrix-postgres') else []) }} matrix_postgres_import_roles_to_ignore: | @@ -1365,6 +1374,32 @@ matrix_postgres_import_databases_to_ignore: | +###################################################################### +# +# matrix-sygnal +# +###################################################################### + +# Most people don't need their own push-server, because they also need their own app to utilize it from. +matrix_sygnal_enabled: false + +# If someone instals Prometheus via the playbook, they most likely wish to monitor Sygnal. +matrix_sygnal_metrics_prometheus_enabled: "{{ matrix_prometheus_enabled }}" + +matrix_sygnal_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_enabled else '127.0.0.1:6000' }}" + +# Postgres is the default, except if not using `matrix_postgres` (internal postgres) +matrix_sygnal_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_sygnal_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'sygnal') | to_uuid }}" + +###################################################################### +# +# /matrix-sygnal +# +###################################################################### + + + ###################################################################### # # matrix-redis diff --git a/roles/matrix-base/defaults/main.yml b/roles/matrix-base/defaults/main.yml index 5f1cf433e..d1d3bf27e 100644 --- a/roles/matrix-base/defaults/main.yml +++ b/roles/matrix-base/defaults/main.yml @@ -27,6 +27,9 @@ matrix_server_fqn_jitsi: "jitsi.{{ matrix_domain }}" # This is where you access Grafana. matrix_server_fqn_grafana: "stats.{{ matrix_domain }}" +# This is where you access the Sygnal push gateway. +matrix_server_fqn_sygnal: "sygnal.{{ matrix_domain }}" + matrix_federation_public_port: 8448 # The architecture that your server runs. diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index c1538c75a..3770bfb1e 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -133,6 +133,10 @@ matrix_nginx_proxy_proxy_jitsi_hostname: "{{ matrix_server_fqn_jitsi }}" matrix_nginx_proxy_proxy_grafana_enabled: false matrix_nginx_proxy_proxy_grafana_hostname: "{{ matrix_server_fqn_grafana }}" +# Controls whether proxying the sygnal domain should be done. +matrix_nginx_proxy_proxy_sygnal_enabled: false +matrix_nginx_proxy_proxy_sygnal_hostname: "{{ matrix_server_fqn_sygnal }}" + # Controls whether proxying for the matrix-corporal API (`/_matrix/corporal`) should be done (on the matrix domain) matrix_nginx_proxy_proxy_matrix_corporal_api_enabled: false matrix_nginx_proxy_proxy_matrix_corporal_api_addr_with_container: "matrix-corporal:41081" @@ -250,6 +254,9 @@ matrix_nginx_proxy_proxy_jitsi_additional_server_configuration_blocks: [] # A list of strings containing additional configuration blocks to add to Grafana's server configuration (matrix-grafana.conf). matrix_nginx_proxy_proxy_grafana_additional_server_configuration_blocks: [] +# A list of strings containing additional configuration blocks to add to Sygnal's server configuration (matrix-sygnal.conf). +matrix_nginx_proxy_proxy_sygnal_additional_server_configuration_blocks: [] + # A list of strings containing additional configuration blocks to add to the base domain server configuration (matrix-base-domain.conf). matrix_nginx_proxy_proxy_domain_additional_server_configuration_blocks: [] diff --git a/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml b/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml index c7e15c95c..7534d28c6 100644 --- a/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml +++ b/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml @@ -100,6 +100,13 @@ mode: 0644 when: matrix_nginx_proxy_proxy_grafana_enabled|bool +- name: Ensure Matrix nginx-proxy configuration for sygnal domain exists + template: + src: "{{ role_path }}/templates/nginx/conf.d/matrix-sygnal.conf.j2" + dest: "{{ matrix_nginx_proxy_confd_path }}/matrix-sygnal.conf" + mode: 0644 + when: matrix_nginx_proxy_proxy_sygnal_enabled|bool + - name: Ensure Matrix nginx-proxy configuration for Matrix domain exists template: src: "{{ role_path }}/templates/nginx/conf.d/matrix-domain.conf.j2" @@ -221,6 +228,12 @@ state: absent when: "not matrix_nginx_proxy_proxy_grafana_enabled|bool" +- name: Ensure Matrix nginx-proxy configuration for sygnal domain deleted + file: + path: "{{ matrix_nginx_proxy_confd_path }}/matrix-sygnal.conf" + state: absent + when: "not matrix_nginx_proxy_proxy_sygnal_enabled|bool" + - name: Ensure Matrix nginx-proxy homepage for base domain deleted file: path: "{{ matrix_nginx_proxy_data_path }}/matrix-domain/index.html" diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-sygnal.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-sygnal.conf.j2 new file mode 100644 index 000000000..e47126578 --- /dev/null +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-sygnal.conf.j2 @@ -0,0 +1,79 @@ +#jinja2: lstrip_blocks: "True" + +{% macro render_vhost_directives() %} + gzip on; + gzip_types text/plain application/json application/javascript text/css image/x-icon font/ttf image/gif; + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Content-Type-Options nosniff; + add_header X-Frame-Options DENY; +{% for configuration_block in matrix_nginx_proxy_proxy_sygnal_additional_server_configuration_blocks %} + {{- configuration_block }} +{% endfor %} + + location / { + {% if matrix_nginx_proxy_enabled %} + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver 127.0.0.11 valid=5s; + set $backend "matrix-sygnal:6000"; + proxy_pass http://$backend; + {% else %} + {# Generic configuration for use outside of our container setup #} + proxy_pass http://127.0.0.1:6000; + {% endif %} + + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + } +{% endmacro %} + +server { + listen {{ 8080 if matrix_nginx_proxy_enabled else 80 }}; + server_name {{ matrix_nginx_proxy_proxy_sygnal_hostname }}; + + server_tokens off; + root /dev/null; + + {% if matrix_nginx_proxy_https_enabled %} + location /.well-known/acme-challenge { + {% if matrix_nginx_proxy_enabled %} + {# Use the embedded DNS resolver in Docker containers to discover the service #} + resolver 127.0.0.11 valid=5s; + set $backend "matrix-certbot:8080"; + proxy_pass http://$backend; + {% else %} + {# Generic configuration for use outside of our container setup #} + proxy_pass http://127.0.0.1:{{ matrix_ssl_lets_encrypt_certbot_standalone_http_port }}; + {% endif %} + } + + location / { + return 301 https://$http_host$request_uri; + } + {% else %} + {{ render_vhost_directives() }} + {% endif %} +} + +{% if matrix_nginx_proxy_https_enabled %} +server { + listen {{ 8443 if matrix_nginx_proxy_enabled else 443 }} ssl http2; + listen [::]:{{ 8443 if matrix_nginx_proxy_enabled else 443 }} ssl http2; + + server_name {{ matrix_nginx_proxy_proxy_sygnal_hostname }}; + + server_tokens off; + root /dev/null; + + ssl_certificate {{ matrix_ssl_config_dir_path }}/live/{{ matrix_nginx_proxy_proxy_sygnal_hostname }}/fullchain.pem; + ssl_certificate_key {{ matrix_ssl_config_dir_path }}/live/{{ matrix_nginx_proxy_proxy_sygnal_hostname }}/privkey.pem; + + ssl_protocols {{ matrix_nginx_proxy_ssl_protocols }}; + {% if matrix_nginx_proxy_ssl_ciphers != '' %} + ssl_ciphers {{ matrix_nginx_proxy_ssl_ciphers }}; + {% endif %} + ssl_prefer_server_ciphers {{ matrix_nginx_proxy_ssl_prefer_server_ciphers }}; + + {{ render_vhost_directives() }} +} +{% endif %} diff --git a/roles/matrix-sygnal/defaults/main.yml b/roles/matrix-sygnal/defaults/main.yml new file mode 100644 index 000000000..50e0e26e7 --- /dev/null +++ b/roles/matrix-sygnal/defaults/main.yml @@ -0,0 +1,95 @@ +# Sygnal is a reference Push Gateway for Matrix. +# To make use of it for delivering push notificatins, you'll need to develop/build your own Matrix app. +# Learn more here: https://github.com/matrix-org/sygnal +matrix_sygnal_enabled: false + +matrix_sygnal_base_path: "{{ matrix_base_data_path }}/sygnal" +matrix_sygnal_config_path: "{{ matrix_sygnal_base_path }}/config" +matrix_sygnal_data_path: "{{ matrix_sygnal_base_path }}/data" + +matrix_sygnal_version: v0.9.0 +matrix_sygnal_docker_image: "docker.io/matrixdotorg/sygnal:{{ matrix_sygnal_version }}" +matrix_sygnal_docker_image_force_pull: "{{ matrix_sygnal_docker_image.endswith(':latest') }}" + +# List of systemd services that matrix-sygnal.service depends on. +matrix_sygnal_systemd_required_services_list: ['docker.service'] + +# List of systemd services that matrix-sygnal.service wants +matrix_sygnal_systemd_wanted_services_list: [] + +# Controls whether the matrix-sygnal container exposes its HTTP port (tcp/6000 in the container). +# +# Takes an ":" or "" value (e.g. "127.0.0.1:6000"), or empty string to not expose. +matrix_sygnal_container_http_host_bind_port: '' + +# A list of extra arguments to pass to the container +matrix_sygnal_container_extra_arguments: [] + +# Database-related configuration fields. +# +# To use SQLite, stick to these defaults. +# +# To use Postgres: +# - change the engine (`matrix_sygnal_database_engine: 'postgres'`) +# - adjust your database credentials via the `matrix_sygnal_postgres_*` variables +matrix_sygnal_database_engine: 'sqlite' + +matrix_sygnal_sqlite_database_path_local: "{{ matrix_sygnal_data_path }}/sygnal.db" +matrix_sygnal_sqlite_database_path_in_container: "/data/sygnal.db" + +matrix_sygnal_database_username: 'matrix_sygnal' +matrix_sygnal_database_password: 'some-password' +matrix_sygnal_database_hostname: 'matrix-postgres' +matrix_sygnal_database_port: 5432 +matrix_sygnal_database_name: 'matrix_sygnal' + +matrix_sygnal_database_connection_string: 'postgres://{{ matrix_sygnal_database_username }}:{{ matrix_sygnal_database_password }}@{{ matrix_sygnal_database_hostname }}:{{ matrix_sygnal_database_port }}/{{ matrix_sygnal_database_name }}' + +# A map (dictionary) of apps instances that this server works with. +# +# Example configuration: +# +# matrix_sygnal_apps: +# com.example.myapp.ios: +# type: apns +# # .. more configuration .. +# com.example.myapp.android: +# type: gcm +# api_key: your_api_key_for_gcm +# # .. more configuration .. +# +# The APNS configuration needs to reference some certificate files. +# One can put these in the `matrix_sygnal_data_path` directory (`/matrix/sygnal/data`), mounted to `/data` in the container. +# The `matrix_sygnal_apps` paths need to use the in-container path (`/data`). +# To install these files via the playbook, one can use the `matrix-aux` role. +# Examples and more details are available in `docs/configuring-playbook-sygnal.md`. +matrix_sygnal_apps: [] + +matrix_sygnal_metrics_prometheus_enabled: false + +# Default Sygnal configuration template which covers the generic use case. +# You can customize it by controlling the various variables inside it. +# +# For a more advanced customization, you can extend the default (see `matrix_sygnal_configuration_extension_yaml`) +# or completely replace this variable with your own template. +matrix_sygnal_configuration_yaml: "{{ lookup('template', 'templates/sygnal.yaml.j2') }}" + +matrix_sygnal_configuration_extension_yaml: | + # Your custom YAML configuration for Sygnal goes here. + # This configuration extends the default starting configuration (`matrix_sygnal_configuration_yaml`). + # + # You can override individual variables from the default configuration, or introduce new ones. + # + # If you need something more special, you can take full control by + # completely redefining `matrix_sygnal_configuration_yaml`. + # + # Example configuration extension follows: + # metrics: + # opentracing: + # enabled: true + +matrix_sygnal_configuration_extension: "{{ matrix_sygnal_configuration_extension_yaml|from_yaml if matrix_sygnal_configuration_extension_yaml|from_yaml is mapping else {} }}" + +# Holds the final sygnal configuration (a combination of the default and its extension). +# You most likely don't need to touch this variable. Instead, see `matrix_sygnal_configuration_yaml`. +matrix_sygnal_configuration: "{{ matrix_sygnal_configuration_yaml|from_yaml|combine(matrix_sygnal_configuration_extension, recursive=True) }}" diff --git a/roles/matrix-sygnal/tasks/init.yml b/roles/matrix-sygnal/tasks/init.yml new file mode 100644 index 000000000..559a3681d --- /dev/null +++ b/roles/matrix-sygnal/tasks/init.yml @@ -0,0 +1,3 @@ +- set_fact: + matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-sygnal.service'] }}" + when: matrix_sygnal_enabled|bool diff --git a/roles/matrix-sygnal/tasks/main.yml b/roles/matrix-sygnal/tasks/main.yml new file mode 100644 index 000000000..c00862a4b --- /dev/null +++ b/roles/matrix-sygnal/tasks/main.yml @@ -0,0 +1,21 @@ +- import_tasks: "{{ role_path }}/tasks/init.yml" + tags: + - always + +- import_tasks: "{{ role_path }}/tasks/validate_config.yml" + when: run_setup|bool + tags: + - setup-all + - setup-sygnal + +- import_tasks: "{{ role_path }}/tasks/setup_install.yml" + when: run_setup|bool and matrix_sygnal_enabled|bool + tags: + - setup-all + - setup-sygnal + +- import_tasks: "{{ role_path }}/tasks/setup_uninstall.yml" + when: run_setup|bool and not matrix_sygnal_enabled|bool + tags: + - setup-all + - setup-sygnal diff --git a/roles/matrix-sygnal/tasks/setup_install.yml b/roles/matrix-sygnal/tasks/setup_install.yml new file mode 100644 index 000000000..afac61c48 --- /dev/null +++ b/roles/matrix-sygnal/tasks/setup_install.yml @@ -0,0 +1,73 @@ +--- + +- set_fact: + matrix_sygnal_requires_restart: false + +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_sygnal_sqlite_database_path_local }}" + register: matrix_sygnal_sqlite_database_path_local_stat_result + + - block: + - set_fact: + matrix_postgres_db_migration_request: + src: "{{ matrix_sygnal_sqlite_database_path_local }}" + dst: "{{ matrix_sygnal_database_connection_string }}" + caller: "{{ role_path|basename }}" + engine_variable_name: 'matrix_sygnal_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-sygnal.service'] + pgloader_options: ['--with "quote identifiers"'] + + - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + + - set_fact: + matrix_sygnal_requires_restart: true + when: "matrix_sygnal_sqlite_database_path_local_stat_result.stat.exists|bool" + when: "matrix_sygnal_database_engine == 'postgres'" + +- name: Ensure Sygnal image is pulled + docker_image: + name: "{{ matrix_sygnal_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_sygnal_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_sygnal_docker_image_force_pull }}" + +- name: Ensure Sygnal paths exists + file: + path: "{{ item }}" + state: directory + mode: 0750 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + with_items: + - "{{ matrix_sygnal_base_path }}" + - "{{ matrix_sygnal_config_path }}" + - "{{ matrix_sygnal_data_path }}" + +- name: Ensure Sygnal config installed + copy: + content: "{{ matrix_sygnal_configuration|to_nice_yaml }}" + dest: "{{ matrix_sygnal_config_path }}/sygnal.yaml" + mode: 0640 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + +- name: Ensure matrix-sygnal.service installed + template: + src: "{{ role_path }}/templates/systemd/matrix-sygnal.service.j2" + dest: "{{ matrix_systemd_path }}/matrix-sygnal.service" + mode: 0644 + register: matrix_sygnal_systemd_service_result + +- name: Ensure systemd reloaded after matrix-sygnal.service installation + service: + daemon_reload: yes + when: "matrix_sygnal_systemd_service_result.changed|bool" + +- name: Ensure matrix-sygnal.service restarted, if necessary + service: + name: "matrix-sygnal.service" + state: restarted + when: "matrix_sygnal_requires_restart|bool" diff --git a/roles/matrix-sygnal/tasks/setup_uninstall.yml b/roles/matrix-sygnal/tasks/setup_uninstall.yml new file mode 100644 index 000000000..dc50078ca --- /dev/null +++ b/roles/matrix-sygnal/tasks/setup_uninstall.yml @@ -0,0 +1,35 @@ +--- + +- name: Check existence of matrix-sygnal service + stat: + path: "{{ matrix_systemd_path }}/matrix-sygnal.service" + register: matrix_sygnal_service_stat + +- name: Ensure matrix-sygnal is stopped + service: + name: matrix-sygnal + state: stopped + daemon_reload: yes + register: stopping_result + when: "matrix_sygnal_service_stat.stat.exists|bool" + +- name: Ensure matrix-sygnal.service doesn't exist + file: + path: "{{ matrix_systemd_path }}/matrix-sygnal.service" + state: absent + when: "matrix_sygnal_service_stat.stat.exists|bool" + +- name: Ensure systemd reloaded after matrix-sygnal.service removal + service: + daemon_reload: yes + when: "matrix_sygnal_service_stat.stat.exists|bool" + +- name: Ensure Sygnal base directory doesn't exist + file: + path: "{{ matrix_sygnal_base_path }}" + state: absent + +- name: Ensure Sygnal Docker image doesn't exist + docker_image: + name: "{{ matrix_sygnal_docker_image }}" + state: absent diff --git a/roles/matrix-sygnal/tasks/validate_config.yml b/roles/matrix-sygnal/tasks/validate_config.yml new file mode 100644 index 000000000..c503e47bf --- /dev/null +++ b/roles/matrix-sygnal/tasks/validate_config.yml @@ -0,0 +1,7 @@ +- name: Fail if no Sygnal apps defined + fail: + msg: >- + Enabling Sygnal requires that you specify at least one app in `matrix_sygnal_apps` + with_items: + - "matrix_sygnal_access_token" + when: "matrix_sygnal_enabled and matrix_sygnal_apps|length == 0" diff --git a/roles/matrix-sygnal/templates/sygnal.yaml.j2 b/roles/matrix-sygnal/templates/sygnal.yaml.j2 new file mode 100644 index 000000000..bb8c521d9 --- /dev/null +++ b/roles/matrix-sygnal/templates/sygnal.yaml.j2 @@ -0,0 +1,288 @@ +## +# This is a configuration for Sygnal, the reference Push Gateway for Matrix +# See: matrix.org +## + +# The 'database' setting defines the database that sygnal uses to store all of +# its data. +# +# 'name' gives the database engine to use: either 'sqlite3' (for SQLite) or +# 'psycopg2' (for PostgreSQL). +# +# 'args' gives options which are passed through to the database engine, +# except for options starting 'cp_', which are used to configure the Twisted +# connection pool. For a reference to valid arguments, see: +# * for sqlite: https://docs.python.org/3/library/sqlite3.html#sqlite3.connect +# * for postgres: https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-PARAMKEYWORDS +# * for the connection pool: https://twistedmatrix.com/documents/current/api/twisted.enterprise.adbapi.ConnectionPool.html#__init__ +# +# +# Example SQLite configuration: +# +#database: +# name: sqlite3 +# args: +# dbfile: /path/to/database.db +# +# +# Example Postgres configuration: +# +#database: +# name: psycopg2 +# args: +# host: localhost +# database: sygnal +# user: sygnal +# password: pass +# cp_min: 1 +# cp_max: 5 +# +{% if matrix_sygnal_database_engine == 'sqlite' %} +database: + name: sqlite3 + args: + dbfile: {{ matrix_sygnal_sqlite_database_path_in_container|to_json }} +{% else %} +database: + name: psycopg2 + args: + host: {{ matrix_sygnal_database_hostname|to_json }} + database: {{ matrix_sygnal_database_name|to_json }} + user: {{ matrix_sygnal_database_username|to_json }} + password: {{ matrix_sygnal_database_password|to_json }} + cp_min: 1 + cp_max: 5 +{% endif %} + +## Logging # +# +log: + # Specify a Python logging 'dictConfig', as described at: + # https://docs.python.org/3.7/library/logging.config.html#logging.config.dictConfig + # + setup: + version: 1 + formatters: + normal: + format: "%(asctime)s [%(process)d] %(levelname)-5s %(name)s %(message)s" + handlers: + # This handler prints to Standard Error + # + stderr: + class: "logging.StreamHandler" + formatter: "normal" + stream: "ext://sys.stderr" + + # This handler prints to Standard Output. + # + stdout: + class: "logging.StreamHandler" + formatter: "normal" + stream: "ext://sys.stdout" + + # This handler demonstrates logging to a text file on the filesystem. + # You can use logrotate(8) to perform log rotation. + # + #file: + # class: "logging.handlers.WatchedFileHandler" + # formatter: "normal" + # filename: "./sygnal.log" + loggers: + # sygnal.access contains the access logging lines. + # Comment out this section if you don't want to give access logging + # any special treatment. + # + sygnal.access: + propagate: false + handlers: ["stdout"] + level: "INFO" + + # sygnal contains log lines from Sygnal itself. + # You can comment out this section to fall back to the root logger. + # + sygnal: + propagate: false + handlers: ["stderr"] + + root: + # Specify the handler(s) to send log messages to. + handlers: ["stderr"] + level: "INFO" + + disable_existing_loggers: false + + + access: + # Specify whether or not to trust the IP address in the `X-Forwarded-For` + # header. In general, you want to enable this if and only if you are using a + # reverse proxy which is configured to emit it. + # + x_forwarded_for: true + +## HTTP Server (Matrix Push Gateway API) # +# +http: + # Specify a list of interface addresses to bind to. + # + # This example listens on the IPv4 loopback device: + #bind_addresses: ['127.0.0.1'] + # This example listens on all IPv4 interfaces: + #bind_addresses: ['0.0.0.0'] + # This example listens on all IPv4 and IPv6 interfaces: + #bind_addresses: ['0.0.0.0', '::'] + bind_addresses: ['::'] + + # Specify the port number to listen on. + # + port: 6000 + +## Proxying for outgoing connections # +# +# Specify the URL of a proxy to use for outgoing traffic +# (e.g. to Apple & Google) if desired. +# Currently only HTTP proxies with CONNECT capability are supported. +# +# If you do not specify a value, the `HTTPS_PROXY` environment variable will +# be used if present. Otherwise, no proxy will be used. +# +# Default is unspecified. +# +#proxy: 'http://user:secret@prox:8080' + +## Metrics # +# +metrics: + ## Prometheus # + # + prometheus: + # Specify whether or not to enable Prometheus. + # + enabled: false + + # Specify an address for the Prometheus HTTP Server to listen on. + # + address: '0.0.0.0' + + # Specify a port for the Prometheus HTTP Server to listen on. + # + port: 8000 + + ## OpenTracing # + # + opentracing: + # Specify whether or not to enable OpenTracing. + # + enabled: false + + # Specify an implementation of OpenTracing to use. Currently only 'jaeger' + # is supported. + # + implementation: jaeger + + # Specify the service name to be reported to the tracer. + # + service_name: sygnal + + # Specify configuration values to pass to jaeger_client. + # + jaeger: + sampler: + type: 'const' + param: 1 +# local_agent: +# reporting_host: '127.0.0.1' +# reporting_port: + logging: true + + ## Sentry # + # + sentry: + # Specify whether or not to enable Sentry. + # + enabled: false + + # Specify your Sentry DSN if you enable Sentry + # + #dsn: "https://@sentry.example.org/" + +## Pushkins/Apps # +# +# Add a section for every push application here. +# Specify the pushkey for the application and also the type. +# For the type, you may specify a fully-qualified Python classname if desired. +# +#apps: + # This is an example APNs push configuration + # + #com.example.myapp.ios: + # type: apns + # + # # Authentication + # # + # # Two methods of authentication to APNs are currently supported. + # # + # # You can authenticate using a key: + # keyfile: my_key.p8 + # key_id: MY_KEY_ID + # team_id: MY_TEAM_ID + # topic: MY_TOPIC + # + # # Or, a certificate can be used instead: + # certfile: com.example.myApp_prod_APNS.pem + # + # # This is the maximum number of in-flight requests *for this pushkin* + # # before additional notifications will be failed. + # # (This is a robustness measure to prevent one pushkin stacking up with + # # queued requests and saturating the inbound connection queue of a load + # # balancer or reverse proxy). + # # Defaults to 512 if unset. + # # + # #inflight_request_limit: 512 + # + # # Specifies whether to use the production or sandbox APNs server. Note that + # # sandbox tokens should only be used with the sandbox server and vice versa. + # # + # # Valid options are: + # # * production + # # * sandbox + # # + # # The default is 'production'. Uncomment to use the sandbox instance. + # #platform: sandbox + + # This is an example GCM/FCM push configuration. + # + #com.example.myapp.android: + # type: gcm + # api_key: your_api_key_for_gcm + # + # # This is the maximum number of connections to GCM servers at any one time + # # the default is 20. + # #max_connections: 20 + # + # # This is the maximum number of in-flight requests *for this pushkin* + # # before additional notifications will be failed. + # # (This is a robustness measure to prevent one pushkin stacking up with + # # queued requests and saturating the inbound connection queue of a load + # # balancer or reverse proxy). + # # Defaults to 512 if unset. + # # + # #inflight_request_limit: 512 + # + # # This allows you to specify additional options to send to Firebase. + # # + # # Of particular interest, admins who wish to support iOS apps using Firebase + # # probably wish to set content_available, and may need to set mutable_content. + # # (content_available allows your iOS app to be woken up by data messages, + # # and mutable_content allows your notification to be modified by a + # # Notification Service app extension). + # # + # # See https://firebase.google.com/docs/cloud-messaging/http-server-ref + # # for the exhaustive list of valid options. + # # + # # Do not specify `data`, `priority`, `to` or `registration_ids` as they may + # # be overwritten or lead to an invalid request. + # # + # #fcm_options: + # # content_available: true + # # mutable_content: true +apps: {{ matrix_sygnal_apps|to_json }} diff --git a/roles/matrix-sygnal/templates/systemd/matrix-sygnal.service.j2 b/roles/matrix-sygnal/templates/systemd/matrix-sygnal.service.j2 new file mode 100644 index 000000000..019ab40c0 --- /dev/null +++ b/roles/matrix-sygnal/templates/systemd/matrix-sygnal.service.j2 @@ -0,0 +1,42 @@ +#jinja2: lstrip_blocks: "True" +[Unit] +Description=Matrix Sygnal +{% for service in matrix_sygnal_systemd_required_services_list %} +Requires={{ service }} +After={{ service }} +{% endfor %} +{% for service in matrix_sygnal_systemd_wanted_services_list %} +Wants={{ service }} +{% endfor %} +DefaultDependencies=no + +[Service] +Type=simple +Environment="HOME={{ matrix_systemd_unit_home_path }}" +ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-sygnal 2>/dev/null' +ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-sygnal 2>/dev/null' + +ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-sygnal \ + --log-driver=none \ + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ + --cap-drop=ALL \ + --env=SYGNAL_CONF=/config/sygnal.yaml \ + --network={{ matrix_docker_network }} \ + {% if matrix_sygnal_container_http_host_bind_port %} + -p {{ matrix_sygnal_container_http_host_bind_port }}:6000 \ + {% endif %} + --mount type=bind,src={{ matrix_sygnal_config_path }},dst=/config \ + --mount type=bind,src={{ matrix_sygnal_data_path }},dst=/data \ + {% for arg in matrix_sygnal_container_extra_arguments %} + {{ arg }} \ + {% endfor %} + {{ matrix_sygnal_docker_image }} + +ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-sygnal 2>/dev/null' +ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-sygnal 2>/dev/null' +Restart=always +RestartSec=30 +SyslogIdentifier=matrix-sygnal + +[Install] +WantedBy=multi-user.target diff --git a/setup.yml b/setup.yml index c87354be6..c4a203ccc 100755 --- a/setup.yml +++ b/setup.yml @@ -46,6 +46,7 @@ - matrix-dimension - matrix-etherpad - matrix-email2matrix + - matrix-sygnal - matrix-nginx-proxy - matrix-coturn - matrix-aux From b824522b331b7c40628702a405720dad66538e70 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sat, 20 Mar 2021 13:34:22 +0200 Subject: [PATCH 11/28] Remove unnecessary with_items statement --- roles/matrix-sygnal/tasks/validate_config.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/roles/matrix-sygnal/tasks/validate_config.yml b/roles/matrix-sygnal/tasks/validate_config.yml index c503e47bf..1cf8357ee 100644 --- a/roles/matrix-sygnal/tasks/validate_config.yml +++ b/roles/matrix-sygnal/tasks/validate_config.yml @@ -2,6 +2,4 @@ fail: msg: >- Enabling Sygnal requires that you specify at least one app in `matrix_sygnal_apps` - with_items: - - "matrix_sygnal_access_token" when: "matrix_sygnal_enabled and matrix_sygnal_apps|length == 0" From 2737ebc29000568f72fc68986c16a4e404633e09 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sat, 20 Mar 2021 13:38:27 +0200 Subject: [PATCH 12/28] Complain if people try to use matrix-sygnal on non-amd64 --- roles/matrix-sygnal/tasks/validate_config.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/roles/matrix-sygnal/tasks/validate_config.yml b/roles/matrix-sygnal/tasks/validate_config.yml index 1cf8357ee..efd64104a 100644 --- a/roles/matrix-sygnal/tasks/validate_config.yml +++ b/roles/matrix-sygnal/tasks/validate_config.yml @@ -3,3 +3,11 @@ msg: >- Enabling Sygnal requires that you specify at least one app in `matrix_sygnal_apps` when: "matrix_sygnal_enabled and matrix_sygnal_apps|length == 0" + +- name: Fail if running on a non-supported architecture + fail: + msg: >- + Sygnal can only be used on the amd64 architecture for now. + Only amd64 container images are pushed for the `docker.io/matrixdotorg/sygnal` container image. + Either use a different image (by redefining `matrix_sygnal_docker_image`) or consider contributing self-building support to this role. + when: "matrix_sygnal_enabled and matrix_architecture != 'amd64' and matrix_sygnal_docker_image.startswith('docker.io/matrixdotorg/sygnal')" From 54fe59f05cbabe34df8b7bc18e1465c941b26073 Mon Sep 17 00:00:00 2001 From: Thom Wiggers Date: Mon, 22 Mar 2021 12:37:35 +0100 Subject: [PATCH 13/28] Update IRC appservice --- roles/matrix-bridge-appservice-irc/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-appservice-irc/defaults/main.yml b/roles/matrix-bridge-appservice-irc/defaults/main.yml index fbfb6fcd5..b8a179727 100644 --- a/roles/matrix-bridge-appservice-irc/defaults/main.yml +++ b/roles/matrix-bridge-appservice-irc/defaults/main.yml @@ -7,7 +7,7 @@ matrix_appservice_irc_container_self_build: false matrix_appservice_irc_docker_repo: "https://github.com/matrix-org/matrix-appservice-irc.git" matrix_appservice_irc_docker_src_files_path: "{{ matrix_base_data_path }}/appservice-irc/docker-src" -matrix_appservice_irc_version: release-0.23.0 +matrix_appservice_irc_version: release-0.25.0 matrix_appservice_irc_docker_image: "docker.io/matrixdotorg/matrix-appservice-irc:{{ matrix_appservice_irc_version }}" matrix_appservice_irc_docker_image_force_pull: "{{ matrix_appservice_irc_docker_image.endswith(':latest') }}" From 74106f2a80615cd9c1b9e72d8644ea2028abd9a0 Mon Sep 17 00:00:00 2001 From: rakshazi Date: Mon, 22 Mar 2021 14:03:42 +0000 Subject: [PATCH 14/28] Updated synapse 1.29.0 -> 1.30.0 --- roles/matrix-synapse/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index d9139da17..7ece0fbf6 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -15,7 +15,7 @@ matrix_synapse_docker_image_name_prefix: "{{ 'localhost/' if matrix_synapse_cont # amd64 gets released first. # arm32 relies on self-building, so the same version can be built immediately. # arm64 users need to wait for a prebuilt image to become available. -matrix_synapse_version: v1.29.0 +matrix_synapse_version: v1.30.0 matrix_synapse_version_arm64: v1.29.0 matrix_synapse_docker_image_tag: "{{ matrix_synapse_version if matrix_architecture in ['arm32', 'amd64'] else matrix_synapse_version_arm64 }}" matrix_synapse_docker_image_force_pull: "{{ matrix_synapse_docker_image.endswith(':latest') }}" From ffe649a2405a25aabda0fbc03e666c83527b8bdd Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 22 Mar 2021 16:41:43 +0200 Subject: [PATCH 15/28] Update homeserver.yaml to keep up with Synapse v1.30.0 Related to https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/958 --- .../templates/synapse/homeserver.yaml.j2 | 67 +++++++++++++------ 1 file changed, 45 insertions(+), 22 deletions(-) diff --git a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 index 63e497d01..075685c4e 100644 --- a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 @@ -65,8 +65,7 @@ use_presence: {{ matrix_synapse_use_presence|to_json }} # Whether to require authentication to retrieve profile data (avatars, # display names) of other users through the client API. Defaults to # 'false'. Note that profile data is also available via the federation -# API, so this setting is of limited value if federation is enabled on -# the server. +# API, unless allow_profile_lookup_over_federation is set to false. # require_auth_for_profile_requests: {{ matrix_synapse_require_auth_for_profile_requests|to_json }} @@ -1777,10 +1776,14 @@ saml2_config: # offer the user a choice of login mechanisms. # # idp_icon: An optional icon for this identity provider, which is presented -# by identity picker pages. If given, must be an MXC URI of the format -# mxc:///. (An easy way to obtain such an MXC URI -# is to upload an image to an (unencrypted) room and then copy the "url" -# from the source of the event.) +# by clients and Synapse's own IdP picker page. If given, must be an +# MXC URI of the format mxc:///. (An easy way to +# obtain such an MXC URI is to upload an image to an (unencrypted) room +# and then copy the "url" from the source of the event.) +# +# idp_brand: An optional brand for this identity provider, allowing clients +# to style the login flow according to the identity provider in question. +# See the spec for possible options here. # # discover: set to 'false' to disable the use of the OIDC discovery mechanism # to discover endpoints. Defaults to true. @@ -1790,7 +1793,26 @@ saml2_config: # # client_id: Required. oauth2 client id to use. # -# client_secret: Required. oauth2 client secret to use. +# client_secret: oauth2 client secret to use. May be omitted if +# client_secret_jwt_key is given, or if client_auth_method is 'none'. +# +# client_secret_jwt_key: Alternative to client_secret: details of a key used +# to create a JSON Web Token to be used as an OAuth2 client secret. If +# given, must be a dictionary with the following properties: +# +# key: a pem-encoded signing key. Must be a suitable key for the +# algorithm specified. Required unless 'key_file' is given. +# +# key_file: the path to file containing a pem-encoded signing key file. +# Required unless 'key' is given. +# +# jwt_header: a dictionary giving properties to include in the JWT +# header. Must include the key 'alg', giving the algorithm used to +# sign the JWT, such as "ES256", using the JWA identifiers in +# RFC7518. +# +# jwt_payload: an optional dictionary giving properties to include in +# the JWT payload. Normally this should include an 'iss' key. # # client_auth_method: auth method to use when exchanging the token. Valid # values are 'client_secret_basic' (default), 'client_secret_post' and @@ -1910,7 +1932,7 @@ oidc_providers: # #- idp_id: github # idp_name: Github - # idp_brand: org.matrix.github + # idp_brand: github # discover: false # issuer: "https://github.com/" # client_id: "your-client-id" # TO BE FILLED @@ -1922,8 +1944,8 @@ oidc_providers: # user_mapping_provider: # config: # subject_claim: "id" - # localpart_template: "{% raw %}{{ user.login }}{% endraw %}" - # display_name_template: "{% raw %}{{ user.name }}{% endraw %}" + # localpart_template: "{{ user.login }}" + # display_name_template: "{{ user.name }}" # Enable Central Authentication Service (CAS) for registration and login. @@ -2675,19 +2697,20 @@ user_directory: -# Local statistics collection. Used in populating the room directory. -# -# 'bucket_size' controls how large each statistics timeslice is. It can -# be defined in a human readable short form -- e.g. "1d", "1y". -# -# 'retention' controls how long historical statistics will be kept for. -# It can be defined in a human readable short form -- e.g. "1d", "1y". -# +# Settings for local room and user statistics collection. See +# docs/room_and_user_statistics.md. # -#stats: -# enabled: true -# bucket_size: 1d -# retention: 1y +stats: + # Uncomment the following to disable room and user statistics. Note that doing + # so may cause certain features (such as the room directory) not to work + # correctly. + # + #enabled: false + + # The size of each timeslice in the room_stats_historical and + # user_stats_historical tables, as a time period. Defaults to "1d". + # + #bucket_size: 1h # Server Notices room configuration From 6a3433fbad0fec15064b514bab393b67be98e450 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 22 Mar 2021 16:42:07 +0200 Subject: [PATCH 16/28] Update Synapse for ARM64 (1.29.0 -> 1.30.0) Related to https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/958 --- roles/matrix-synapse/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index 7ece0fbf6..b46224783 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -16,7 +16,7 @@ matrix_synapse_docker_image_name_prefix: "{{ 'localhost/' if matrix_synapse_cont # arm32 relies on self-building, so the same version can be built immediately. # arm64 users need to wait for a prebuilt image to become available. matrix_synapse_version: v1.30.0 -matrix_synapse_version_arm64: v1.29.0 +matrix_synapse_version_arm64: v1.30.0 matrix_synapse_docker_image_tag: "{{ matrix_synapse_version if matrix_architecture in ['arm32', 'amd64'] else matrix_synapse_version_arm64 }}" matrix_synapse_docker_image_force_pull: "{{ matrix_synapse_docker_image.endswith(':latest') }}" From d09609daa800537c49f530227d9dfb4469629ff3 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 22 Mar 2021 17:13:10 +0200 Subject: [PATCH 17/28] Fix Jinja2 syntax error Fixes a regression introduced in ffe649a2405a25 --- roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 index 075685c4e..4be505246 100644 --- a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 @@ -1944,8 +1944,8 @@ oidc_providers: # user_mapping_provider: # config: # subject_claim: "id" - # localpart_template: "{{ user.login }}" - # display_name_template: "{{ user.name }}" + # localpart_template: "{% raw %}{{ user.login }}{% endraw %}" + # display_name_template: "{% raw %}{{ user.name }}{% endraw %}" # Enable Central Authentication Service (CAS) for registration and login. From 06c74728eb54d864855713cf2fab77cab3f20ffb Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 23 Mar 2021 10:28:32 +0200 Subject: [PATCH 18/28] Move matrix_nginx_proxy_proxy_synapse_federation_api_enabled definition to the role This variable was previously undefined in the role and was only getting defined via `group_vars/matrix_servers`. We now properly initialize it (and its good default value) in the role itself. --- group_vars/matrix_servers | 1 - roles/matrix-nginx-proxy/defaults/main.yml | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index a6c433cb1..cbd18faad 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -1113,7 +1113,6 @@ matrix_nginx_proxy_proxy_matrix_federation_api_addr_sans_container: "127.0.0.1:1 # Settings controlling matrix-synapse-proxy.conf matrix_nginx_proxy_proxy_synapse_enabled: "{{ matrix_synapse_enabled }}" -matrix_nginx_proxy_proxy_synapse_federation_api_enabled: "{{ matrix_nginx_proxy_proxy_matrix_federation_api_enabled }}" # When matrix-nginx-proxy is disabled, the actual port number that the vhost uses may begin to matter. matrix_nginx_proxy_proxy_matrix_federation_port: "{{ matrix_federation_public_port }}" diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index 3770bfb1e..ef431b72c 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -104,6 +104,10 @@ matrix_nginx_proxy_proxy_riot_compat_redirect_hostname: "riot.{{ matrix_domain } # Controls whether proxying the Synapse domain should be done. matrix_nginx_proxy_proxy_synapse_enabled: false matrix_nginx_proxy_proxy_synapse_hostname: "matrix-nginx-proxy" +matrix_nginx_proxy_proxy_synapse_federation_api_enabled: "{{ matrix_nginx_proxy_proxy_matrix_federation_api_enabled }}" +# The addresses where the Federation API is, when using Synapse. +matrix_nginx_proxy_proxy_synapse_federation_api_addr_with_container: "matrix-synapse:8048" +matrix_nginx_proxy_proxy_synapse_federation_api_addr_sans_container: "localhost:8048" # Controls whether proxying the Element domain should be done. matrix_nginx_proxy_proxy_element_enabled: false @@ -220,10 +224,6 @@ matrix_nginx_proxy_proxy_matrix_federation_api_client_max_body_size_mb: "{{ (mat matrix_nginx_proxy_proxy_matrix_federation_api_ssl_certificate: "{{ matrix_ssl_config_dir_path }}/live/{{ matrix_nginx_proxy_proxy_matrix_hostname }}/fullchain.pem" matrix_nginx_proxy_proxy_matrix_federation_api_ssl_certificate_key: "{{ matrix_ssl_config_dir_path }}/live/{{ matrix_nginx_proxy_proxy_matrix_hostname }}/privkey.pem" -# The addresses where the Federation API is, when using Synapse. -matrix_nginx_proxy_proxy_synapse_federation_api_addr_with_container: "matrix-synapse:8048" -matrix_nginx_proxy_proxy_synapse_federation_api_addr_sans_container: "localhost:8048" - # The tmpfs at /tmp needs to be large enough to handle multiple concurrent file uploads. matrix_nginx_proxy_tmp_directory_size_mb: "{{ (matrix_nginx_proxy_proxy_matrix_federation_api_client_max_body_size_mb | int) * 50 }}" From c6fb7acf5fdaf986a9f8e73dce21bce0d0dd476c Mon Sep 17 00:00:00 2001 From: Aaron Raimist Date: Tue, 23 Mar 2021 23:05:48 -0500 Subject: [PATCH 19/28] Make ma1sd's SRV record optional --- docs/configuring-dns.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/configuring-dns.md b/docs/configuring-dns.md index 82bef8a2d..285ded424 100644 --- a/docs/configuring-dns.md +++ b/docs/configuring-dns.md @@ -15,22 +15,22 @@ As we discuss in [Server Delegation](howto-server-delegation.md), there are 2 di This playbook mostly discusses the well-known file method, because it's easier to manage with regard to certificates. If you decide to go with the alternative method ([Server Delegation via a DNS SRV record (advanced)](howto-server-delegation.md#server-delegation-via-a-dns-srv-record-advanced)), please be aware that the general flow that this playbook guides you through may not match what you need to do. -## Required DNS settings for services enabled by default +## DNS settings for services enabled by default | Type | Host | Priority | Weight | Port | Target | | ----- | ---------------------------- | -------- | ------ | ---- | ---------------------- | | A | `matrix` | - | - | - | `matrix-server-IP` | | CNAME | `element` | - | - | - | `matrix.` | -| SRV | `_matrix-identity._tcp` | 10 | 0 | 443 | `matrix.` | Be mindful as to how long it will take for the DNS records to propagate. If you are using Cloudflare DNS, make sure to disable the proxy and set all records to `DNS only`. Otherwise, fetching certificates will fail. -## Required DNS settings for optional services +## DNS settings for optional services/features | Type | Host | Priority | Weight | Port | Target | | ----- | ---------------------------- | -------- | ------ | ---- | ---------------------- | +| SRV | `_matrix-identity._tcp` | 10 | 0 | 443 | `matrix.` | | CNAME | `dimension` (*) | - | - | - | `matrix.` | | CNAME | `jitsi` (*) | - | - | - | `matrix.` | | CNAME | `stats` (*) | - | - | - | `matrix.` | @@ -41,7 +41,7 @@ If you are using Cloudflare DNS, make sure to disable the proxy and set all reco As the table above illustrates, you need to create 2 subdomains (`matrix.` and `element.`) and point both of them to your new server's IP address (DNS `A` record or `CNAME` record is fine). -The `element.` subdomain is necessary, because this playbook installs the [Element](https://github.com/vector-im/element-web) web client for you. +The `element.` subdomain may be necessary, because this playbook installs the [Element](https://github.com/vector-im/element-web) web client for you. If you'd rather instruct the playbook not to install Element (`matrix_client_element_enabled: false` when [Configuring the playbook](configuring-playbook.md) later), feel free to skip the `element.` DNS record. The `dimension.` subdomain may be necessary, because this playbook could install the [Dimension integrations manager](http://dimension.t2bot.io/) for you. Dimension installation is disabled by default, because it's only possible to install it after the other Matrix services are working (see [Setting up Dimension](configuring-playbook-dimension.md) later). If you do not wish to set up Dimension, feel free to skip the `dimension.` DNS record. @@ -57,9 +57,12 @@ The `sygnal.` subdomain may be necessary, because this playbook cou ## `_matrix-identity._tcp` SRV record setup -To make the [ma1sd](https://github.com/ma1uta/ma1sd) Identity Server (which this playbook installs for you) be authoritative for your domain name, set up one more SRV record that looks like this: +To make the [ma1sd](https://github.com/ma1uta/ma1sd) Identity Server (which this playbook installs for you) enable its federation features, set up an SRV record that looks like this: - Name: `_matrix-identity._tcp` (use this text as-is) - Content: `10 0 443 matrix.` (replace `` with your own) +This is an optional feature. See [ma1sd's documentation](https://github.com/ma1uta/ma1sd/wiki/mxisd-and-your-privacy#choices-are-never-easy) for information on the privacy implications of setting up this SRV record. + +Note: This `_matrix-identity._tcp` SRV record for the identity server is different from the `_matrix._tcp` that can be used for Synapse delegation. See [howto-server-delegation.md](howto-server-delegation.md) for more information about delegation. When you're done with the DNS configuration and ready to proceed, continue with [Configuring this Ansible playbook](configuring-playbook.md). From b89936afba9f922f83dfe771d725518bb1c6b35f Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 24 Mar 2021 13:59:46 +0200 Subject: [PATCH 20/28] Make bridges depend on matrix-nginx-proxy We don't point them straight to Synapse anymore, but rather to matrix-nginx-proxy, which forwards to Synapse (or something else). --- group_vars/matrix_servers | 44 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index cbd18faad..24fda5d5d 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -69,6 +69,8 @@ matrix_appservice_discord_systemd_required_services_list: | (['matrix-synapse.service'] if matrix_synapse_enabled else []) + (['matrix-postgres.service'] if matrix_postgres_enabled else []) + + + (['matrix-nginx-proxy.service'] if matrix_nginx_proxy_enabled else []) }} matrix_appservice_discord_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'discord.as.token') | to_uuid }}" @@ -112,6 +114,8 @@ matrix_appservice_webhooks_systemd_required_services_list: | ['docker.service'] + (['matrix-synapse.service'] if matrix_synapse_enabled else []) + + + (['matrix-nginx-proxy.service'] if matrix_nginx_proxy_enabled else []) }} ###################################################################### @@ -148,6 +152,8 @@ matrix_appservice_slack_systemd_required_services_list: | ['docker.service'] + (['matrix-synapse.service'] if matrix_synapse_enabled else []) + + + (['matrix-nginx-proxy.service'] if matrix_nginx_proxy_enabled else []) }} # Postgres is the default, except if not using `matrix_postgres` (internal postgres) @@ -185,6 +191,8 @@ matrix_appservice_irc_systemd_required_services_list: | ['docker.service'] + (['matrix-synapse.service'] if matrix_synapse_enabled else []) + + + (['matrix-nginx-proxy.service'] if matrix_nginx_proxy_enabled else []) }} matrix_appservice_irc_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'irc.as.token') | to_uuid }}" @@ -220,6 +228,8 @@ matrix_mautrix_facebook_systemd_required_services_list: | (['matrix-synapse.service'] if matrix_synapse_enabled else []) + (['matrix-postgres.service'] if matrix_postgres_enabled else []) + + + (['matrix-nginx-proxy.service'] if matrix_nginx_proxy_enabled else []) }} matrix_mautrix_facebook_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'fb.as.token') | to_uuid }}" @@ -260,6 +270,8 @@ matrix_mautrix_hangouts_systemd_required_services_list: | (['matrix-synapse.service'] if matrix_synapse_enabled else []) + (['matrix-postgres.service'] if matrix_postgres_enabled else []) + + + (['matrix-nginx-proxy.service'] if matrix_nginx_proxy_enabled else []) }} matrix_mautrix_hangouts_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'ho.as.token') | to_uuid }}" @@ -299,6 +311,8 @@ matrix_mautrix_instagram_systemd_required_services_list: | (['matrix-synapse.service'] if matrix_synapse_enabled else []) + (['matrix-postgres.service'] if matrix_postgres_enabled else []) + + + (['matrix-nginx-proxy.service'] if matrix_nginx_proxy_enabled else []) }} matrix_mautrix_instagram_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'ig.as.token') | to_uuid }}" @@ -338,6 +352,8 @@ matrix_mautrix_signal_systemd_required_services_list: | + (['matrix-postgres.service'] if matrix_postgres_enabled else []) + + (['matrix-nginx-proxy.service'] if matrix_nginx_proxy_enabled else []) + + ['matrix-mautrix-signal-daemon.service'] }} @@ -380,6 +396,8 @@ matrix_mautrix_telegram_systemd_required_services_list: | (['matrix-synapse.service'] if matrix_synapse_enabled else []) + (['matrix-postgres.service'] if matrix_postgres_enabled else []) + + + (['matrix-nginx-proxy.service'] if matrix_nginx_proxy_enabled else []) }} matrix_mautrix_telegram_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'telegr.as.token') | to_uuid }}" @@ -418,6 +436,8 @@ matrix_mautrix_whatsapp_systemd_required_services_list: | (['matrix-synapse.service'] if matrix_synapse_enabled else []) + (['matrix-postgres.service'] if matrix_postgres_enabled else []) + + + (['matrix-nginx-proxy.service'] if matrix_nginx_proxy_enabled else []) }} matrix_mautrix_whatsapp_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'whats.as.token') | to_uuid }}" @@ -450,6 +470,8 @@ matrix_sms_bridge_systemd_required_services_list: | ['docker.service'] + (['matrix-synapse.service'] if matrix_synapse_enabled else []) + + + (['matrix-nginx-proxy.service'] if matrix_nginx_proxy_enabled else []) }} matrix_sms_bridge_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'sms.as.token') | to_uuid }}" @@ -480,6 +502,8 @@ matrix_mx_puppet_skype_systemd_required_services_list: | (['matrix-synapse.service'] if matrix_synapse_enabled else []) + (['matrix-postgres.service'] if matrix_postgres_enabled else []) + + + (['matrix-nginx-proxy.service'] if matrix_nginx_proxy_enabled else []) }} matrix_mx_puppet_skype_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'skype.as.tok') | to_uuid }}" @@ -517,6 +541,8 @@ matrix_mx_puppet_slack_systemd_required_services_list: | (['matrix-synapse.service'] if matrix_synapse_enabled else []) + (['matrix-postgres.service'] if matrix_postgres_enabled else []) + + + (['matrix-nginx-proxy.service'] if matrix_nginx_proxy_enabled else []) }} matrix_mx_puppet_slack_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxslk.as.tok') | to_uuid }}" @@ -553,6 +579,8 @@ matrix_mx_puppet_twitter_systemd_required_services_list: | (['matrix-synapse.service'] if matrix_synapse_enabled else []) + (['matrix-postgres.service'] if matrix_postgres_enabled else []) + + + (['matrix-nginx-proxy.service'] if matrix_nginx_proxy_enabled else []) }} matrix_mx_puppet_twitter_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxtwt.as.tok') | to_uuid }}" @@ -592,6 +620,8 @@ matrix_mx_puppet_instagram_systemd_required_services_list: | (['matrix-synapse.service'] if matrix_synapse_enabled else []) + (['matrix-postgres.service'] if matrix_postgres_enabled else []) + + + (['matrix-nginx-proxy.service'] if matrix_nginx_proxy_enabled else []) }} matrix_mx_puppet_instagram_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxig.as.tok') | to_uuid }}" @@ -628,6 +658,8 @@ matrix_mx_puppet_discord_systemd_required_services_list: | (['matrix-synapse.service'] if matrix_synapse_enabled else []) + (['matrix-postgres.service'] if matrix_postgres_enabled else []) + + + (['matrix-nginx-proxy.service'] if matrix_nginx_proxy_enabled else []) }} matrix_mx_puppet_discord_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxdsc.as.tok') | to_uuid }}" @@ -664,6 +696,8 @@ matrix_mx_puppet_steam_systemd_required_services_list: | (['matrix-synapse.service'] if matrix_synapse_enabled else []) + (['matrix-postgres.service'] if matrix_postgres_enabled else []) + + + (['matrix-nginx-proxy.service'] if matrix_nginx_proxy_enabled else []) }} matrix_mx_puppet_steam_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxste.as.tok') | to_uuid }}" @@ -700,6 +734,8 @@ matrix_mx_puppet_groupme_systemd_required_services_list: | (['matrix-synapse.service'] if matrix_synapse_enabled else []) + (['matrix-postgres.service'] if matrix_postgres_enabled else []) + + + (['matrix-nginx-proxy.service'] if matrix_nginx_proxy_enabled else []) }} matrix_mx_puppet_groupme_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxgro.as.tok') | to_uuid }}" @@ -732,6 +768,10 @@ matrix_bot_matrix_reminder_bot_systemd_required_services_list: | ['docker.service'] + (['matrix-postgres.service'] if matrix_postgres_enabled else []) + + + (['matrix-synapse.service'] if matrix_synapse_enabled else []) + + + (['matrix-nginx-proxy.service'] if matrix_nginx_proxy_enabled else []) }} # Postgres is the default, except if not using `matrix_postgres` (internal postgres) @@ -757,6 +797,10 @@ matrix_bot_go_neb_enabled: false matrix_bot_go_neb_systemd_required_services_list: | {{ ['docker.service'] + + + (['matrix-synapse.service'] if matrix_synapse_enabled else []) + + + (['matrix-nginx-proxy.service'] if matrix_nginx_proxy_enabled else []) }} matrix_bot_go_neb_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_enabled else '127.0.0.1:4050' }}" From 5a4ea5f866cc8680b58975c933907df12e91cdf1 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 24 Mar 2021 14:02:53 +0200 Subject: [PATCH 21/28] Make AWX enabling/disabling consistent with other playbook roles That is: - enabled in the role by default - disabled in the compilation (playbook), if considered an optional component --- group_vars/matrix_servers | 3 +++ roles/matrix-awx/defaults/main.yml | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 24fda5d5d..b135cfcb0 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -35,6 +35,9 @@ matrix_homeserver_container_url: "{{ 'http://matrix-nginx-proxy:12080' if matrix # ###################################################################### +# We don't enable AWX support by default. +matrix_awx_enabled: false + matrix_nginx_proxy_data_path: "{{ '/chroot/website' if (matrix_awx_enabled and not matrix_nginx_proxy_base_domain_homepage_enabled) else (matrix_nginx_proxy_base_path + '/data') }}" matrix_nginx_proxy_data_path_in_container: "{{ '/nginx-data/matrix-domain' if (matrix_awx_enabled and not matrix_nginx_proxy_base_domain_homepage_enabled) else '/nginx-data' }}" diff --git a/roles/matrix-awx/defaults/main.yml b/roles/matrix-awx/defaults/main.yml index 9394ecab9..fb9f56ae0 100755 --- a/roles/matrix-awx/defaults/main.yml +++ b/roles/matrix-awx/defaults/main.yml @@ -1,2 +1 @@ - -matrix_awx_enabled: false +matrix_awx_enabled: true From 97d8527e00312fd90f3d744305e4edaebd145a64 Mon Sep 17 00:00:00 2001 From: Dan Arnfield Date: Wed, 24 Mar 2021 09:42:08 -0500 Subject: [PATCH 22/28] Update nginx (1.19.6 -> 1.19.8) --- roles/matrix-nginx-proxy/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index ef431b72c..004a1ec14 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -1,5 +1,5 @@ matrix_nginx_proxy_enabled: true -matrix_nginx_proxy_version: 1.19.6-alpine +matrix_nginx_proxy_version: 1.19.8-alpine # We use an official nginx image, which we fix-up to run unprivileged. # An alternative would be an `nginxinc/nginx-unprivileged` image, but From 5b7dbe82750d53e653fba072f6d258d367fb2dcc Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 25 Mar 2021 08:51:15 +0200 Subject: [PATCH 23/28] Make matrix-synapse wanted by matrix-nginx-proxy only if enabled --- group_vars/matrix_servers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index b135cfcb0..d5b7e9c0f 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -1191,7 +1191,7 @@ matrix_nginx_proxy_synapse_frontend_proxy_locations: "{{ matrix_synapse_workers matrix_nginx_proxy_systemd_wanted_services_list: | {{ - (['matrix-synapse.service']) + (['matrix-synapse.service'] if matrix_synapse_enabled else []) + (['matrix-corporal.service'] if matrix_corporal_enabled else []) + From efbeb428577d7dc69573b41169bdd20811e876ad Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 25 Mar 2021 09:02:32 +0200 Subject: [PATCH 24/28] Fix YAML indentation in docs --- docs/configuring-playbook-sygnal.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuring-playbook-sygnal.md b/docs/configuring-playbook-sygnal.md index b8dc11f0d..3eb626ebc 100644 --- a/docs/configuring-playbook-sygnal.md +++ b/docs/configuring-playbook-sygnal.md @@ -23,7 +23,7 @@ matrix_sygnal_enabled: true matrix_sygnal_apps: com.example.myapp.ios: type: apns - keyfile: /data/my_key.p8 + keyfile: /data/my_key.p8 # .. more configuration .. com.example.myapp.android: type: gcm From 59ac8a59210814ecd2834b4603ae61cfc91ff413 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 25 Mar 2021 11:56:34 +0200 Subject: [PATCH 25/28] Make Dimension require matrix-nginx-proxy, if enabled Related to https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/966 --- group_vars/matrix_servers | 2 ++ 1 file changed, 2 insertions(+) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index d5b7e9c0f..81ddc2194 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -911,6 +911,8 @@ matrix_dimension_systemd_required_services_list: | ['docker.service'] + (['matrix-postgres.service'] if matrix_postgres_enabled else []) + + + (['matrix-nginx-proxy.service'] if matrix_nginx_proxy_enabled else []) }} # Postgres is the default, except if not using `matrix_postgres` (internal postgres) From dbae18fd6aafc2781c8b58d25b11174832d23846 Mon Sep 17 00:00:00 2001 From: transcaffeine Date: Thu, 25 Mar 2021 18:49:43 +0100 Subject: [PATCH 26/28] feat: push ephemeral events to appservices This adds https://github.com/matrix-org/matrix-doc/pull/2409 to the appservice registrations, enabling synapse to push EDUs to appservices. --- roles/matrix-bridge-mautrix-facebook/defaults/main.yml | 1 + roles/matrix-bridge-mautrix-hangouts/defaults/main.yml | 1 + roles/matrix-bridge-mautrix-instagram/defaults/main.yml | 1 + .../matrix-bridge-mautrix-signal/templates/registration.yaml.j2 | 1 + roles/matrix-bridge-mautrix-telegram/defaults/main.yml | 1 + roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml | 1 + roles/matrix-bridge-mx-puppet-discord/defaults/main.yml | 1 + roles/matrix-bridge-mx-puppet-groupme/defaults/main.yml | 1 + roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml | 1 + roles/matrix-bridge-mx-puppet-skype/defaults/main.yml | 1 + roles/matrix-bridge-mx-puppet-slack/defaults/main.yml | 1 + roles/matrix-bridge-mx-puppet-steam/defaults/main.yml | 1 + roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml | 1 + 13 files changed, 13 insertions(+) diff --git a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml index 7c6b58cac..71a225f7c 100644 --- a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml @@ -110,5 +110,6 @@ matrix_mautrix_facebook_registration_yaml: | # See https://github.com/tulir/mautrix-signal/issues/43 sender_localpart: _bot_{{ matrix_mautrix_facebook_appservice_bot_username }} rate_limited: false + de.sorunome.msc2409.push_ephemeral: true matrix_mautrix_facebook_registration: "{{ matrix_mautrix_facebook_registration_yaml|from_yaml }}" diff --git a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml index a029b9e82..48b66b8d6 100644 --- a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml @@ -110,5 +110,6 @@ matrix_mautrix_hangouts_registration_yaml: | # See https://github.com/tulir/mautrix-signal/issues/43 sender_localpart: _bot_{{ matrix_mautrix_hangouts_appservice_bot_username }} rate_limited: false + de.sorunome.msc2409.push_ephemeral: true matrix_mautrix_hangouts_registration: "{{ matrix_mautrix_hangouts_registration_yaml|from_yaml }}" diff --git a/roles/matrix-bridge-mautrix-instagram/defaults/main.yml b/roles/matrix-bridge-mautrix-instagram/defaults/main.yml index 936df4c5f..5204386d5 100644 --- a/roles/matrix-bridge-mautrix-instagram/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-instagram/defaults/main.yml @@ -100,5 +100,6 @@ matrix_mautrix_instagram_registration_yaml: | # See https://github.com/tulir/mautrix-signal/issues/43 sender_localpart: _bot_{{ matrix_mautrix_instagram_appservice_bot_username }} rate_limited: false + de.sorunome.msc2409.push_ephemeral: true matrix_mautrix_instagram_registration: "{{ matrix_mautrix_instagram_registration_yaml|from_yaml }}" diff --git a/roles/matrix-bridge-mautrix-signal/templates/registration.yaml.j2 b/roles/matrix-bridge-mautrix-signal/templates/registration.yaml.j2 index 6891c2b59..54df82da8 100644 --- a/roles/matrix-bridge-mautrix-signal/templates/registration.yaml.j2 +++ b/roles/matrix-bridge-mautrix-signal/templates/registration.yaml.j2 @@ -15,3 +15,4 @@ url: {{ matrix_mautrix_signal_appservice_address }} # See https://github.com/tulir/mautrix-signal/issues/43 sender_localpart: _bot_{{ matrix_mautrix_signal_appservice_bot_username }} rate_limited: false +de.sorunome.msc2409.push_ephemeral: true diff --git a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml index 2d5b52642..e4c7696aa 100644 --- a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml @@ -121,5 +121,6 @@ matrix_mautrix_telegram_registration_yaml: | sender_localpart: _bot_{{ matrix_mautrix_telegram_appservice_bot_username }} url: {{ matrix_mautrix_telegram_appservice_address }} rate_limited: false + de.sorunome.msc2409.push_ephemeral: true matrix_mautrix_telegram_registration: "{{ matrix_mautrix_telegram_registration_yaml|from_yaml }}" diff --git a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml index b3aee68a9..f8ae7d873 100644 --- a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml @@ -105,5 +105,6 @@ matrix_mautrix_whatsapp_registration_yaml: | exclusive: true - exclusive: true regex: '^@{{ matrix_mautrix_whatsapp_appservice_bot_username|regex_escape }}:{{ matrix_mautrix_whatsapp_homeserver_domain|regex_escape }}$' + de.sorunome.msc2409.push_ephemeral: true matrix_mautrix_whatsapp_registration: "{{ matrix_mautrix_whatsapp_registration_yaml|from_yaml }}" diff --git a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml index 252fb9e13..34a8ee366 100644 --- a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml @@ -108,5 +108,6 @@ matrix_mx_puppet_discord_registration_yaml: | rate_limited: false sender_localpart: _discordpuppet_bot url: {{ matrix_mx_puppet_discord_appservice_address }} + de.sorunome.msc2409.push_ephemeral: true matrix_mx_puppet_discord_registration: "{{ matrix_mx_puppet_discord_registration_yaml|from_yaml }}" diff --git a/roles/matrix-bridge-mx-puppet-groupme/defaults/main.yml b/roles/matrix-bridge-mx-puppet-groupme/defaults/main.yml index 53676e832..cfc2aa0d5 100644 --- a/roles/matrix-bridge-mx-puppet-groupme/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-groupme/defaults/main.yml @@ -107,5 +107,6 @@ matrix_mx_puppet_groupme_registration_yaml: | rate_limited: false sender_localpart: _groupmepuppet_bot url: {{ matrix_mx_puppet_groupme_appservice_address }} + de.sorunome.msc2409.push_ephemeral: true matrix_mx_puppet_groupme_registration: "{{ matrix_mx_puppet_groupme_registration_yaml|from_yaml }}" diff --git a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml index b14ed0f8e..24377fb3f 100644 --- a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml @@ -98,5 +98,6 @@ matrix_mx_puppet_instagram_registration_yaml: | rate_limited: false sender_localpart: _instagrampuppet_bot url: {{ matrix_mx_puppet_instagram_appservice_address }} + de.sorunome.msc2409.push_ephemeral: true matrix_mx_puppet_instagram_registration: "{{ matrix_mx_puppet_instagram_registration_yaml|from_yaml }}" diff --git a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml index c68c41609..0f603df0c 100644 --- a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml @@ -106,5 +106,6 @@ matrix_mx_puppet_skype_registration_yaml: | rate_limited: false sender_localpart: _skypepuppet_bot url: {{ matrix_mx_puppet_skype_appservice_address }} + de.sorunome.msc2409.push_ephemeral: true matrix_mx_puppet_skype_registration: "{{ matrix_mx_puppet_skype_registration_yaml|from_yaml }}" diff --git a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml index c5b046f4b..32df72366 100644 --- a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml @@ -110,5 +110,6 @@ matrix_mx_puppet_slack_registration_yaml: | rate_limited: false sender_localpart: _slackpuppet_bot url: {{ matrix_mx_puppet_slack_appservice_address }} + de.sorunome.msc2409.push_ephemeral: true matrix_mx_puppet_slack_registration: "{{ matrix_mx_puppet_slack_registration_yaml|from_yaml }}" diff --git a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml index aee4da2cc..7123e0403 100644 --- a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml @@ -107,5 +107,6 @@ matrix_mx_puppet_steam_registration_yaml: | rate_limited: false sender_localpart: _steampuppet_bot url: {{ matrix_mx_puppet_steam_appservice_address }} + de.sorunome.msc2409.push_ephemeral: true matrix_mx_puppet_steam_registration: "{{ matrix_mx_puppet_steam_registration_yaml|from_yaml }}" diff --git a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml index a8b811d8c..c10f521d3 100644 --- a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml @@ -117,5 +117,6 @@ matrix_mx_puppet_twitter_registration_yaml: | rate_limited: false sender_localpart: "{{ matrix_mx_puppet_twitter_bot_localpart }}" url: {{ matrix_mx_puppet_twitter_appservice_address }} + de.sorunome.msc2409.push_ephemeral: true matrix_mx_puppet_twitter_registration: "{{ matrix_mx_puppet_twitter_registration_yaml|from_yaml }}" From 94487dc6a725ac7b51f9d2f1c905c716229a150c Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 26 Mar 2021 15:37:11 +0200 Subject: [PATCH 27/28] Upgrade Synapse for amd64 (1.30.0 -> 1.30.1) --- roles/matrix-synapse/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index b46224783..32463639c 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -15,7 +15,7 @@ matrix_synapse_docker_image_name_prefix: "{{ 'localhost/' if matrix_synapse_cont # amd64 gets released first. # arm32 relies on self-building, so the same version can be built immediately. # arm64 users need to wait for a prebuilt image to become available. -matrix_synapse_version: v1.30.0 +matrix_synapse_version: v1.30.1 matrix_synapse_version_arm64: v1.30.0 matrix_synapse_docker_image_tag: "{{ matrix_synapse_version if matrix_architecture in ['arm32', 'amd64'] else matrix_synapse_version_arm64 }}" matrix_synapse_docker_image_force_pull: "{{ matrix_synapse_docker_image.endswith(':latest') }}" From 49868db3dec30e2aabf20b84c6cca9cdd85be462 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 26 Mar 2021 16:48:15 +0200 Subject: [PATCH 28/28] Upgrade Synapse for ARM64 (1.30.0 -> 1.30.1) --- roles/matrix-synapse/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index 32463639c..daa335b60 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -16,7 +16,7 @@ matrix_synapse_docker_image_name_prefix: "{{ 'localhost/' if matrix_synapse_cont # arm32 relies on self-building, so the same version can be built immediately. # arm64 users need to wait for a prebuilt image to become available. matrix_synapse_version: v1.30.1 -matrix_synapse_version_arm64: v1.30.0 +matrix_synapse_version_arm64: v1.30.1 matrix_synapse_docker_image_tag: "{{ matrix_synapse_version if matrix_architecture in ['arm32', 'amd64'] else matrix_synapse_version_arm64 }}" matrix_synapse_docker_image_force_pull: "{{ matrix_synapse_docker_image.endswith(':latest') }}"