diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c68ed63d..b7800da08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,31 @@ +# 2022-04-19 + +## Borg backup support + +Thanks to [Aine](https://gitlab.com/etke.cc) of [etke.cc](https://etke.cc/), the playbook can now set up [Borg](https://www.borgbackup.org/) backups with [borgmatic](https://torsion.org/borgmatic/) of your Matrix server. + +See our [Setting up borg backup](docs/configuring-playbook-backup-borg.md) documentation to get started. + + +## (Compatibility Break) Upgrading to Synapse v1.57 on setups using workers may require manual action + +If you're running a worker setup for Synapse (`matrix_synapse_workers_enabled: true`), the [Synapse v1.57 upgrade notes](https://github.com/matrix-org/synapse/blob/v1.57.0rc1/docs/upgrade.md#changes-to-database-schema-for-application-services) say that you may need to take special care when upgrading: + +> Synapse v1.57.0 includes a change to the way transaction IDs are managed for application services. If your deployment uses a dedicated worker for application service traffic, **it must be stopped** when the database is upgraded (which normally happens when the main process is upgraded), to ensure the change is made safely without any risk of reusing transaction IDs. + +If you're not running an `appservice` worker (`matrix_synapse_workers_preset: little-federation-helper` or `matrix_synapse_workers_appservice_workers_count: 0`), you are probably safe to upgrade as per normal, without taking any special care. + +If you are running a setup with an `appservice` worker, or otherwise want to be on the safe side, we recommend the following upgrade path: + +0. Pull the latest playbook changes +1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) +2. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all`) +3. Start Postgres (`systemctl start matrix-postgres` on the server) +4. Start the main Synapse process (`systemctl start matrix-synapse` on the server) +5. Wait a while so that Synapse can start and complete the database migrations. You can use `journalctl -fu matrix-synapse` on the server to get a clue. Waiting a few minutes should also be enough. +6. It should now be safe to start all other services. `ansible-playbook -i inventory/hosts setup.yml --tags=start` will do it for you + + # 2022-04-14 ## (Compatibility Break) Changes to `docker-src` permissions necessitating manual action diff --git a/docs/configuring-playbook-backup-borg.md b/docs/configuring-playbook-backup-borg.md index 4177c561e..44c970af9 100644 --- a/docs/configuring-playbook-backup-borg.md +++ b/docs/configuring-playbook-backup-borg.md @@ -8,6 +8,9 @@ You will need a remote server where borg will store the backups. There are hoste The backup will run based on `matrix_backup_borg_schedule` var (systemd timer calendar), default: 4am every day. +By default, if you're using the integrated Postgres database server (as opposed to [an external Postgres server](configuring-playbook-external-postgres.md)), Borg backups will also include dumps of your Postgres database. An alternative solution for backing up the Postgres database is [postgres backup](configuring-playbook-postgres-backup.md). If you decide to go with another solution, you can disable Postgres-backup support for Borg using the `matrix_backup_borg_postgresql_enabled` variable. + + ## Prerequisites 1. Create a new SSH key: diff --git a/docs/configuring-playbook-postgres-backup.md b/docs/configuring-playbook-postgres-backup.md index 2d878c11d..75b599c88 100644 --- a/docs/configuring-playbook-postgres-backup.md +++ b/docs/configuring-playbook-postgres-backup.md @@ -2,6 +2,9 @@ The playbook can install and configure [docker-postgres-backup-local](https://github.com/prodrigestivill/docker-postgres-backup-local) for you. +For a more complete backup solution (one that includes not only Postgres, but also other configuration/data files), you may wish to look into [borg backup](configuring-playbook-backup-borg.md) instead. + + ## Adjusting the playbook configuration Minimal working configuration (`inventory/host_vars/matrix.DOMAIN/vars.yml`) to enable Postgres backup: diff --git a/docs/configuring-playbook.md b/docs/configuring-playbook.md index 3d5e6c2c7..c842a8703 100644 --- a/docs/configuring-playbook.md +++ b/docs/configuring-playbook.md @@ -152,6 +152,13 @@ When you're done with all the configuration you'd like to do, continue with [Ins - [Setting up Mjolnir](configuring-playbook-bot-mjolnir.md) - a moderation tool/bot (optional) +### Backups + +- [Setting up borg backup](configuring-playbook-backup-borg.md) - a full Matrix server backup solution, including the Postgres database (optional) + +- [Setting up postgres backup](configuring-playbook-postgres-backup.md) - a Postgres-database backup solution (note: does not include other files) (optional) + + ### Other specialized services - [Setting up the Sygnal push gateway](configuring-playbook-sygnal.md) (optional) diff --git a/roles/matrix-backup-borg/tasks/init.yml b/roles/matrix-backup-borg/tasks/init.yml index 0a90a2e88..a82fb7b8e 100644 --- a/roles/matrix-backup-borg/tasks/init.yml +++ b/roles/matrix-backup-borg/tasks/init.yml @@ -1,4 +1,4 @@ --- - set_fact: - matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-backup-borg.service', 'matrix-backup-borg.timer'] }}" + matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-backup-borg.timer'] }}" when: matrix_backup_borg_enabled|bool diff --git a/roles/matrix-base/defaults/main.yml b/roles/matrix-base/defaults/main.yml index ae39d00a5..498a6c321 100644 --- a/roles/matrix-base/defaults/main.yml +++ b/roles/matrix-base/defaults/main.yml @@ -71,6 +71,10 @@ matrix_container_global_registry_prefix: "docker.io/" matrix_container_retries_count: 10 matrix_container_retries_delay: 10 +# Each get_url will retry on failed attempt 10 times with delay of 10 seconds between each attempt. +matrix_geturl_retries_count: 10 +matrix_geturl_retries_delay: 10 + matrix_user_username: "matrix" matrix_user_groupname: "matrix" diff --git a/roles/matrix-grafana/tasks/setup.yml b/roles/matrix-grafana/tasks/setup.yml index 95a0ba53f..16b9fa659 100644 --- a/roles/matrix-grafana/tasks/setup.yml +++ b/roles/matrix-grafana/tasks/setup.yml @@ -70,6 +70,10 @@ group: "{{ matrix_user_groupname }}" with_items: "{{ matrix_grafana_dashboard_download_urls_all }}" when: matrix_grafana_enabled|bool + register: result + retries: "{{ matrix_geturl_retries_count }}" + delay: "{{ matrix_geturl_retries_delay }}" + until: result is not failed - name: Ensure matrix-grafana.service installed template: diff --git a/roles/matrix-prometheus/tasks/setup_install.yml b/roles/matrix-prometheus/tasks/setup_install.yml index e0fe8cf62..06989e7ef 100644 --- a/roles/matrix-prometheus/tasks/setup_install.yml +++ b/roles/matrix-prometheus/tasks/setup_install.yml @@ -32,6 +32,10 @@ owner: "{{ matrix_user_username }}" group: "{{ matrix_user_groupname }}" when: "matrix_prometheus_scraper_synapse_rules_enabled|bool" + register: result + retries: "{{ matrix_geturl_retries_count }}" + delay: "{{ matrix_geturl_retries_delay }}" + until: result is not failed - name: Ensure prometheus.yml installed copy: diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index 5f501dbc3..58b4dd8c8 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -9,7 +9,7 @@ matrix_synapse_container_image_self_build_repo: "https://github.com/matrix-org/s matrix_synapse_docker_image: "{{ matrix_synapse_docker_image_name_prefix }}matrixdotorg/synapse:{{ matrix_synapse_docker_image_tag }}" matrix_synapse_docker_image_name_prefix: "{{ 'localhost/' if matrix_synapse_container_image_self_build else matrix_container_global_registry_prefix }}" -matrix_synapse_version: v1.57.0 +matrix_synapse_version: v1.57.1 matrix_synapse_docker_image_tag: "{{ matrix_synapse_version }}" matrix_synapse_docker_image_force_pull: "{{ matrix_synapse_docker_image.endswith(':latest') }}" diff --git a/roles/matrix-synapse/tasks/ext/encryption-disabler/setup_install.yml b/roles/matrix-synapse/tasks/ext/encryption-disabler/setup_install.yml index dfc15a207..90411a34b 100644 --- a/roles/matrix-synapse/tasks/ext/encryption-disabler/setup_install.yml +++ b/roles/matrix-synapse/tasks/ext/encryption-disabler/setup_install.yml @@ -8,6 +8,10 @@ mode: 0440 owner: "{{ matrix_user_username }}" group: "{{ matrix_user_groupname }}" + register: result + retries: "{{ matrix_geturl_retries_count }}" + delay: "{{ matrix_geturl_retries_delay }}" + until: result is not failed - set_fact: matrix_synapse_modules: | diff --git a/roles/matrix-synapse/tasks/ext/rest-auth/setup_install.yml b/roles/matrix-synapse/tasks/ext/rest-auth/setup_install.yml index 634b1ca5e..13a64c58c 100644 --- a/roles/matrix-synapse/tasks/ext/rest-auth/setup_install.yml +++ b/roles/matrix-synapse/tasks/ext/rest-auth/setup_install.yml @@ -13,6 +13,10 @@ mode: 0440 owner: "{{ matrix_user_username }}" group: "{{ matrix_user_groupname }}" + register: result + retries: "{{ matrix_geturl_retries_count }}" + delay: "{{ matrix_geturl_retries_delay }}" + until: result is not failed - set_fact: matrix_synapse_password_providers_enabled: true diff --git a/roles/matrix-synapse/tasks/ext/shared-secret-auth/setup_install.yml b/roles/matrix-synapse/tasks/ext/shared-secret-auth/setup_install.yml index f408e2f9f..843f03703 100644 --- a/roles/matrix-synapse/tasks/ext/shared-secret-auth/setup_install.yml +++ b/roles/matrix-synapse/tasks/ext/shared-secret-auth/setup_install.yml @@ -18,6 +18,10 @@ mode: 0440 owner: "{{ matrix_user_username }}" group: "{{ matrix_user_groupname }}" + register: result + retries: "{{ matrix_geturl_retries_count }}" + delay: "{{ matrix_geturl_retries_delay }}" + until: result is not failed - set_fact: matrix_synapse_modules: |