diff --git a/docs/configuring-playbook-backup.md b/docs/configuring-playbook-backup.md new file mode 100644 index 000000000..f9106eafd --- /dev/null +++ b/docs/configuring-playbook-backup.md @@ -0,0 +1,141 @@ +# Setting up Matrix Synapse backups (optional) + +This playbook installs a weekly cron backup. + +## Variable Table + +| Variables | Default | Example | +|-----------|---------|---------| +| matrix_backup_enabled | false | True | +| matrix_backup_bucket | "" | "s3//bucketname/prefix/" | +| matrix_backup_bucket_endpoint | "" | "https://nyc3.digitaloceanspaces.com" | +| matrix_backup_bucket_awscli_docker_image_latest | "amazon/aws-cli:2.0.10" | "amazon/aws-cli:latest" | +| matrix_backup_bucket_key_id | "" | "AKIAQIOAVK3Q4HMXL272" | +| matrix_backup_bucket_key_secret | "" | "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" | +| matrix_backup_rsync_target | "" | ?? | +| matrix_backup_cron_day | "*/7" (Weekly) | "*/2" Biweekly | + + +## Method 1: Rsync + +?? + +## Method 2: S3 Compatible object store + +Setup: S3 compatible buckets + +### S3 compatible services https://en.wikipedia.org/wiki/Amazon_S3#S3_API_and_competing_services + +| Service Provider | Costs | Compatibility | Endpoint | +|------------------|-------|---------------|----------| +| AWS S3 | https://aws.amazon.com/s3/pricing/ | N/A | N/A | +| Digital Ocean Spaces | https://www.digitalocean.com/pricing/#Storage | https://developers.digitalocean.com/documentation/spaces/ | `https://.digitaloceanspaces.com` | +| Azure Blob | https://azure.microsoft.com/en-us/pricing/details/storage/blobs/ | https://cloudblogs.microsoft.com/opensource/2017/11/09/s3cmd-amazon-s3-compatible-apps-azure-storage/ | Requires minio | +| Blackblaze B2 | https://www.backblaze.com/b2/cloud-storage-pricing.html | https://www.backblaze.com/b2/docs/s3_compatible_api.html | `https://s3..backblazeb2.com/` | +| Google Cloud Storage | https://cloud.google.com/storage/pricing | https://cloud.google.com/storage/docs/interoperability | `https://storage.googleapis.com` | +| Wasbi | https://wasabi.com/s3-compatible-cloud-storage/ | https://wasabi-support.zendesk.com/hc/en-us/articles/115001910791-How-do-I-use-AWS-CLI-with-Wasabi- | `https://s3.wasabisys.com` | +| IBM Cloud Object Storage | https://cloud.ibm.com/catalog/services/cloud-object-storage | https://cloud.ibm.com/docs/cloud-object-storage?topic=cloud-object-storage-aws-cli | `s3..cloud-object-storage.appdomain.cloud` | +| Linode Object Storage | https://www.linode.com/pricing/#row--storage | https://www.linode.com/docs/platform/object-storage/bucket-versioning/ | `http://.linodeobjects.com` | +| Dream Hosts | https://www.dreamhost.com/cloud/storage/ | https://help.dreamhost.com/hc/en-us/articles/360022654971-AWS-CLI-commands-to-manage-your-DreamObjects-data | https://objects-us-east-1.dream.io | + +### Preparation + +Select a S3 compatible provider. +Create S3 Bucket +Create a specialized IAM users with the permissions recorded below. For users who deployed their postgres instance on an AWS EC2, you can create attachable IAM roles instead for password less S3 access. + + + + +Backup-acl.json +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor1", + "Effect": "Allow", + "Action": "s3:ListBucket", + "Resource": "arn:aws:s3:::", + "Condition": { + "ForAnyValue:IpAddress": { + "aws:SourceIp": [ + "" + ] + } + } + }, + { + "Sid": "VisualEditor3", + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:AbortMultipartUpload" + ], + "Resource": [ + "arn:aws:s3:::/matrix/*", + "arn:aws:s3:::/matrix" + ], + "Condition": { + "IpAddress": { + "aws:SourceIp": "" + } + } + } + ] +} +``` + +Restore-acl.json +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor1", + "Effect": "Allow", + "Action": "s3:ListBucket", + "Resource": "arn:aws:s3:::", + "Condition": { + "ForAnyValue:IpAddress": { + "aws:SourceIp": [ + "" + ] + } + } + }, + { + "Sid": "VisualEditor3", + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:AbortMultipartUpload" + ], + "Resource": [ + "arn:aws:s3:::/matrix/*", + "arn:aws:s3:::/matrix" + ], + "Condition": { + "IpAddress": { + "aws:SourceIp": "" + } + } + } + ] +} +``` + +### Deploy Matrix S3 Backup +#### Using AWS IAM Role +Set `matrix_backup_enabled` and `matrix_backup_bucket`. +#### Using AWS IAM User +Set `matrix_backup_enabled`, `matrix_backup_bucket`, `matrix_backup_bucket_key_id`, and `matrix_backup_bucket_key_secret` + +#### S3 Compatible Services +Set `matrix_backup_enabled`, `matrix_backup_bucket`, `matrix_backup_bucket_key_id`, `matrix_backup_bucket_key_secret`, and `matrix_backup_bucket_endpoint` + +#### Run +```bash +ansible-playbook -i inventory/hosts setup.yml --tags=setup-matrix-backup,start +``` \ No newline at end of file diff --git a/roles/matrix-backup/defaults/main.yml b/roles/matrix-backup/defaults/main.yml new file mode 100644 index 000000000..5b135d9c0 --- /dev/null +++ b/roles/matrix-backup/defaults/main.yml @@ -0,0 +1,33 @@ + + +matrix_backup_enabled: false + +# Configure daily values within the cron tab +matrix_backup_cron_day: "*/7" + +matrix_backup_base_path: "{{ matrix_base_data_path }}/matrix-backup" + +# Enable Remote AWS s3 backups +# matrix_backup_bucket value should include the prefix +# Example value +# matrix_backup_bucket: s3://example-bucket/matrix-prefix/ +# s3://bucketname/prefix/ +matrix_backup_bucket: "" + +# Set this variable for any s3 compatible service +# "https://nyc3.digitaloceanspaces.com" +matrix_backup_bucket_endpoint: "" +# AWS Access credentials +matrix_backup_bucket_key_id: "" +matrix_backup_bucket_key_secret: "" + +# Official utility from AWS +# https://hub.docker.com/r/amazon/aws-cli + +# This variable is assigned at runtime. Overriding its value has no effect. +matrix_backup_awscli_docker_image_v2: "amazon/aws-cli:2.0.16" +matrix_backup_awscli_docker_image_latest: "{{ matrix_backup_awscli_docker_image_v2 }}" +matrix_backup_awscli_docker_image: '{{ matrix_backup_awscli_docker_image_latest }}' + +# Use Rsync instead +matrix_backup_rsync_target: "" \ No newline at end of file diff --git a/roles/matrix-backup/tasks/main.yml b/roles/matrix-backup/tasks/main.yml new file mode 100644 index 000000000..5c34b6f6a --- /dev/null +++ b/roles/matrix-backup/tasks/main.yml @@ -0,0 +1,5 @@ +- import_tasks: "{{ role_path }}/tasks/setup_matrix_backup.yml" + when: run_setup|bool + tags: + - setup-all + - setup-matrix-backup \ No newline at end of file diff --git a/roles/matrix-backup/tasks/setup_matrix_backup.yml b/roles/matrix-backup/tasks/setup_matrix_backup.yml new file mode 100644 index 000000000..2d453658a --- /dev/null +++ b/roles/matrix-backup/tasks/setup_matrix_backup.yml @@ -0,0 +1,105 @@ +- name: Ensure matrix-backup.service is installed + template: + src: "{{ role_path }}/templates/systemd/{{ item }}.j2" + dest: "{{ matrix_systemd_path }}/{{ item }}" + mode: 0644 + with_items: + - "matrix-backup.service" + register: matrix_backup_service_result + when: matrix_postgres_enabled|bool and matrix_backup_enabled|bool + +- name: Ensure systemd reloaded after matrix-backup.service installation + service: + daemon_reload: yes + when: "matrix_postgres_enabled|bool and matrix_backup_service_result.changed" + +- name: Ensure matrix backup paths exist + file: + path: "{{ item }}" + state: directory + mode: 0700 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + with_items: + - "{{ matrix_backup_base_path }}" + when: matrix_postgres_enabled|bool and matrix_backup_enabled|bool + +- name: Ensure matrix backup environment variables file created + template: + src: "{{ role_path }}/templates/{{ item }}.j2" + dest: "{{ matrix_backup_base_path }}/{{ item }}" + mode: 0640 + with_items: + - "env-backup" + when: matrix_postgres_enabled|bool and matrix_backup_enabled|bool + +- name: Creates a matrix synapse backup cron file under /etc/cron.d + cron: + name: Matrix Backup Service + state: present + day: "{{ matrix_backup_cron_day }}" + minute: "0" + hour: "2" + user: root + job: "systemctl start matrix-backup.service" + cron_file: matrix-backup + +# +# Tasks related to removing matrix-backup (if it was previously enabled) +# + +- name: Check existence of matrix-backup service + stat: + path: "{{ matrix_systemd_path }}/matrix-backup.service" + register: matrix_backup_service_stat + when: "not matrix_postgres_enabled|bool or not matrix_backup_enabled|bool" + +- name: Ensure matrix-backup is stopped + service: + name: matrix-backup + state: stopped + daemon_reload: yes + when: + - (not matrix_postgres_enabled|bool or not matrix_backup_enabled|bool) + - matrix_backup_service_stat.stat.exists|bool + +- name: Ensure matrix-backup.service doesn't exist + file: + path: "{{ matrix_systemd_path }}/{{ item }}" + state: absent + with_items: + - "matrix-backup.service" + when: + - (not matrix_postgres_enabled|bool or not matrix_backup_enabled|bool) + - matrix_backup_service_stat.stat.exists|bool + +- name: Ensure matrix-backup cronjob removed + cron: + name: Matrix Backup Service + user: root + cron_file: matrix-backup + state: absent + when: + - (not matrix_postgres_enabled|bool or not matrix_backup_enabled|bool) + - matrix_backup_service_stat.stat.exists|bool + +- name: Ensure matrix backup environment file doesn't exist + file: + path: "{{ matrix_backup_base_path }}" + state: absent + when: + - (not matrix_postgres_enabled|bool or not matrix_backup_enabled|bool) + - matrix_backup_service_stat.stat.exists|bool + +- name: Ensure systemd reloaded after matrix-backup timer + service: + daemon_reload: yes + when: + - (not matrix_postgres_enabled|bool or not matrix_backup_enabled|bool) + - matrix_backup_service_stat.stat.exists|bool + +- name: Ensure awscli Docker image doesn't exist + docker_image: + name: "{{ matrix_backup_awscli_docker_image }}" + state: absent + when: "not matrix_postgres_enabled|bool or not matrix_backup_enabled|bool" \ No newline at end of file diff --git a/roles/matrix-backup/templates/env-backup.j2 b/roles/matrix-backup/templates/env-backup.j2 new file mode 100644 index 000000000..3bd394aea --- /dev/null +++ b/roles/matrix-backup/templates/env-backup.j2 @@ -0,0 +1,5 @@ +#jinja2: lstrip_blocks: "True" +{% if matrix_backup_bucket_key_id %} +AWS_ACCESS_KEY_ID={{ matrix_backup_bucket_key_id }} +AWS_SECRET_ACCESS_KEY={{ matrix_backup_bucket_key_secret }} +{% endif %} \ No newline at end of file diff --git a/roles/matrix-backup/templates/systemd/matrix-backup.service.j2 b/roles/matrix-backup/templates/systemd/matrix-backup.service.j2 new file mode 100644 index 000000000..af0042922 --- /dev/null +++ b/roles/matrix-backup/templates/systemd/matrix-backup.service.j2 @@ -0,0 +1,31 @@ +#jinja2: lstrip_blocks: "True" + +[Unit] +Description=Backup service for Matrix Postgres Server + +[Service] +{% if matrix_backup_bucket %} +Environment=AWS_BUCKET={{ matrix_backup_bucket }} +Environment=TMP_OUTPUT=/tmp/postgres-backup.sql.gz +{% endif %} +Type=oneshot +ExecStartPre=/bin/sh -c '/usr/bin/docker run --rm \ + --network={{ matrix_docker_network }} \ + --env-file={{ matrix_postgres_base_path }}/env-postgres-psql \ + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ + {{ matrix_postgres_docker_image_to_use }} \ + pg_dumpall -h matrix-postgres | gzip -c > ${TMP_OUTPUT}' +{% if matrix_backup_bucket %} +# Mounting files host files into amazon/aws-cli +# The /aws working directory is user controlled. +# The image will not write to this directory, unless instructed by the user in running a AWS CLI command. +ExecStart=/bin/sh -c 'docker run --rm --name matrix-backup \ + --env-file={{ matrix_backup_base_path }}/env-backup \ + -v ${TMP_OUTPUT}:/aws/postgres.sql.gz \ + {{ matrix_backup_awscli_docker_image }} \ + s3 cp /aws/postgres.sql.gz ${AWS_BUCKET}$$(date +%%m-%%d-%%Y)/ \ + {% if matrix_backup_bucket_endpoint %} + --endpoint-url {{ matrix_backup_bucket_endpoint }} \ + {% endif %} && rm ${TMP_OUTPUT}' +{% endif %} +Group=systemd-journal \ No newline at end of file diff --git a/setup.yml b/setup.yml index ff80320ab..5b2e5a3d4 100755 --- a/setup.yml +++ b/setup.yml @@ -7,6 +7,7 @@ - matrix-base - matrix-mailer - matrix-postgres + - matrix-backup - matrix-corporal - matrix-bridge-appservice-discord - matrix-bridge-appservice-slack