add borg backup (#1727)

* add borg backup

* lint fix

* add exlclude patterns

* missed in the #1726 fix for honoroit

* feedback

* Fix indentation

* feedback

* feedback

* feedback

Co-authored-by: Slavi Pantaleev <slavi@devture.com>

README.md

@@ -121,6 +121,8 @@ Using this playbook, you can get the following services configured on your serve
 
 - (optional) the [Cinny](https://github.com/ajbura/cinny) web client - see [docs/configuring-playbook-client-cinny.md](docs/configuring-playbook-client-cinny.md) for setup documentation
 
+- (optional) the [Borg](https://borgbackup.org) backup - see [docs/configuring-playbook-backup-borg.md](docs/configuring-playbook-backup-borg.md) for setup documentation
+
 Basically, this playbook aims to get you up-and-running with all the necessities around Matrix, without you having to do anything else.
 
 **Note**: the list above is exhaustive. It includes optional or even some advanced components that you will most likely not need.

docs/configuring-playbook-backup-borg.md

@@ -0,0 +1,56 @@
+# Setting up borg backup (optional)
+
+The playbook can install and configure [borgbackup](https://www.borgbackup.org/) with [borgmatic](https://torsion.org/borgmatic/) for you.
+BorgBackup is a deduplicating backup program with optional compression and encryption.
+That means your daily incremental backups can be stored in a fraction of the space and is safe whether you store it at home or on a cloud service.
+
+The backup will run based on `matrix_backup_borg_schedule` var (systemd timer calendar), default: 4am every day
+
+## Prerequisites
+
+1. Create ssh key on any machine:
+
+```bash
+ssh-keygen -t ed25519 -N '' -f matrix-borg-backup -C matrix
+```
+
+2. Add public part of that ssh key to your borg provider / server:
+
+```bash
+# example to append the new PUBKEY contents, where:
+# PUBKEY is path to the public key,
+# USER is a ssh user on a provider / server
+# HOST is a ssh host of a provider / server
+cat PUBKEY | ssh USER@HOST 'dd of=.ssh/authorized_keys oflag=append conv=notrunc'
+```
+
+## Adjusting the playbook configuration
+
+Minimal working configuration (`inventory/host_vars/matrix.DOMAIN/vars.yml`) to enable borg backup:
+
+```yaml
+matrix_backup_borg_enabled: true
+matrix_backup_borg_location_repositories:
+ - USER@HOST:REPO
+matrix_backup_borg_storage_encryption_passphrase: "PASSPHRASE"
+matrix_backup_borg_ssh_key_private: |
+	PRIVATE KEY
+```
+
+where:
+
+* USER - ssh user of a provider / server
+* HOST - ssh host of a provider / server
+* REPO - borg repository name, it will be initialized on backup start, eg: `matrix`
+* PASSPHRASE - super-secret borg passphrase, you may generate it with `pwgen -s 64 1` or use any password manager
+* PRIVATE KEY - the content of the public part of the ssh key you created before
+
+Check the `roles/matrix-backup-borg/defaults/main.yml` for the full list of available options
+
+## Installing
+
+After configuring the playbook, run the [installation](installing.md) command again:
+
+```
+ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start
+```

group_vars/matrix_servers

@@ -1095,6 +1095,27 @@ matrix_bot_mjolnir_systemd_required_services_list: |
 #
 ######################################################################
 
+######################################################################
+#
+# matrix-backup-borg
+#
+######################################################################
+
+matrix_backup_borg_enabled: false
+matrix_backup_borg_location_source_directories:
+  - "{{ matrix_base_data_path }}"
+matrix_backup_borg_location_exclude_patterns: |
+  {{
+    {
+      'synapse': ["{{ matrix_synapse_media_store_path }}/local_thumbnails", "{{ matrix_synapse_media_store_path }}/remote_thumbnail", "{{ matrix_synapse_media_store_path }}/url_cache", "{{ matrix_synapse_media_store_path }}/url_cache_thumbnails"],
+    }[matrix_homeserver_implementation]
+  }}
+
+######################################################################
+#
+# /matrix-backup-borg
+#
+######################################################################
 
 ######################################################################
 #

roles/matrix-backup-borg/defaults/main.yml

@@ -0,0 +1,63 @@
+---
+matrix_backup_borg_enabled: true
+
+matrix_backup_borg_container_image_self_build: false
+matrix_backup_borg_docker_repo: "https://github.com/borgmatic-collective/docker-borgmatic"
+matrix_backup_borg_docker_src_files_path: "{{ matrix_base_data_path }}/borg/docker-src"
+
+matrix_backup_borg_version: latest
+matrix_backup_borg_docker_image: "{{ matrix_backup_borg_docker_image_name_prefix }}etke.cc/borgmatic:{{ matrix_backup_borg_version }}"
+matrix_backup_borg_docker_image_name_prefix: "{{ 'localhost/' if matrix_backup_borg_container_image_self_build else 'registry.gitlab.com/' }}"
+matrix_backup_borg_docker_image_force_pull: "{{ matrix_backup_borg_docker_image.endswith(':latest') }}"
+
+matrix_backup_borg_base_path: "{{ matrix_base_data_path }}/backup-borg"
+matrix_backup_borg_config_path: "{{ matrix_backup_borg_base_path }}/config"
+
+# A list of extra arguments to pass to the container
+matrix_backup_borg_container_extra_arguments: []
+
+# List of systemd services that matrix-backup-borg.service depends on
+matrix_backup_borg_systemd_required_services_list: ['docker.service']
+
+# List of systemd services that matrix-backup-borg.service wants
+matrix_backup_borg_systemd_wanted_services_list: []
+
+# systemd calendar configuration for backup job
+matrix_backup_borg_schedule: "*-*-* 04:00:00"
+
+# what directories should be added to backup
+matrix_backup_borg_location_source_directories: []
+
+# target repositories
+matrix_backup_borg_location_repositories: []
+
+# exclude following paths:
+matrix_backup_borg_location_exclude_patterns: []
+
+# borg encryption mode, only repokey-* is supported
+matrix_backup_borg_encryption: repokey-blake2
+
+# private ssh key used to connect to the borg repo
+matrix_backup_borg_ssh_key_private: ""
+
+# borg ssh command with ssh key
+matrix_backup_borg_storage_ssh_command: ssh -o "StrictHostKeyChecking accept-new" -i /etc/borgmatic.d/sshkey
+
+# compression algorithm
+matrix_backup_borg_storage_compression: lz4
+
+# archive name format
+matrix_backup_borg_storage_archive_name_format: "matrix-{now:%Y-%m-%d-%H%M%S}"
+
+# repository passphrase
+matrix_backup_borg_storage_encryption_passphrase: ""
+
+# retention configuration
+matrix_backup_borg_retention_keep_hourly: 0
+matrix_backup_borg_retention_keep_daily: 7
+matrix_backup_borg_retention_keep_weekly: 4
+matrix_backup_borg_retention_keep_monthly: 12
+matrix_backup_borg_retention_keep_yearly: 2
+
+# retention prefix
+matrix_backup_borg_retention_prefix: "matrix-"

roles/matrix-backup-borg/tasks/init.yml

@@ -0,0 +1,4 @@
+---
+- set_fact:
+    matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-backup-borg.service', 'matrix-backup-borg.timer'] }}"
+  when: matrix_backup_borg_enabled|bool

roles/matrix-backup-borg/tasks/main.yml

@@ -0,0 +1,23 @@
+---
+
+- import_tasks: "{{ role_path }}/tasks/init.yml"
+  tags:
+    - always
+
+- import_tasks: "{{ role_path }}/tasks/validate_config.yml"
+  when: "run_setup|bool and matrix_backup_borg_enabled|bool"
+  tags:
+    - setup-all
+    - setup-backup-borg
+
+- import_tasks: "{{ role_path }}/tasks/setup_install.yml"
+  when: "run_setup|bool and matrix_backup_borg_enabled|bool"
+  tags:
+    - setup-all
+    - setup-backup-borg
+
+- import_tasks: "{{ role_path }}/tasks/setup_uninstall.yml"
+  when: "run_setup|bool and not matrix_backup_borg_enabled|bool"
+  tags:
+    - setup-all
+    - setup-backup-borg

roles/matrix-backup-borg/tasks/setup_install.yml

@@ -0,0 +1,97 @@
+---
+- name: Ensure borg paths exist
+  file:
+    path: "{{ item.path }}"
+    state: directory
+    mode: 0750
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+  with_items:
+    - {path: "{{ matrix_backup_borg_config_path }}", when: true}
+    - {path: "{{ matrix_backup_borg_docker_src_files_path }}", when: true}
+  when: "item.when|bool"
+
+- name: Ensure borg config is created
+  template:
+    src: "{{ role_path }}/templates/config.yaml.j2"
+    dest: "{{ matrix_backup_borg_config_path }}/config.yaml"
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+    mode: 0640
+
+- name: Ensure borg passwd is created
+  template:
+    src: "{{ role_path }}/templates/passwd.j2"
+    dest: "{{ matrix_backup_borg_config_path }}/passwd"
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+    mode: 0640
+
+- name: Ensure borg ssh key is created
+  template:
+    src: "{{ role_path }}/templates/sshkey.j2"
+    dest: "{{ matrix_backup_borg_config_path }}/sshkey"
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+    mode: 0600
+
+- name: Ensure borg image is pulled
+  docker_image:
+    name: "{{ matrix_backup_borg_docker_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_backup_borg_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_backup_borg_docker_image_force_pull }}"
+  when: "not matrix_backup_borg_container_image_self_build|bool"
+  register: result
+  retries: "{{ matrix_container_retries_count }}"
+  delay: "{{ matrix_container_retries_delay }}"
+  until: result is not failed
+
+- name: Ensure borg repository is present on self-build
+  git:
+    repo: "{{ matrix_backup_borg_docker_repo }}"
+    dest: "{{ matrix_backup_borg_docker_src_files_path }}"
+    force: "yes"
+  register: matrix_backup_borg_git_pull_results
+  when: "matrix_backup_borg_container_image_self_build|bool"
+
+- name: Ensure borg image is built
+  docker_image:
+    name: "{{ matrix_backup_borg_docker_image }}"
+    source: build
+    force_source: "{{ matrix_backup_borg_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mailer_git_pull_results.changed }}"
+    build:
+      dockerfile: Dockerfile
+      path: "{{ matrix_backup_borg_docker_src_files_path }}"
+      pull: true
+  when: "matrix_backup_borg_container_image_self_build|bool"
+
+- name: Ensure matrix-backup-borg.service installed
+  template:
+    src: "{{ role_path }}/templates/systemd/matrix-backup-borg.service.j2"
+    dest: "{{ matrix_systemd_path }}/matrix-backup-borg.service"
+    mode: 0644
+  register: matrix_backup_borg_systemd_service_result
+
+- name: Ensure matrix-backup-borg.timer installed
+  template:
+    src: "{{ role_path }}/templates/systemd/matrix-backup-borg.timer.j2"
+    dest: "{{ matrix_systemd_path }}/matrix-backup-borg.timer"
+    mode: 0644
+  register: matrix_backup_borg_systemd_timer_result
+
+- name: Ensure systemd reloaded after matrix-backup-borg.service installation
+  service:
+    daemon_reload: true
+  when: "matrix_backup_borg_systemd_service_result.changed|bool"
+
+- name: Ensure matrix-backup-borg.service enabled
+  service:
+    enabled: true
+    name: matrix-backup-borg.service
+
+- name: Ensure matrix-backup-borg.timer enabled
+  service:
+    enabled: true
+    name: matrix-backup-borg.timer

roles/matrix-backup-borg/tasks/setup_uninstall.yml

@@ -0,0 +1,41 @@
+---
+- name: Check existence of matrix-backup-borg service
+  stat:
+    path: "{{ matrix_systemd_path }}/matrix-backup-borg.service"
+  register: matrix_backup_borg_service_stat
+
+- name: Ensure matrix-backup-borg is stopped
+  service:
+    name: matrix-backup-borg
+    state: stopped
+    enabled: false
+    daemon_reload: true
+  register: stopping_result
+  when: "matrix_backup_borg_service_stat.stat.exists|bool"
+
+- name: Ensure matrix-backup-borg.service doesn't exist
+  file:
+    path: "{{ matrix_systemd_path }}/matrix-backup-borg.service"
+    state: absent
+  when: "matrix_backup_borg_service_stat.stat.exists|bool"
+
+- name: Ensure matrix-backup-borg.timer doesn't exist
+  file:
+    path: "{{ matrix_systemd_path }}/matrix-backup-borg.timer"
+    state: absent
+  when: "matrix_backup_borg_service_stat.stat.exists|bool"
+
+- name: Ensure systemd reloaded after matrix-backup-borg.service removal
+  service:
+    daemon_reload: true
+  when: "matrix_backup_borg_service_stat.stat.exists|bool"
+
+- name: Ensure Matrix borg paths don't exist
+  file:
+    path: "{{ matrix_backup_borg_base_path }}"
+    state: absent
+
+- name: Ensure borg Docker image doesn't exist
+  docker_image:
+    name: "{{ matrix_backup_borg_docker_image }}"
+    state: absent

roles/matrix-backup-borg/tasks/validate_config.yml

@@ -0,0 +1,10 @@
+---
+- name: Fail if required settings not defined
+  fail:
+    msg: >-
+      You need to define a required configuration setting (`{{ item }}`).
+  when: "vars[item] == ''"
+  with_items:
+    - "matrix_backup_borg_ssh_key_private"
+    - "matrix_backup_borg_location_repositories"
+    - "matrix_backup_borg_storage_encryption_passphrase"

roles/matrix-backup-borg/templates/config.yaml.j2

@@ -0,0 +1,32 @@
+#jinja2: lstrip_blocks: "True", trim_blocks: "True"
+
+location:
+  source_directories: {{ matrix_backup_borg_location_source_directories|to_json }}
+  repositories: {{ matrix_backup_borg_location_repositories|to_json }}
+  one_file_system: true
+  exclude_patterns: {{ matrix_backup_borg_location_exclude_patterns|to_json }}
+
+storage:
+  compression: {{ matrix_backup_borg_storage_compression }}
+  ssh_command: {{ matrix_backup_borg_storage_ssh_command }}
+  archive_name_format: '{{ matrix_backup_borg_storage_archive_name_format }}'
+  encryption_passphrase: {{ matrix_backup_borg_storage_encryption_passphrase }}
+
+retention:
+  keep_hourly: {{ matrix_backup_borg_retention_keep_hourly }}
+  keep_daily: {{ matrix_backup_borg_retention_keep_daily }}
+  keep_weekly: {{ matrix_backup_borg_retention_keep_weekly }}
+  keep_monthly: {{ matrix_backup_borg_retention_keep_monthly }}
+  keep_yearly: {{ matrix_backup_borg_retention_keep_yearly }}
+  prefix: '{{ matrix_backup_borg_retention_prefix }}'
+
+consistency:
+  checks:
+    - repository
+    - archives
+
+hooks:
+  after_backup:
+    - echo "Backup created."
+  on_error:
+    - echo "Error while creating a backup."

roles/matrix-backup-borg/templates/passwd.j2

@@ -0,0 +1,29 @@
+{# the passwd file with correct username, UID and GID is mandatory to work with borg over ssh, otherwise ssh connections will fail #}
+root:x:0:0:root:/root:/bin/ash
+bin:x:1:1:bin:/bin:/sbin/nologin
+daemon:x:2:2:daemon:/sbin:/sbin/nologin
+adm:x:3:4:adm:/var/adm:/sbin/nologin
+lp:x:4:7:lp:/var/spool/lpd:/sbin/nologin
+sync:x:5:0:sync:/sbin:/bin/sync
+shutdown:x:6:0:shutdown:/sbin:/sbin/shutdown
+halt:x:7:0:halt:/sbin:/sbin/halt
+mail:x:8:12:mail:/var/mail:/sbin/nologin
+news:x:9:13:news:/usr/lib/news:/sbin/nologin
+uucp:x:10:14:uucp:/var/spool/uucppublic:/sbin/nologin
+operator:x:11:0:operator:/root:/sbin/nologin
+man:x:13:15:man:/usr/man:/sbin/nologin
+postmaster:x:14:12:postmaster:/var/mail:/sbin/nologin
+cron:x:16:16:cron:/var/spool/cron:/sbin/nologin
+ftp:x:21:21::/var/lib/ftp:/sbin/nologin
+sshd:x:22:22:sshd:/dev/null:/sbin/nologin
+at:x:25:25:at:/var/spool/cron/atjobs:/sbin/nologin
+squid:x:31:31:Squid:/var/cache/squid:/sbin/nologin
+xfs:x:33:33:X Font Server:/etc/X11/fs:/sbin/nologin
+games:x:35:35:games:/usr/games:/sbin/nologin
+cyrus:x:85:12::/usr/cyrus:/sbin/nologin
+vpopmail:x:89:89::/var/vpopmail:/sbin/nologin
+ntp:x:123:123:NTP:/var/empty:/sbin/nologin
+smmsp:x:209:209:smmsp:/var/spool/mqueue:/sbin/nologin
+guest:x:405:100:guest:/dev/null:/sbin/nologin
+{{ matrix_user_username }}:x:{{ matrix_user_uid }}:{{ matrix_user_gid }}:Matrix:/tmp:/bin/ash
+nobody:x:65534:65534:nobody:/:/sbin/nologin

roles/matrix-backup-borg/templates/sshkey.j2

@@ -0,0 +1 @@
+{{ matrix_backup_borg_ssh_key_private }}

roles/matrix-backup-borg/templates/systemd/matrix-backup-borg.service.j2

@@ -0,0 +1,58 @@
+#jinja2: lstrip_blocks: "True"
+[Unit]
+Description=Matrix Borg Backup
+{% for service in matrix_backup_borg_systemd_required_services_list %}
+Requires={{ service }}
+After={{ service }}
+{% endfor %}
+{% for service in matrix_backup_borg_systemd_wanted_services_list %}
+Wants={{ service }}
+{% endfor %}
+DefaultDependencies=no
+
+[Service]
+Type=oneshot
+Environment="HOME={{ matrix_systemd_unit_home_path }}"
+ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-backup-borg 2>/dev/null'
+ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-backup-borg 2>/dev/null'
+ExecStartPre=-{{ matrix_host_command_docker }} run --rm --name matrix-backup-borg \
+			--log-driver=none \
+			--cap-drop=ALL \
+			--read-only \
+			--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
+			--network={{ matrix_docker_network }} \
+			--tmpfs=/tmp:rw,noexec,nosuid,size=100m \
+			--mount type=bind,src={{ matrix_backup_borg_config_path }}/passwd,dst=/etc/passwd,ro \
+			--mount type=bind,src={{ matrix_backup_borg_config_path }},dst=/etc/borgmatic.d,ro \
+			{% for source in matrix_backup_borg_location_source_directories %}
+			--mount type=bind,src={{ source }},dst={{ source }},ro \
+			{% endfor %}
+			{% for arg in matrix_backup_borg_container_extra_arguments %}
+			{{ arg }} \
+			{% endfor %}
+			{{ matrix_backup_borg_docker_image }} \
+			sh -c "borgmatic --init --encryption {{ matrix_backup_borg_encryption }}"
+
+ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-backup-borg \
+			--log-driver=none \
+			--cap-drop=ALL \
+			--read-only \
+			--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
+			--network={{ matrix_docker_network }} \
+			--tmpfs=/tmp:rw,noexec,nosuid,size=100m \
+			--mount type=bind,src={{ matrix_backup_borg_config_path }}/passwd,dst=/etc/passwd,ro \
+			--mount type=bind,src={{ matrix_backup_borg_config_path }},dst=/etc/borgmatic.d,ro \
+			{% for source in matrix_backup_borg_location_source_directories %}
+			--mount type=bind,src={{ source }},dst={{ source }},ro \
+			{% endfor %}
+			{% for arg in matrix_backup_borg_container_extra_arguments %}
+			{{ arg }} \
+			{% endfor %}
+			{{ matrix_backup_borg_docker_image }}
+
+ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-backup-borg 2>/dev/null'
+ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-backup-borg 2>/dev/null'
+SyslogIdentifier=matrix-backup-borg
+
+[Install]
+WantedBy=multi-user.target

roles/matrix-backup-borg/templates/systemd/matrix-backup-borg.timer.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=Matrix Borg Backup timer
+
+[Timer]
+Unit=matrix-backup-borg.service
+OnCalendar={{ matrix_backup_borg_schedule }}
+RandomizedDelaySec=2h
+
+[Install]
+WantedBy=timers.target

roles/matrix-bot-honoroit/tasks/setup_install.yml

@@ -43,6 +43,8 @@
   template:
     src: "{{ role_path }}/templates/env.j2"
     dest: "{{ matrix_bot_honoroit_config_path }}/env"
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
     mode: 0640
 
 - name: Ensure honoroit image is pulled

setup.yml

@@ -14,6 +14,7 @@
     - matrix-postgres
     - matrix-redis
     - matrix-corporal
+    - matrix-backup-borg
     - matrix-bridge-appservice-discord
     - matrix-bridge-appservice-slack
     - matrix-bridge-appservice-webhooks