Add support for using an external PostgreSQL server

README.md

@@ -12,7 +12,7 @@ Using this playbook, you can get the following services configured on your serve
 
 - (optional) [Amazon S3](https://aws.amazon.com/s3/) storage for your Matrix Synapse's content repository (`media_store`) files using [s3fs-fuse](https://github.com/s3fs-fuse/s3fs-fuse)
 
-- a [PostgreSQL](https://www.postgresql.org/) database for Matrix Synapse - providing better performance than the default [SQLite](https://sqlite.org/) database
+- (optional default) [PostgreSQL](https://www.postgresql.org/) database for Matrix Synapse - providing better performance than the default [SQLite](https://sqlite.org/) database. Using an external PostgreSQL server [is possible](#using-an-external-postgresql-server-optional) as well
 
 - a [STUN/TURN server](https://github.com/coturn/coturn) for WebRTC audio/video calls
 
@@ -35,7 +35,9 @@ This is similar to the [EMnify/matrix-synapse-auto-deploy](https://github.com/EM
 
 - this one retrieves and automatically renews free [Let's Encrypt](https://letsencrypt.org/) **SSL certificates** for you
 
-- this one optionally can store the `media_store` content repository files on [Amazon S3](https://aws.amazon.com/s3/)
+- this one optionally can store the `media_store` content repository files on [Amazon S3](https://aws.amazon.com/s3/) (but defaults to storing files on the server's filesystem)
+
+- this one optionally allows you to use an external PostgreSQL server for Matrix Synapse's database (but defaults to running one in a container)
 
 Special thanks goes to:
 
@@ -97,6 +99,9 @@ You can follow these steps:
 
 ## Amazon S3 configuration (optional)
 
+By default, this playbook configures your server to store Matrix Synapse's content repository (`media_store`) files on the local filesystem.
+If that's alright, you can skip ahead.
+
 If you'd like to store Matrix Synapse's content repository (`media_store`) files on Amazon S3,
 you can let this playbook configure [s3fs-fuse](https://github.com/s3fs-fuse/s3fs-fuse) for you.
 
@@ -131,6 +136,26 @@ matrix_s3_media_store_aws_secret_key: "secret-key-goes-here"
 ```
 
 
+## Using an external PostgreSQL server (optional)
+
+By default, this playbook would set up a PostgreSQL database server on your machine, running in a Docker container.
+If that's alright, you can skip ahead.
+
+If you'd like to use an external PostgreSQL server that you manage, you can edit your configuration file  (`inventory/matrix.<your-domain>/vars.yml`).
+It should be something like this:
+
+```
+matrix_postgres_use_external: true
+matrix_postgres_connection_hostname: "your-postgres-server-hostname"
+matrix_postgres_connection_username: "your-postgres-server-username"
+matrix_postgres_connection_password: "your-postgres-server-password"
+matrix_postgres_db_name: "your-postgres-server-database-name"
+```
+
+The database (as specified in `matrix_postgres_db_name`) must exist and be accessible with the given credentials.
+It must be empty or contain a valid Matrix Synapse database. If empty, Matrix Synapse would populate it the first time it runs.
+
+
 ## Installing
 
 Once you have your server and you have [configured your DNS records](#configuring-dns), you can proceed with installing.

roles/matrix-server/defaults/main.yml

@@ -18,6 +18,10 @@ matrix_user_username: "matrix"
 matrix_user_uid: 991
 matrix_user_gid: 991
 
+# The defaults below cause a postgres server to be configured (running within a container).
+# Using an external server is possible by tweaking all of the parameters below.
+matrix_postgres_use_external: false
+matrix_postgres_connection_hostname: "postgres"
 matrix_postgres_connection_username: "synapse"
 matrix_postgres_connection_password: "synapse-password"
 matrix_postgres_db_name: "homeserver"
@@ -57,4 +61,4 @@ matrix_max_log_files_count: 10
 matrix_s3_media_store_enabled: false
 matrix_s3_media_store_bucket_name: "your-bucket-name"
 matrix_s3_media_store_aws_access_key: "your-aws-access-key"
-matrix_s3_media_store_aws_secret_key: "your-aws-secret-key"
+matrix_s3_media_store_aws_secret_key: "your-aws-secret-key"

roles/matrix-server/tasks/import_sqlite_db.yml

@@ -72,7 +72,7 @@
       - "{{ matrix_scratchpad_dir }}:/scratchpad"
       - "{{ matrix_scratchpad_dir }}/synapse_port_db_with_patch:/usr/local/bin/synapse_port_db_with_patch"
     links:
-      - "matrix-postgres:postgres"
+      - "matrix-postgres:{{ matrix_postgres_connection_hostname }}"
 
 - name: Ensure scratchpad directory is deleted
   file:

roles/matrix-server/tasks/setup_postgres.yml

@@ -1,13 +1,10 @@
 ---
 
-- name: Ensure postgres data path exists
-  file:
-    path: "{{ matrix_postgres_data_path }}"
-    state: directory
-    mode: 0700
-    owner: "{{ matrix_user_username }}"
-    group: "{{ matrix_user_username }}"
+#
+# Generic tasks, no matter what kind of server we're using (internal/external)
+#
 
+# Even if we don't run the internal server, we still need this for running the CLI
 - name: Ensure postgres Docker image is pulled
   docker_image:
     name: "{{ docker_postgres_image }}"
@@ -27,8 +24,52 @@
     dest: "/usr/local/bin/matrix-postgres-cli"
     mode: 0750
 
+#
+# Tasks related to setting up an internal postgres server
+#
+
+- name: Ensure postgres data path exists
+  file:
+    path: "{{ matrix_postgres_data_path }}"
+    state: directory
+    mode: 0700
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_username }}"
+  when: "not matrix_postgres_use_external"
+
 - name: Ensure matrix-postgres.service installed
   template:
     src: "{{ role_path }}/templates/systemd/matrix-postgres.service.j2"
     dest: "/etc/systemd/system/matrix-postgres.service"
-    mode: 0644
+    mode: 0644
+  when: "not matrix_postgres_use_external"
+
+#
+# Tasks related to getting rid of the internal postgres server (if it was previously enabled)
+#
+
+- name: Check existance of matrix-postgres service
+  stat: path="/etc/systemd/system/matrix-postgres.service"
+  register: matrix_postgres_service_stat
+  when: matrix_postgres_use_external
+
+- name: Ensure matrix-postgres is stopped
+  service: name=matrix-postgres state=stopped daemon_reload=yes
+  when: "matrix_postgres_use_external and matrix_postgres_service_stat.stat.exists"
+
+- name: Ensure matrix-postgres.service doesn't exist
+  file:
+    path: "/etc/systemd/system/matrix-postgres.service"
+    state: absent
+  when: "matrix_postgres_use_external and matrix_postgres_service_stat.stat.exists"
+
+- name: Check existance of matrix-postgres local data path
+  stat: path="{{ matrix_postgres_data_path }}"
+  register: matrix_postgres_data_path_stat
+  when: matrix_postgres_use_external
+
+# We just want to notify the user. Deleting data is too destructive.
+- name: Notify if matrix-postgres local data remains
+  debug:
+    msg: "Note: You are not using a local PostgreSQL database, but some old data remains from before in {{ matrix_postgres_data_path }}. Feel free to delete that."
+  when: "matrix_postgres_use_external and matrix_postgres_data_path_stat.stat.exists"

roles/matrix-server/tasks/setup_synapse.yml

@@ -109,12 +109,11 @@
     line: '\1name: "psycopg2"'
     backrefs: yes
 
-- name: Augment Matrix config (add the Postgres connection parameters)
-  lineinfile:
+- name: Augment Matrix config (set the Postgres connection parameters)
+  replace:
     dest: "{{ matrix_synapse_config_dir_path }}/homeserver.yaml"
-    regexp: '(.*)database: "(.*)homeserver.db"'
-    line: '\1user: "{{ matrix_postgres_connection_username }}"\n\1password: "{{ matrix_postgres_connection_password }}"\n\1database: "homeserver"\n\1host: "postgres"\n\1cp_min: 5\n\1cp_max: 10'
-    backrefs: yes
+    regexp: '(.*)name: "psycopg2"((?:.|\n)*?)\n\n'
+    replace: '\1name: "psycopg2"\n\1args:\n\1\1user: "{{ matrix_postgres_connection_username }}"\n\1\1password: "{{ matrix_postgres_connection_password }}"\n\1\1database: "{{ matrix_postgres_db_name }}"\n\1\1host: "{{ matrix_postgres_connection_hostname }}"\n\1\1cp_min: 5\n\1\1cp_max: 10\n\n'
 
 - name: Augment Matrix config (configure Coturn)
   lineinfile: "dest={{ matrix_synapse_config_dir_path }}/turnserver.conf"

roles/matrix-server/templates/systemd/matrix-synapse.service.j2

@@ -2,8 +2,10 @@
 Description=Matrix Synapse server
 After=docker.service
 Requires=docker.service
+{% if not matrix_postgres_use_external %}
 Requires=matrix-postgres.service
 After=matrix-postgres.service
+{% endif %}
 {% if matrix_s3_media_store_enabled %}
 After=matrix-s3fs.service
 Requires=matrix-s3fs.service
@@ -15,7 +17,9 @@ ExecStartPre=-/usr/bin/docker kill matrix-synapse
 ExecStartPre=-/usr/bin/docker rm matrix-synapse
 ExecStartPre=-/usr/bin/chown {{ matrix_user_username }}:{{ matrix_user_username }} {{ ssl_certs_path }} -R
 ExecStart=/usr/bin/docker run --rm --name matrix-synapse \
-			--link matrix-postgres:postgres \
+			{% if not matrix_postgres_use_external %}
+			--link matrix-postgres:{{ matrix_postgres_connection_hostname }} \
+			{% endif %}
 			-p 8448:8448 \
 			-p 3478:3478 \
 			-p 3478:3478/udp \

roles/matrix-server/templates/usr-local-bin/matrix-postgres-cli.j2

@@ -4,6 +4,8 @@ docker run \
 	-it \
 	--rm \
 	--env-file={{ matrix_environment_variables_data_path }}/env-postgres-pgsql-docker \
-	--link=matrix-postgres:postgres \
+	{% if not matrix_postgres_use_external %}
+	--link=matrix-postgres:{{ matrix_postgres_connection_hostname }} \
+	{% endif %}
 	{{ docker_postgres_image }} \
-	psql -h postgres
+	psql -h {{ matrix_postgres_connection_hostname }}