Switch from acmetool to certbot for SSL certificate retrieval

CHANGELOG.md

@@ -1,3 +1,21 @@
+# 2018-08-29
+
+## Changing the way SSL certificates are retrieved
+
+We've been using [acmetool](https://github.com/hlandau/acme) (with the [willwill/acme-docker](https://hub.docker.com/r/willwill/acme-docker/) Docker image) until now.
+
+Due to the Docker image being deprecated, and for things looking bleak for acmetool's support of the newer ACME v2 API endpoint, we've switched to using [certbot](https://certbot.eff.org/) (with the [certbot/certbot](https://hub.docker.com/r/certbot/certbot/) Docker image).
+
+Simply re-running the playbook will retrieve new certificates for you.
+To ensure you don't leave any old files behind, though, you'd better do this:
+
+- `systemctl stop matrix*`
+- stop your custom webserver, if you're running one (only affects you if you've installed with `matrix_nginx_proxy_enabled: false`)
+- `mv /matrix/ssl /matrix/ssl-acmetool-delete-later`
+- re-run the playbook's [installation](docs/installing.md)
+- possibly delete `/matrix/ssl-acmetool-delete-later`
+
+
 # 2018-08-21
 
 ## Matrix Corporal support

roles/matrix-server/defaults/main.yml

@@ -24,8 +24,7 @@ matrix_postgres_connection_password: "synapse-password"
 matrix_postgres_db_name: "homeserver"
 
 matrix_base_data_path: "/matrix"
-matrix_ssl_certs_path: "{{ matrix_base_data_path }}/ssl"
-matrix_ssl_support_email: "{{ host_specific_matrix_ssl_support_email }}"
+
 matrix_environment_variables_data_path: "{{ matrix_base_data_path }}/environment-variables"
 
 matrix_synapse_base_path: "{{ matrix_base_data_path }}/synapse"
@@ -217,9 +216,18 @@ matrix_nginx_proxy_matrix_client_api_addr_with_proxy_container: "matrix-synapse:
 matrix_nginx_proxy_matrix_client_api_addr_sans_proxy_container: "localhost:8008"
 
 
+matrix_ssl_base_path: "{{ matrix_base_data_path }}/ssl"
+matrix_ssl_config_dir_path: "{{ matrix_ssl_base_path }}/config"
+matrix_ssl_log_dir_path: "{{ matrix_ssl_base_path }}/log"
+matrix_ssl_support_email: "{{ host_specific_matrix_ssl_support_email }}"
+matrix_ssl_certbot_docker_image: "certbot/certbot:v0.26.1"
+matrix_ssl_certbot_standalone_http_port: 2402
+matrix_ssl_use_staging: false
+
 # Specifies when to attempt to retrieve new SSL certificates from Let's Encrypt.
 matrix_ssl_renew_cron_time_definition: "15 4 */5 * *"
 
+
 # Specifies when to reload the matrix-nginx-proxy service so that
 # a new SSL certificate could go into effect.
 matrix_nginx_proxy_reload_cron_time_definition: "20 4 */5 * *"

roles/matrix-server/tasks/setup_ssl.yml

@@ -20,46 +20,32 @@
     - https
   when: ansible_os_family == 'RedHat'
 
-- name: Ensure acmetool Docker image is pulled
+- name: Ensure certbot Docker image is pulled
   docker_image:
-    name: willwill/acme-docker
+    name: "{{ matrix_ssl_certbot_docker_image }}"
 
-# Granting +rx to others as well, because the `nginx` user from within
-# matrix-nginx-proxy needs to be able to read the acme-challenge files inside
-# for renewal purposes.
-#
-# This should not be causing security trouble outside of the container,
-# as the parent directory (/matrix) does not allow "others" to access it or any of its children.
-# Still, it works when the /ssl subtree is mounted in the container.
-- name: Ensure SSL certificates path exists
+- name: Ensure SSL certificate paths exists
   file:
-    path: "{{ matrix_ssl_certs_path }}"
+    path: "{{ item }}"
     state: directory
-    mode: 0775
+    mode: 0770
     owner: "{{ matrix_user_username }}"
     group: "{{ matrix_user_username }}"
+  with_items:
+    - "{{ matrix_ssl_log_dir_path }}"
+    - "{{ matrix_ssl_config_dir_path }}"
 
-- name: Check matrix-nginx-proxy state
-  service: name=matrix-nginx-proxy
-  register: matrix_nginx_proxy_state
-
-- name: Ensure matrix-nginx-proxy is stopped (if previously installed & started)
-  service: name=matrix-nginx-proxy state=stopped
-  when: "matrix_nginx_proxy_state.status.ActiveState|default('missing') == 'active'"
-
-- name: Ensure SSL certificates are marked as wanted in acmetool
-  shell: >-
-    /usr/bin/docker run --rm --name acmetool --net=host
-    -v {{ matrix_ssl_certs_path }}:/certs
-    -v {{ matrix_ssl_certs_path }}/run:/var/run/acme
-    -e ACME_EMAIL={{ matrix_ssl_support_email }}
-    willwill/acme-docker
-    acmetool want {{ item }} --xlog.severity=debug
+- name: Obtain initial certificates
+  include_tasks: "setup_ssl_for_domain.yml"
   with_items: "{{ domains_to_obtain_certificate_for }}"
+  loop_control:
+    loop_var: domain_name
 
-- name: Ensure matrix-nginx-proxy is started (if previously installed & started)
-  service: name=matrix-nginx-proxy state=started
-  when: "matrix_nginx_proxy_state.status.ActiveState|default('missing') == 'active'"
+- name: Ensure SSL renewal script installed
+  template:
+    src: "{{ role_path }}/templates/usr-local-bin/matrix-ssl-certificates-renew.j2"
+    dest: "/usr/local/bin/matrix-ssl-certificates-renew"
+    mode: 0750
 
 - name: Ensure periodic SSL renewal cronjob configured
   template:

roles/matrix-server/tasks/setup_ssl_for_domain.yml

@@ -0,0 +1,70 @@
+- debug:
+    msg: "Dealing with SSL certificate retrieval for domain: {{ domain_name }}"
+
+- set_fact:
+    domain_name_certificate_path: "{{ matrix_ssl_config_dir_path }}/live/{{ domain_name }}/cert.pem"
+
+- name: Check if a certificate for the domain already exists
+  stat:
+    path: "{{ domain_name_certificate_path }}"
+  register: domain_name_certificate_path_stat
+
+- set_fact:
+    domain_name_needs_cert: "{{ not domain_name_certificate_path_stat.stat.exists }}"
+
+# This will fail if there is something running on port 80 (like matrix-nginx-proxy).
+# We suppress the error, as we'll try another method below.
+- name: Attempt initial SSL certificate retrieval with standalone authenticator (directly)
+  shell: >-
+    /usr/bin/docker run
+    --rm
+    --name=matrix-certbot
+    --net=host
+    -v {{ matrix_ssl_config_dir_path }}:/etc/letsencrypt
+    -v {{ matrix_ssl_log_dir_path }}:/var/log/letsencrypt
+    {{ matrix_ssl_certbot_docker_image }}
+    certonly
+    --non-interactive
+    {% if matrix_ssl_use_staging %}--staging{% endif %}
+    --standalone
+    --preferred-challenges http
+    --agree-tos
+    --email={{ matrix_ssl_support_email }}
+    -d {{ domain_name }}
+  when: "domain_name_needs_cert"
+  register: result_certbot_direct
+  ignore_errors: true
+
+# If matrix-nginx-proxy is configured from a previous run of this playbook,
+# and it's running now, it may be able to proxy requests to `matrix_ssl_certbot_standalone_http_port`.
+- name: Attempt initial SSL certificate retrieval with standalone authenticator (via proxy)
+  shell: >-
+    /usr/bin/docker run
+    --rm
+    --name=matrix-certbot
+    -p 127.0.0.1:{{ matrix_ssl_certbot_standalone_http_port }}:80
+    --network={{ matrix_docker_network }}
+    -v {{ matrix_ssl_config_dir_path }}:/etc/letsencrypt
+    -v {{ matrix_ssl_log_dir_path }}:/var/log/letsencrypt
+    {{ matrix_ssl_certbot_docker_image }}
+    certonly
+    --non-interactive
+    {% if matrix_ssl_use_staging %}--staging{% endif %}
+    --standalone
+    --preferred-challenges http
+    --agree-tos
+    --email={{ matrix_ssl_support_email }}
+    -d {{ domain_name }}
+  when: "domain_name_needs_cert and result_certbot_direct.failed"
+  register: result_certbot_proxy
+  ignore_errors: true
+
+- name: Fail if all SSL certificate retrieval attempts failed
+  fail:
+    msg: |
+      Failed to obtain a certificate directly (by listening on port 80)
+      and also failed to obtain by relying on the server at port 80 to proxy the request.
+      See above for details.
+      You may wish to set up proxying of /.well-known/acme-challenge to {{ matrix_ssl_certbot_standalone_http_port }} or,
+      more easily, stop the server on port 80 while this playbook runs.
+  when: "domain_name_needs_cert and result_certbot_direct.failed and result_certbot_proxy.failed"

roles/matrix-server/templates/cron.d/matrix-ssl-certificate-renewal.j2

@@ -1,24 +1,11 @@
 MAILTO="{{ matrix_ssl_support_email }}"
 
-# The goal of this cronjob is to ask acmetool to check
+# The goal of this cronjob is to ask certbot to check
 # the current SSL certificates and to see if some need renewal.
 # If so, it would attempt to renew.
 #
 # Various services depend on these certificates and would need to be restarted.
 # This is not our concern here. We simply make sure the certificates are up to date.
 # Restarting of services happens on its own different schedule (other cronjobs).
-#
-#
-# How renewal works?
-#
-# acmetool will fail to bind to port :80 (because matrix-nginx-proxy or some other server is running there),
-# and will fall back to its "webroot" validation method.
-#
-# Thus, it would put validation files in `/var/run/acme/acme-challenge`.
-# These files can be retrieved via any vhost on port 80 of matrix-nginx-proxy,
-# because it aliases `/.well-known/acme-challenge` to that same directory.
-#
-# When a custom proxy server (not matrix-nginx-proxy provided by this playbook),
-# you'd need to make sure you alias these files correctly or SSL renewal would not work.
 
-{{ matrix_ssl_renew_cron_time_definition }} root /usr/bin/docker run --rm --net=host -v {{ matrix_ssl_certs_path }}:/certs -v {{ matrix_ssl_certs_path }}/run:/var/run/acme -e ACME_EMAIL={{ matrix_ssl_support_email }} willwill/acme-docker acmetool --batch reconcile # --xlog.severity=debug
+{{ matrix_ssl_renew_cron_time_definition }} root /bin/bash /usr/local/bin/matrix-ssl-certificates-renew

roles/matrix-server/templates/nginx-conf.d/matrix-riot-web.conf.j2

@@ -5,17 +5,14 @@ server {
 	server_tokens off;
 
 	location /.well-known/acme-challenge {
-		{#
-			The proxy can access the files directly.
-			An external server likely does not have permission to read these files,
-			so we'll just proxy to acme's :402 port.
-		#}
-
-		{%- if matrix_nginx_proxy_enabled -%}
-		default_type "text/plain";
-		alias {{ matrix_ssl_certs_path }}/run/acme-challenge;
-		{%- else -%}
-		proxy_pass http://localhost:402;
+		{% if matrix_nginx_proxy_enabled %}
+			{# Use the embedded DNS resolver in Docker containers to discover the service #}
+			resolver 127.0.0.11 valid=5s;
+			set $backend "matrix-certbot:80";
+			proxy_pass http://$backend;
+		{% else %}
+			{# Generic configuration for use outside of our container setup #}
+			proxy_pass http://localhost:{{ matrix_ssl_certbot_standalone_http_port }};
 		{% endif %}
 	}
 
@@ -36,8 +33,8 @@ server {
 	gzip on;
 	gzip_types text/plain application/json application/javascript text/css image/x-icon font/ttf image/gif;
 
-	ssl_certificate {{ matrix_ssl_certs_path }}/live/{{ hostname_riot }}/fullchain;
-	ssl_certificate_key {{ matrix_ssl_certs_path }}/live/{{ hostname_riot }}/privkey;
+	ssl_certificate {{ matrix_ssl_config_dir_path }}/live/{{ hostname_riot }}/fullchain.pem;
+	ssl_certificate_key {{ matrix_ssl_config_dir_path }}/live/{{ hostname_riot }}/privkey.pem;
 	ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
 	ssl_prefer_server_ciphers on;
 	ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH";

roles/matrix-server/templates/nginx-conf.d/matrix-synapse.conf.j2

@@ -5,17 +5,14 @@ server {
 	server_tokens off;
 
 	location /.well-known/acme-challenge {
-		{#
-			The proxy can access the files directly.
-			An external server likely does not have permission to read these files,
-			so we'll just proxy to acme's :402 port.
-		#}
-
-		{%- if matrix_nginx_proxy_enabled -%}
-		default_type "text/plain";
-		alias {{ matrix_ssl_certs_path }}/run/acme-challenge;
-		{%- else -%}
-		proxy_pass http://localhost:402;
+		{% if matrix_nginx_proxy_enabled %}
+			{# Use the embedded DNS resolver in Docker containers to discover the service #}
+			resolver 127.0.0.11 valid=5s;
+			set $backend "matrix-certbot:80";
+			proxy_pass http://$backend;
+		{% else %}
+			{# Generic configuration for use outside of our container setup #}
+			proxy_pass http://localhost:{{ matrix_ssl_certbot_standalone_http_port }};
 		{% endif %}
 	}
 
@@ -36,8 +33,8 @@ server {
 	gzip on;
 	gzip_types text/plain application/json;
 
-	ssl_certificate {{ matrix_ssl_certs_path }}/live/{{ hostname_matrix }}/fullchain;
-	ssl_certificate_key {{ matrix_ssl_certs_path }}/live/{{ hostname_matrix }}/privkey;
+	ssl_certificate {{ matrix_ssl_config_dir_path }}/live/{{ hostname_matrix }}/fullchain.pem;
+	ssl_certificate_key {{ matrix_ssl_config_dir_path }}/live/{{ hostname_matrix }}/privkey.pem;
 	ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
 	ssl_prefer_server_ciphers on;
 	ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH";

roles/matrix-server/templates/systemd/matrix-nginx-proxy.service.j2

@@ -22,7 +22,7 @@ ExecStart=/usr/bin/docker run --rm --name matrix-nginx-proxy \
 			-p 80:80 \
 			-p 443:443 \
 			-v {{ matrix_nginx_proxy_confd_path }}:/etc/nginx/conf.d:ro \
-			-v {{ matrix_ssl_certs_path }}:{{ matrix_ssl_certs_path }}:ro \
+			-v {{ matrix_ssl_config_dir_path }}:{{ matrix_ssl_config_dir_path }}:ro \
 			{{ matrix_docker_image_nginx }}
 ExecStop=-/usr/bin/docker kill matrix-nginx-proxy
 ExecStop=-/usr/bin/docker rm matrix-nginx-proxy

roles/matrix-server/templates/usr-local-bin/matrix-ssl-certificates-renew.j2

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# For renewal to work, matrix-nginx-proxy (or another webserver, if matrix-nginx-proxy is disabled)
+# need to forward requests for `/.well-known/acme-challenge` to the certbot container.
+#
+# This can happen inside the container network by proxying to `http://matrix-certbot:80`
+# or outside (on the host) by proxying to `http://localhost:{{ matrix_ssl_certbot_standalone_http_port }}`.
+
+docker run \
+	--rm \
+	--name=matrix-certbot \
+	--network="{{ matrix_docker_network }}" \
+	-p 127.0.0.1:{{ matrix_ssl_certbot_standalone_http_port }}:80 \
+	-v {{ matrix_ssl_config_dir_path }}:/etc/letsencrypt \
+	-v {{ matrix_ssl_log_dir_path }}:/var/log/letsencrypt \
+	{{ matrix_ssl_certbot_docker_image }} \
+	renew \
+		--non-interactive \
+		{% if matrix_ssl_use_staging %}
+			--staging \
+		{% endif %}
+		--quiet \
+		--standalone \
+		--preferred-challenges http \
+		--agree-tos \
+		--email={{ matrix_ssl_support_email }}