Enable customizations for Maubot docker image

roles/custom/matrix-bot-maubot/defaults/main.yml

@@ -6,9 +6,88 @@
 matrix_bot_maubot_enabled: true
 matrix_bot_maubot_container_image_self_build: false
 matrix_bot_maubot_docker_repo: "https://mau.dev/maubot/maubot.git"
+matrix_bot_maubot_container_image_self_build_repo: "https://mau.dev/maubot/maubot.git"
+
+# matrix_bot_maubot_container_image_customizations_enabled controls whether a customized Maubot image will be built.
+#
+# We toggle this variable to `true` when certain features which require a custom build are enabled.
+# Feel free to toggle this to `true` yourself and specify build steps in `matrix_bot_maubot_container_image_customizations_dockerfile_body_custom`.
+#
+# See:
+# - `roles/custom/matrix-bot-maubot/templates/customizations/Dockerfile.j2`
+# - `matrix_bot_maubot_container_image_customizations_dockerfile_body_custom`
+# - `matrix_bot_maubot_docker_image_customized`
+# - `matrix_bot_maubot_docker_image_final`
+
+matrix_bot_maubot_container_image_customizations_enabled: |-
+  {{
+    matrix_bot_maubot_container_image_customizations_azure_dependencies_installation_enabled
+    or
+    matrix_bot_maubot_container_image_customizations_encryption_enabled
+    or
+    matrix_bot_maubot_container_image_customizations_templates_enabled
+  }}
+
+# Controls whether custom build steps will be added to the Dockerfile for installing Azure dependencies.
+# The version that will be installed is specified in `matrix_bot_maubot_azure_dependencies_version`.
+matrix_bot_maubot_container_image_customizations_azure_dependencies_installation_enabled: "{{ matrix_bot_maubot_azure_dependencies_installation_enabled }}"
+
+# Controls whether custom build steps will be added to the Dockerfile for customizing the email templates used by Maubot.
+#
+# Example usage:
+#
+# ```yml
+# matrix_bot_maubot_container_image_customizations_templates_enabled: true
+# # The templates are expected to be in a `templates/` subdirectory in
+# matrix_bot_maubot_container_image_customizations_templates_in_container_template_files_relative_path: templates/
+# matrix_bot_maubot_container_image_customizations_templates_git_repository_url: git@github.com:organization/repository.git
+# matrix_bot_maubot_container_image_customizations_templates_git_repository_branch: main
+# matrix_bot_maubot_container_image_customizations_templates_git_repository_keyscan_enabled: true
+# matrix_bot_maubot_container_image_customizations_templates_git_repository_keyscan_hostname: github.com
+# ```
+#
+# See: https://github.com/matrix-org/synapse/blob/develop/docs/templates.md
+matrix_bot_maubot_container_image_customizations_templates_enabled: false
+matrix_bot_maubot_container_image_customizations_templates_in_container_base_path: /custom-templates
+matrix_bot_maubot_container_image_customizations_templates_in_container_template_files_relative_path: ''
+matrix_bot_maubot_container_image_customizations_templates_in_container_full_path: "{{ matrix_bot_maubot_container_image_customizations_templates_in_container_base_path }}/{{ matrix_bot_maubot_container_image_customizations_templates_in_container_template_files_relative_path }}"
+matrix_bot_maubot_container_image_customizations_templates_git_repository_url: ''
+matrix_bot_maubot_container_image_customizations_templates_git_repository_branch: main
+matrix_bot_maubot_container_image_customizations_templates_git_repository_keyscan_enabled: false
+matrix_bot_maubot_container_image_customizations_templates_git_repository_keyscan_hostname: ''
+
+# matrix_bot_maubot_container_image_customizations_dockerfile_body contains your custom Dockerfile steps
+# for building your customized Maubot image based on the original (upstream) image (`matrix_bot_maubot_docker_image`).
+# A `FROM ...` clause is included automatically so you don't have to.
+#
+# Example:
+# matrix_bot_maubot_container_image_customizations_dockerfile_body_custom: |
+#  RUN echo 'This is a custom step for building the customized Docker image for Maubot.'
+#  RUN echo 'You can override matrix_bot_maubot_container_image_customizations_dockerfile_body_custom to add your own steps.'
+#  RUN echo 'You do NOT need to include a FROM clause yourself.'
+matrix_bot_maubot_container_image_customizations_dockerfile_body_custom: ''
+
+# matrix_bot_maubot_docker_image_customized is the name of the locally built Maubot image
+# which adds various customizations on top of the original (upstream) Maubot image.
+# This image will be based on the upstream `matrix_bot_maubot_docker_image` image, only if `matrix_bot_maubot_container_image_customizations_enabled: true`.
+matrix_bot_maubot_docker_image_customized: "localhost/matrixdotorg/bot_maubot:{{ matrix_bot_maubot_docker_image_tag }}-customized"
+
+# Controls whether the customized image (`matrix_bot_maubot_docker_image_customized`) is to be force-built without layer caching enabled.
+# This is useful if you've enabled customizations (e.g. `matrix_bot_maubot_container_image_customizations_templates_enabled`),
+# which clone some branch of some repository, and you'd like for each Ansible run to pull new revisions from that branch.
+matrix_bot_maubot_docker_image_customized_build_nocache: true
+
+# Controls whether the customized image (`matrix_bot_maubot_docker_image_customized`) is to be built, even if it already exists.
+# Related to: matrix_bot_maubot_docker_image_customized_build_nocache
+matrix_bot_maubot_docker_image_customized_force_source: "{{ matrix_bot_maubot_docker_image_customized_build_nocache }}"
+
+# matrix_bot_maubot_docker_image_final holds the name of the Maubot image to run depending on whether or not customizations are enabled.
+matrix_bot_maubot_docker_image_final: "{{ matrix_bot_maubot_docker_image_customized if matrix_bot_maubot_container_image_customizations_enabled else matrix_bot_maubot_docker_image }} "
+
 matrix_bot_maubot_docker_src_files_path: "{{ matrix_bot_maubot_base_path }}/docker-src"
 matrix_bot_maubot_docker_repo_version: "{{ 'master' if matrix_bot_maubot_version == 'latest' else matrix_bot_maubot_version }}"
 
+matrix_bot_maubot_customized_docker_src_files_path: "{{ matrix_bot_maubot_base_path }}/docker-src"
 
 matrix_bot_maubot_version: v0.4.1
 matrix_bot_maubot_docker_image: "{{ matrix_bot_maubot_docker_image_name_prefix }}maubot/maubot:{{ matrix_bot_maubot_version }}"

roles/custom/matrix-bot-maubot/tasks/setup_install.yml

@@ -17,13 +17,42 @@
     - {path: "{{ matrix_bot_maubot_docker_src_files_path }}", when: "{{ matrix_bot_maubot_container_image_self_build }}"}
   when: "item.when|bool"
 
-- name: Ensure maubot configuration file created
-  ansible.builtin.template:
-    src: "{{ role_path }}/templates/config/config.yaml.j2"
-    dest: "{{ matrix_bot_maubot_config_path }}/config.yaml"
-    owner: "{{ matrix_user_username }}"
-    group: "{{ matrix_user_groupname }}"
-    mode: "u=rwx"
+- when: "matrix_bot_maubot_container_image_self_build | bool"
+  block:
+    - name: Determine the Git branch or tag to use
+      set_fact:
+        git_version_to_use: "{{ 'master' if matrix_bot_maubot_docker_image.split(':')[1] == 'latest' else matrix_bot_maubot_docker_image.split(':')[1] }}"
+
+    - name: Ensure Maubot repository is present on self-build
+      ansible.builtin.git:
+        repo: "{{ matrix_bot_maubot_docker_repo }}"
+        dest: "{{ matrix_bot_maubot_docker_src_files_path }}"
+        version: "{{ git_version_to_use }}"
+        force: "yes"
+      become: true
+      become_user: "{{ matrix_user_username }}"
+      register: matrix_bot_maubot_git_pull_results
+
+    - name: Check if Maubot Docker image exists
+      ansible.builtin.command: "{{ devture_systemd_docker_base_host_command_docker }} images --quiet --filter 'reference={{ matrix_bot_maubot_docker_image }}'"
+      register: matrix_bot_maubot_docker_image_check_result
+      changed_when: false
+
+    # Invoking the `docker build` command here, instead of calling the `docker_image` Ansible module,
+    # because the latter does not support BuildKit.
+    # See: https://github.com/ansible-collections/community.general/issues/514
+    - name: Ensure Maubot Docker image is built
+      ansible.builtin.shell:
+        chdir: "{{ matrix_bot_maubot_docker_src_files_path }}"
+        cmd: |
+          {{ devture_systemd_docker_base_host_command_docker }} build \
+          -t "{{ matrix_bot_maubot_docker_image }}" \
+          -f Dockerfile \
+          .
+      environment:
+        DOCKER_BUILDKIT: 1
+      changed_when: true
+      when: "matrix_bot_maubot_git_pull_results.changed | bool or matrix_bot_maubot_docker_image_check_result.stdout == ''"
 
 - name: Ensure maubot image is pulled
   community.docker.docker_image:
@@ -37,28 +66,34 @@
   delay: "{{ devture_playbook_help_container_retries_delay }}"
   until: result is not failed
 
-- name: Ensure maubot repository is present on self-build
-  ansible.builtin.git:
-    repo: "{{ matrix_bot_maubot_docker_repo }}"
-    version: "{{ matrix_bot_maubot_docker_repo_version }}"
-    dest: "{{ matrix_bot_maubot_docker_src_files_path }}"
-    force: "yes"
-  become: true
-  become_user: "{{ matrix_user_username }}"
-  register: matrix_bot_maubot_git_pull_results
-  when: "matrix_bot_maubot_container_image_self_build|bool"
+- when: "matrix_bot_maubot_container_image_customizations_enabled | bool"
+  block:
+    - name: Ensure customizations Dockerfile is created
+      ansible.builtin.template:
+        src: "{{ role_path }}/templates/customizations/Dockerfile.j2"
+        dest: "{{ matrix_bot_maubot_customized_docker_src_files_path }}/Dockerfile"
+        owner: "{{ matrix_user_username }}"
+        group: "{{ matrix_user_groupname }}"
+        mode: 0640
+      register: matrix_bot_maubot_container_image_customizations_dockerfile_result
 
-- name: Ensure maubot image is built
-  community.docker.docker_image:
-    name: "{{ matrix_bot_maubot_docker_image }}"
-    source: build
-    force_source: "{{ matrix_bot_maubot_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
-    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_bot_maubot_git_pull_results.changed }}"
-    build:
-      dockerfile: Dockerfile
-      path: "{{ matrix_bot_maubot_docker_src_files_path }}"
-      pull: true
-  when: "matrix_bot_maubot_container_image_self_build|bool"
+    - name: Ensure customized Docker image for Maubot is built
+      community.docker.docker_image:
+        name: "{{ matrix_bot_maubot_docker_image_customized }}"
+        source: build
+        force_source: "{{ matrix_bot_maubot_container_image_customizations_dockerfile_result.changed or matrix_bot_maubot_docker_image_customized_force_source }}"
+        build:
+          dockerfile: Dockerfile
+          path: "{{ matrix_bot_maubot_customized_docker_src_files_path }}"
+          nocache: "{{ matrix_bot_maubot_docker_image_customized_build_nocache }}"
+
+- name: Ensure maubot configuration file created
+  ansible.builtin.template:
+    src: "{{ role_path }}/templates/config/config.yaml.j2"
+    dest: "{{ matrix_bot_maubot_config_path }}/config.yaml"
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+    mode: "u=rwx"
 
 - name: Ensure matrix-bot-maubot.service installed
   ansible.builtin.template:

roles/custom/matrix-bot-maubot/templates/customizations/Dockerfile.j2

@@ -0,0 +1,57 @@
+#jinja2: lstrip_blocks: "True"
+FROM {{ matrix_bot_maubot_docker_image }}
+
+{% if matrix_bot_maubot_container_image_customizations_azure_dependencies_installation_enabled %}
+RUN apk add gcc
+RUN pip install --upgrade pip
+RUN pip install azure-identity msgraph-core ldap3
+{% endif %}
+
+{% if matrix_bot_maubot_container_image_customizations_encryption_enabled %}
+RUN pip install --upgrade maubot[encryption]
+{% endif %}
+
+{% if matrix_bot_maubot_container_image_customizations_templates_enabled %}
+{#
+This ugly script below does quite a lot:
+	- installs git and other dependencies temporarily, just so we could do a shallow-clone
+	- prepare the SSH config: keyscanning (if enabled), private key (if enabled)
+	- performs a git shallow clone with just the branch we need
+	- makes sure the files are owned by the user that will actually run the container later
+	- removes the `.git` directory to save space, but keeps git revision in `git-revision.txt`, should we need it for debugging
+	- finally, verifies that the templates path can indeed be found within the base path (sanity check)
+#}
+{% set dependencies = ['git', 'ssh', 'openssh-client'] %}
+RUN \
+	{% if matrix_bot_maubot_container_image_customizations_templates_git_repository_ssh_private_key %}
+	echo '{{ matrix_bot_maubot_container_image_customizations_templates_git_repository_ssh_private_key | b64encode }}' | base64 -d > /custom-templates-private-key && \
+	chmod 400 /custom-templates-private-key && \
+	{% endif %}
+	apt-get update && \
+	apt-get install --no-install-recommends -y {{ dependencies | join(' ') }} && \
+	{% if matrix_bot_maubot_container_image_customizations_templates_git_repository_keyscan_enabled %}
+	mkdir ~/.ssh && \
+	chmod 700 ~/.ssh && \
+	ssh-keyscan -t rsa {{ matrix_bot_maubot_container_image_customizations_templates_git_repository_keyscan_hostname }} >> ~/.ssh/known_hosts && \
+	{% endif %}
+	{% if matrix_bot_maubot_container_image_customizations_templates_git_repository_ssh_private_key %}GIT_SSH_COMMAND='ssh -i /custom-templates-private-key'{% endif %} git \
+		clone \
+			--branch={{ matrix_bot_maubot_container_image_customizations_templates_git_repository_branch }} \
+			--depth=1 \
+			--single-branch \
+			--no-tags \
+			{{ matrix_bot_maubot_container_image_customizations_templates_git_repository_url }} \
+			{{ matrix_bot_maubot_container_image_customizations_templates_in_container_base_path }} && \
+	/bin/sh -c 'cd {{ matrix_bot_maubot_container_image_customizations_templates_in_container_base_path }} && git rev-parse HEAD > git-revision.txt' && \
+	rm -rf {{ matrix_bot_maubot_container_image_customizations_templates_in_container_base_path }}/.git && \
+	chown -R {{ matrix_bot_maubot_uid }}:{{ matrix_bot_maubot_gid }} {{ matrix_bot_maubot_container_image_customizations_templates_in_container_base_path }} && \
+	apt-get autoremove -y {{ dependencies | join(' ') }} && \
+	{% if matrix_bot_maubot_container_image_customizations_templates_git_repository_ssh_private_key %}
+	rm /custom-templates-private-key && \
+	{% endif %}
+	true
+
+RUN /bin/sh -c 'stat {{ matrix_bot_maubot_container_image_customizations_templates_in_container_base_path }}/{{ matrix_bot_maubot_container_image_customizations_templates_in_container_template_files_relative_path }} || exit 1'
+{% endif %}
+
+{{ matrix_bot_maubot_container_image_customizations_dockerfile_body_custom }}

roles/custom/matrix-bot-maubot/templates/systemd/matrix-bot-maubot.service.j2

@@ -30,7 +30,11 @@ ExecStart={{ devture_systemd_docker_base_host_command_docker }} run --rm --name
 			{% if matrix_bot_maubot_management_interface_http_bind_port %}
 			-p {{ matrix_bot_maubot_management_interface_http_bind_port }}:{{ matrix_bot_maubot_management_interface_port }} \
 			{% endif %}
+			{% if matrix_bot_maubot_container_image_customizations_enabled %}
+			{{ matrix_bot_maubot_docker_image_customized }} \
+			{% else %}
 			{{ matrix_bot_maubot_docker_image }} \
+			{% endif %}
  			python3 -m maubot -c /config/config.yaml --no-update
 
 ExecStop=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} kill matrix-bot-maubot 2>/dev/null || true'

roles/custom/matrix-synapse/templates/synapse/homeserver.yaml.j2

@@ -2499,7 +2499,7 @@ email:
 #        #bind_password:
 #        #filter: "(objectClass=posixAccount)"
 {% if matrix_synapse_password_providers_enabled %}
-password_providers:
+modules:
 {% if matrix_synapse_ext_password_provider_rest_auth_enabled %}
   - module: "rest_auth_provider.RestAuthProvider"
     config: