From 33cbad74f51936a0faa8d3a0b6b5ae9575576161 Mon Sep 17 00:00:00 2001 From: Tomas Kracmar Date: Wed, 13 Sep 2023 11:22:12 +0200 Subject: [PATCH] Enable customizations for Maubot docker image --- .../matrix-bot-maubot/defaults/main.yml | 79 ++++++++++++++++ .../matrix-bot-maubot/tasks/setup_install.yml | 91 +++++++++++++------ .../templates/customizations/Dockerfile.j2 | 57 ++++++++++++ .../systemd/matrix-bot-maubot.service.j2 | 4 + .../templates/synapse/homeserver.yaml.j2 | 2 +- 5 files changed, 204 insertions(+), 29 deletions(-) create mode 100644 roles/custom/matrix-bot-maubot/templates/customizations/Dockerfile.j2 diff --git a/roles/custom/matrix-bot-maubot/defaults/main.yml b/roles/custom/matrix-bot-maubot/defaults/main.yml index 5b35b9d95..5db6ed263 100644 --- a/roles/custom/matrix-bot-maubot/defaults/main.yml +++ b/roles/custom/matrix-bot-maubot/defaults/main.yml @@ -6,9 +6,88 @@ matrix_bot_maubot_enabled: true matrix_bot_maubot_container_image_self_build: false matrix_bot_maubot_docker_repo: "https://mau.dev/maubot/maubot.git" +matrix_bot_maubot_container_image_self_build_repo: "https://mau.dev/maubot/maubot.git" + +# matrix_bot_maubot_container_image_customizations_enabled controls whether a customized Maubot image will be built. +# +# We toggle this variable to `true` when certain features which require a custom build are enabled. +# Feel free to toggle this to `true` yourself and specify build steps in `matrix_bot_maubot_container_image_customizations_dockerfile_body_custom`. +# +# See: +# - `roles/custom/matrix-bot-maubot/templates/customizations/Dockerfile.j2` +# - `matrix_bot_maubot_container_image_customizations_dockerfile_body_custom` +# - `matrix_bot_maubot_docker_image_customized` +# - `matrix_bot_maubot_docker_image_final` + +matrix_bot_maubot_container_image_customizations_enabled: |- + {{ + matrix_bot_maubot_container_image_customizations_azure_dependencies_installation_enabled + or + matrix_bot_maubot_container_image_customizations_encryption_enabled + or + matrix_bot_maubot_container_image_customizations_templates_enabled + }} + +# Controls whether custom build steps will be added to the Dockerfile for installing Azure dependencies. +# The version that will be installed is specified in `matrix_bot_maubot_azure_dependencies_version`. +matrix_bot_maubot_container_image_customizations_azure_dependencies_installation_enabled: "{{ matrix_bot_maubot_azure_dependencies_installation_enabled }}" + +# Controls whether custom build steps will be added to the Dockerfile for customizing the email templates used by Maubot. +# +# Example usage: +# +# ```yml +# matrix_bot_maubot_container_image_customizations_templates_enabled: true +# # The templates are expected to be in a `templates/` subdirectory in +# matrix_bot_maubot_container_image_customizations_templates_in_container_template_files_relative_path: templates/ +# matrix_bot_maubot_container_image_customizations_templates_git_repository_url: git@github.com:organization/repository.git +# matrix_bot_maubot_container_image_customizations_templates_git_repository_branch: main +# matrix_bot_maubot_container_image_customizations_templates_git_repository_keyscan_enabled: true +# matrix_bot_maubot_container_image_customizations_templates_git_repository_keyscan_hostname: github.com +# ``` +# +# See: https://github.com/matrix-org/synapse/blob/develop/docs/templates.md +matrix_bot_maubot_container_image_customizations_templates_enabled: false +matrix_bot_maubot_container_image_customizations_templates_in_container_base_path: /custom-templates +matrix_bot_maubot_container_image_customizations_templates_in_container_template_files_relative_path: '' +matrix_bot_maubot_container_image_customizations_templates_in_container_full_path: "{{ matrix_bot_maubot_container_image_customizations_templates_in_container_base_path }}/{{ matrix_bot_maubot_container_image_customizations_templates_in_container_template_files_relative_path }}" +matrix_bot_maubot_container_image_customizations_templates_git_repository_url: '' +matrix_bot_maubot_container_image_customizations_templates_git_repository_branch: main +matrix_bot_maubot_container_image_customizations_templates_git_repository_keyscan_enabled: false +matrix_bot_maubot_container_image_customizations_templates_git_repository_keyscan_hostname: '' + +# matrix_bot_maubot_container_image_customizations_dockerfile_body contains your custom Dockerfile steps +# for building your customized Maubot image based on the original (upstream) image (`matrix_bot_maubot_docker_image`). +# A `FROM ...` clause is included automatically so you don't have to. +# +# Example: +# matrix_bot_maubot_container_image_customizations_dockerfile_body_custom: | +# RUN echo 'This is a custom step for building the customized Docker image for Maubot.' +# RUN echo 'You can override matrix_bot_maubot_container_image_customizations_dockerfile_body_custom to add your own steps.' +# RUN echo 'You do NOT need to include a FROM clause yourself.' +matrix_bot_maubot_container_image_customizations_dockerfile_body_custom: '' + +# matrix_bot_maubot_docker_image_customized is the name of the locally built Maubot image +# which adds various customizations on top of the original (upstream) Maubot image. +# This image will be based on the upstream `matrix_bot_maubot_docker_image` image, only if `matrix_bot_maubot_container_image_customizations_enabled: true`. +matrix_bot_maubot_docker_image_customized: "localhost/matrixdotorg/bot_maubot:{{ matrix_bot_maubot_docker_image_tag }}-customized" + +# Controls whether the customized image (`matrix_bot_maubot_docker_image_customized`) is to be force-built without layer caching enabled. +# This is useful if you've enabled customizations (e.g. `matrix_bot_maubot_container_image_customizations_templates_enabled`), +# which clone some branch of some repository, and you'd like for each Ansible run to pull new revisions from that branch. +matrix_bot_maubot_docker_image_customized_build_nocache: true + +# Controls whether the customized image (`matrix_bot_maubot_docker_image_customized`) is to be built, even if it already exists. +# Related to: matrix_bot_maubot_docker_image_customized_build_nocache +matrix_bot_maubot_docker_image_customized_force_source: "{{ matrix_bot_maubot_docker_image_customized_build_nocache }}" + +# matrix_bot_maubot_docker_image_final holds the name of the Maubot image to run depending on whether or not customizations are enabled. +matrix_bot_maubot_docker_image_final: "{{ matrix_bot_maubot_docker_image_customized if matrix_bot_maubot_container_image_customizations_enabled else matrix_bot_maubot_docker_image }} " + matrix_bot_maubot_docker_src_files_path: "{{ matrix_bot_maubot_base_path }}/docker-src" matrix_bot_maubot_docker_repo_version: "{{ 'master' if matrix_bot_maubot_version == 'latest' else matrix_bot_maubot_version }}" +matrix_bot_maubot_customized_docker_src_files_path: "{{ matrix_bot_maubot_base_path }}/docker-src" matrix_bot_maubot_version: v0.4.1 matrix_bot_maubot_docker_image: "{{ matrix_bot_maubot_docker_image_name_prefix }}maubot/maubot:{{ matrix_bot_maubot_version }}" diff --git a/roles/custom/matrix-bot-maubot/tasks/setup_install.yml b/roles/custom/matrix-bot-maubot/tasks/setup_install.yml index 60b87861e..1d98d26ea 100644 --- a/roles/custom/matrix-bot-maubot/tasks/setup_install.yml +++ b/roles/custom/matrix-bot-maubot/tasks/setup_install.yml @@ -17,13 +17,42 @@ - {path: "{{ matrix_bot_maubot_docker_src_files_path }}", when: "{{ matrix_bot_maubot_container_image_self_build }}"} when: "item.when|bool" -- name: Ensure maubot configuration file created - ansible.builtin.template: - src: "{{ role_path }}/templates/config/config.yaml.j2" - dest: "{{ matrix_bot_maubot_config_path }}/config.yaml" - owner: "{{ matrix_user_username }}" - group: "{{ matrix_user_groupname }}" - mode: "u=rwx" +- when: "matrix_bot_maubot_container_image_self_build | bool" + block: + - name: Determine the Git branch or tag to use + set_fact: + git_version_to_use: "{{ 'master' if matrix_bot_maubot_docker_image.split(':')[1] == 'latest' else matrix_bot_maubot_docker_image.split(':')[1] }}" + + - name: Ensure Maubot repository is present on self-build + ansible.builtin.git: + repo: "{{ matrix_bot_maubot_docker_repo }}" + dest: "{{ matrix_bot_maubot_docker_src_files_path }}" + version: "{{ git_version_to_use }}" + force: "yes" + become: true + become_user: "{{ matrix_user_username }}" + register: matrix_bot_maubot_git_pull_results + + - name: Check if Maubot Docker image exists + ansible.builtin.command: "{{ devture_systemd_docker_base_host_command_docker }} images --quiet --filter 'reference={{ matrix_bot_maubot_docker_image }}'" + register: matrix_bot_maubot_docker_image_check_result + changed_when: false + + # Invoking the `docker build` command here, instead of calling the `docker_image` Ansible module, + # because the latter does not support BuildKit. + # See: https://github.com/ansible-collections/community.general/issues/514 + - name: Ensure Maubot Docker image is built + ansible.builtin.shell: + chdir: "{{ matrix_bot_maubot_docker_src_files_path }}" + cmd: | + {{ devture_systemd_docker_base_host_command_docker }} build \ + -t "{{ matrix_bot_maubot_docker_image }}" \ + -f Dockerfile \ + . + environment: + DOCKER_BUILDKIT: 1 + changed_when: true + when: "matrix_bot_maubot_git_pull_results.changed | bool or matrix_bot_maubot_docker_image_check_result.stdout == ''" - name: Ensure maubot image is pulled community.docker.docker_image: @@ -37,28 +66,34 @@ delay: "{{ devture_playbook_help_container_retries_delay }}" until: result is not failed -- name: Ensure maubot repository is present on self-build - ansible.builtin.git: - repo: "{{ matrix_bot_maubot_docker_repo }}" - version: "{{ matrix_bot_maubot_docker_repo_version }}" - dest: "{{ matrix_bot_maubot_docker_src_files_path }}" - force: "yes" - become: true - become_user: "{{ matrix_user_username }}" - register: matrix_bot_maubot_git_pull_results - when: "matrix_bot_maubot_container_image_self_build|bool" +- when: "matrix_bot_maubot_container_image_customizations_enabled | bool" + block: + - name: Ensure customizations Dockerfile is created + ansible.builtin.template: + src: "{{ role_path }}/templates/customizations/Dockerfile.j2" + dest: "{{ matrix_bot_maubot_customized_docker_src_files_path }}/Dockerfile" + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + mode: 0640 + register: matrix_bot_maubot_container_image_customizations_dockerfile_result -- name: Ensure maubot image is built - community.docker.docker_image: - name: "{{ matrix_bot_maubot_docker_image }}" - source: build - force_source: "{{ matrix_bot_maubot_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" - force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_bot_maubot_git_pull_results.changed }}" - build: - dockerfile: Dockerfile - path: "{{ matrix_bot_maubot_docker_src_files_path }}" - pull: true - when: "matrix_bot_maubot_container_image_self_build|bool" + - name: Ensure customized Docker image for Maubot is built + community.docker.docker_image: + name: "{{ matrix_bot_maubot_docker_image_customized }}" + source: build + force_source: "{{ matrix_bot_maubot_container_image_customizations_dockerfile_result.changed or matrix_bot_maubot_docker_image_customized_force_source }}" + build: + dockerfile: Dockerfile + path: "{{ matrix_bot_maubot_customized_docker_src_files_path }}" + nocache: "{{ matrix_bot_maubot_docker_image_customized_build_nocache }}" + +- name: Ensure maubot configuration file created + ansible.builtin.template: + src: "{{ role_path }}/templates/config/config.yaml.j2" + dest: "{{ matrix_bot_maubot_config_path }}/config.yaml" + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + mode: "u=rwx" - name: Ensure matrix-bot-maubot.service installed ansible.builtin.template: diff --git a/roles/custom/matrix-bot-maubot/templates/customizations/Dockerfile.j2 b/roles/custom/matrix-bot-maubot/templates/customizations/Dockerfile.j2 new file mode 100644 index 000000000..70aa571ce --- /dev/null +++ b/roles/custom/matrix-bot-maubot/templates/customizations/Dockerfile.j2 @@ -0,0 +1,57 @@ +#jinja2: lstrip_blocks: "True" +FROM {{ matrix_bot_maubot_docker_image }} + +{% if matrix_bot_maubot_container_image_customizations_azure_dependencies_installation_enabled %} +RUN apk add gcc +RUN pip install --upgrade pip +RUN pip install azure-identity msgraph-core ldap3 +{% endif %} + +{% if matrix_bot_maubot_container_image_customizations_encryption_enabled %} +RUN pip install --upgrade maubot[encryption] +{% endif %} + +{% if matrix_bot_maubot_container_image_customizations_templates_enabled %} +{# +This ugly script below does quite a lot: + - installs git and other dependencies temporarily, just so we could do a shallow-clone + - prepare the SSH config: keyscanning (if enabled), private key (if enabled) + - performs a git shallow clone with just the branch we need + - makes sure the files are owned by the user that will actually run the container later + - removes the `.git` directory to save space, but keeps git revision in `git-revision.txt`, should we need it for debugging + - finally, verifies that the templates path can indeed be found within the base path (sanity check) +#} +{% set dependencies = ['git', 'ssh', 'openssh-client'] %} +RUN \ + {% if matrix_bot_maubot_container_image_customizations_templates_git_repository_ssh_private_key %} + echo '{{ matrix_bot_maubot_container_image_customizations_templates_git_repository_ssh_private_key | b64encode }}' | base64 -d > /custom-templates-private-key && \ + chmod 400 /custom-templates-private-key && \ + {% endif %} + apt-get update && \ + apt-get install --no-install-recommends -y {{ dependencies | join(' ') }} && \ + {% if matrix_bot_maubot_container_image_customizations_templates_git_repository_keyscan_enabled %} + mkdir ~/.ssh && \ + chmod 700 ~/.ssh && \ + ssh-keyscan -t rsa {{ matrix_bot_maubot_container_image_customizations_templates_git_repository_keyscan_hostname }} >> ~/.ssh/known_hosts && \ + {% endif %} + {% if matrix_bot_maubot_container_image_customizations_templates_git_repository_ssh_private_key %}GIT_SSH_COMMAND='ssh -i /custom-templates-private-key'{% endif %} git \ + clone \ + --branch={{ matrix_bot_maubot_container_image_customizations_templates_git_repository_branch }} \ + --depth=1 \ + --single-branch \ + --no-tags \ + {{ matrix_bot_maubot_container_image_customizations_templates_git_repository_url }} \ + {{ matrix_bot_maubot_container_image_customizations_templates_in_container_base_path }} && \ + /bin/sh -c 'cd {{ matrix_bot_maubot_container_image_customizations_templates_in_container_base_path }} && git rev-parse HEAD > git-revision.txt' && \ + rm -rf {{ matrix_bot_maubot_container_image_customizations_templates_in_container_base_path }}/.git && \ + chown -R {{ matrix_bot_maubot_uid }}:{{ matrix_bot_maubot_gid }} {{ matrix_bot_maubot_container_image_customizations_templates_in_container_base_path }} && \ + apt-get autoremove -y {{ dependencies | join(' ') }} && \ + {% if matrix_bot_maubot_container_image_customizations_templates_git_repository_ssh_private_key %} + rm /custom-templates-private-key && \ + {% endif %} + true + +RUN /bin/sh -c 'stat {{ matrix_bot_maubot_container_image_customizations_templates_in_container_base_path }}/{{ matrix_bot_maubot_container_image_customizations_templates_in_container_template_files_relative_path }} || exit 1' +{% endif %} + +{{ matrix_bot_maubot_container_image_customizations_dockerfile_body_custom }} diff --git a/roles/custom/matrix-bot-maubot/templates/systemd/matrix-bot-maubot.service.j2 b/roles/custom/matrix-bot-maubot/templates/systemd/matrix-bot-maubot.service.j2 index 34c856350..ca588066b 100644 --- a/roles/custom/matrix-bot-maubot/templates/systemd/matrix-bot-maubot.service.j2 +++ b/roles/custom/matrix-bot-maubot/templates/systemd/matrix-bot-maubot.service.j2 @@ -30,7 +30,11 @@ ExecStart={{ devture_systemd_docker_base_host_command_docker }} run --rm --name {% if matrix_bot_maubot_management_interface_http_bind_port %} -p {{ matrix_bot_maubot_management_interface_http_bind_port }}:{{ matrix_bot_maubot_management_interface_port }} \ {% endif %} + {% if matrix_bot_maubot_container_image_customizations_enabled %} + {{ matrix_bot_maubot_docker_image_customized }} \ + {% else %} {{ matrix_bot_maubot_docker_image }} \ + {% endif %} python3 -m maubot -c /config/config.yaml --no-update ExecStop=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} kill matrix-bot-maubot 2>/dev/null || true' diff --git a/roles/custom/matrix-synapse/templates/synapse/homeserver.yaml.j2 b/roles/custom/matrix-synapse/templates/synapse/homeserver.yaml.j2 index 7b1c1dfd1..b5d092ba2 100644 --- a/roles/custom/matrix-synapse/templates/synapse/homeserver.yaml.j2 +++ b/roles/custom/matrix-synapse/templates/synapse/homeserver.yaml.j2 @@ -2499,7 +2499,7 @@ email: # #bind_password: # #filter: "(objectClass=posixAccount)" {% if matrix_synapse_password_providers_enabled %} -password_providers: +modules: {% if matrix_synapse_ext_password_provider_rest_auth_enabled %} - module: "rest_auth_provider.RestAuthProvider" config: