Merge pull request #339 from netresearch/master

Add matrix-appservice-webhooks role.

README.md

@@ -50,6 +50,8 @@ Using this playbook, you can get the following services configured on your serve
 
 - (optional) the [matrix-appservice-slack](https://github.com/matrix-org/matrix-appservice-slack) bridge for bridging your Matrix server to [Slack](https://slack.com/)
 
+- (optional) the [matrix-appservice-webhooks](https://github.com/turt2live/matrix-appservice-webhooks) bridge for slack compatible webhooks ([ConcourseCI](https://concourse-ci.org/), [Slack](https://slack.com/) etc. pp.)
+
 - (optional) [Email2Matrix](https://github.com/devture/email2matrix) for relaying email messages to Matrix rooms
 
 - (optional) [Dimension](https://github.com/turt2live/matrix-dimension), an open source integrations manager for matrix clients

docs/configuring-playbook-bridge-appservice-webhooks.md

@@ -0,0 +1,58 @@
+# Setting up Appservice Webhooks (optional)
+
+The playbook can install and configure [matrix-appservice-webhooks](https://github.com/turt2live/matrix-appservice-webhooks) for you.
+
+Setup Instructions:
+
+loosely based on [this](https://github.com/turt2live/matrix-appservice-webhooks/blob/master/README.md)
+
+1. All you basically need is to adjust your `inventory/host_vars/matrix.<domain-name>/vars.yml`:
+    ```yaml
+    matrix_appservice_webhooks_enabled: true
+    matrix_appservice_webhooks_api_secret: '<your_secret>'
+    ```
+
+2. In case you want to change the verbosity of logging via `journalctl -fu matrix-appservice-webhooks.service`
+you can adjust this in `inventory/host_vars/matrix.<domain-name>/vars.yml` as well.
+<br>
+*Note*: default value is: `info` and availabe log levels are : `info, verbose`
+
+    ```yaml
+    matrix_appservice_webhooks_log_level: '<log_level>'
+    ```
+
+3. If you've already installed Matrix services using the playbook before, you'll need to re-run it (`--tags=setup-all,start`). If not, proceed with [configuring other playbook services](configuring-playbook.md) and then with [Installing](installing.md). Get back to this guide once ready.
+
+4. Invite the bridge bot user to your room:
+    - either with:
+      ```
+      /invite @_webhook:<domain.name>
+      ```
+        *Note*: Make sure you have administration permissions in your room
+    - or simply add the bridge bot to a private channel (personal channels imply you being an administrator)
+
+5. Send a message to the bridge bot in order to receive a private message including the webhook link.
+    ```
+    !webhook
+    ```
+
+6. The JSON body for posting messages will have to look like this:
+    ```
+    {
+        "text": "Hello world!",
+        "format": "plain",
+        "displayName": "My Cool Webhook",
+        "avatarUrl": "http://i.imgur.com/IDOBtEJ.png"
+    }
+    ```
+   - You can test this via curl like so:
+     ```
+     curl --header "Content-Type: application/json" \
+     --data '{
+     "text": "Hello world!",
+     "format": "plain",
+     "displayName": "My Cool Webhook",
+     "avatarUrl": "http://i.imgur.com/IDOBtEJ.png"
+     }' \
+     <the link you've gotten in 5.>
+     ```

group_vars/matrix_servers

@@ -60,6 +60,40 @@ matrix_appservice_discord_homeserver_token: "{{ matrix_synapse_macaroon_secret_k
 ######################################################################
 
 
+######################################################################
+#
+# matrix-appservice-webhooks
+#
+######################################################################
+
+# We don't enable bridges by default.
+matrix_appservice_webhooks_enabled: false
+
+# Normally, matrix-nginx-proxy is enabled and nginx can reach matrix-appservice-webhooks over the container network.
+# If matrix-nginx-proxy is not enabled, or you otherwise have a need for it, you can expose
+# matrix-appservice-webhooks' client-server port to the local host.
+matrix_appservice_webhooks_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_enabled else '127.0.0.1:{{ matrix_appservice_webhooks_webhooks_port }}' }}"
+
+matrix_appservice_webhooks_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'webhooks-appservice-token') | to_uuid }}"
+
+matrix_appservice_webhooks_homeserver_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'webhooks-homeserver-token') | to_uuid }}"
+
+matrix_appservice_webhooks_id_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'webhooks-id-token') | to_uuid }}"
+
+matrix_appservice_webhooks_systemd_required_services_list: |
+  {{
+    ['docker.service']
+    +
+    (['matrix-synapse.service'] if matrix_synapse_enabled else [])
+  }}
+
+######################################################################
+#
+# /matrix-appservice-webhooks
+#
+######################################################################
+
+
 ######################################################################
 #
 # matrix-appservice-slack

roles/matrix-bridge-appservice-webhooks/defaults/main.yml

@@ -0,0 +1,105 @@
+# matrix-appservice-webhooks is a Matrix <-> webhook bridge
+# See: https://github.com/turt2live/matrix-appservice-webhooks
+
+matrix_appservice_webhooks_enabled: true
+
+matrix_appservice_webhooks_docker_image: "turt2live/matrix-appservice-webhooks:latest"
+matrix_appservice_webhooks_docker_image_force_pull: "{{ matrix_appservice_webhooks_docker_image.endswith(':latest') }}"
+
+matrix_appservice_webhooks_base_path: "{{ matrix_base_data_path }}/appservice-webhooks"
+matrix_appservice_webhooks_config_path: "{{ matrix_appservice_webhooks_base_path }}/config"
+matrix_appservice_webhooks_data_path: "{{ matrix_appservice_webhooks_base_path }}/data"
+
+# If nginx-proxy is disabled, the bridge itself expects its endpoint to be on its own domain (e.g. "localhost:6789")
+matrix_appservice_webhooks_public_endpoint: /appservice-webhooks
+matrix_appservice_webhooks_inbound_uri_prefix: "{{ matrix_homeserver_url }}{{ matrix_appservice_webhooks_public_endpoint }}"
+
+# Once you make a control room in Matrix, you can get its ID by typing any message and checking its source
+matrix_appservice_webhooks_control_room_id: ''
+matrix_appservice_webhooks_bot_name: 'webhookbot'
+matrix_appservice_webhooks_user_prefix: '_webhook'
+
+# Controls the webhooks_PORT and MATRIX_PORT of the installation
+matrix_appservice_webhooks_matrix_port: 6789
+matrix_appservice_webhooks_webhooks_port: 6788
+
+# Controls whether the appservice-webhooks container exposes its HTTP port (tcp/6788 in the container).
+#
+# Takes an "<ip>:<port>" or "<port>" value (e.g. "127.0.0.1:9999"), or empty string to not expose.
+matrix_appservice_webhooks_container_http_host_bind_port: ''
+
+matrix_appservice_webhooks_homeserver_media_url: "matrix.{{ matrix_domain }}"
+matrix_appservice_webhooks_homeserver_url: "http://matrix-synapse:8008"
+matrix_appservice_webhooks_homeserver_domain: "{{ matrix_domain }}"
+matrix_appservice_webhooks_appservice_url: 'http://matrix-appservice-webhooks'
+
+# A list of extra arguments to pass to the container
+matrix_appservice_webhooks_container_extra_arguments: []
+
+# List of systemd services that matrix-appservice-webhooks.service depends on.
+matrix_appservice_webhooks_systemd_required_services_list: ['docker.service', 'matrix-synapse.service']
+
+# List of systemd services that matrix-appservice-webhooks.service wants
+matrix_appservice_webhooks_systemd_wanted_services_list: []
+
+matrix_appservice_webhooks_appservice_token: ''
+matrix_appservice_webhooks_homeserver_token: ''
+matrix_appservice_webhooks_id_token: ''
+matrix_appservice_webhooks_api_secret: ''
+
+# Logging information (info and verbose is available) default is: info
+matrix_appservice_webhooks_log_level: 'info'
+
+matrix_appservice_webhooks_configuration_yaml: |
+
+  # Configuration specific to the application service. All fields (unless otherwise marked) are required.
+  homeserver:
+    # The domain for the client-server API calls.
+    url: "{{ matrix_appservice_webhooks_homeserver_url }}"
+
+    # The domain part for user IDs on this home server. Usually, but not always, this is the same as the
+    # home server's URL.
+    domain: "{{ matrix_domain }}"
+
+  # Configuration specific to the bridge. All fields (unless otherwise marked) are required.
+  webhookBot:
+    # The localpart to use for the bot. May require re-registering the application service.
+    localpart: "_webhook"
+
+  # Provisioning API options
+  provisioning:
+    # Your secret for the API. Required for all provisioning API requests.
+    secret: '{{ matrix_appservice_webhooks_api_secret }}'
+
+  # Configuration related to the web portion of the bridge. Handles the inbound webhooks
+  web:
+    hookUrlBase:  "{{ matrix_appservice_webhooks_inbound_uri_prefix }}"
+
+  logging:
+    console: true
+    consoleLevel: {{ matrix_appservice_webhooks_log_level }}
+    writeFiles: false
+
+matrix_appservice_webhooks_configuration_extension_yaml: |
+  #
+
+matrix_appservice_webhooks_configuration_extension: "{{ matrix_appservice_webhooks_configuration_extension_yaml|from_yaml if matrix_appservice_webhooks_configuration_extension_yaml|from_yaml else {} }}"
+
+matrix_appservice_webhooks_configuration: "{{ matrix_appservice_webhooks_configuration_yaml|from_yaml|combine(matrix_appservice_webhooks_configuration_extension, recursive=True) }}"
+
+matrix_appservice_webhooks_registration_yaml: |
+  id: "{{ matrix_appservice_webhooks_id_token }}"
+  hs_token: "{{ matrix_appservice_webhooks_homeserver_token }}"
+  as_token: "{{ matrix_appservice_webhooks_appservice_token }}"
+  namespaces:
+    users:
+      - exclusive: true
+        regex: '@{{ matrix_appservice_webhooks_user_prefix }}*:{{ matrix_domain }}'
+    aliases: []
+    rooms: []
+  url: "{{ matrix_appservice_webhooks_appservice_url }}:{{ matrix_appservice_webhooks_matrix_port }}"
+  sender_localpart: _webhook
+  rate_limited: false
+  protocols: null
+
+matrix_appservice_webhooks_registration: "{{ matrix_appservice_webhooks_registration_yaml|from_yaml }}"

roles/matrix-bridge-appservice-webhooks/tasks/init.yml

@@ -0,0 +1,78 @@
+# If the matrix-synapse role is not used, `matrix_synapse_role_executed` won't exist.
+# We don't want to fail in such cases.
+- name: Fail if matrix-synapse role already executed
+  fail:
+    msg: >-
+      The matrix-bridge-appservice-webhooks role needs to execute before the matrix-synapse role.
+  when: "matrix_synapse_role_executed|default(False)"
+
+- set_fact:
+    matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-appservice-webhooks'] }}"
+  when: matrix_appservice_webhooks_enabled|bool
+
+# If the matrix-synapse role is not used, these variables may not exist.
+- set_fact:
+    matrix_synapse_container_extra_arguments: >
+      {{ matrix_synapse_container_extra_arguments|default([]) }}
+      +
+      ["--mount type=bind,src={{ matrix_appservice_webhooks_config_path }}/webhooks-registration.yaml,dst=/matrix-appservice-webhooks-registration.yaml,ro"]
+
+    matrix_synapse_app_service_config_files: >
+      {{ matrix_synapse_app_service_config_files|default([]) }}
+      +
+      {{ ["/matrix-appservice-webhooks-registration.yaml"] }}
+  when: matrix_appservice_webhooks_enabled|bool
+
+# If the matrix-synapse role is not used, `matrix_synapse_role_executed` won't exist.
+# We don't want to fail in such cases.
+- name: Fail if matrix-synapse role already executed
+  fail:
+    msg: >-
+      The matrix-bridge-appservice-webhooks role needs to execute before the matrix-synapse role.
+  when: "matrix_synapse_role_executed|default(False)"
+
+- block:
+  - name: Fail if matrix-nginx-proxy role already executed
+    fail:
+      msg: >-
+        Trying to append webhooks Appservice's reverse-proxying configuration to matrix-nginx-proxy,
+        but it's pointless since the matrix-nginx-proxy role had already executed.
+        To fix this, please change the order of roles in your plabook,
+        so that the matrix-nginx-proxy role would run after the matrix-bridge-appservice-webhooks role.
+    when: matrix_nginx_proxy_role_executed|default(False)|bool
+
+  - name: Generate Matrix Appservice webhooks proxying configuration for matrix-nginx-proxy
+    set_fact:
+      matrix_appservice_webhooks_matrix_nginx_proxy_configuration: |
+        location {{ matrix_appservice_webhooks_public_endpoint }}/ {
+        {% if matrix_nginx_proxy_enabled|default(False) %}
+        {# Use the embedded DNS resolver in Docker containers to discover the service #}
+          resolver 127.0.0.11 valid=5s;
+          proxy_pass {{ matrix_appservice_webhooks_appservice_url }}:{{ matrix_appservice_webhooks_matrix_port }}/;
+        {% else %}
+          {# Generic configuration for use outside of our container setup #}
+          proxy_pass http://127.0.0.1:{{ matrix_appservice_webhooks_matrix_port }}/;
+        {% endif %}
+        }
+
+  - name: Register webhooks Appservice proxying configuration with matrix-nginx-proxy
+    set_fact:
+      matrix_nginx_proxy_proxy_matrix_additional_server_configuration_blocks: |
+        {{
+          matrix_nginx_proxy_proxy_matrix_additional_server_configuration_blocks|default([])
+          +
+          [matrix_appservice_webhooks_matrix_nginx_proxy_configuration]
+        }}
+  tags:
+   - always
+  when: matrix_appservice_webhooks_enabled|bool
+
+- name: Warn about reverse-proxying if matrix-nginx-proxy not used
+  debug:
+    msg: >-
+      NOTE: You've enabled the Matrix webhooks bridge but are not using the matrix-nginx-proxy
+      reverse proxy.
+      Please make sure that you're proxying the `{{ matrix_appservice_webhooks_public_endpoint }}`
+      URL endpoint to the matrix-appservice-webhooks container.
+      You can expose the container's port using the `matrix_appservice_webhooks_container_http_host_bind_port` variable.
+  when: "matrix_appservice_webhooks_enabled|bool and matrix_nginx_proxy_enabled is not defined"

roles/matrix-bridge-appservice-webhooks/tasks/main.yml

@@ -0,0 +1,21 @@
+- import_tasks: "{{ role_path }}/tasks/init.yml"
+  tags:
+    - always
+
+- import_tasks: "{{ role_path }}/tasks/validate_config.yml"
+  when: "run_setup|bool and matrix_appservice_webhooks_enabled|bool"
+  tags:
+    - setup-all
+    - setup-appservice-webhooks
+
+- import_tasks: "{{ role_path }}/tasks/setup_install.yml"
+  when: "run_setup|bool and matrix_appservice_webhooks_enabled|bool"
+  tags:
+    - setup-all
+    - setup-appservice-webhooks
+
+- import_tasks: "{{ role_path }}/tasks/setup_uninstall.yml"
+  when: "run_setup|bool and not matrix_appservice_webhooks_enabled|bool"
+  tags:
+    - setup-all
+    - setup-appservice-webhooks

roles/matrix-bridge-appservice-webhooks/tasks/setup_install.yml

@@ -0,0 +1,64 @@
+---
+
+- name: Ensure Appservice webhooks image is pulled
+  docker_image:
+    name: "{{ matrix_appservice_webhooks_docker_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_appservice_webhooks_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_appservice_webhooks_docker_image_force_pull }}"
+
+- name: Ensure AppService webhooks paths exist
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: 0750
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_username }}"
+  with_items:
+    - "{{ matrix_appservice_webhooks_base_path }}"
+    - "{{ matrix_appservice_webhooks_config_path }}"
+    - "{{ matrix_appservice_webhooks_data_path }}"
+
+- name: Ensure Matrix Appservice webhooks config is installed
+  copy:
+    content: "{{ matrix_appservice_webhooks_configuration|to_nice_yaml }}"
+    dest: "{{ matrix_appservice_webhooks_config_path }}/config.yaml"
+    mode: 0644
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_username }}"
+
+- name: Ensure Matrix Appservice webhooks schema.yml template exists
+  template:
+    src: "{{ role_path }}/templates/schema.yml.j2"
+    dest: "{{ matrix_appservice_webhooks_config_path }}/schema.yml"
+    mode: 0644
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_username }}"
+
+- name: Ensure Matrix Appservice webhooks database.json template exists
+  template:
+    src: "{{ role_path }}/templates/database.json.j2"
+    dest: "{{ matrix_appservice_webhooks_data_path }}/database.json"
+    mode: 0644
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_username }}"
+
+- name: Ensure appservice-webhooks registration.yaml installed
+  copy:
+    content: "{{ matrix_appservice_webhooks_registration|to_nice_yaml }}"
+    dest: "{{ matrix_appservice_webhooks_config_path }}/webhooks-registration.yaml"
+    mode: 0644
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_username }}"
+
+- name: Ensure matrix-appservice-webhooks.service installed
+  template:
+    src: "{{ role_path }}/templates/systemd/matrix-appservice-webhooks.service.j2"
+    dest: "/etc/systemd/system/matrix-appservice-webhooks.service"
+    mode: 0644
+  register: matrix_appservice_webhooks_systemd_service_result
+
+- name: Ensure systemd reloaded after matrix-appservice-webhooks.service installation
+  service:
+    daemon_reload: yes
+  when: "matrix_appservice_webhooks_systemd_service_result.changed"

roles/matrix-bridge-appservice-webhooks/tasks/setup_uninstall.yml

@@ -0,0 +1,24 @@
+---
+
+- name: Check existence of matrix-appservice-webhooks service
+  stat:
+    path: "/etc/systemd/system/matrix-appservice-webhooks.service"
+  register: matrix_appservice_webhooks_service_stat
+
+- name: Ensure matrix-appservice-webhooks is stopped
+  service:
+    name: matrix-appservice-webhooks
+    state: stopped
+    daemon_reload: yes
+  when: "matrix_appservice_webhooks_service_stat.stat.exists"
+
+- name: Ensure matrix-appservice-webhooks.service doesn't exist
+  file:
+    path: "/etc/systemd/system/matrix-appservice-webhooks.service"
+    state: absent
+  when: "matrix_appservice_webhooks_service_stat.stat.exists"
+
+- name: Ensure systemd reloaded after matrix-appservice-webhooks.service removal
+  service:
+    daemon_reload: yes
+  when: "matrix_appservice_webhooks_service_stat.stat.exists"

roles/matrix-bridge-appservice-webhooks/tasks/validate_config.yml

@@ -0,0 +1,12 @@
+---
+
+- name: Fail if required settings not defined
+  fail:
+    msg: >-
+      You need to define a required configuration setting (`{{ item }}`).
+  when: "vars[item] == ''"
+  with_items:
+    - "matrix_appservice_webhooks_appservice_token"
+    - "matrix_appservice_webhooks_homeserver_token"
+    - "matrix_appservice_webhooks_id_token"
+    - "matrix_appservice_webhooks_api_secret"

roles/matrix-bridge-appservice-webhooks/templates/database.json.j2

@@ -0,0 +1,13 @@
+{
+  "defaultEnv": {
+    "ENV": "NODE_ENV"
+  },
+  "development": {
+    "driver": "sqlite3",
+    "filename": "/data/development.db"
+  },
+  "production": {
+    "driver": "sqlite3",
+    "filename": "/data/production.db"
+  }
+}

roles/matrix-bridge-appservice-webhooks/templates/schema.yml.j2

@@ -0,0 +1,54 @@
+"$schema": "http://json-schema.org/draft-04/schema#"
+type: "object"
+properties:
+  provisioning:
+    type: "object"
+    properties:
+      secret:
+        type: "string"
+  homeserver:
+    type: "object"
+    properties:
+      domain:
+        type: "string"
+      url:
+        type: "string"
+      mediaUrl:
+        type: "string"
+  web:
+    type: "object"
+    properties:
+      hookUrlBase:
+        type: "string"
+  webhookBot:
+    type: "object"
+    properties:
+      localpart:
+        type: "string"
+      appearance:
+        type: "object"
+        properties:
+          displayName:
+            type: "string"
+          avatarUrl:
+            type: "string"
+  logging:
+    type: "object"
+    properties:
+      file:
+        type: "string"
+      console:
+        type: "boolean"
+      consoleLevel:
+        type: "string"
+      fileLevel:
+        type: "string"
+      writeFiles:
+        type: "boolean"
+      rotate:
+        type: "object"
+        properties:
+          size:
+            type: "number"
+          count:
+            type: "number"

roles/matrix-bridge-appservice-webhooks/templates/systemd/matrix-appservice-webhooks.service.j2

@@ -0,0 +1,43 @@
+#jinja2: lstrip_blocks: "True"
+[Unit]
+Description=Matrix Appservice webhooks server
+{% for service in matrix_appservice_webhooks_systemd_required_services_list %}
+Requires={{ service }}
+After={{ service }}
+{% endfor %}
+{% for service in matrix_appservice_webhooks_systemd_wanted_services_list %}
+Wants={{ service }}
+{% endfor %}
+
+[Service]
+Type=simple
+ExecStartPre=-/usr/bin/docker kill matrix-appservice-webhooks
+ExecStartPre=-/usr/bin/docker rm matrix-appservice-webhooks
+
+# Intentional delay, so that the homeserver (we likely depend on) can manage to start.
+ExecStartPre=/bin/sleep 5
+
+ExecStart=/usr/bin/docker run --rm --name matrix-appservice-webhooks \
+			--log-driver=none \
+			--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
+			--cap-drop=ALL \
+			--network={{ matrix_docker_network }} \
+			{% if matrix_appservice_webhooks_container_http_host_bind_port %}
+			-p {{ matrix_appservice_webhooks_container_http_host_bind_port }}:{{matrix_appservice_webhooks_matrix_port}} \
+			{% endif %}
+			-v {{ matrix_appservice_webhooks_config_path }}:/config:z \
+			-v {{ matrix_appservice_webhooks_data_path }}:/data:z \
+			{% for arg in matrix_appservice_webhooks_container_extra_arguments %}
+			{{ arg }} \
+			{% endfor %}
+			{{ matrix_appservice_webhooks_docker_image }} \
+			node index.js -p {{ matrix_appservice_webhooks_matrix_port }} -c /config/config.yaml -f /config/webhooks-registration.yaml
+
+ExecStop=-/usr/bin/docker kill matrix-appservice-webhooks
+ExecStop=-/usr/bin/docker rm matrix-appservice-webhooks
+Restart=always
+RestartSec=30
+SyslogIdentifier=matrix-appservice-webhooks
+
+[Install]
+WantedBy=multi-user.target

setup.yml

@@ -10,6 +10,7 @@
     - matrix-corporal
     - matrix-bridge-appservice-discord
     - matrix-bridge-appservice-slack
+    - matrix-bridge-appservice-webhooks
     - matrix-bridge-appservice-irc
     - matrix-bridge-mautrix-facebook
     - matrix-bridge-mautrix-hangouts