* Enable Internal Admin API Access separately from Public access. * Add Config variable for Draupnir Hijack command And also make the internal admin API be automatically activated when this capability is used. * Apply suggestions from code review Co-authored-by: Slavi Pantaleev <slavi@devture.com> * Further Refine Internal Admin API * Add Non Worker Labels for Internal Admin API * Variable Rename * Add validation rules for Internal Synapse admin API * Add Draupnir Admin API required config validation. * Override `matrix_synapse_reverse_proxy_companion_container_labels_internal_client_synapse_admin_api_traefik_entrypoints` via group vars * Wire `matrix_bot_draupnir_admin_api_enabled` to `matrix_bot_draupnir_config_admin_enableMakeRoomAdminCommand` in Draupnir's `defaults/main.yml` * Remove unnecessary `matrix_bot_draupnir_admin_api_enabled` override from `group_vars/matrix_servers` The same value is now (more appropriately) defined in Draupnir's `defaults/main.yml` file anyway. * Add additional condition (`matrix_bot_draupnir_enabled`) for enabling `matrix_synapse_container_labels_internal_client_synapse_admin_api_enabled` * Use a separate task for validating `matrix_bot_draupnir_admin_api_enabled` when `matrix_bot_draupnir_config_admin_enableMakeRoomAdminCommand` The other task deals with checking for null and not-blank and can't handle booleans properly. --------- Co-authored-by: Slavi Pantaleev <slavi@devture.com>pull/4181/head
| @@ -4855,6 +4855,8 @@ matrix_synapse_container_labels_public_client_root_redirection_enabled: "{{ matr | |||||
| matrix_synapse_container_labels_public_client_root_redirection_url: "{{ (('https://' if matrix_playbook_ssl_enabled else 'http://') + matrix_server_fqn_element) if matrix_client_element_enabled else '' }}" | matrix_synapse_container_labels_public_client_root_redirection_url: "{{ (('https://' if matrix_playbook_ssl_enabled else 'http://') + matrix_server_fqn_element) if matrix_client_element_enabled else '' }}" | ||||
| matrix_synapse_container_labels_public_client_synapse_admin_api_enabled: "{{ matrix_synapse_admin_enabled }}" | matrix_synapse_container_labels_public_client_synapse_admin_api_enabled: "{{ matrix_synapse_admin_enabled }}" | ||||
| matrix_synapse_container_labels_internal_client_synapse_admin_api_enabled: "{{ (matrix_bot_draupnir_enabled and matrix_bot_draupnir_admin_api_enabled) }}" | |||||
| matrix_synapse_container_labels_internal_client_synapse_admin_api_traefik_entrypoints: "{{ matrix_playbook_internal_matrix_client_api_traefik_entrypoint_name }}" | |||||
| matrix_synapse_container_labels_public_federation_api_traefik_hostname: "{{ matrix_server_fqn_matrix_federation }}" | matrix_synapse_container_labels_public_federation_api_traefik_hostname: "{{ matrix_server_fqn_matrix_federation }}" | ||||
| matrix_synapse_container_labels_public_federation_api_traefik_entrypoints: "{{ matrix_federation_traefik_entrypoint_name }}" | matrix_synapse_container_labels_public_federation_api_traefik_entrypoints: "{{ matrix_federation_traefik_entrypoint_name }}" | ||||
| @@ -5041,6 +5043,9 @@ matrix_synapse_reverse_proxy_companion_container_labels_traefik_compression_midd | |||||
| matrix_synapse_reverse_proxy_companion_container_labels_public_client_synapse_client_api_enabled: "{{ matrix_synapse_container_labels_public_client_synapse_client_api_enabled }}" | matrix_synapse_reverse_proxy_companion_container_labels_public_client_synapse_client_api_enabled: "{{ matrix_synapse_container_labels_public_client_synapse_client_api_enabled }}" | ||||
| matrix_synapse_reverse_proxy_companion_container_labels_public_client_synapse_admin_api_enabled: "{{ matrix_synapse_container_labels_public_client_synapse_admin_api_enabled }}" | matrix_synapse_reverse_proxy_companion_container_labels_public_client_synapse_admin_api_enabled: "{{ matrix_synapse_container_labels_public_client_synapse_admin_api_enabled }}" | ||||
| matrix_synapse_reverse_proxy_companion_container_labels_internal_client_synapse_admin_api_enabled: "{{ matrix_synapse_container_labels_internal_client_synapse_admin_api_enabled }}" | |||||
| matrix_synapse_reverse_proxy_companion_container_labels_internal_client_synapse_admin_api_traefik_entrypoints: "{{ matrix_playbook_internal_matrix_client_api_traefik_entrypoint_name }}" | |||||
| matrix_synapse_reverse_proxy_companion_container_labels_public_federation_api_traefik_entrypoints: "{{ matrix_synapse_container_labels_public_federation_api_traefik_entrypoints }}" | matrix_synapse_reverse_proxy_companion_container_labels_public_federation_api_traefik_entrypoints: "{{ matrix_synapse_container_labels_public_federation_api_traefik_entrypoints }}" | ||||
| matrix_synapse_reverse_proxy_companion_container_labels_public_federation_api_traefik_tls: "{{ matrix_synapse_container_labels_public_federation_api_traefik_tls }}" | matrix_synapse_reverse_proxy_companion_container_labels_public_federation_api_traefik_tls: "{{ matrix_synapse_container_labels_public_federation_api_traefik_tls }}" | ||||
| @@ -117,6 +117,13 @@ matrix_bot_draupnir_config_rawHomeserverUrl: "" # noqa var-naming | |||||
| # It is exposed here because it is common enough to be valid to expose. | # It is exposed here because it is common enough to be valid to expose. | ||||
| matrix_bot_draupnir_config_disableServerACL: false # noqa var-naming | matrix_bot_draupnir_config_disableServerACL: false # noqa var-naming | ||||
| # Control if Draupnir wants for the Synapse Admin API to be exposed internally to containers, therefore giving Draupnir Access. | |||||
| matrix_bot_draupnir_admin_api_enabled: "{{ matrix_bot_draupnir_config_admin_enableMakeRoomAdminCommand }}" | |||||
| # Controls if the Draupnir room hijack command is activated or not. | |||||
| # Also see `matrix_bot_draupnir_admin_api_enabled`. | |||||
| matrix_bot_draupnir_config_admin_enableMakeRoomAdminCommand: false # noqa var-naming | |||||
| # Controls if the room state backing store is activated. | # Controls if the room state backing store is activated. | ||||
| # Room state backing store makes restarts of the bot lightning fast as the bot does not suffer from amnesia. | # Room state backing store makes restarts of the bot lightning fast as the bot does not suffer from amnesia. | ||||
| # This config option has diminished improvements for bots on extremely fast homeservers or very very small bots on fast homeservers. | # This config option has diminished improvements for bots on extremely fast homeservers or very very small bots on fast homeservers. | ||||
| @@ -39,6 +39,11 @@ | |||||
| - {'name': 'matrix_bot_draupnir_pantalaimon_password', when: "{{ matrix_bot_draupnir_pantalaimon_use }}"} | - {'name': 'matrix_bot_draupnir_pantalaimon_password', when: "{{ matrix_bot_draupnir_pantalaimon_use }}"} | ||||
| when: "item.when | bool and (vars[item.name] == '' or vars[item.name] is none)" | when: "item.when | bool and (vars[item.name] == '' or vars[item.name] is none)" | ||||
| - name: Fail if Draupnir room hijacking enabled without enabling the Synapse Admin API | |||||
| ansible.builtin.fail: | |||||
| msg: "When matrix_bot_draupnir_config_admin_enableMakeRoomAdminCommand is enabled, matrix_bot_draupnir_admin_api_enabled must also be enabled" | |||||
| when: "matrix_bot_draupnir_config_admin_enableMakeRoomAdminCommand | bool and not matrix_bot_draupnir_admin_api_enabled | bool" | |||||
| - name: Fail if inappropriate variables are defined | - name: Fail if inappropriate variables are defined | ||||
| ansible.builtin.fail: | ansible.builtin.fail: | ||||
| msg: "The `{{ item.name }}` variable must be undefined or have a null value." | msg: "The `{{ item.name }}` variable must be undefined or have a null value." | ||||
| @@ -131,18 +131,15 @@ protectAllJoinedRooms: false | |||||
| # of the homeserver may be more impacted. | # of the homeserver may be more impacted. | ||||
| backgroundDelayMS: 500 | backgroundDelayMS: 500 | ||||
| # FIXME: This configuration option is currently broken in the playbook as admin APIs cannot | |||||
| # be accessed from containers. See https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/3389 | |||||
| # and https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/3308 | |||||
| # Server administration commands, these commands will only work if Draupnir is | # Server administration commands, these commands will only work if Draupnir is | ||||
| # a global server administrator, and the bot's server is a Synapse instance. | # a global server administrator, and the bot's server is a Synapse instance. | ||||
| #admin: | |||||
| # # Whether or not Draupnir can temporarily take control of any eligible account from the local homeserver who's in the room | |||||
| # # (with enough permissions) to "make" a user an admin. | |||||
| # # | |||||
| # # This only works if a local user with enough admin permissions is present in the room. | |||||
| # enableMakeRoomAdminCommand: false | |||||
| # | |||||
| admin: | |||||
| # Whether or not Draupnir can temporarily take control of any eligible account from the local homeserver who's in the room | |||||
| # (with enough permissions) to "make" a user an admin. | |||||
| # | |||||
| # This only works if a local user with enough admin permissions is present in the room. | |||||
| enableMakeRoomAdminCommand: {{ matrix_bot_draupnir_config_enableMakeRoomAdminCommand | to_json }} | |||||
| # Misc options for command handling and commands | # Misc options for command handling and commands | ||||
| commands: | commands: | ||||
| # Whether or not the `!draupnir` prefix is necessary to submit commands. | # Whether or not the `!draupnir` prefix is necessary to submit commands. | ||||
| @@ -5,6 +5,7 @@ | |||||
| # SPDX-FileCopyrightText: 2024 Charles Wright | # SPDX-FileCopyrightText: 2024 Charles Wright | ||||
| # SPDX-FileCopyrightText: 2024 David Mehren | # SPDX-FileCopyrightText: 2024 David Mehren | ||||
| # SPDX-FileCopyrightText: 2024 Michael Hollister | # SPDX-FileCopyrightText: 2024 Michael Hollister | ||||
| # SPDX-FileCopyrightText: 2024 - 2025 Catalan Lover <catalanlover@protonmail.com> | |||||
| # | # | ||||
| # SPDX-License-Identifier: AGPL-3.0-or-later | # SPDX-License-Identifier: AGPL-3.0-or-later | ||||
| @@ -119,6 +120,14 @@ matrix_synapse_reverse_proxy_companion_container_labels_public_client_synapse_ad | |||||
| matrix_synapse_reverse_proxy_companion_container_labels_public_client_synapse_admin_api_traefik_tls: "{{ matrix_synapse_reverse_proxy_companion_container_labels_public_client_synapse_admin_api_traefik_entrypoints != 'web' }}" | matrix_synapse_reverse_proxy_companion_container_labels_public_client_synapse_admin_api_traefik_tls: "{{ matrix_synapse_reverse_proxy_companion_container_labels_public_client_synapse_admin_api_traefik_entrypoints != 'web' }}" | ||||
| matrix_synapse_reverse_proxy_companion_container_labels_public_client_synapse_admin_api_traefik_tls_certResolver: "{{ matrix_synapse_reverse_proxy_companion_container_labels_traefik_tls_certResolver }}" # noqa var-naming | matrix_synapse_reverse_proxy_companion_container_labels_public_client_synapse_admin_api_traefik_tls_certResolver: "{{ matrix_synapse_reverse_proxy_companion_container_labels_traefik_tls_certResolver }}" # noqa var-naming | ||||
| # Controls whether labels will be added that expose the /_synapse/admin paths on the internal Traefik entrypoint. | |||||
| # This is similar to `matrix_synapse_container_labels_public_client_api_enabled`, but the entrypoint and intent is different. | |||||
| matrix_synapse_reverse_proxy_companion_container_labels_internal_client_synapse_admin_api_enabled: false | |||||
| matrix_synapse_reverse_proxy_companion_container_labels_internal_client_synapse_admin_api_traefik_path_prefix: "{{ matrix_synapse_container_labels_internal_client_synapse_admin_api_traefik_path_prefix }}" | |||||
| matrix_synapse_reverse_proxy_companion_container_labels_internal_client_synapse_admin_api_traefik_rule: "PathPrefix(`{{ matrix_synapse_reverse_proxy_companion_container_labels_internal_client_synapse_admin_api_traefik_path_prefix }}`)" | |||||
| matrix_synapse_reverse_proxy_companion_container_labels_internal_client_synapse_admin_api_traefik_priority: 0 | |||||
| matrix_synapse_reverse_proxy_companion_container_labels_internal_client_synapse_admin_api_traefik_entrypoints: "" | |||||
| # Controls whether labels will be added that expose the Server-Server API (Federation API). | # Controls whether labels will be added that expose the Server-Server API (Federation API). | ||||
| matrix_synapse_reverse_proxy_companion_container_labels_public_federation_api_enabled: "{{ matrix_synapse_reverse_proxy_companion_federation_api_enabled }}" | matrix_synapse_reverse_proxy_companion_container_labels_public_federation_api_enabled: "{{ matrix_synapse_reverse_proxy_companion_federation_api_enabled }}" | ||||
| matrix_synapse_reverse_proxy_companion_container_labels_public_federation_api_traefik_hostname: "{{ matrix_synapse_reverse_proxy_companion_container_labels_traefik_hostname }}" | matrix_synapse_reverse_proxy_companion_container_labels_public_federation_api_traefik_hostname: "{{ matrix_synapse_reverse_proxy_companion_container_labels_traefik_hostname }}" | ||||
| @@ -21,5 +21,7 @@ | |||||
| - {'name': 'matrix_synapse_reverse_proxy_companion_container_labels_public_federation_api_traefik_entrypoints', when: "{{ matrix_synapse_reverse_proxy_companion_container_labels_public_federation_api_enabled }}"} | - {'name': 'matrix_synapse_reverse_proxy_companion_container_labels_public_federation_api_traefik_entrypoints', when: "{{ matrix_synapse_reverse_proxy_companion_container_labels_public_federation_api_enabled }}"} | ||||
| - {'name': 'matrix_synapse_reverse_proxy_companion_container_labels_internal_client_api_traefik_entrypoints', when: "{{ matrix_synapse_reverse_proxy_companion_container_labels_internal_client_api_enabled }}"} | - {'name': 'matrix_synapse_reverse_proxy_companion_container_labels_internal_client_api_traefik_entrypoints', when: "{{ matrix_synapse_reverse_proxy_companion_container_labels_internal_client_api_enabled }}"} | ||||
| - {'name': 'matrix_synapse_reverse_proxy_companion_container_labels_internal_client_synapse_admin_api_traefik_entrypoints', when: "{{ matrix_synapse_container_labels_internal_client_synapse_admin_api_enabled }}"} | |||||
| - {'name': 'matrix_synapse_reverse_proxy_companion_container_labels_traefik_compression_middleware_name', when: "{{ matrix_synapse_reverse_proxy_companion_container_labels_traefik_compression_middleware_enabled }}"} | - {'name': 'matrix_synapse_reverse_proxy_companion_container_labels_traefik_compression_middleware_name', when: "{{ matrix_synapse_reverse_proxy_companion_container_labels_traefik_compression_middleware_enabled }}"} | ||||
| @@ -1,5 +1,6 @@ | |||||
| {# | {# | ||||
| SPDX-FileCopyrightText: 2024 Slavi Pantaleev | SPDX-FileCopyrightText: 2024 Slavi Pantaleev | ||||
| SPDX-FileCopyrightText: 2024 - 2025 Catalan Lover <catalanlover@protonmail.com> | |||||
| SPDX-License-Identifier: AGPL-3.0-or-later | SPDX-License-Identifier: AGPL-3.0-or-later | ||||
| #} | #} | ||||
| @@ -155,6 +156,29 @@ traefik.http.routers.matrix-synapse-reverse-proxy-companion-public-client-synaps | |||||
| ############################################################ | ############################################################ | ||||
| {% endif %} | {% endif %} | ||||
| {% if matrix_synapse_reverse_proxy_companion_container_labels_internal_client_synapse_admin_api_enabled %} | |||||
| ############################################################ | |||||
| # # | |||||
| # Internal Synapse Admin API (/_synapse/admin) # | |||||
| # # | |||||
| ############################################################ | |||||
| traefik.http.routers.matrix-synapse-reverse-proxy-companion-internal-client-synapse-admin-api.rule={{ matrix_synapse_reverse_proxy_companion_container_labels_internal_admin_api_traefik_rule }} | |||||
| {% if matrix_synapse_reverse_proxy_companion_container_labels_internal_client_synapse_admin_api_traefik_priority | int > 0 %} | |||||
| traefik.http.routers.matrix-synapse-reverse-proxy-companion-public-client-synapse-admin-api.priority={{ matrix_synapse_reverse_proxy_companion_container_labels_internal_client_synapse_admin_api_traefik_priority }} | |||||
| {% endif %} | |||||
| traefik.http.routers.matrix-synapse-reverse-proxy-companion-internal-client-synapse-admin-api.service=matrix-synapse-reverse-proxy-companion-client-api | |||||
| traefik.http.routers.matrix-synapse-reverse-proxy-companion-internal-client-synapse-admin-api.entrypoints={{ matrix_synapse_reverse_proxy_companion_container_labels_internal_client_synapse_admin_api_traefik_entrypoints }} | |||||
| ############################################################ | |||||
| # # | |||||
| # /Internal Synapse Admin API (/_synapse/admin) # | |||||
| # # | |||||
| ############################################################ | |||||
| {% endif %} | |||||
| {% if matrix_synapse_reverse_proxy_companion_container_labels_public_federation_api_enabled %} | {% if matrix_synapse_reverse_proxy_companion_container_labels_public_federation_api_enabled %} | ||||
| ############################################################ | ############################################################ | ||||
| @@ -268,6 +268,16 @@ matrix_synapse_container_labels_public_client_synapse_admin_api_traefik_entrypoi | |||||
| matrix_synapse_container_labels_public_client_synapse_admin_api_traefik_tls: "{{ matrix_synapse_container_labels_public_client_synapse_admin_api_traefik_entrypoints != 'web' }}" | matrix_synapse_container_labels_public_client_synapse_admin_api_traefik_tls: "{{ matrix_synapse_container_labels_public_client_synapse_admin_api_traefik_entrypoints != 'web' }}" | ||||
| matrix_synapse_container_labels_public_client_synapse_admin_api_traefik_tls_certResolver: "{{ matrix_synapse_container_labels_traefik_tls_certResolver }}" # noqa var-naming | matrix_synapse_container_labels_public_client_synapse_admin_api_traefik_tls_certResolver: "{{ matrix_synapse_container_labels_traefik_tls_certResolver }}" # noqa var-naming | ||||
| # Controls whether labels will be added that expose the /_synapse/admin paths on the internal Traefik entrypoint. | |||||
| # Following these recommendations (https://github.com/element-hq/synapse/blob/master/docs/reverse_proxy.md), by default, we don't. | |||||
| # Regardless of whether this is enabled, it may or may not take effect due to the value of other variables. | |||||
| matrix_synapse_container_labels_internal_client_synapse_admin_api_enabled: false | |||||
| matrix_synapse_container_labels_internal_client_synapse_admin_api_traefik_path_prefix: /_synapse/admin | |||||
| matrix_synapse_container_labels_internal_client_synapse_admin_api_traefik_rule: "PathPrefix(`{{ matrix_synapse_container_labels_internal_client_synapse_admin_api_traefik_path_prefix }}`)" | |||||
| matrix_synapse_container_labels_internal_client_synapse_admin_api_traefik_priority: 0 | |||||
| matrix_synapse_container_labels_internal_client_synapse_admin_api_traefik_entrypoints: "" | |||||
| # Controls whether labels will be added that expose the Server-Server API (Federation API). | # Controls whether labels will be added that expose the Server-Server API (Federation API). | ||||
| # Regardless of whether this is enabled, it may or may not take effect due to the value of other variables. | # Regardless of whether this is enabled, it may or may not take effect due to the value of other variables. | ||||
| # See `matrix_synapse_container_labels_traefik_enabled` or `matrix_synapse_container_labels_matrix_related_labels_enabled` | # See `matrix_synapse_container_labels_traefik_enabled` or `matrix_synapse_container_labels_matrix_related_labels_enabled` | ||||
| @@ -35,5 +35,6 @@ SPDX-FileCopyrightText: 2023 Luke D Iremadze | |||||
| SPDX-FileCopyrightText: 2023 Samuel Meenzen | SPDX-FileCopyrightText: 2023 Samuel Meenzen | ||||
| SPDX-FileCopyrightText: 2024 - 2025 Suguru Hirahara | SPDX-FileCopyrightText: 2024 - 2025 Suguru Hirahara | ||||
| SPDX-FileCopyrightText: 2024 Charles Wright | SPDX-FileCopyrightText: 2024 Charles Wright | ||||
| SPDX-FileCopyrightText: 2025 Catalan Lover <catalanlover@protonmail.com> | |||||
| SPDX-License-Identifier: AGPL-3.0-or-later | SPDX-License-Identifier: AGPL-3.0-or-later | ||||
| @@ -28,6 +28,7 @@ | |||||
| - {'name': 'matrix_synapse_container_labels_public_client_api_traefik_hostname', when: "{{ matrix_synapse_container_labels_public_client_api_enabled }}"} | - {'name': 'matrix_synapse_container_labels_public_client_api_traefik_hostname', when: "{{ matrix_synapse_container_labels_public_client_api_enabled }}"} | ||||
| - {'name': 'matrix_synapse_container_labels_internal_client_api_traefik_entrypoints', when: "{{ matrix_synapse_container_labels_internal_client_api_enabled }}"} | - {'name': 'matrix_synapse_container_labels_internal_client_api_traefik_entrypoints', when: "{{ matrix_synapse_container_labels_internal_client_api_enabled }}"} | ||||
| - {'name': 'matrix_synapse_container_labels_internal_client_synapse_admin_api_traefik_entrypoints', when: "{{ matrix_synapse_container_labels_internal_client_synapse_admin_api_enabled }}"} | |||||
| - {'name': 'matrix_synapse_container_labels_public_client_synapse_client_api_traefik_hostname', when: "{{ matrix_synapse_container_labels_public_client_synapse_client_api_enabled }}"} | - {'name': 'matrix_synapse_container_labels_public_client_synapse_client_api_traefik_hostname', when: "{{ matrix_synapse_container_labels_public_client_synapse_client_api_enabled }}"} | ||||
| - {'name': 'matrix_synapse_container_labels_public_client_synapse_admin_api_traefik_hostname', when: "{{ matrix_synapse_container_labels_public_client_synapse_admin_api_enabled }}"} | - {'name': 'matrix_synapse_container_labels_public_client_synapse_admin_api_traefik_hostname', when: "{{ matrix_synapse_container_labels_public_client_synapse_admin_api_enabled }}"} | ||||
| @@ -211,6 +211,30 @@ traefik.http.routers.matrix-synapse-public-client-synapse-admin-api.tls.certReso | |||||
| {% endif %} | {% endif %} | ||||
| {% if matrix_synapse_container_labels_internal_client_synapse_admin_api_enabled %} | |||||
| ############################################################ | |||||
| # # | |||||
| # Internal Synapse Admin API (/_synapse/admin) # | |||||
| # # | |||||
| ############################################################ | |||||
| traefik.http.routers.matrix-synapse-internal-client-synapse-admin-api.rule={{ matrix_synapse_container_labels_internal_client_synapse_admin_api_traefik_rule }} | |||||
| {% if matrix_synapse_container_labels_internal_client_synapse_admin_api_traefik_priority | int > 0 %} | |||||
| traefik.http.routers.matrix-synapse-internal-client-synapse-admin-api.priority={{ matrix_synapse_container_labels_internal_client_synapse_admin_api_traefik_priority }} | |||||
| {% endif %} | |||||
| traefik.http.routers.matrix-synapse-internal-client-synapse-admin-api.service=matrix-synapse-client-api | |||||
| traefik.http.routers.matrix-synapse-internal-client-synapse-admin-api.entrypoints={{ matrix_synapse_container_labels_internal_client_synapse_admin_api_traefik_entrypoints }} | |||||
| ############################################################ | |||||
| # # | |||||
| # /Internal Synapse Admin API (/_synapse/admin) # | |||||
| # # | |||||
| ############################################################ | |||||
| {% endif %} | |||||
| {% if matrix_synapse_container_labels_public_federation_api_enabled %} | {% if matrix_synapse_container_labels_public_federation_api_enabled %} | ||||
| ############################################################ | ############################################################ | ||||
| # # | # # | ||||