Share the Molecule scaffolding and add `just molecule`

Three things that would not have scaled to 70 roles:

- The Python and Ansible dependency pins were about to be copied into every
  role. They now live once in molecule-shared/, which scenarios reference
  relatively, so they cannot drift apart.
- The helper container images used for probing were hardcoded inline. They
  are pinned once in molecule-shared/vars.yml, carry `# renovate:`
  annotations, and a custom manager in .github/renovate.json keeps them
  current - verified with a local Renovate dry run, which offers
  curl 8.11.1 -> 8.21.0 and python 3.13 -> 3.14-alpine. Seventy invisible
  hardcodes is the blindness class we have been removing elsewhere.
- Running a scenario meant knowing the venv and cd incantation. `just
  molecule <role>` does it, and with no argument lists the roles that have
  a scenario.

Molecule is deliberately not wired into prek: a run takes minutes, pulls
images and needs Docker, which is fine on request and not fine per commit.

docs/molecule-testing.md covers how to run and write these, including the
four things a role here needs that a standalone role does not. AGENTS.md
points at it rather than carrying the detail.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

.github/renovate.json

@@ -1,72 +1,84 @@
 {
-	"$schema": "https://docs.renovatebot.com/renovate-schema.json",
-	"extends": [
-		"config:recommended",
-		"helpers:pinGitHubActionDigests"
-	],
-	"labels": [
-		"dependencies"
-	],
-	"customManagers": [
-		{
-			"customType": "regex",
-			"managerFilePatterns": [
-				"/defaults/main.yml$/"
-			],
-			"matchStrings": [
-				"# renovate: datasource=(?<datasource>[a-z-.]+?) depName=(?<depName>[^\\s]+?)(?: (?:lookupName|packageName)=(?<packageName>[^\\s]+?))?(?: versioning=(?<versioning>[a-z-0-9]+?))?\\s+[A-Za-z0-9_]+?(?:_version|_tag)\\s*:\\s*[\"']?(?<currentValue>.+?)[\"']?\\s"
-			]
-		}
-	],
-	"packageRules": [
-		{
-			"ignoreUnstable": false,
-			"versioning": "loose",
-			"matchSourceUrls": [
-				"https://github.com/devture/com.devture.ansible.role{/,}**",
-				"https://github.com/mother-of-all-self-hosting{/,}**"
-			]
-		},
-		{
-			"description": "mautrix images are dual-tagged (v0.YYMM.PATCH and vYY.MM[.PATCH]). Stick to the v0 scheme: it matches the git tags (needed for self-building) and has a consistent number of components (the calver tags do not, which makes Renovate stop offering updates).",
-			"matchPackageNames": [
-				"dock.mau.dev/mautrix/**"
-			],
-			"allowedVersions": "/^v0\\./"
-		},
-		{
-			"description": "Automerge version bumps of roles maintained in the MASH organization and of devture roles (via branch push - no PR)",
-			"matchManagers": ["ansible-galaxy"],
-			"matchSourceUrls": [
-				"https://github.com/devture/com.devture.ansible.role{/,}**",
-				"https://github.com/mother-of-all-self-hosting{/,}**"
-			],
-			"automerge": true,
-			"automergeType": "branch"
-		},
-		{
-			"description": "Housekeeping updates merge via branch push (no PR, no email); a failure on the branch still surfaces as a PR. i18n/requirements.txt bumps are exercised by the Matrix i18n workflow before merging.",
-			"matchFileNames": [
-				".github/workflows/close-stale-issues.yml",
-				".github/workflows/i18n.yml",
-				".github/workflows/lock-threads.yml",
-				".github/workflows/matrix.yml",
-				".github/workflows/update-translations.yml",
-				"flake.lock",
-				"i18n/requirements.txt",
-				"mise.toml"
-			],
-			"automerge": true,
-			"automergeType": "branch"
-		}
-	],
-	"pre-commit": {
-		"enabled": true
-	},
-	"nix": {
-		"enabled": true,
-		"lockFileMaintenance": {
-			"enabled": true
-		}
-	}
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:recommended",
+    "helpers:pinGitHubActionDigests"
+  ],
+  "labels": [
+    "dependencies"
+  ],
+  "customManagers": [
+    {
+      "customType": "regex",
+      "managerFilePatterns": [
+        "/defaults/main.yml$/"
+      ],
+      "matchStrings": [
+        "# renovate: datasource=(?<datasource>[a-z-.]+?) depName=(?<depName>[^\\s]+?)(?: (?:lookupName|packageName)=(?<packageName>[^\\s]+?))?(?: versioning=(?<versioning>[a-z-0-9]+?))?\\s+[A-Za-z0-9_]+?(?:_version|_tag)\\s*:\\s*[\"']?(?<currentValue>.+?)[\"']?\\s"
+      ]
+    },
+    {
+      "description": "Helper container images used by the Molecule scenarios. They are pinned once in molecule-shared/vars.yml rather than inline in each role's verify.yml, so this manager keeps that single pin current.",
+      "customType": "regex",
+      "managerFilePatterns": [
+        "/^molecule-shared/vars\\.yml$/"
+      ],
+      "matchStrings": [
+        "# renovate: datasource=(?<datasource>[a-z-.]+?) depName=(?<depName>[^\\s]+?)\\s+[A-Za-z0-9_]+?\\s*:\\s*[\"'][^:]+:(?<currentValue>[^\"']+)[\"']"
+      ]
+    }
+  ],
+  "packageRules": [
+    {
+      "ignoreUnstable": false,
+      "versioning": "loose",
+      "matchSourceUrls": [
+        "https://github.com/devture/com.devture.ansible.role{/,}**",
+        "https://github.com/mother-of-all-self-hosting{/,}**"
+      ]
+    },
+    {
+      "description": "mautrix images are dual-tagged (v0.YYMM.PATCH and vYY.MM[.PATCH]). Stick to the v0 scheme: it matches the git tags (needed for self-building) and has a consistent number of components (the calver tags do not, which makes Renovate stop offering updates).",
+      "matchPackageNames": [
+        "dock.mau.dev/mautrix/**"
+      ],
+      "allowedVersions": "/^v0\\./"
+    },
+    {
+      "description": "Automerge version bumps of roles maintained in the MASH organization and of devture roles (via branch push - no PR)",
+      "matchManagers": [
+        "ansible-galaxy"
+      ],
+      "matchSourceUrls": [
+        "https://github.com/devture/com.devture.ansible.role{/,}**",
+        "https://github.com/mother-of-all-self-hosting{/,}**"
+      ],
+      "automerge": true,
+      "automergeType": "branch"
+    },
+    {
+      "description": "Housekeeping updates merge via branch push (no PR, no email); a failure on the branch still surfaces as a PR. i18n/requirements.txt bumps are exercised by the Matrix i18n workflow before merging.",
+      "matchFileNames": [
+        ".github/workflows/close-stale-issues.yml",
+        ".github/workflows/i18n.yml",
+        ".github/workflows/lock-threads.yml",
+        ".github/workflows/matrix.yml",
+        ".github/workflows/update-translations.yml",
+        "flake.lock",
+        "i18n/requirements.txt",
+        "mise.toml"
+      ],
+      "automerge": true,
+      "automergeType": "branch"
+    }
+  ],
+  "pre-commit": {
+    "enabled": true
+  },
+  "nix": {
+    "enabled": true,
+    "lockFileMaintenance": {
+      "enabled": true
+    }
+  }
 }

AGENTS.md

@@ -19,6 +19,7 @@ An Ansible playbook that installs and manages a Matrix homeserver and dozens of
 - `roles/galaxy/`: external roles, downloaded according to `requirements.yml` via [agru](https://github.com/etkecc/agru) (preferred) or `ansible-galaxy`. Run `just roles` to install them (or `just update` to also pull the playbook itself). Editing these roles locally is fine while preparing or testing a fix, but the changes get wiped on the next roles update, so they must be synced back to the role's upstream repository, followed by a version pin update in `requirements.yml`.
 - `group_vars/matrix_servers`: wires roles together (feeding one role's variables into another). Values a role can construct by itself belong in the role's `defaults/main.yml`, not here.
 - `docs/`: user-facing documentation, one page per component.
+- `molecule-shared/`: files shared by the roles' Molecule scenarios (Python and Ansible dependencies, pinned helper container images).
 - `i18n/`: translation infrastructure. Do not edit locale files by hand; they are managed by automation.
 - `CHANGELOG.md`: user-facing announcements, newest first.
 
@@ -30,6 +31,7 @@ Follow the [style guide for playbook developers](docs/style-guide.md). In partic
 - Playbook-extensible list variables use the `_auto` + `_custom` split; `_custom` is reserved for users.
 - Renamed or removed variables get a validation entry, so stale user configuration produces an error instead of being silently ignored. Each role deprecates its own variables in its `validate_config.yml`; the `matrix_playbook_migration` role covers eliminated roles and very-early validation, and also gates breaking changes via `matrix_playbook_migration_expected_version` (see the style guide).
 - Every file carries SPDX license headers ([REUSE](https://reuse.software/) specification).
+- Roles may carry a Molecule scenario, proving the component starts and does not choke on the configuration the role rendered. Run one with `just molecule <role>` (no argument lists the roles that have one); CI runs only the scenarios of roles a push touched. See [Molecule testing for roles](docs/molecule-testing.md) before writing one - roles here need context a standalone role does not.
 - New components must be registered in `setup.yml`, `group_vars/matrix_servers`, `docs/README.md`, `README.md`, `docs/container-images.md`, and get a `CHANGELOG.md` entry.
 
 ## Other notes

bin/molecule.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+# Runs a role's Molecule scenario locally.
+#
+# Called through `just molecule [role] [args...]`. With no role, lists the roles
+# that have a scenario.
+#
+# The same scenarios run in CI (.github/workflows/molecule.yml), but running one
+# here is the faster loop while writing or fixing a role: CI only tells you after
+# a push, and only about the roles that push touched.
+#
+# Deliberately NOT wired into prek. A run takes minutes, pulls container images
+# and needs a working Docker - which is fine when you ask for it, and not fine on
+# every commit.
+#
+# Usage:
+#   just molecule                          # list roles that have a scenario
+#   just molecule matrix-alertmanager-receiver
+#   just molecule matrix-alertmanager-receiver converge   # any molecule subcommand
+#
+# Environment:
+#   MOLECULE_DISTRO   base image to test on (default: ubuntu2604)
+
+set -euo pipefail
+
+repo_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+roles_dir="${repo_dir}/roles/custom"
+venv_dir="${repo_dir}/var/molecule-venv"
+
+list_roles() {
+	find "${roles_dir}" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' \
+		| while read -r candidate; do
+			if [ -f "${roles_dir}/${candidate}/molecule/default/molecule.yml" ]; then
+				echo "${candidate}"
+			fi
+		done \
+		| sort
+}
+
+role="${1:-}"
+
+if [ -z "${role}" ]; then
+	echo "Roles with a Molecule scenario:"
+	found="$(list_roles)"
+	if [ -z "${found}" ]; then
+		echo "  (none yet)"
+	else
+		printf '  %s\n' ${found}
+	fi
+	echo
+	echo "Run one with: just molecule <role>"
+	exit 0
+fi
+
+shift || true
+
+if [ ! -f "${roles_dir}/${role}/molecule/default/molecule.yml" ]; then
+	echo "No Molecule scenario at roles/custom/${role}/molecule/default" >&2
+	echo >&2
+	echo "Roles that have one:" >&2
+	list_roles | sed 's/^/  /' >&2
+	exit 1
+fi
+
+# The virtualenv lives under var/, which is gitignored, and is shared by every
+# role - the dependencies are the same for all of them.
+if [ ! -x "${venv_dir}/bin/molecule" ]; then
+	echo "Creating the Molecule virtualenv in ${venv_dir/#$HOME/\~} ..."
+	python3 -m venv "${venv_dir}"
+	"${venv_dir}/bin/pip" install --quiet --upgrade pip
+	"${venv_dir}/bin/pip" install --quiet -r "${repo_dir}/molecule-shared/requirements.txt"
+fi
+
+export MOLECULE_DISTRO="${MOLECULE_DISTRO:-ubuntu2604}"
+export PY_COLORS="${PY_COLORS:-1}"
+export ANSIBLE_FORCE_COLOR="${ANSIBLE_FORCE_COLOR:-1}"
+
+echo "Running Molecule for ${role} on ${MOLECULE_DISTRO} ..."
+cd "${roles_dir}/${role}"
+
+if [ $# -eq 0 ]; then
+	exec "${venv_dir}/bin/molecule" test --scenario-name default
+fi
+
+exec "${venv_dir}/bin/molecule" "$@" --scenario-name default

docs/molecule-testing.md

@@ -0,0 +1,71 @@
+<!--
+SPDX-FileCopyrightText: 2026 Slavi Pantaleev
+
+SPDX-License-Identifier: AGPL-3.0-or-later
+-->
+
+# Molecule testing for roles
+
+Roles in `roles/custom/` can carry a [Molecule](https://ansible.readthedocs.io/projects/molecule/) scenario, which installs the role into a container and then checks that the component actually came up with the configuration the role rendered.
+
+Not every role has one yet. Roles without a scenario are simply not tested.
+
+## Running a scenario
+
+```sh
+just molecule                                    # list roles that have a scenario
+just molecule matrix-alertmanager-receiver       # run one
+just molecule matrix-alertmanager-receiver converge   # any molecule subcommand
+```
+
+The first run creates a virtualenv in `var/molecule-venv/` (gitignored) from `molecule-shared/requirements.txt`. Docker must be working, and a run takes minutes because it pulls container images.
+
+`MOLECULE_DISTRO` selects the base image; it defaults to `ubuntu2604`.
+
+Molecule is deliberately **not** part of the `prek` hooks. A run is far too slow to sit in front of a commit, and it needs Docker. Run it when you have touched a role; CI runs it too, asynchronously.
+
+## What CI runs
+
+`.github/workflows/molecule.yml` does not run every scenario on every push — with one repository holding every role, that would be unaffordable. Its first job works out which roles the push actually touched, keeps the ones that have a scenario, and builds the job matrix from those. A documentation change runs nothing.
+
+When the diff base cannot be determined (a new branch, a force push), it falls back to running every scenario, which errs toward testing too much rather than too little. `workflow_dispatch` accepts an optional role name.
+
+## Writing a scenario
+
+Start from `roles/custom/matrix-alertmanager-receiver/molecule/default/` — it is the reference. Four things differ from a standalone role's scenario, all of them consequences of these roles living inside a playbook:
+
+### The playbook's context has to be supplied
+
+The role reads variables that `matrix-base` and `group_vars/matrix_servers` would normally provide. The set is small — `matrix_base_data_path`, `matrix_domain`, `matrix_user_name`, `matrix_group_name`, `matrix_user_uid`, `matrix_user_gid` — and belongs in the scenario's `group_vars`, rather than including `matrix-base`, which does much more than a role scenario needs.
+
+### The `matrix` user and group must exist first
+
+The roles' file tasks set `owner:` and `group:` by name, and Ansible resolves those through the passwd database, so `prepare.yml` has to create them before the role runs.
+
+### Most components need a homeserver to be present
+
+Many of these components contact the homeserver while starting up, and exit if it is unreachable — `matrix-alertmanager-receiver`, for example, fetches `/_matrix/client/v3/joined_rooms` to resolve its room mapping and exits with a failure if that call fails.
+
+A stub is enough, and is what the reference scenario stands up. The point of these scenarios is to prove that **the component starts and does not choke on the configuration the role rendered** — not to exercise real bridging. A scenario should never need a credential or an account on a third-party network; that is the line where it stops being a test of this repository.
+
+### `verify.yml` is a separate play
+
+Role defaults are out of scope there, so any path it reads has to be pinned in the scenario's `group_vars`. Deliberately do **not** pin the component's version that way: read it from the role's `defaults/main.yml` with `include_vars`, so the assertion compares the running image against what the role ships rather than against the scenario itself.
+
+## Shared files
+
+`molecule-shared/` holds what would otherwise be duplicated into every role:
+
+- `requirements.txt` — the Python packages, for both CI and `just molecule`.
+- `requirements.yml` — the external Ansible roles and collections the scenarios need.
+- `vars.yml` — helper container images used for probing, pinned once. They carry `# renovate:` annotations and a custom manager in `.github/renovate.json` keeps them current.
+
+A helper image is used to reach a role's container over its own container network. That indirection is deliberate: the roles publish no host port, matching a real deployment, and publishing one for the test would collide between scenarios running in parallel.
+
+## Making a scenario worth having
+
+A suite that only waits for the systemd unit to become `active` proves very little: these units carry `Restart=always`, so a container crash-looping on a bad configuration still reports `active`. Check the restart counter alongside it, and probe something the component can only answer correctly if the role's configuration reached it.
+
+Give the scenario values that differ from both the role's defaults and the component's own defaults. Otherwise a passing assertion cannot distinguish "the role configured this" from "it would have happened anyway".
+
+Then try to break it. If a scenario cannot be made to fail by deliberately breaking the thing it checks, it is not testing that thing.

justfile

@@ -135,6 +135,10 @@ stop-all *extra_args: (run-tags "stop-all" extra_args)
 stop-group group *extra_args:
     @{{ just_executable() }} --justfile "{{ justfile() }}" run-tags stop-group --extra-vars="group={{ group }}" {{ extra_args }}
 
+# Runs a role's Molecule scenario locally (no argument lists the roles that have one)
+molecule *args:
+    @{{ justfile_directory() }}/bin/molecule.sh {{ args }}
+
 # Internal - ensures var/mise and var/prek directories exist
 _ensure_mise_data_directory:
     @mkdir -p "{{ mise_data_dir }}"

roles/custom/matrix-alertmanager-receiver/molecule/default/requirements.yml → molecule-shared/requirements.yml

@@ -1,4 +1,7 @@
 ---
+# Shared by every role scenario under roles/custom/*/molecule/, referenced from
+# each scenario's molecule.yml. Kept in one place so the pins cannot drift
+# apart across roles.
 roles:
   - name: ansible-role-docker
     src: https://github.com/geerlingguy/ansible-role-docker

molecule-shared/requirements.yml.license

@@ -0,0 +1,3 @@
+SPDX-FileCopyrightText: 2026 Slavi Pantaleev
+
+SPDX-License-Identifier: AGPL-3.0-or-later

molecule-shared/vars.yml

@@ -0,0 +1,16 @@
+---
+# Helper container images the scenarios use for probing. They live here rather
+# than inline in each verify.yml so that there is one pin per image instead of
+# one per role, and so Renovate can see them (see the customManager in
+# .github/renovate.json).
+
+# Used to reach a role's container over its own container network. A helper is
+# needed because the role publishes no host port - exactly as in a real
+# deployment - and publishing one for the test would collide between scenarios
+# running in parallel.
+# renovate: datasource=docker depName=docker.io/curlimages/curl
+molecule_shared_image_curl: "docker.io/curlimages/curl:8.11.1"
+
+# Used for small stub services (a fake homeserver, and so on).
+# renovate: datasource=docker depName=docker.io/library/python
+molecule_shared_image_python: "docker.io/library/python:3.13-alpine"

molecule-shared/vars.yml.license

@@ -0,0 +1,3 @@
+SPDX-FileCopyrightText: 2026 Slavi Pantaleev
+
+SPDX-License-Identifier: AGPL-3.0-or-later

roles/custom/matrix-alertmanager-receiver/molecule/default/molecule.yml

@@ -6,7 +6,7 @@
 dependency:
   name: galaxy
   options:
-    requirements-file: requirements.yml
+    requirements-file: ../../../../../molecule-shared/requirements.yml
     force: true
 driver:
   name: docker

roles/custom/matrix-alertmanager-receiver/molecule/default/prepare.yml

@@ -6,6 +6,8 @@
 - name: Prepare matrix-alertmanager-receiver Molecule tests
   hosts: all
   become: true
+  vars_files:
+    - "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}/../../../molecule-shared/vars.yml"
   gather_facts: true
   tasks:
     - name: Ensure apt cache is updated
@@ -116,7 +118,7 @@
           - --network={{ matrix_alertmanager_receiver_container_network }}
           - --network-alias=matrix.molecule.local
           - --volume=/root/matrix-homeserver-stub.py:/stub.py:ro
-          - docker.io/library/python:3.13-alpine
+          - "{{ molecule_shared_image_python }}"
           - python3
           - /stub.py
       register: matrix_alertmanager_receiver_molecule_stub

roles/custom/matrix-alertmanager-receiver/molecule/default/verify.yml

@@ -6,6 +6,8 @@
 - name: Verify matrix-alertmanager-receiver
   hosts: all
   become: true
+  vars_files:
+    - "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}/../../../molecule-shared/vars.yml"
   gather_facts: false
 
   tasks:
@@ -54,7 +56,7 @@
           - run
           - --rm
           - --network={{ matrix_alertmanager_receiver_container_network }}
-          - docker.io/curlimages/curl:8.11.1
+          - "{{ molecule_shared_image_curl }}"
           - --silent
           - --show-error
           - --write-out
@@ -100,7 +102,7 @@
           - run
           - --rm
           - --network={{ matrix_alertmanager_receiver_container_network }}
-          - docker.io/curlimages/curl:8.11.1
+          - "{{ molecule_shared_image_curl }}"
           - --silent
           - --output
           - /dev/null