hint supported automatic migration nedb->postgres

roles/matrix-bridge-appservice-irc/tasks/validate_config.yml

@@ -49,7 +49,7 @@
         To continue using neDB, opt into it explicitly: add `matrix_appservice_irc_database_engine: nedb` to your vars.yml file and re-run this same command.
         Alternatively, to migrate your existing neDB database to Postgres:
         1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the neDB database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_irc_data_path }} postgres_connection_string_variable_name=matrix_appservice_irc_database_connString'`)
+        2. Import the neDB database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-irc-nedb --extra-vars='nedb_database_path={{ matrix_appservice_irc_data_path }} postgres_connection_string_variable_name=matrix_appservice_irc_database_connString'`)
         3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
     when: "matrix_appservice_irc_nedb_stat_result.stat.exists"
   when: "matrix_appservice_irc_database_engine == 'postgres'"

roles/matrix-postgres/tasks/import_nedb.yml

@@ -0,0 +1,117 @@
+---
+
+# Pre-checks
+
+- name: Fail if Postgres not enabled
+  fail:
+    msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot import."
+  when: "not matrix_postgres_enabled|bool"
+
+- name: Fail if playbook called incorrectly
+  fail:
+    msg: "The `nedb_database_path` variable needs to be provided to this playbook, via --extra-vars"
+  when: "nedb_database_path is not defined or nedb_database_path.startswith('<')"
+
+- name: Check if the provided nedb database file exists
+  stat:
+    path: "{{ nedb_database_path }}"
+  register: nedb_database_path_stat_result
+
+- name: Fail if provided SQLite database file doesn't exist
+  fail:
+    msg: "File cannot be found on the server at {{ nedb_database_path }}"
+  when: "not nedb_database_path_stat_result.stat.exists"
+
+# We either expect `postgres_db_connection_string` specifying a full Postgres database connection string,
+# or `postgres_connection_string_variable_name`, specifying a name of a variable, which contains a valid connection string.
+
+- block:
+    - name: Fail if postgres_connection_string_variable_name points to an undefined variable
+      fail: msg="postgres_connection_string_variable_name is defined, but there is no variable with the name `{{ postgres_connection_string_variable_name }}`"
+      when: "postgres_connection_string_variable_name not in vars"
+
+    - name: Get Postgres connection string from variable
+      set_fact:
+        postgres_db_connection_string: "{{ lookup('vars', postgres_connection_string_variable_name) }}"
+  when: 'postgres_connection_string_variable_name is defined'
+
+- name: Fail if playbook called incorrectly
+  fail:
+    msg: >-
+      Either a `postgres_db_connection_string` variable or a `postgres_connection_string_variable_name` needs to be provided to this playbook, via `--extra-vars`.
+      Example: `--extra-vars="postgres_db_connection_string=postgresql://username:password@localhost:<port>/database_name"` or `--extra-vars="postgres_connection_string_variable_name=matrix_appservice_discord_database_connString"`
+  when: "postgres_db_connection_string is not defined or not postgres_db_connection_string.startswith('postgresql://')"
+
+
+# Defaults
+
+- name: Set postgres_start_wait_time, if not provided
+  set_fact:
+    postgres_start_wait_time: 15
+  when: "postgres_start_wait_time|default('') == ''"
+
+
+# Actual import work
+
+- name: Ensure matrix-postgres is started
+  service:
+    name: matrix-postgres
+    state: started
+    daemon_reload: yes
+  register: matrix_postgres_service_start_result
+
+- name: Wait a bit, so that Postgres can start
+  wait_for:
+    timeout: "{{ postgres_start_wait_time }}"
+  delegate_to: 127.0.0.1
+  become: false
+  when: "matrix_postgres_service_start_result.changed|bool"
+
+# See https://github.com/matrix-org/matrix-appservice-irc/wiki/Migrating-from-NEdB-to-PostgreSQL
+- name: Import appservice_irc NeDB database from {{ sqlite_database_path }} into Postgres
+  when: database == 'appservice_irc'
+  command:
+    cmd: >-
+      {{ matrix_host_command_docker }} run
+      --rm
+      --user={{ matrix_user_uid }}:{{ matrix_user_gid }}
+      --cap-drop=ALL
+      --network={{ matrix_docker_network }}
+      --mount type=bind,src={{ matrix_appservice_irc_data_path }}:/data:ro
+      --entrypoint=/bin/sh
+      {{ matrix_appservice_irc_docker_image }}
+      -c
+      './scripts/migrate-db-to-pgres.sh -d /data -p passkey.pem -c {{ postgres_db_connection_string }}'
+
+# No migration.sh available, but found this:
+# https://github.com/matrix-org/matrix-appservice-slack/blob/develop/src/scripts/migrateToPostgres.ts
+# Usage should be similar to appservice_irc
+- name: Import appservice_slack NeDB database from {{ sqlite_database_path }} into Postgres
+  when: database == 'appservice_slack'
+  command:
+    cmd: >-
+      {{ matrix_host_command_docker }} run
+      --rm
+      --user={{ matrix_user_uid }}:{{ matrix_user_gid }}
+      --cap-drop=ALL
+      --network={{ matrix_docker_network }}
+      --mount type=bind,src={{ matrix_appservice_irc_data_path }}:/data:ro
+      --entrypoint=/bin/sh
+      {{ matrix_appservice_slack_docker_image }}
+      -c
+      'node /lib/scripts/migrate-db-to-pgres.js -d /data -p passkey.pem -c {{ postgres_db_connection_string }}'
+
+- name: Archive NeDB database ({{ sqlite_database_path }} -> {{ sqlite_database_path }}.backup)
+  command:
+    cmd: "mv {{ sqlite_database_path }} {{ sqlite_database_path }}.backup"
+
+- name: Inject result
+  set_fact:
+    matrix_playbook_runtime_results: |
+      {{
+        matrix_playbook_runtime_results|default([])
+        +
+        [
+          "NOTE: Your NeDB database file has been imported into Postgres. The original directory has been moved from `{{ nedb_database_path }}` to `{{ nedb_database_path }}.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete this file."
+        ]
+      }}

roles/matrix-postgres/tasks/main.yml

@@ -32,6 +32,22 @@
   tags:
     - import-generic-sqlite-db
 
+# Imports appservice-irc NeDB into postgres
+- import_tasks: "{{ role_path }}/tasks/import_nedb.yml"
+  vars:
+    database: appservice_irc
+  when: run_postgres_import_nedb|bool
+  tags:
+    - import-irc-nedb
+
+# Imports slacks neDB to postgres.
+- import_tasks: "{{ role_path }}/tasks/import_nedb.yml"
+  vars:
+    database: appservice_slack
+  when: run_postgres_import_nedb|bool
+  tags:
+    - import-slack-nedb
+
 - import_tasks: "{{ role_path }}/tasks/upgrade_postgres.yml"
   when: run_postgres_upgrade|bool
   tags: