From be3cbe995b0d7a80b18acd00feb605088c77772c Mon Sep 17 00:00:00 2001 From: Mathieu Garcia Date: Sat, 27 Jun 2026 13:10:55 +0200 Subject: [PATCH 1/7] feat(nomad): install and configure exec2 driver - Add nomad-driver-exec2 package installation - Configure exec2.hcl plugin directory - Add plugin_dir to nomad.hcl template - Disable drain on shutdown (force and ignore_system_jobs) - Make ExecStop conditional in override.conf.j2 --- ansible/playbooks/paas/roles/nomad/defaults/main.yml | 8 ++++---- ansible/playbooks/paas/roles/nomad/tasks/05_install.yml | 5 +++++ .../paas/roles/nomad/tasks/06_configuration.yml | 9 +++++++++ .../playbooks/paas/roles/nomad/templates/exec2.hcl.j2 | 7 +++++++ .../playbooks/paas/roles/nomad/templates/nomad.hcl.j2 | 2 ++ .../paas/roles/nomad/templates/override.conf.j2 | 2 ++ 6 files changed, 29 insertions(+), 4 deletions(-) create mode 100644 ansible/playbooks/paas/roles/nomad/templates/exec2.hcl.j2 diff --git a/ansible/playbooks/paas/roles/nomad/defaults/main.yml b/ansible/playbooks/paas/roles/nomad/defaults/main.yml index 53d05854..895ecb30 100644 --- a/ansible/playbooks/paas/roles/nomad/defaults/main.yml +++ b/ansible/playbooks/paas/roles/nomad/defaults/main.yml @@ -38,8 +38,8 @@ nomad_job_files_dir: "/var/tmp" nomad_disable_anonymous_signature: false nomad_disable_update_check: false -nomad_leave_on_terminate: true -nomad_leave_on_interrupt: true +nomad_leave_on_terminate: false +nomad_leave_on_interrupt: false nomad_client_auto_join: true nomad_server_auto_join: true @@ -170,8 +170,8 @@ nomad_client_server_join_retry_max: 3 nomad_client_server_join_retry_interval: 15s nomad_client_drain_on_shutdown_deadline: 1m -nomad_client_drain_on_shutdown_force: true -nomad_client_drain_on_shutdown_ignore_system_jobs: true +nomad_client_drain_on_shutdown_force: false +nomad_client_drain_on_shutdown_ignore_system_jobs: false nomad_client_cpu_total_compute: 0 nomad_client_memory_total_mb: 0 diff --git a/ansible/playbooks/paas/roles/nomad/tasks/05_install.yml b/ansible/playbooks/paas/roles/nomad/tasks/05_install.yml index 8970d792..97ef2f66 100644 --- a/ansible/playbooks/paas/roles/nomad/tasks/05_install.yml +++ b/ansible/playbooks/paas/roles/nomad/tasks/05_install.yml @@ -7,6 +7,11 @@ update_cache: true when: nomad_version is not defined +- name: "Nomad Install | Install nomad-driver-exec2" + ansible.builtin.apt: + name: nomad-driver-exec2 + state: latest + - name: "Nomad Install | Install binary" ansible.builtin.apt: name: "nomad={{ nomad_version }}-1" diff --git a/ansible/playbooks/paas/roles/nomad/tasks/06_configuration.yml b/ansible/playbooks/paas/roles/nomad/tasks/06_configuration.yml index 6f51bae2..b84920ef 100644 --- a/ansible/playbooks/paas/roles/nomad/tasks/06_configuration.yml +++ b/ansible/playbooks/paas/roles/nomad/tasks/06_configuration.yml @@ -31,6 +31,15 @@ mode: '0644' when: nomad_node_role == 'client' or nomad_node_role == 'both' +- name: "Nomad Configuration | Insert Nomad exec2 configuration" + ansible.builtin.template: + src: exec2.hcl.j2 + dest: "{{ nomad_config_dir }}/exec2.hcl" + owner: nomad + group: nomad + mode: '0644' + when: nomad_node_role == 'client' or nomad_node_role == 'both' + - name: "Nomad Install | Copy configurations files" ansible.builtin.template: src: nomad.hcl.j2 diff --git a/ansible/playbooks/paas/roles/nomad/templates/exec2.hcl.j2 b/ansible/playbooks/paas/roles/nomad/templates/exec2.hcl.j2 new file mode 100644 index 00000000..f14b13ca --- /dev/null +++ b/ansible/playbooks/paas/roles/nomad/templates/exec2.hcl.j2 @@ -0,0 +1,7 @@ +plugin "nomad-driver-exec2" { + config { + unveil_defaults = true + unveil_paths = [] + unveil_by_task = true + } +} diff --git a/ansible/playbooks/paas/roles/nomad/templates/nomad.hcl.j2 b/ansible/playbooks/paas/roles/nomad/templates/nomad.hcl.j2 index 63e31586..01d08918 100644 --- a/ansible/playbooks/paas/roles/nomad/templates/nomad.hcl.j2 +++ b/ansible/playbooks/paas/roles/nomad/templates/nomad.hcl.j2 @@ -7,6 +7,8 @@ disable_update_check = {{ nomad_disable_update_check | lower }} data_dir = "{{ nomad_data_dir }}" +plugin_dir = "{{ nomad_data_dir }}/data/plugins" + bind_addr = "{{ nomad_bind_address }}" advertise { diff --git a/ansible/playbooks/paas/roles/nomad/templates/override.conf.j2 b/ansible/playbooks/paas/roles/nomad/templates/override.conf.j2 index a9767b14..e5502eaf 100644 --- a/ansible/playbooks/paas/roles/nomad/templates/override.conf.j2 +++ b/ansible/playbooks/paas/roles/nomad/templates/override.conf.j2 @@ -5,5 +5,7 @@ After=docker.service ExecReload=/bin/kill --signal HUP $MAINPID {% if nomad_node_role in ['both', 'client'] %} ExecStartPost=/usr/bin/nomad node eligibility -enable -address={{ nomad_http_scheme }}://{{ hostvars[nomad_primary_master_node | default(inventory_hostname)]['ansible_' + nomad_iface].ipv4.address | default('127.0.0.1') }}:{{ nomad_http_port }} -ca-cert={{ nomad_tls_host_certificate_dir }}/{{ nomad_tls_ca_pubkey }} -client-cert={{ nomad_tls_host_certificate_dir }}/{{ inventory_hostname }}-dc1-client-nomad.pem -client-key={{ nomad_tls_host_certificate_dir }}/{{ inventory_hostname }}-dc1-client-nomad.key -token={{ lookup('simple-stack-ui', type='secret', key=nomad_primary_master_node | default(inventory_hostname), subkey='nomad_management_token', missing='error') }} {{ node_id }} +{% if nomad_client_drain_on_shutdown_force %} ExecStop=/usr/bin/nomad node drain -enable -address={{ nomad_http_scheme }}://{{ hostvars[nomad_primary_master_node | default(inventory_hostname)]['ansible_' + nomad_iface].ipv4.address | default('127.0.0.1') }}:{{ nomad_http_port }} -ca-cert={{ nomad_tls_host_certificate_dir }}/{{ nomad_tls_ca_pubkey }} -client-cert={{ nomad_tls_host_certificate_dir }}/{{ inventory_hostname }}-dc1-client-nomad.pem -client-key={{ nomad_tls_host_certificate_dir }}/{{ inventory_hostname }}-dc1-client-nomad.key -token={{ lookup('simple-stack-ui', type='secret', key=nomad_primary_master_node | default(inventory_hostname), subkey='nomad_management_token', missing='error') }} {{ node_id }} {% endif %} +{% endif %} From 1559a6060d23c9a9f335b5d396dda39e1022420e Mon Sep 17 00:00:00 2001 From: Mathieu Garcia Date: Sat, 27 Jun 2026 13:11:05 +0200 Subject: [PATCH 2/7] fix(postgresql): switch to official postgres image and fix paths - Change from bitnami/postgresql to postgres official image - Update backup script to use /bin/pg_dumpall instead of bitnami path - Update restore script to use /bin/psql - Create /var/run/postgresql directory (owner 999:999, mode 3775) - Create /var/backup directory for backups - Mount /var/run/postgresql volume in nomad.hcl - Add host_network = "public" for network exposure - Update volumes path to use var/lib/postgresql structure --- ansible/playbooks/saas/roles/postgresql/files/backup | 2 +- ansible/playbooks/saas/roles/postgresql/files/restore | 2 +- ansible/playbooks/saas/roles/postgresql/tasks/main.yml | 7 +++++++ .../playbooks/saas/roles/postgresql/templates/nomad.hcl | 2 ++ ansible/playbooks/saas/roles/postgresql/vars/actions.yml | 5 +++-- 5 files changed, 14 insertions(+), 4 deletions(-) diff --git a/ansible/playbooks/saas/roles/postgresql/files/backup b/ansible/playbooks/saas/roles/postgresql/files/backup index 9ffc0f2e..9a0d2520 100644 --- a/ansible/playbooks/saas/roles/postgresql/files/backup +++ b/ansible/playbooks/saas/roles/postgresql/files/backup @@ -2,4 +2,4 @@ set +eu echo "Backup database..." -PGPASSWORD="${POSTGRESQL_PASSWORD}" /opt/bitnami/postgresql/bin/pg_dumpall -U postgres -f /var/backup/dump.sql +PGPASSWORD="${POSTGRESQL_PASSWORD}" /bin/pg_dumpall -U postgres -f /var/backup/dump.sql diff --git a/ansible/playbooks/saas/roles/postgresql/files/restore b/ansible/playbooks/saas/roles/postgresql/files/restore index 253ced65..2e8f4a53 100644 --- a/ansible/playbooks/saas/roles/postgresql/files/restore +++ b/ansible/playbooks/saas/roles/postgresql/files/restore @@ -2,4 +2,4 @@ set +eu echo "Import database..." -PGPASSWORD="${POSTGRESQL_PASSWORD}" /opt/bitnami/postgresql/bin/psql -U postgres -f /var/backup/dump.sql +PGPASSWORD="${POSTGRESQL_PASSWORD}" bin/psql -U postgres -f /var/backup/dump.sql diff --git a/ansible/playbooks/saas/roles/postgresql/tasks/main.yml b/ansible/playbooks/saas/roles/postgresql/tasks/main.yml index 32e47681..2bfed899 100644 --- a/ansible/playbooks/saas/roles/postgresql/tasks/main.yml +++ b/ansible/playbooks/saas/roles/postgresql/tasks/main.yml @@ -10,6 +10,13 @@ - path: "{{ software_path }}/var/lib/postgresql" owner: root group: root + - path: "{{ software_path }}/var/run/postgresql" + owner: 999 + group: 999 + mode: '3775' + - path: "{{ software_path }}/var/backup" + owner: 999 + group: 999 delegate_to: "{{ software.instance }}" - name: Copy script to operate software diff --git a/ansible/playbooks/saas/roles/postgresql/templates/nomad.hcl b/ansible/playbooks/saas/roles/postgresql/templates/nomad.hcl index 94761abf..6160f8ef 100644 --- a/ansible/playbooks/saas/roles/postgresql/templates/nomad.hcl +++ b/ansible/playbooks/saas/roles/postgresql/templates/nomad.hcl @@ -24,6 +24,7 @@ job "{{ domain }}" { {% if software.static_port is defined %} static = {{ software.static_port }} {% endif %} + host_network = "public" } } @@ -48,6 +49,7 @@ job "{{ domain }}" { ports = ["postgresql"] volumes = [ "{{ software_path }}/var/lib/postgresql:/var/lib/postgresql:rw", + "{{ software_path }}/var/run/postgresql:/var/run/postgresql:rw", "{{ software_path }}/tmp:/tmp:rw" ] } diff --git a/ansible/playbooks/saas/roles/postgresql/vars/actions.yml b/ansible/playbooks/saas/roles/postgresql/vars/actions.yml index b75c17a2..f0d6fcd9 100644 --- a/ansible/playbooks/saas/roles/postgresql/vars/actions.yml +++ b/ansible/playbooks/saas/roles/postgresql/vars/actions.yml @@ -3,10 +3,11 @@ postgresql_actions: environment: | POSTGRESQL_PASSWORD = "{{ lookup('simple-stack-ui', type='secret', key=domain, subkey='passwd', missing='error') }}" volumes: - - "{{ software_path }}/bitnami/postgresql:/bitnami/postgresql:ro" + - "{{ software_path }}/var/lib/postgresql/{{ catalogs.postgresql.version }}/docker:/var/lib/postgresql/{{ catalogs.postgresql.version }}/docker:ro" + - "{{ software_path }}/var/run/postgresql:/var/run/postgresql:ro" - "{{ software_path }}/var/backup:/var/backup:rw" - "{{ software_path }}/tmp:/tmp:rw" - "/usr/local/bin/postgresql-backup:/usr/local/bin/postgresql-backup:ro" - "/usr/local/bin/postgresql-restore:/usr/local/bin/postgresql-restore:ro" - image: "bitnami/postgresql:{{ catalogs.postgresql.version }}" + image: "postgres:{{ catalogs.postgresql.version }}" user: postgres From d79b4637564f101c6ed8815ee13e735d8efae485 Mon Sep 17 00:00:00 2001 From: Mathieu Garcia Date: Sat, 27 Jun 2026 13:11:17 +0200 Subject: [PATCH 3/7] fix(minio): simplify build and add minio-client - Simplify download task: direct path to binary instead of find-binary loop - Remove include_role for upstream/find-binary (no longer needed) - Add minio-client (mc) package to Dockerfile - Add host_network = "public" to nomad.hcl - Fix traefik_tag template indentation --- ansible/playbooks/saas/roles/minio/tasks/build.yml | 11 ++--------- .../saas/roles/minio/templates/Dockerfile.j2 | 2 ++ .../playbooks/saas/roles/minio/templates/nomad.hcl | 3 ++- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/ansible/playbooks/saas/roles/minio/tasks/build.yml b/ansible/playbooks/saas/roles/minio/tasks/build.yml index 55d164e0..c940d0c4 100644 --- a/ansible/playbooks/saas/roles/minio/tasks/build.yml +++ b/ansible/playbooks/saas/roles/minio/tasks/build.yml @@ -14,18 +14,11 @@ - name: Download latest release ansible.builtin.get_url: url: "{{ upstream_file_url }}" - dest: "{{ build_work_dir }}/download/" - mode: '0644' + dest: "{{ build_work_dir }}/{{ upstream_default_arch }}/minio" + mode: '0755' force: no register: download_result -- name: Find binary - ansible.builtin.include_role: - name: upstream - tasks_from: find-binary - loop: - - "{{ image.upstream.binary }}" - - name: Copy dockerfile ansible.builtin.template: src: Dockerfile.j2 diff --git a/ansible/playbooks/saas/roles/minio/templates/Dockerfile.j2 b/ansible/playbooks/saas/roles/minio/templates/Dockerfile.j2 index 07074814..fd798c3b 100644 --- a/ansible/playbooks/saas/roles/minio/templates/Dockerfile.j2 +++ b/ansible/playbooks/saas/roles/minio/templates/Dockerfile.j2 @@ -4,6 +4,8 @@ FROM {{ image.origin }} ARG TARGETARCH +RUN apk add --no-cache minio-client + COPY ./${TARGETARCH}/minio /usr/local/bin/minio CMD ["/usr/local/bin/minio", "server", "/data", "--console-address", ":9001"] diff --git a/ansible/playbooks/saas/roles/minio/templates/nomad.hcl b/ansible/playbooks/saas/roles/minio/templates/nomad.hcl index 5240d4fb..20a3923f 100644 --- a/ansible/playbooks/saas/roles/minio/templates/nomad.hcl +++ b/ansible/playbooks/saas/roles/minio/templates/nomad.hcl @@ -24,6 +24,7 @@ job "{{ domain }}" { {% if software.static_port is defined %} static = {{ software.static_port }} {% endif %} + host_network = "public" } } @@ -41,7 +42,7 @@ job "{{ domain }}" { port = "minio" provider = "nomad" tags = [ - {{ lookup('template', '../../traefik/templates/traefik_tag.j2') | indent(8) }} + {{ lookup('template', '../../traefik/templates/traefik_tag.j2') }} ] } From 58eeb1b3cfa821202f48492f5984e83d0b5e75e5 Mon Sep 17 00:00:00 2001 From: Mathieu Garcia Date: Sat, 27 Jun 2026 13:11:26 +0200 Subject: [PATCH 4/7] fix(grafana): use GF_PLUGINS_PREINSTALL environment variable - Change GF_INSTALL_PLUGINS to GF_PLUGINS_PREINSTALL (correct env var) - Comment out llm.yaml plugin configuration (not needed) --- ansible/playbooks/saas/roles/grafana/tasks/main.yml | 4 ++-- ansible/playbooks/saas/roles/grafana/templates/nomad.hcl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/playbooks/saas/roles/grafana/tasks/main.yml b/ansible/playbooks/saas/roles/grafana/tasks/main.yml index 281547e3..6ae8b882 100644 --- a/ansible/playbooks/saas/roles/grafana/tasks/main.yml +++ b/ansible/playbooks/saas/roles/grafana/tasks/main.yml @@ -32,8 +32,8 @@ loop: - path: datasources file: prometheus.yaml - - path: plugins - file: llm.yaml + # - path: plugins + # file: llm.yaml delegate_to: "{{ software.instance }}" - name: Copy nomad job diff --git a/ansible/playbooks/saas/roles/grafana/templates/nomad.hcl b/ansible/playbooks/saas/roles/grafana/templates/nomad.hcl index c25c1530..ac2c7a60 100644 --- a/ansible/playbooks/saas/roles/grafana/templates/nomad.hcl +++ b/ansible/playbooks/saas/roles/grafana/templates/nomad.hcl @@ -41,7 +41,7 @@ job "{{ domain }}" { env { GF_LOG_MODE = "console" GF_SERVER_HTTP_PORT = "3000" - GF_INSTALL_PLUGINS = "grafana-piechart-panel,grafana-llm-app" + GF_PLUGINS_PREINSTALL = "grafana-piechart-panel,grafana-llm-app" GF_SECURITY_ADMIN_USER = "{{ lookup('simple-stack-ui', type='secret', key=domain, subkey='user', missing='create', nosymbols=true, length=8) }}" GF_SECURITY_ADMIN_PASSWORD = "{{ lookup('simple-stack-ui', type='secret', key=domain, subkey='passwd', missing='create', length=12) }}" DS_PROMETHEUS = "prometheus" From f6fc88c2917c4d6ffa1a921ea25e5c3adc48462f Mon Sep 17 00:00:00 2001 From: Mathieu Garcia Date: Sat, 27 Jun 2026 13:11:41 +0200 Subject: [PATCH 5/7] feat(matrix): add complete Matrix stack roles Add coturn role: - TURN/STUN server for Matrix media relay - Auto-generated turn_shared_secret - Configurable relay port range (49152-49252) - README with firewall rules documentation Add synapse role: - Matrix homeserver deployment - Auto-generated secrets (registration, macaroon, form) - Integration with PostgreSQL, Redis, TURN - User management documentation Add synapse_admin role: - Synapse Admin UI for user management - Docker-based deployment Add element_web role: - Web client for Matrix - Configurable server connection Add matrix_migrate role: - Migration tools for Matrix data - Python-based migration script All roles include: - backup/restore/destroy tasks - Nomad job templates - Upstream variables --- ansible/playbooks/saas/roles/coturn/README.md | 47 + .../saas/roles/coturn/defaults/main.yml | 1 + .../saas/roles/coturn/tasks/backup.yml | 1 + .../saas/roles/coturn/tasks/build.yml | 12 + .../saas/roles/coturn/tasks/destroy.yml | 5 + .../saas/roles/coturn/tasks/main.yml | 14 + .../saas/roles/coturn/tasks/restore.yml | 1 + .../saas/roles/coturn/templates/nomad.hcl | 80 ++ .../roles/coturn/templates/turnserver.conf.j2 | 15 + .../playbooks/saas/roles/coturn/vars/main.yml | 11 + .../saas/roles/coturn/vars/upstream.yml | 2 + .../saas/roles/element_web/defaults/main.yml | 1 + .../saas/roles/element_web/tasks/backup.yml | 1 + .../saas/roles/element_web/tasks/build.yml | 12 + .../saas/roles/element_web/tasks/destroy.yml | 5 + .../saas/roles/element_web/tasks/main.yml | 14 + .../saas/roles/element_web/tasks/restore.yml | 1 + .../element_web/templates/config.json.j2 | 21 + .../roles/element_web/templates/nomad.hcl | 82 ++ .../saas/roles/element_web/vars/main.yml | 11 + .../saas/roles/element_web/vars/upstream.yml | 2 + .../saas/roles/matrix_migrate/README.md | 143 ++++ .../roles/matrix_migrate/defaults/main.yml | 1 + .../roles/matrix_migrate/files/migrate.py | 807 ++++++++++++++++++ .../matrix_migrate/files/requirements.txt | 4 + .../roles/matrix_migrate/tasks/destroy.yml | 5 + .../saas/roles/matrix_migrate/tasks/main.yml | 32 + .../templates/app-service.yaml.j2 | 14 + .../matrix_migrate/templates/config.yaml.j2 | 3 + .../roles/matrix_migrate/templates/nomad.hcl | 91 ++ .../saas/roles/matrix_migrate/vars/main.yml | 6 + .../playbooks/saas/roles/synapse/README.md | 51 ++ .../saas/roles/synapse/defaults/main.yml | 1 + .../saas/roles/synapse/tasks/backup.yml | 1 + .../saas/roles/synapse/tasks/build.yml | 12 + .../saas/roles/synapse/tasks/destroy.yml | 5 + .../saas/roles/synapse/tasks/main.yml | 59 ++ .../saas/roles/synapse/tasks/restore.yml | 1 + .../synapse/templates/homeserver.yaml.j2 | 103 +++ .../roles/synapse/templates/log.config.j2 | 20 + .../saas/roles/synapse/templates/nomad.hcl | 94 ++ .../saas/roles/synapse/vars/main.yml | 11 + .../saas/roles/synapse/vars/upstream.yml | 2 + .../roles/synapse_admin/defaults/main.yml | 1 + .../saas/roles/synapse_admin/tasks/backup.yml | 1 + .../saas/roles/synapse_admin/tasks/build.yml | 12 + .../roles/synapse_admin/tasks/destroy.yml | 5 + .../saas/roles/synapse_admin/tasks/main.yml | 14 + .../roles/synapse_admin/tasks/restore.yml | 1 + .../synapse_admin/templates/config.json.j2 | 23 + .../roles/synapse_admin/templates/nomad.hcl | 86 ++ .../saas/roles/synapse_admin/vars/main.yml | 11 + .../roles/synapse_admin/vars/upstream.yml | 2 + 53 files changed, 1961 insertions(+) create mode 100644 ansible/playbooks/saas/roles/coturn/README.md create mode 100644 ansible/playbooks/saas/roles/coturn/defaults/main.yml create mode 100644 ansible/playbooks/saas/roles/coturn/tasks/backup.yml create mode 100644 ansible/playbooks/saas/roles/coturn/tasks/build.yml create mode 100644 ansible/playbooks/saas/roles/coturn/tasks/destroy.yml create mode 100644 ansible/playbooks/saas/roles/coturn/tasks/main.yml create mode 100644 ansible/playbooks/saas/roles/coturn/tasks/restore.yml create mode 100644 ansible/playbooks/saas/roles/coturn/templates/nomad.hcl create mode 100644 ansible/playbooks/saas/roles/coturn/templates/turnserver.conf.j2 create mode 100644 ansible/playbooks/saas/roles/coturn/vars/main.yml create mode 100644 ansible/playbooks/saas/roles/coturn/vars/upstream.yml create mode 100644 ansible/playbooks/saas/roles/element_web/defaults/main.yml create mode 100644 ansible/playbooks/saas/roles/element_web/tasks/backup.yml create mode 100644 ansible/playbooks/saas/roles/element_web/tasks/build.yml create mode 100644 ansible/playbooks/saas/roles/element_web/tasks/destroy.yml create mode 100644 ansible/playbooks/saas/roles/element_web/tasks/main.yml create mode 100644 ansible/playbooks/saas/roles/element_web/tasks/restore.yml create mode 100644 ansible/playbooks/saas/roles/element_web/templates/config.json.j2 create mode 100644 ansible/playbooks/saas/roles/element_web/templates/nomad.hcl create mode 100644 ansible/playbooks/saas/roles/element_web/vars/main.yml create mode 100644 ansible/playbooks/saas/roles/element_web/vars/upstream.yml create mode 100644 ansible/playbooks/saas/roles/matrix_migrate/README.md create mode 100644 ansible/playbooks/saas/roles/matrix_migrate/defaults/main.yml create mode 100644 ansible/playbooks/saas/roles/matrix_migrate/files/migrate.py create mode 100644 ansible/playbooks/saas/roles/matrix_migrate/files/requirements.txt create mode 100644 ansible/playbooks/saas/roles/matrix_migrate/tasks/destroy.yml create mode 100644 ansible/playbooks/saas/roles/matrix_migrate/tasks/main.yml create mode 100644 ansible/playbooks/saas/roles/matrix_migrate/templates/app-service.yaml.j2 create mode 100644 ansible/playbooks/saas/roles/matrix_migrate/templates/config.yaml.j2 create mode 100644 ansible/playbooks/saas/roles/matrix_migrate/templates/nomad.hcl create mode 100644 ansible/playbooks/saas/roles/matrix_migrate/vars/main.yml create mode 100644 ansible/playbooks/saas/roles/synapse/README.md create mode 100644 ansible/playbooks/saas/roles/synapse/defaults/main.yml create mode 100644 ansible/playbooks/saas/roles/synapse/tasks/backup.yml create mode 100644 ansible/playbooks/saas/roles/synapse/tasks/build.yml create mode 100644 ansible/playbooks/saas/roles/synapse/tasks/destroy.yml create mode 100644 ansible/playbooks/saas/roles/synapse/tasks/main.yml create mode 100644 ansible/playbooks/saas/roles/synapse/tasks/restore.yml create mode 100644 ansible/playbooks/saas/roles/synapse/templates/homeserver.yaml.j2 create mode 100644 ansible/playbooks/saas/roles/synapse/templates/log.config.j2 create mode 100644 ansible/playbooks/saas/roles/synapse/templates/nomad.hcl create mode 100644 ansible/playbooks/saas/roles/synapse/vars/main.yml create mode 100644 ansible/playbooks/saas/roles/synapse/vars/upstream.yml create mode 100644 ansible/playbooks/saas/roles/synapse_admin/defaults/main.yml create mode 100644 ansible/playbooks/saas/roles/synapse_admin/tasks/backup.yml create mode 100644 ansible/playbooks/saas/roles/synapse_admin/tasks/build.yml create mode 100644 ansible/playbooks/saas/roles/synapse_admin/tasks/destroy.yml create mode 100644 ansible/playbooks/saas/roles/synapse_admin/tasks/main.yml create mode 100644 ansible/playbooks/saas/roles/synapse_admin/tasks/restore.yml create mode 100644 ansible/playbooks/saas/roles/synapse_admin/templates/config.json.j2 create mode 100644 ansible/playbooks/saas/roles/synapse_admin/templates/nomad.hcl create mode 100644 ansible/playbooks/saas/roles/synapse_admin/vars/main.yml create mode 100644 ansible/playbooks/saas/roles/synapse_admin/vars/upstream.yml diff --git a/ansible/playbooks/saas/roles/coturn/README.md b/ansible/playbooks/saas/roles/coturn/README.md new file mode 100644 index 00000000..d915cdab --- /dev/null +++ b/ansible/playbooks/saas/roles/coturn/README.md @@ -0,0 +1,47 @@ +# Role: `coturn` + +## How to use this Ansible role? + +1. In your `host_vars` directory, create a subdirectory with the name of your instance. +2. Inside this subdirectory, create a YAML file (e.g., `turn.domain.com.yml`) and define the following variables: + +```yaml +turn.domain.com: + software: coturn + size: small + realm: domain.com + external_ip: 1.2.3.4 +``` + +## Ports to open (UFW / firewall) + +| Port | Protocol | Usage | +|------|----------|-------| +| 3478 | TCP/UDP | STUN/TURN | +| 49152-49252 | UDP | Media relay (configurable) | + +Add the following to the host variables (`ufw_custom_rules`) of the instance where coturn is deployed: + +```yaml +ufw_custom_rules: + - port: 3478 + proto: tcp + rule: allow + - port: 3478 + proto: udp + rule: allow + - port: 49152:49252 + proto: udp + rule: allow +``` + +The relay port range is configurable via: + +- `min_relay_port` (default: `49152`) +- `max_relay_port` (default: `49252`) + +Adjust the port range accordingly depending on the number of simultaneous calls needed (1 port = 1 relayed call). + +## Secret + +The `turn_shared_secret` is auto-generated and must be shared with the Synapse role to enable TURN authentication. diff --git a/ansible/playbooks/saas/roles/coturn/defaults/main.yml b/ansible/playbooks/saas/roles/coturn/defaults/main.yml new file mode 100644 index 00000000..ed97d539 --- /dev/null +++ b/ansible/playbooks/saas/roles/coturn/defaults/main.yml @@ -0,0 +1 @@ +--- diff --git a/ansible/playbooks/saas/roles/coturn/tasks/backup.yml b/ansible/playbooks/saas/roles/coturn/tasks/backup.yml new file mode 100644 index 00000000..ed97d539 --- /dev/null +++ b/ansible/playbooks/saas/roles/coturn/tasks/backup.yml @@ -0,0 +1 @@ +--- diff --git a/ansible/playbooks/saas/roles/coturn/tasks/build.yml b/ansible/playbooks/saas/roles/coturn/tasks/build.yml new file mode 100644 index 00000000..c8c8cb30 --- /dev/null +++ b/ansible/playbooks/saas/roles/coturn/tasks/build.yml @@ -0,0 +1,12 @@ +--- +- name: Include upstream variables + ansible.builtin.include_vars: upstream.yml + +- name: Set custom variables + ansible.builtin.set_fact: + image_version: "{{ latest_version }}" + image_definition: "{{ image }}" + +- name: End playbook if no new version + ansible.builtin.meta: end_host + when: catalogs[catalog_image_name] is defined and catalogs[catalog_image_name].version == image_version diff --git a/ansible/playbooks/saas/roles/coturn/tasks/destroy.yml b/ansible/playbooks/saas/roles/coturn/tasks/destroy.yml new file mode 100644 index 00000000..a246acc0 --- /dev/null +++ b/ansible/playbooks/saas/roles/coturn/tasks/destroy.yml @@ -0,0 +1,5 @@ +--- +- name: Destroy service + ansible.builtin.include_role: + name: common + tasks_from: destroy.yml diff --git a/ansible/playbooks/saas/roles/coturn/tasks/main.yml b/ansible/playbooks/saas/roles/coturn/tasks/main.yml new file mode 100644 index 00000000..4ad6f032 --- /dev/null +++ b/ansible/playbooks/saas/roles/coturn/tasks/main.yml @@ -0,0 +1,14 @@ +--- +- name: Copy nomad job to destination + ansible.builtin.template: + src: nomad.hcl + dest: "/var/tmp/{{ domain }}.nomad" + owner: root + group: root + mode: '0600' + delegate_to: "{{ software.instance }}" + +- name: Run nomad job + ansible.builtin.include_role: + name: nomad + tasks_from: job_start.yml diff --git a/ansible/playbooks/saas/roles/coturn/tasks/restore.yml b/ansible/playbooks/saas/roles/coturn/tasks/restore.yml new file mode 100644 index 00000000..ed97d539 --- /dev/null +++ b/ansible/playbooks/saas/roles/coturn/tasks/restore.yml @@ -0,0 +1 @@ +--- diff --git a/ansible/playbooks/saas/roles/coturn/templates/nomad.hcl b/ansible/playbooks/saas/roles/coturn/templates/nomad.hcl new file mode 100644 index 00000000..956d20e0 --- /dev/null +++ b/ansible/playbooks/saas/roles/coturn/templates/nomad.hcl @@ -0,0 +1,80 @@ +job "{{ domain }}" { + region = "{{ fact_instance.region }}" + datacenters = ["{{ fact_instance.datacenter }}"] + type = "service" + +{% if software.constraints is defined and software.constraints.location is defined %} + constraint { + attribute = "${meta.location}" + set_contains = "{{ software.constraints.location }}" + } +{% endif %} + + constraint { + attribute = "${meta.instance}" + set_contains = "{{ software.instance }}" + } + + group "coturn" { + + count = 1 + + restart { + attempts = 2 + interval = "10m" + delay = "15s" + mode = "fail" + } + + network { + port "stun" { + static = 3478 + to = 3478 + } + } + + service { + name = "{{ service_name }}" + port = "stun" + provider = "nomad" + tags = [] + check { + name = "{{ service_name }}" + type = "tcp" + interval = "60s" + timeout = "30s" + check_restart { + limit = 3 + grace = "90s" + ignore_warnings = false + } + } + } + + task "{{ domain }}-coturn" { + + driver = "docker" + + config { + image = "coturn/coturn:{{ catalogs.coturn.version }}" + network_mode = "host" + volumes = [ + "local/turnserver.conf:/etc/coturn/turnserver.conf:ro" + ] + } + + template { + change_mode = "restart" + destination = "local/turnserver.conf" + data = < --port --collection=rocketchat_message --db=parties --out=rocketchat_message.json +mongoexport --host --port --collection=rocketchat_room --db=parties --out=rocketchat_room.json +mongoexport --host --port --collection=users --db=parties --out=users.json +``` + +Place these files in `{{ software_path }}/inputs/` on the target instance. + +### 2. Synapse (target) + +- Synapse must be running with PostgreSQL +- An admin account must exist on Synapse +- **Disable rate limiting** temporarily in `homeserver.yaml` for migration: + +```yaml +rc_joins: + local: + per_second: 1024 + burst_count: 2048 +rc_joins_per_room: + per_second: 1024 + burst_count: 2048 +rc_message: + per_second: 1024 + burst_count: 2048 +rc_invites: + per_room: + per_second: 1024 + burst_count: 2048 + per_user: + per_second: 1024 + burst_count: 2048 + per_issuer: + per_second: 1024 + burst_count: 2048 +``` + +- **Register an Application Service** by adding to `homeserver.yaml`: + +```yaml +app_service_config_files: + - /data/app-service.yaml +``` + +### 3. Network + +The migration container must be able to reach: + +- Synapse API (port 8008 or via public domain) + +## Step-by-step procedure + +### Step 1: Create an admin account on Synapse + +```bash +docker exec -it synapse register_new_matrix_user \ + http://localhost:8008 \ + -c /data/homeserver.yaml \ + -u admin -p monmotdepasse -a +``` + +### Step 2: Export Rocket.Chat data + +```bash +mongosh --eval "db = db.getSiblingDB('parties'); db.rocketchat_message.countDocuments({})" +``` + +Expected output: a number > 0 (e.g. `188819`). + +Then export: + +```bash +mongoexport --collection=rocketchat_message --db=parties --out=rocketchat_message.json +mongoexport --collection=rocketchat_room --db=parties --out=rocketchat_room.json +mongoexport --collection=users --db=parties --out=users.json +``` + +### Step 3: Place export files on the instance + +Copy the JSON files to `{{ software_path }}/inputs/` on the host where the migration job will run. + +### Step 4: Verify Synapse admin API + +```bash +curl -X POST "https://matrix.domain.com/_matrix/client/r0/login" \ + -H "Content-Type: application/json" \ + -d '{"type": "m.login.password", "user": "admin", "password": "monmotdepasse"}' +``` + +### Step 5: Backup before migration + +```bash +# Backup Synapse PostgreSQL +pg_dump -h -U synapse synapse > /var/backup/synapse.sql +``` + +### Step 6: Run the migration + +Configure the following software variables: + +```yaml +migrate.domain.com: + software: matrix_migrate + size: medium + matrix_domain: matrix.domain.com + matrix_admin_user: admin +``` + +Secret: + +```yaml +matrix_admin_password: monmotdepasse +``` + +The migration job runs as a Nomad batch job (one-shot). Monitor progress via Nomad logs. + +### Step 7: Post-migration + +- Remove rate limiting overrides from `homeserver.yaml` +- Restart Synapse + +## Checklist + +- [ ] MongoDB data exported to JSON files +- [ ] JSON files placed in `{{ software_path }}/inputs/` +- [ ] Synapse running with PostgreSQL +- [ ] Admin account created on Synapse +- [ ] Rate limiting disabled temporarily +- [ ] Application Service configured +- [ ] Synapse PostgreSQL backup done +- [ ] Migration job executed successfully +- [ ] Rate limiting re-enabled post-migration diff --git a/ansible/playbooks/saas/roles/matrix_migrate/defaults/main.yml b/ansible/playbooks/saas/roles/matrix_migrate/defaults/main.yml new file mode 100644 index 00000000..ed97d539 --- /dev/null +++ b/ansible/playbooks/saas/roles/matrix_migrate/defaults/main.yml @@ -0,0 +1 @@ +--- diff --git a/ansible/playbooks/saas/roles/matrix_migrate/files/migrate.py b/ansible/playbooks/saas/roles/matrix_migrate/files/migrate.py new file mode 100644 index 00000000..b1b7fcff --- /dev/null +++ b/ansible/playbooks/saas/roles/matrix_migrate/files/migrate.py @@ -0,0 +1,807 @@ +#!/usr/bin/python3 + +import argparse +import sys +import os +import pprint as ppprint +import json +import requests +from datetime import datetime +import re +import markdown +import errno +import time +# for retries +from requests.adapters import HTTPAdapter +from urllib3.util import Retry +import magic + +import emoji # for reactions + +## TODO +# Handle topic/announcement/announcementDetails/md of original room + +# globals +roomsfile = "rocketchat_rooms.json" +usersfile = "rocketchat_users.json" +histfile = "rocketchat_messages.json" +verbose = False +dry_run = False +messages_cachefile = "messages_cache.txt" +users_cachefile = "users_cache.txt" +rooms_cachefile = "rooms_cache.txt" + + +class DryRunResponse: + """Mock response for dry-run mode""" + def __init__(self, method, url, context=""): + self.status_code = 200 + self._method = method + self._url = url + self._context = context + self._fake_room_id = "!dryrun_" + str(DryRunResponse._counter) + ":localhost" + self._fake_event_id = "$dryrun_" + str(DryRunResponse._counter) + self._fake_mxc = "mxc://localhost/dryrun_" + str(DryRunResponse._counter) + DryRunResponse._counter += 1 + + _counter = 0 + + def json(self): + return { + "access_token": "dryrun_token", + "room_id": self._fake_room_id, + "event_id": self._fake_event_id, + "content_uri": self._fake_mxc, + "creator": "@admin:localhost", + } + + +class DryRunSession: + """Wraps a requests.Session to intercept writes in dry-run mode""" + def __init__(self, real_session): + self._session = real_session + + def _handle_rate_limit(self, response, method, url, kwargs): + """Retry on 429 M_LIMIT_EXCEEDED and transient 502/503/504""" + retries = 0 + while response.status_code == 429 and retries < 10: + try: + wait_ms = response.json().get('retry_after_ms', 5000) + except Exception: + wait_ms = 5000 + wait_s = (wait_ms / 1000) + 1 + print(f" [RATE-LIMITED] waiting {wait_s:.0f}s before retry...") + time.sleep(wait_s) + response = method(url, **kwargs) + retries += 1 + gateway_retries = 0 + while response.status_code in (502, 503, 504) and gateway_retries < 3: + wait_s = 5 * (gateway_retries + 1) + print(f" [GATEWAY ERROR {response.status_code}] waiting {wait_s}s before retry...") + time.sleep(wait_s) + response = method(url, **kwargs) + gateway_retries += 1 + return response + + def get(self, url, **kwargs): + if dry_run: + print(f" [DRY-RUN] GET {url}") + return DryRunResponse("GET", url) + response = self._session.get(url, **kwargs) + return self._handle_rate_limit(response, self._session.get, url, kwargs) + + def post(self, url, **kwargs): + if dry_run: + print(f" [DRY-RUN] POST {url}") + return DryRunResponse("POST", url) + response = self._session.post(url, **kwargs) + return self._handle_rate_limit(response, self._session.post, url, kwargs) + + def put(self, url, **kwargs): + if dry_run: + print(f" [DRY-RUN] PUT {url}") + return DryRunResponse("PUT", url) + response = self._session.put(url, **kwargs) + return self._handle_rate_limit(response, self._session.put, url, kwargs) + + def mount(self, prefix, adapter): + self._session.mount(prefix, adapter) + + +# pretty printing functions, switched by verbose argument +def terminal_size(): + import fcntl + import termios + import struct + h, w, hp, wp = struct.unpack('HHHH', fcntl.ioctl(0, termios.TIOCGWINSZ, struct.pack('HHHH', 0, 0, 0, 0))) + return w, h + +def pprint(name, data): + if verbose: + w, h = terminal_size() + pp = ppprint.PrettyPrinter(indent=2, width=w) + print(name + ": ") + pp.pprint(data) + print("\n\n") + +def vprint(data): + if verbose: + print(str(data)) + print("\n\n") + +def safe_json(response): + """Safely parse JSON from response, return empty dict on failure""" + try: + return response.json() + except Exception: + return {} + +def is_forbidden(response): + """Check if response is a 403 M_FORBIDDEN""" + return response.status_code == 403 and safe_json(response).get('errcode') == 'M_FORBIDDEN' + +# Arguments parser +def createArgParser(): + parser = argparse.ArgumentParser(description='Launches RC2Matrix migration') + parser.add_argument("-i", type=str, help='inputs folder, defaults to inputs/', dest="inputs", default="inputs/") + parser.add_argument("-n", type=str, help='Matrix server', dest="hostname", default="localhost") + parser.add_argument("-u", type=str, help='Admin username', dest="username", default="admin") + parser.add_argument("-p", type=str, help='Admin password', dest="password", default="password") + parser.add_argument("-t", type=str, help='Admin token', dest="token", default=None ) + parser.add_argument("-a", type=str, help='Application token', dest="apptoken", default=None ) + parser.add_argument("-f", type=str, help='Only import messages from this date (ISO format, e.g. 2024-01-01)', dest="fromdate", default=None) + parser.add_argument("-k", help='Disable TLS certificate check', dest="nocertcheck", action="store_true") + parser.add_argument("-d", help='Dry-run mode (no API calls)', dest="dryrun", action="store_true") + parser.add_argument("-v", help='verbose', dest="verbose", action="store_true") + + return parser + +# Try to format a markdown message into html +def format_message(raw): + #formatted = raw + #formatted = re.sub("```(.+)```", "\\1", formatted) + #formatted = re.sub("`(.+)`", "\\1", formatted) + formatted = markdown.markdown(raw) + if len(formatted) <= len(raw)+7: # markdown adds

tags + api_params = {'msgtype': 'm.text', 'body': raw} + else: + api_params = {'msgtype': 'm.text', 'body': raw, + "format": "org.matrix.custom.html", + "formatted_body": formatted} + + return api_params + +# Add a related event, currently unused +def relate_message(raw, ancestor): + api_params = {'msgtype': 'm.text', 'body': raw, + "m.relates_to": { + "m.in_reply_to": { + "event_id": ancestor + } + } + } + + return api_params + +def invite(api_base, api_headers_admin, tgtroom, tgtuser): + + # Method 1, use admin user (possible for public rooms) + api_endpoint = api_base + "_synapse/admin/v1/join/" + tgtroom + api_params = {'user_id': tgtuser} + response = session.post(api_endpoint, json=api_params, headers=api_headers_admin) + vprint(response.json()) + + if response.status_code != 200: + # Method 2, use AS to invite as creator, then join as tgtuser via AS + + # Get creator + api_endpoint = api_base + "/_synapse/admin/v1/rooms/" + tgtroom + response = session.get(api_endpoint, headers=api_headers_admin) + vprint(response.json()) + creator = response.json().get("creator") + + if creator: + # Invite tgtuser as creator via AS + api_endpoint = api_base + "_matrix/client/v3/rooms/" + tgtroom + "/invite?user_id=" + requests.utils.quote(creator) + api_params = {'user_id': tgtuser} + response = session.post(api_endpoint, json=api_params, headers=api_headers_as) + vprint(response.json()) + + # Join as tgtuser via AS + api_endpoint = api_base + "_matrix/client/v3/rooms/" + tgtroom + "/join?user_id=" + requests.utils.quote(tgtuser) + response = session.post(api_endpoint, json={}, headers=api_headers_as) + vprint(response.json()) + + # Check join + if response.status_code != 200: + print("error inviting " + tgtuser + " to " + tgtroom) + print(response.json()) + exit(1) + + +if __name__ == '__main__': + parser = createArgParser() + args = parser.parse_args() + verbose = args.verbose + dry_run = args.dryrun + mime = magic.Magic(mime=True) + + if dry_run: + print("=== DRY-RUN MODE — no changes will be made ===") + + if (verbose): + print("Arguments are: ", args) + + if (args.nocertcheck): + import ssl + ssl._create_default_https_context = ssl._create_unverified_context + ssl.SSLContext.verify_mode = property(lambda self: ssl.CERT_NONE, lambda self, newval: None) + from urllib3.exceptions import InsecureRequestWarning + requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning) + + api_base = "https://" + args.hostname + "/" + + # Obtain an admin token if not provided + if args.token is None and os.environ.get('MATRIX_ADMIN_TOKEN') is None and not dry_run: + api_endpoint = api_base + "_matrix/client/v3/login" + api_params = {"type": "m.login.password","user": args.username,"password": args.password,"device_id": "rc2m"} + response = requests.post(api_endpoint, json=api_params) + vprint(response.json()) + if response.status_code == 200: + token=response.json()["access_token"] + vprint("Token is " + token) + exit(0) + else: + exit("failed to connect") + + # admin allows to connect as the admin user, as allows to connect as the application service + token = args.token or os.environ.get('MATRIX_ADMIN_TOKEN') or 'dryrun' + apptoken = args.apptoken or os.environ.get('MATRIX_APP_TOKEN') or 'dryrun' + api_headers_admin = {"Authorization":"Bearer " + token} + api_headers_as = {"Authorization":"Bearer " + apptoken} + + print(f"Target: {api_base}") + + # retry in case of error + real_session = requests.Session() + retry = Retry(connect=3, backoff_factor=0.5) + adapter = HTTPAdapter(max_retries=retry) + real_session.mount('http://', adapter) + real_session.mount('https://', adapter) + session = DryRunSession(real_session) + + # Dry-run counters + dryrun_users = 0 + dryrun_rooms = 0 + dryrun_messages = 0 + + # Import users + print("Importing users...") + users = set() + # load cache + nbcache = 0 + try: + with open(users_cachefile, encoding='utf8') as f: + for line in f: + nbcache+=1 + users.add(line.rstrip('\n')) + f.close() + print("Restored " + str(nbcache) + " user ids from cache") + except FileNotFoundError: + print("No user cache to restore") + cache = open(os.devnull, 'a') if dry_run else open(users_cachefile, 'a') + # import new users + with open(args.inputs + usersfile, 'r') as jsonfile: + # Each line is a JSON representing a RC user + for line in jsonfile: + currentuser = json.loads(line) + pprint("current user", currentuser) + if ("username" not in currentuser): + continue + # Skip bot users (rocket.cat, etc.) + if currentuser.get('type') == 'bot': + print(f" Skipping bot user: {currentuser['username']}") + continue + username=currentuser['username'].lower() + if "name" in currentuser and isinstance(currentuser['name'], str): + displayname=currentuser['name'] + else: + displayname=username + if username in users: + print("user " + username + " already processed (in cache), skipping") + continue + print(f" Creating user: {username} ({displayname})") + # matrix username will be @username:server + api_endpoint = api_base + "_synapse/admin/v2/users/@" + username + ":" + args.hostname + api_params = {"admin": False, "displayname": displayname} + response = session.put(api_endpoint, json=api_params, headers=api_headers_admin) + if response.status_code == 500: + # Synapse bug with displayname on fresh DB, retry without displayname + print(f" Retrying without displayname for {username}") + api_params = {"admin": False} + response = session.put(api_endpoint, json=api_params, headers=api_headers_admin) + if response.status_code < 200 or response.status_code > 299: #2xx + print("error adding user") + print(currentuser) + print(response.json()) + print(response.status_code) + exit(1) + + # avatar + if "avatarETag" in currentuser: + try: # try to find the file in the export + api_endpoint = api_base + "_matrix/media/v3/upload?user_id=@" + username + ":" + args.hostname + "&filename=" + requests.utils.quote(username) + api_headers_file = api_headers_as.copy() + localfile=currentuser["avatarETag"] + with open(args.inputs + "avatars_users/" + localfile, 'rb') as f: + # upload as a media to matrix + api_headers_file['Content-Type'] = mime.from_file(args.inputs + "avatars_users/" + localfile) + response = session.post(api_endpoint, headers=api_headers_file, data=f) + vprint(response.json()) + if response.status_code != 200: # Upload problem + vprint(response) + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), localfile) + mxcurl=response.json()['content_uri'] # URI of the uploaded media + # Then post a user update referencing this media + api_endpoint = api_base + "_synapse/admin/v2/users/@" + username + ":" + args.hostname + api_params = {"admin": False, "avatar_url": mxcurl} + response = session.put(api_endpoint, json=api_params, headers=api_headers_admin) + if response.status_code < 200 or response.status_code > 299: #2xx + print("error adding avatar for user") + print(currentuser) + print(response.json()) + print(response.status_code) + exit(1) + + except FileNotFoundError: # We do not have the linked attachment + print("Avatar not found for " + username + ", in " + localfile) + + cache.write(username + "\n") + dryrun_users += 1 + vprint(response.json()) + cache.close() + + # Import rooms + print("Importing rooms...") + roomids = {} # Map RC_roomID to Matrix_roomID + # load cache + nbcache = 0 + try: + with open(rooms_cachefile, encoding='utf8') as f: + for line in f: + nbcache+=1 + atoms = line.rstrip('\n').split('$') + roomids[atoms[0]] = atoms[1] + f.close() + print("Restored " + str(nbcache) + " room ids from cache") + except FileNotFoundError: + print("No room cache to restore") + cache = open(os.devnull, 'a') if dry_run else open(rooms_cachefile, 'a') + dm_pairs = set() # Track DM pairs to avoid duplicates + # Import new rooms + with open(args.inputs + roomsfile, 'r') as jsonfile: + # Each line is a JSON representing a RC room + for line in jsonfile: + currentroom = json.loads(line) + if currentroom['_id'] in roomids: + print("room " + currentroom.get('name', currentroom['_id']) + " already processed (in cache), skipping") + continue + pprint("current room", currentroom) + + # Use AS token with user_id to bypass rate limiting + api_headers_create = api_headers_as + create_user_id = None + if 'u' in currentroom: + try: + owner_id = "@" + currentroom['u']['username'].lower() + ":" + args.hostname + api_endpoint = api_base + "_synapse/admin/v2/users/" + owner_id + vprint(api_endpoint) + response = session.get(api_endpoint, headers=api_headers_admin) + vprint(response.json()) + if not 'errcode' in response.json(): + create_user_id = owner_id + except: + pass + + api_endpoint = api_base + "_matrix/client/v3/createRoom" + if create_user_id: + api_endpoint += "?user_id=" + requests.utils.quote(create_user_id) + + # Check if room already exists before creating (avoid duplicates on re-run) + if currentroom['t'] == 'c' or currentroom['t'] == 'p': + roomname = currentroom['name'] + alias = "#" + roomname + ":" + args.hostname + api_endpoint_check = api_base + "_matrix/client/v3/directory/room/" + requests.utils.quote(alias) + response_check = session.get(api_endpoint_check, headers=api_headers_admin) + if response_check.status_code == 200: + print(f" Room already exists (alias {alias}), reusing") + roomids[currentroom['_id']] = response_check.json()['room_id'] + cache.write(currentroom['_id'] + "$" + response_check.json()['room_id'] + "\n") + continue + elif currentroom['t'] == 'd': + dm_key = tuple(sorted(u.lower() for u in currentroom['usernames'])) + roomname = "ZZ-" + "-".join(dm_key) + # Search for existing DM by name (search all pages) + search_url = api_base + "_synapse/admin/v1/rooms?search_term=" + requests.utils.quote(roomname) + "&limit=100" + response_check = session.get(search_url, headers=api_headers_admin) + if response_check.status_code == 200: + for room in response_check.json().get('rooms', []): + if room['name'].lower() == roomname.lower(): + print(f" DM already exists ({roomname}), reusing") + roomids[currentroom['_id']] = room['room_id'] + cache.write(currentroom['_id'] + "$" + room['room_id'] + "\n") + break + if currentroom['_id'] in roomids: + continue + + if currentroom['t'] == 'd': # DM, create a private chatroom + dm_key = tuple(sorted(u.lower() for u in currentroom['usernames'])) + if dm_key in dm_pairs: + print("Skipping duplicate DM: " + str(dm_key)) + continue + dm_pairs.add(dm_key) + roomname="ZZ-" + "-".join(dm_key) + api_params = {"visibility": "private", "name": roomname, "join_rules": "invite", 'is_direct': 'true'} + elif currentroom['t'] == 'c': # public chatroom + roomname=currentroom['name'] + if 'announcement' in currentroom: # there is a topic + api_params = {"visibility": "public", "name": roomname, "room_alias_name": roomname, 'topic': currentroom['announcement']} + else: + api_params = {"visibility": "public", "name": roomname, "room_alias_name": roomname} + elif currentroom['t'] == 'p': # private chatroom + roomname=currentroom['name'] + if 'announcement' in currentroom: # there is a topic + api_params = {"visibility": "private", "join_rules": "invite", "name": roomname, "room_alias_name": roomname, 'topic': currentroom['announcement']} + else: + api_params = {"visibility": "private", "join_rules": "invite", "name": roomname, "room_alias_name": roomname} + else: + print("Skipping unsupported room type: " + currentroom['t'] + " (" + currentroom.get('name', currentroom['_id']) + ")") + continue + print(f" Creating room: {roomname} (type={currentroom['t']})") + response = session.post(api_endpoint, json=api_params, headers=api_headers_create) + vprint(response.json()) + if response.status_code == 200: # room created successfully + roomids[currentroom['_id']] = response.json()['room_id'] # map RC_roomID to Matrix_roomID + cache.write(currentroom['_id'] + "$" + response.json()['room_id'] + "\n") + dryrun_rooms += 1 + elif response.status_code == 400 and safe_json(response).get('errcode') == 'M_ROOM_IN_USE': # room already existing, we search it + #api_endpoint = api_base + "/_matrix/client/v3/publicRooms" + api_endpoint = api_base + "_synapse/admin/v1/rooms?search_term=" + roomname + #api_params = {"filter": { "generic_search_term": roomname}} + vprint(api_endpoint) + response = session.get(api_endpoint, headers=api_headers_admin) + if response.status_code != 200: + print("error getting room") + print("current room", currentroom) + print(response.json()) + exit(1) + vprint(response.json()) + found = False + for room in response.json()['rooms']: + if room['name'].lower() == roomname.lower(): + found = True + roomids[currentroom['_id']] = room['room_id'] # map RC_roomID to Matrix_roomID + cache.write(currentroom['_id'] + "$" + room['room_id'] + "\n") + if not found: + # Fallback: resolve alias directly + alias = "#" + roomname + ":" + args.hostname + api_endpoint_alias = api_base + "_matrix/client/v3/directory/room/" + requests.utils.quote(alias) + response_alias = session.get(api_endpoint_alias, headers=api_headers_admin) + if response_alias.status_code == 200: + found = True + roomids[currentroom['_id']] = response_alias.json()['room_id'] + cache.write(currentroom['_id'] + "$" + response_alias.json()['room_id'] + "\n") + print(f" Resolved via alias: {alias} -> {response_alias.json()['room_id']}") + if not found: + print("error finding room") + print("current room", currentroom) + print(response.json()) + exit(1) + # roomids[currentroom['_id']] = response.json()['rooms'][0]['room_id'] # map RC_roomID to Matrix_roomID + else: + print("current room", currentroom) + print(response.json()) + exit("Unsupported fail for room creation") + # rooms.append(json.loads(line)) + + # Make old owner admin of this new room + try: + api_endpoint = api_base + "_synapse/admin/v1/rooms/" + roomids[currentroom['_id']] + "/make_room_admin" + api_params = {"user_id": "@" + currentroom['u']['username'].lower() + ":" + args.hostname} + response = session.post(api_endpoint, json=api_params, headers=api_headers_admin) + if response.status_code != 200: + print("error setting admin") + print("current room", currentroom) + print(response.json()) + exit(1) + vprint(response.json()) + except: + pass + + # avatar + if "avatarETag" in currentroom: + try: # try to find the file in the export + api_endpoint = api_base + "_matrix/media/v3/upload?user_id=@" + currentroom['u']['username'].lower() + ":" + args.hostname + "&filename=" + requests.utils.quote(roomids[currentroom['_id']]) + api_headers_file = api_headers_as.copy() + localfile=currentroom["avatarETag"] + with open(args.inputs + "avatars_rooms/" + localfile, 'rb') as f: + # upload as a media to matrix + api_headers_file['Content-Type'] = mime.from_file(args.inputs + "avatars_rooms/" + localfile) + response = session.post(api_endpoint, headers=api_headers_file, data=f) + vprint(response.json()) + if response.status_code != 200: # Upload problem + vprint(response) + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), localfile) + mxcurl=response.json()['content_uri'] # URI of the uploaded media + # Then post a room update referencing this media + invite(api_base, api_headers_admin, roomids[currentroom['_id']], '@' + currentroom['u']['username'].lower() + ":" + args.hostname) + api_endpoint = api_base + "_matrix/client/v3/rooms/" + roomids[currentroom['_id']] + '/state/m.room.avatar/?user_id=@' + currentroom['u']['username'].lower() + ":" + args.hostname + api_params = {"url": mxcurl} + response = session.put(api_endpoint, json=api_params, headers=api_headers_as) + vprint(response.json()) + if response.status_code < 200 or response.status_code > 299: #2xx + print("error adding avatar for room") + print(currentroom) + print(response.json()) + print(response.status_code) + exit(1) + + except FileNotFoundError: # We do not have the linked attachment + print("Avatar not found for " + roomname + ", in " + localfile) + + cache.close() + pprint("room ids", roomids) + + # Messages + print("Importing messages...") + # Load and sort messages by timestamp + print(" Loading messages file...") + messages = [] + with open(args.inputs + histfile, 'r') as jsonfile: + for line in jsonfile: + messages.append(json.loads(line)) + print(f" Sorting {len(messages)} messages by timestamp...") + messages.sort(key=lambda m: m.get('ts', {}).get('$date', '')) + # Filter messages by --fromdate if specified + if args.fromdate: + fromdate_dt = datetime.fromisoformat(args.fromdate) + print(f" Filtering messages from {args.fromdate} onwards...") + messages = [m for m in messages if m.get('ts', {}).get('$date', '') >= args.fromdate] + print(f" {len(messages)} messages after filtering") + nblines = len(messages) + lastts = 0 + currentline = 0 + idmaps = {} # map RC_messageID to Matrix_messageID for threads, replies, ... + + # load cache + nbcache = 0 + try: + with open(messages_cachefile, encoding='utf8') as f: + for line in f: + nbcache+=1 + atoms = line.rstrip('\n').split(':') + idmaps[atoms[0]] = atoms[1] + f.close() + print("Restored " + str(nbcache) + " message ids from cache") + except FileNotFoundError: + print("No message cache to restore") + cache = open(os.devnull, 'a') if dry_run else open(messages_cachefile, 'a') + + for currentmsg in messages: + currentline+=1 + print("Importing message " + str(currentline) + "/" + str(nblines), end='') + pprint("current message", currentmsg) + finished=False # set to true to not (re)print the message in the final step + response=None + if 'rid' not in currentmsg: + print(", skipping (no rid)") + continue + if currentmsg['rid'] in roomids: + tgtroom = roomids[currentmsg['rid']] # tgtroom is the matrix room + tgtuser = "@" + currentmsg['u']['username'].lower() + ":" + args.hostname # tgtuser is the matrix user + + # Skip messages from bot users + if currentmsg['u']['username'].lower() == 'rocket.cat': + print(", skipping (bot user rocket.cat)") + continue + + dateTimeObj = datetime.fromisoformat(currentmsg['ts']['$date']) + tgtts = int(dateTimeObj.timestamp()*1000) # tgtts is the message timestamp + if currentmsg['_id'] in idmaps: + print(", already processed (in cache), skipping") + continue + print(", timestamp=" + str(tgtts)) + lastts = tgtts + + # Pinned messages, unhandled + if 't' in currentmsg and currentmsg['t']=="message_pinned": + print(", timestamp=" + str(tgtts) + ", message pinning event, skipping") + continue + + # Jitsi start messages, unhandled + if 't' in currentmsg and currentmsg['t']=="jitsi_call_started": + print(", timestamp=" + str(tgtts) + ", jitsi_call event, skipping") + continue + + # First, iterate attachments + # https://developer.rocket.chat/reference/api/rest-api/endpoints/messaging/chat-endpoints/send-message#attachment-field-objects + if 'attachments' in currentmsg and hasattr(currentmsg['attachments'], '__iter__'): + for attachment in currentmsg['attachments']: + if ('type' in attachment and attachment['type'] == 'file') or ('title_link' in attachment and '/file-upload/' in attachment['title_link']): # A file + vprint("a file") + api_endpoint = api_base + "_matrix/media/v3/upload?user_id=" + tgtuser + "&ts=" + str(tgtts) + "&filename=" + requests.utils.quote(attachment['title']) + api_headers_file = api_headers_as.copy() + if 'image_type' in attachment: # we have a content-type + vprint("an image") + api_headers_file['Content-Type'] = attachment['image_type'] + try: # try to find the file in the export + localfile=attachment['title_link'] + localfile=re.sub("/file-upload/", "", localfile) + localfile=re.sub("/.*", "", localfile) + filepath = args.inputs + "files/" + localfile + if not os.path.exists(filepath): + print(f" [MISSING FILE] {filepath} (title_link={attachment['title_link']})") + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), localfile) + with open(filepath, 'rb') as f: + # upload as a media to matrix + if 'Content-Type' not in api_headers_file: + api_headers_file['Content-Type'] = mime.from_file(filepath) + response = session.post(api_endpoint, headers=api_headers_file, data=f) + try: + vprint(response.json()) + except Exception: + vprint(f"Response status={response.status_code}, no JSON body") + if response.status_code != 200: # Upload problem + print(f" [UPLOAD FAILED] {response.status_code} {safe_json(response)}") + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), localfile) + upload_data = safe_json(response) + if 'content_uri' not in upload_data: + print(f" [UPLOAD FAILED] no content_uri in response: {upload_data}") + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), localfile) + mxcurl=upload_data['content_uri'] # URI of the uploaded media + # Then post a message referencing this media + api_endpoint = api_base + "_matrix/client/v3/rooms/" + tgtroom + '/send/m.room.message?user_id=' + tgtuser + "&ts=" + str(tgtts) # ts, ?user_id=@_irc_user:example.org + if 'image_type' in attachment: # attachment is an image + api_params = {'msgtype': 'm.image', 'body': attachment['title'], 'url': mxcurl} + else: # other files + api_params = {'msgtype': 'm.file', 'body': attachment['title'], 'url': mxcurl} + response = session.post(api_endpoint, json=api_params, headers=api_headers_as) + if is_forbidden(response): # not in the room + invite(api_base, api_headers_admin, tgtroom, tgtuser) + response = session.post(api_endpoint, json=api_params, headers=api_headers_as) + if response.status_code != 200: + print("error posting attachment") + print(attachment['title']) + print(safe_json(response)) + exit(1) + vprint(safe_json(response)) + except FileNotFoundError: # We do not have the linked attachment + api_endpoint = api_base + "_matrix/client/v3/rooms/" + tgtroom + '/send/m.room.message?user_id=' + tgtuser + "&ts=" + str(tgtts) # ts, ?user_id=@_irc_user:example.org + api_params = {'msgtype': 'm.text', 'body': "<<< A file named \"" + attachment['title'] + "\" was lost during the migration to Matrix >>>"} + response = session.post(api_endpoint, json=api_params, headers=api_headers_as) + if is_forbidden(response): # not in the room + invite(api_base, api_headers_admin, tgtroom, tgtuser) + response = session.post(api_endpoint, json=api_params, headers=api_headers_as) + if response.status_code != 200: + print(f" [WARN] could not post placeholder for missing attachment: {attachment['title']} (status={response.status_code})") + else: + vprint(safe_json(response)) + if 'description' in attachment: # Matrix does not support descriptions, we post as a message + api_endpoint = api_base + "_matrix/client/v3/rooms/" + tgtroom + '/send/m.room.message?user_id=' + tgtuser + "&ts=" + str(tgtts) # ts, ?user_id=@_irc_user:example.org + api_params = {'msgtype': 'm.text', 'body': attachment['description']} + response = session.post(api_endpoint, json=api_params, headers=api_headers_as) + if is_forbidden(response): # not in the room + invite(api_base, api_headers_admin, tgtroom, tgtuser) + response = session.post(api_endpoint, json=api_params, headers=api_headers_as) + if response.status_code != 200: + print("error posting description") + print(attachment['description']) + print(safe_json(response)) + exit(1) + vprint(safe_json(response)) + + elif 'message_link' in attachment: # This is a citation + vprint("A citation") + if 'msg' in currentmsg: + textmsg = emoji.emojize(currentmsg['msg'], language='alias') + else: + textmsg = "" + html = markdown.markdown(textmsg) # render the markdown + related = re.sub(r".*\?msg=", "", attachment['message_link']) # find related Matrix_messageID + api_endpoint = api_base + "_matrix/client/v3/rooms/" + tgtroom + '/send/m.room.message?user_id=' + tgtuser + "&ts=" + str(tgtts) # ts, ?user_id=@_irc_user:example.org + if related in idmaps: + api_params = {'msgtype': 'm.text', 'body': "> <" + attachment['author_name'] + ">" + attachment['text'] + "\n\n" + textmsg, + "format": "org.matrix.custom.html", + "formatted_body": "
In reply to " + attachment['author_name'] + "
" + attachment['text'] + "
" + html, + "m.relates_to": { + "m.in_reply_to": { + "event_id": idmaps[related] + } + }} + else: + api_params = {'msgtype': 'm.text', 'body': "> <" + attachment['author_name'] + ">" + attachment['text'] + "\n\n" + textmsg, + "format": "org.matrix.custom.html", + "formatted_body": "
In reply to " + attachment['author_name'] + "
" + attachment['text'] + "
" + html, + } + response = session.post(api_endpoint, json=api_params, headers=api_headers_as) + if is_forbidden(response): # not in the room + invite(api_base, api_headers_admin, tgtroom, tgtuser) + response = session.post(api_endpoint, json=api_params, headers=api_headers_as) + if response.status_code != 200: + print("error posting related") + print(textmsg) + print(safe_json(response)) + exit(1) + vprint(safe_json(response)) + finished=True # do not repost this message in the final step + elif 'image_url' in attachment: # This is an external image + vprint("An external image") + api_endpoint = api_base + "_matrix/client/v3/rooms/" + tgtroom + '/send/m.room.message?user_id=' + tgtuser + "&ts=" + str(tgtts) # ts, ?user_id=@_irc_user:example.org + api_params = {'msgtype': 'm.text', 'body': attachment['image_url']} + response = session.post(api_endpoint, json=api_params, headers=api_headers_as) + if is_forbidden(response): # not in the room + invite(api_base, api_headers_admin, tgtroom, tgtuser) + response = session.post(api_endpoint, json=api_params, headers=api_headers_as) + if response.status_code != 200: + print("error posting image url") + print(attachment['image_url']) + print(safe_json(response)) + exit(1) + vprint(safe_json(response)) + else: + print("Skipping unsupported attachment: " + str(attachment)) + + # Finally post the message + if 'msg' in currentmsg: + if currentmsg['msg'] != "" and not finished: + api_endpoint = api_base + "_matrix/client/v3/rooms/" + tgtroom + '/send/m.room.message?user_id=' + tgtuser + "&ts=" + str(tgtts) # ts, ?user_id=@_irc_user:example.org + api_params = format_message(emoji.emojize(currentmsg['msg'], language='alias')) + response = session.post(api_endpoint, json=api_params, headers=api_headers_as) + vprint(safe_json(response)) + + if is_forbidden(response): # not in the room + invite(api_base, api_headers_admin, tgtroom, tgtuser) + response = session.post(api_endpoint, json=api_params, headers=api_headers_as) + vprint(safe_json(response)) + + if response.status_code != 200: + print("error posting message") + print(currentmsg['msg']) + print(safe_json(response)) + exit(1) + + # We keep track of messageIDs to link future references + if response is not None: # is None if no message has been posted, nothing to keep in idmaps in this case + idmaps[currentmsg['_id']]=response.json()['event_id'] + cache.write(currentmsg['_id'] + ":" + response.json()['event_id'] + "\n") + dryrun_messages += 1 + else: + vprint("No response to get an event_id from") + continue + + if 'reactions' in currentmsg and currentmsg['reactions']: + for reaction in currentmsg['reactions']: + tgtreaction = emoji.emojize(reaction, language='alias') + for username in currentmsg['reactions'][reaction]['usernames']: + tgtusername = "@" + username.lower() + ":" + args.hostname + vprint(tgtusername + ":" + tgtreaction) + api_endpoint = api_base + "_matrix/client/v3/rooms/" + tgtroom + '/send/m.reaction?user_id=' + tgtusername + "&ts=" + str(tgtts) + api_params = {"m.relates_to": { + "event_id": idmaps[currentmsg['_id']], + "key": tgtreaction, + "rel_type": "m.annotation" + }} + response = session.post(api_endpoint, json=api_params, headers=api_headers_as) + vprint(response.json()) + else: + print(", skipping (room not imported)") + continue + + cache.close() + + if dry_run: + print("\n=== DRY-RUN SUMMARY ===") + print(f" Users to migrate: {dryrun_users}") + print(f" Rooms to create: {dryrun_rooms}") + print(f" Messages to import: {dryrun_messages}") + print("=== No changes were made ===") diff --git a/ansible/playbooks/saas/roles/matrix_migrate/files/requirements.txt b/ansible/playbooks/saas/roles/matrix_migrate/files/requirements.txt new file mode 100644 index 00000000..949528f1 --- /dev/null +++ b/ansible/playbooks/saas/roles/matrix_migrate/files/requirements.txt @@ -0,0 +1,4 @@ +requests +markdown +python-magic +emoji diff --git a/ansible/playbooks/saas/roles/matrix_migrate/tasks/destroy.yml b/ansible/playbooks/saas/roles/matrix_migrate/tasks/destroy.yml new file mode 100644 index 00000000..a246acc0 --- /dev/null +++ b/ansible/playbooks/saas/roles/matrix_migrate/tasks/destroy.yml @@ -0,0 +1,5 @@ +--- +- name: Destroy service + ansible.builtin.include_role: + name: common + tasks_from: destroy.yml diff --git a/ansible/playbooks/saas/roles/matrix_migrate/tasks/main.yml b/ansible/playbooks/saas/roles/matrix_migrate/tasks/main.yml new file mode 100644 index 00000000..0e36dc18 --- /dev/null +++ b/ansible/playbooks/saas/roles/matrix_migrate/tasks/main.yml @@ -0,0 +1,32 @@ +--- +- name: Create migration directory + ansible.builtin.file: + path: "{{ software_path }}/tmp/migration" + state: directory + owner: root + group: root + mode: '0755' + delegate_to: "{{ software.instance }}" + +- name: Deploy Application Service config for migration + ansible.builtin.template: + src: app-service.yaml.j2 + dest: "/data/{{ software.synapse_domain }}/data/app-service.yaml" + owner: 991 + group: 991 + mode: '0600' + delegate_to: "{{ software.instance }}" + +- name: Copy nomad job to destination + ansible.builtin.template: + src: nomad.hcl + dest: "/var/tmp/{{ domain }}.nomad" + owner: root + group: root + mode: '0600' + delegate_to: "{{ software.instance }}" + +- name: Run nomad job + ansible.builtin.include_role: + name: nomad + tasks_from: job_start.yml diff --git a/ansible/playbooks/saas/roles/matrix_migrate/templates/app-service.yaml.j2 b/ansible/playbooks/saas/roles/matrix_migrate/templates/app-service.yaml.j2 new file mode 100644 index 00000000..2ffe2e6d --- /dev/null +++ b/ansible/playbooks/saas/roles/matrix_migrate/templates/app-service.yaml.j2 @@ -0,0 +1,14 @@ +id: rc2matrix +hs_token: "{{ lookup('simple-stack-ui', type='secret', key=domain, subkey='matrix_app_token', missing='create', length=64) }}" +as_token: "{{ lookup('simple-stack-ui', type='secret', key=domain, subkey='matrix_app_token', missing='create', length=64) }}" +url: null +sender_localpart: rc2matrix +namespaces: + users: + - exclusive: false + regex: "@.*:{{ software.matrix_server_name }}" + rooms: [] + aliases: + - exclusive: false + regex: "#.*:{{ software.matrix_server_name }}" +rate_limited: false diff --git a/ansible/playbooks/saas/roles/matrix_migrate/templates/config.yaml.j2 b/ansible/playbooks/saas/roles/matrix_migrate/templates/config.yaml.j2 new file mode 100644 index 00000000..f2c0c6c7 --- /dev/null +++ b/ansible/playbooks/saas/roles/matrix_migrate/templates/config.yaml.j2 @@ -0,0 +1,3 @@ +SYNAPSE_URL=https://{{ software.matrix_domain }} +ADMIN_USERNAME={{ software.matrix_admin_user | default('admin') }} +ADMIN_PASSWORD={{ lookup('simple-stack-ui', type='secret', key=domain, subkey='matrix_admin_password', missing='error') }} diff --git a/ansible/playbooks/saas/roles/matrix_migrate/templates/nomad.hcl b/ansible/playbooks/saas/roles/matrix_migrate/templates/nomad.hcl new file mode 100644 index 00000000..d048e63d --- /dev/null +++ b/ansible/playbooks/saas/roles/matrix_migrate/templates/nomad.hcl @@ -0,0 +1,91 @@ +job "{{ domain }}" { + region = "{{ fact_instance.region }}" + datacenters = ["{{ fact_instance.datacenter }}"] + type = "batch" + +{% if software.constraints is defined and software.constraints.location is defined %} + constraint { + attribute = "${meta.location}" + set_contains = "{{ software.constraints.location }}" + } +{% endif %} + + constraint { + attribute = "${meta.instance}" + set_contains = "{{ software.instance }}" + } + + group "migrate" { + + count = 1 + + restart { + attempts = 0 + mode = "fail" + } + + task "{{ domain }}-migrate" { + + driver = "docker" + + config { + image = "python:3.12-slim" + command = "bash" + args = ["/app/run.sh"] + work_dir = "/tmp/migration" + volumes = [ + "{{ software_path }}/migration:/tmp/migration", + "local/migrate.py:/app/migrate.py:ro", + "local/requirements.txt:/app/requirements.txt:ro", + "local/run.sh:/app/run.sh:ro" + ] + } + + env { + MATRIX_ADMIN_TOKEN = "{{ lookup('simple-stack-ui', type='secret', key=domain, subkey='matrix_admin_token', missing='error') }}" + MATRIX_APP_TOKEN = "{{ lookup('simple-stack-ui', type='secret', key=domain, subkey='matrix_app_token', missing='error') }}" + } + + template { + change_mode = "restart" + destination = "local/migrate.py" + data = < The `-a` flag grants admin privileges. Omit it for a regular user. diff --git a/ansible/playbooks/saas/roles/synapse/defaults/main.yml b/ansible/playbooks/saas/roles/synapse/defaults/main.yml new file mode 100644 index 00000000..ed97d539 --- /dev/null +++ b/ansible/playbooks/saas/roles/synapse/defaults/main.yml @@ -0,0 +1 @@ +--- diff --git a/ansible/playbooks/saas/roles/synapse/tasks/backup.yml b/ansible/playbooks/saas/roles/synapse/tasks/backup.yml new file mode 100644 index 00000000..ed97d539 --- /dev/null +++ b/ansible/playbooks/saas/roles/synapse/tasks/backup.yml @@ -0,0 +1 @@ +--- diff --git a/ansible/playbooks/saas/roles/synapse/tasks/build.yml b/ansible/playbooks/saas/roles/synapse/tasks/build.yml new file mode 100644 index 00000000..c8c8cb30 --- /dev/null +++ b/ansible/playbooks/saas/roles/synapse/tasks/build.yml @@ -0,0 +1,12 @@ +--- +- name: Include upstream variables + ansible.builtin.include_vars: upstream.yml + +- name: Set custom variables + ansible.builtin.set_fact: + image_version: "{{ latest_version }}" + image_definition: "{{ image }}" + +- name: End playbook if no new version + ansible.builtin.meta: end_host + when: catalogs[catalog_image_name] is defined and catalogs[catalog_image_name].version == image_version diff --git a/ansible/playbooks/saas/roles/synapse/tasks/destroy.yml b/ansible/playbooks/saas/roles/synapse/tasks/destroy.yml new file mode 100644 index 00000000..a246acc0 --- /dev/null +++ b/ansible/playbooks/saas/roles/synapse/tasks/destroy.yml @@ -0,0 +1,5 @@ +--- +- name: Destroy service + ansible.builtin.include_role: + name: common + tasks_from: destroy.yml diff --git a/ansible/playbooks/saas/roles/synapse/tasks/main.yml b/ansible/playbooks/saas/roles/synapse/tasks/main.yml new file mode 100644 index 00000000..e326115b --- /dev/null +++ b/ansible/playbooks/saas/roles/synapse/tasks/main.yml @@ -0,0 +1,59 @@ +--- +- name: Install mandatories packages + ansible.builtin.apt: + pkg: + - python3-psycopg2 + +- name: Create default directories + ansible.builtin.file: + path: "{{ item.path }}" + state: directory + owner: "{{ item.owner | default('root') }}" + group: "{{ item.group | default('root') }}" + mode: '0755' + loop: + - path: "{{ software_path }}/data" + owner: 991 + group: 991 + - path: "{{ software_path }}/data/media_store" + owner: 991 + group: 991 + delegate_to: "{{ software.instance }}" + +- name: Create postgresql user + community.postgresql.postgresql_user: + login_user: postgres + login_password: "{{ lookup('simple-stack-ui', type='secret', key=software.db_host, subkey='passwd', missing='error') }}" + login_host: "{{ software.db_host | replace('-', '') | replace('.', '') }}.default.service.nomad" + login_port: "{{ software.db_port | default(5432) }}" + name: "{{ software.db_user | default('synapse') }}" + password: "{{ lookup('simple-stack-ui', type='secret', key=domain, subkey='db_password', missing='create', length=16) }}" + state: present + +- name: Create postgresql database + community.postgresql.postgresql_db: + login_user: postgres + login_password: "{{ lookup('simple-stack-ui', type='secret', key=software.db_host, subkey='passwd', missing='error') }}" + login_host: "{{ software.db_host | replace('-', '') | replace('.', '') }}.default.service.nomad" + login_port: "{{ software.db_port | default(5432) }}" + name: "{{ software.db_name | default('synapse') }}" + encoding: UTF8 + lc_collate: C + lc_ctype: C + template: template0 + owner: "{{ software.db_user | default('synapse') }}" + state: present + +- name: Copy nomad job to destination + ansible.builtin.template: + src: nomad.hcl + dest: "/var/tmp/{{ domain }}.nomad" + owner: root + group: root + mode: '0600' + delegate_to: "{{ software.instance }}" + +- name: Run nomad job + ansible.builtin.include_role: + name: nomad + tasks_from: job_start.yml diff --git a/ansible/playbooks/saas/roles/synapse/tasks/restore.yml b/ansible/playbooks/saas/roles/synapse/tasks/restore.yml new file mode 100644 index 00000000..ed97d539 --- /dev/null +++ b/ansible/playbooks/saas/roles/synapse/tasks/restore.yml @@ -0,0 +1 @@ +--- diff --git a/ansible/playbooks/saas/roles/synapse/templates/homeserver.yaml.j2 b/ansible/playbooks/saas/roles/synapse/templates/homeserver.yaml.j2 new file mode 100644 index 00000000..4d591def --- /dev/null +++ b/ansible/playbooks/saas/roles/synapse/templates/homeserver.yaml.j2 @@ -0,0 +1,103 @@ +server_name: "{{ software.domain }}" +pid_file: /data/homeserver.pid +public_baseurl: "https://{{ software.domain }}" +web_client_location: "https://{{ software.domain }}" + +listeners: + - port: 8008 + tls: false + type: http + x_forwarded: true + resources: + - names: [client, federation] + compress: false + +database: + name: psycopg2 + txn_limit: 10000 + args: + host: "{{ software.db_host | replace('-', '') | replace('.', '') }}.default.service.nomad" + port: {{ software.db_port | default(5432) }} + user: "{{ software.db_user | default('synapse') }}" + password: "{{ lookup('simple-stack-ui', type='secret', key=domain, subkey='db_password', missing='create', length=16) }}" + database: "{{ software.db_name | default('synapse') }}" + cp_min: 5 + cp_max: 10 + +log_config: "/local/log.config" +media_store_path: /data/media_store +max_upload_size: 50M + +registration_shared_secret: "{{ lookup('simple-stack-ui', type='secret', key=domain, subkey='registration_shared_secret', missing='create', length=32) }}" +enable_registration: {{ software.enable_registration | default(false) | lower }} + +report_stats: false + +macaroon_secret_key: "{{ lookup('simple-stack-ui', type='secret', key=domain, subkey='macaroon_secret_key', missing='create', length=32) }}" +form_secret: "{{ lookup('simple-stack-ui', type='secret', key=domain, subkey='form_secret', missing='create', length=32) }}" + +signing_key_path: "/data/signing.key" + +trusted_key_servers: + - server_name: "matrix.org" + +{% if software.cache_host is defined %} +redis: + enabled: true + host: "{{ software.cache_host }}" + port: {{ software.cache_port | default(6379) }} +{% endif %} + +{% if software.turn_domain is defined %} +turn_uris: + - "turn:{{ software.turn_domain }}:3478?transport=udp" + - "turn:{{ software.turn_domain }}:3478?transport=tcp" +turn_shared_secret: "{{ lookup('simple-stack-ui', type='secret', key=software.turn_domain, subkey='turn_shared_secret', missing='error') }}" +turn_user_lifetime: 86400000 +turn_allow_guests: true +{% endif %} + +suppress_key_server_warning: true + +{% if software.disable_ratelimiting | default(false) %} +rc_message: + per_second: 1000 + burst_count: 2000 +rc_joins: + local: + per_second: 1000 + burst_count: 2000 +rc_joins_per_room: + per_second: 1000 + burst_count: 2000 +rc_invites: + per_room: + per_second: 1000 + burst_count: 2000 + per_user: + per_second: 1000 + burst_count: 2000 + per_issuer: + per_second: 1000 + burst_count: 2000 +rc_login: + address: + per_second: 1000 + burst_count: 2000 + account: + per_second: 1000 + burst_count: 2000 + failed_attempts: + per_second: 1000 + burst_count: 2000 +rc_admin_redaction: + per_second: 1000 + burst_count: 2000 +{% endif %} + +{% if software.app_service_config_files is defined %} +app_service_config_files: +{% for asfile in software.app_service_config_files %} + - "{{ asfile }}" +{% endfor %} +{% endif %} diff --git a/ansible/playbooks/saas/roles/synapse/templates/log.config.j2 b/ansible/playbooks/saas/roles/synapse/templates/log.config.j2 new file mode 100644 index 00000000..ca411cf0 --- /dev/null +++ b/ansible/playbooks/saas/roles/synapse/templates/log.config.j2 @@ -0,0 +1,20 @@ +version: 1 + +formatters: + precise: + format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s' + +handlers: + console: + class: logging.StreamHandler + formatter: precise + +loggers: + synapse.storage.SQL: + level: INFO + +root: + level: INFO + handlers: [console] + +disable_existing_loggers: false diff --git a/ansible/playbooks/saas/roles/synapse/templates/nomad.hcl b/ansible/playbooks/saas/roles/synapse/templates/nomad.hcl new file mode 100644 index 00000000..c34181de --- /dev/null +++ b/ansible/playbooks/saas/roles/synapse/templates/nomad.hcl @@ -0,0 +1,94 @@ +job "{{ domain }}" { + region = "{{ fact_instance.region }}" + datacenters = ["{{ fact_instance.datacenter }}"] + type = "service" + +{% if software.constraints is defined and software.constraints.location is defined %} + constraint { + attribute = "${meta.location}" + set_contains = "{{ software.constraints.location }}" + } +{% endif %} + + constraint { + attribute = "${meta.instance}" + set_contains = "{{ software.instance }}" + } + + group "synapse" { + + count = 1 + + restart { + attempts = 2 + interval = "10m" + delay = "15s" + mode = "fail" + } + + network { + port "synapse" { + to = 8008 + } + } + + service { + name = "{{ service_name }}" + port = "synapse" + provider = "nomad" + tags = [ + {{ lookup('template', '../../traefik/templates/traefik_tag.j2') | indent(8) }} + ] + check { + name = "{{ service_name }}" + type = "http" + path = "/health" + interval = "60s" + timeout = "30s" + check_restart { + limit = 3 + grace = "90s" + ignore_warnings = false + } + } + } + + task "{{ domain }}-synapse" { + + driver = "docker" + + env { + SYNAPSE_CONFIG_PATH = "/local/homeserver.yaml" + } + + config { + image = "matrixdotorg/synapse:v{{ catalogs.synapse.version }}" + ports = ["synapse"] + volumes = [ + "{{ software_path }}/data:/data:rw" + ] + } + + template { + change_mode = "restart" + destination = "local/homeserver.yaml" + data = < Date: Sat, 27 Jun 2026 13:11:50 +0200 Subject: [PATCH 6/7] chore(roles-variables): update PaaS and SaaS variables JSON PaaS changes: - Add ufw_blocklist_enabled and ufw_blocklist_url - Remove ufw_packages array - Disable fail2ban_ratelimit_enabled - Update nomad_client_drain_on_shutdown_force/ignore_system_jobs to false - Update nomad_leave_on_terminate/interrupt to false SaaS changes: - Add coturn role (empty config) - Add element_web role (empty config) - Add matrix_migrate role (empty config) - Add synapse role (empty config) - Add synapse_admin role (empty config) --- ui-next/public/roles-variables-paas.json | 17 ++++++++--------- ui-next/public/roles-variables-saas.json | 7 ++++++- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/ui-next/public/roles-variables-paas.json b/ui-next/public/roles-variables-paas.json index 2ab9f1b9..32b0b19a 100644 --- a/ui-next/public/roles-variables-paas.json +++ b/ui-next/public/roles-variables-paas.json @@ -54,6 +54,8 @@ }, "ansible-ufw": { "ufw_applications": [], + "ufw_blocklist_enabled": true, + "ufw_blocklist_url": "https://cdn.jsdelivr.net/gh/duggytuxy/Data-Shield_IPv4_Blocklist@refs/heads/main/prod_data-shield_ipv4_blocklist.txt", "ufw_custom_rules": [], "ufw_default_application_policy": "SKIP", "ufw_default_forward_policy": "DROP", @@ -61,9 +63,6 @@ "ufw_default_output_policy": "ACCEPT", "ufw_ipv6": "yes", "ufw_logging": "off", - "ufw_packages": [ - "ufw" - ], "ufw_reset": false, "ufw_rules": [ { @@ -266,7 +265,7 @@ "conntrack" ], "fail2ban_ratelimit_bantime": 600, - "fail2ban_ratelimit_enabled": true, + "fail2ban_ratelimit_enabled": false, "fail2ban_ratelimit_findtime": 30, "fail2ban_ratelimit_maxretry": 50, "fail2ban_traefik_access_log": "/var/log/traefik/traefik-access.log", @@ -303,8 +302,8 @@ "nomad_client_disk_free_mb": 0, "nomad_client_disk_total_mb": 0, "nomad_client_drain_on_shutdown_deadline": "1m", - "nomad_client_drain_on_shutdown_force": true, - "nomad_client_drain_on_shutdown_ignore_system_jobs": true, + "nomad_client_drain_on_shutdown_force": false, + "nomad_client_drain_on_shutdown_ignore_system_jobs": false, "nomad_client_enabled": true, "nomad_client_gc_disk_usage_threshold": 80, "nomad_client_gc_inode_usage_threshold": 70, @@ -386,8 +385,8 @@ "nomad_http_scheme": "https", "nomad_iface": "ens3", "nomad_job_files_dir": "/var/tmp", - "nomad_leave_on_interrupt": true, - "nomad_leave_on_terminate": true, + "nomad_leave_on_interrupt": false, + "nomad_leave_on_terminate": false, "nomad_location": "{{ fact_instance.region }}", "nomad_log_file": "/var/log/nomad/nomad.log", "nomad_log_level": "WARN", @@ -529,4 +528,4 @@ "unattended_syslog_facility": "daemon" }, "upstream": {} -} +} \ No newline at end of file diff --git a/ui-next/public/roles-variables-saas.json b/ui-next/public/roles-variables-saas.json index 1747b3a9..5fc24b03 100644 --- a/ui-next/public/roles-variables-saas.json +++ b/ui-next/public/roles-variables-saas.json @@ -10,7 +10,9 @@ "timezone": "Europe/Paris" }, "common": null, + "coturn": {}, "dolibarr": {}, + "element_web": {}, "forgejo": {}, "freqtrade": {}, "freshrss": {}, @@ -65,6 +67,7 @@ }, "loki": {}, "mariadb": {}, + "matrix_migrate": {}, "milvus": {}, "mimir": { "mimir_install_node": "{{ software.install_node |default('singlenode') }}" @@ -92,6 +95,8 @@ "rocketchat": {}, "simplestack_ansible": null, "simplestack_ui": null, + "synapse": {}, + "synapse_admin": {}, "traefik": { "nomad_http_ip": "127.0.0.1", "traefik_email": "{{ software.email | default(traefik_email_default) }}" @@ -104,4 +109,4 @@ "zigbee2mqtt": { "zigbee2mqtt_config": "{{ software.config | default(zigbee2mqtt_config_default) }}" } -} +} \ No newline at end of file From fa724c758eb54a7d12f22275dfda665f15b157f9 Mon Sep 17 00:00:00 2001 From: Mathieu Garcia Date: Sat, 27 Jun 2026 13:12:04 +0200 Subject: [PATCH 7/7] fix(fail2ban): disable rate limit by default - Set fail2ban_ratelimit_enabled to false (was true) - Prevents false positives on high-traffic servers --- ansible/playbooks/paas/roles/fail2ban/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/paas/roles/fail2ban/defaults/main.yml b/ansible/playbooks/paas/roles/fail2ban/defaults/main.yml index 12cad1c4..d0c617f9 100644 --- a/ansible/playbooks/paas/roles/fail2ban/defaults/main.yml +++ b/ansible/playbooks/paas/roles/fail2ban/defaults/main.yml @@ -37,7 +37,7 @@ fail2ban_404_findtime: 600 fail2ban_404_bantime: 600 # Rate limit -fail2ban_ratelimit_enabled: true +fail2ban_ratelimit_enabled: false fail2ban_ratelimit_maxretry: 50 fail2ban_ratelimit_findtime: 30 fail2ban_ratelimit_bantime: 600