diff --git a/docs/habref_update.md b/docs/habref_update.md new file mode 100644 index 0000000..d7d35fd --- /dev/null +++ b/docs/habref_update.md @@ -0,0 +1,68 @@ +# Mise à jour du référentiel HABREF + +Scripts permettant de mettre à jour le référentiel des habitats (HABREF) vers une nouvelle version. + +## Avant de commencer + +> [!WARNING] +> La mise à jour du référentiel HABREF efface les données de la version du référentiel précédemment installée. + +> [!WARNING] +> Il est fortement recommandé de **faire une sauvegarde de la base de données** avant de commencer. + +## 1. Importer la nouvelle version et détecter les orphelins + +Dans la première étape, il faut télécharger les données du référentiels, stocker ces dernières dans une table temporaire (ref*habitats.tmp*). Pour cela, on lance la commande : + +```bash +geonature habref import-v07 +``` + +> [!DANGER] +> Il se peut que certaines entrées du référentiels soient supprimées lors d'une mise à jours. Dans ce cas, le fichier `tmp/habref/orphans_habref.csv` liste l'ensemble des données dans votre base utilisant ces entrées Habref. + +Si la commande est relancée, les tables temporaires existantes sont automatiquement supprimées et recréées. + +--- + +## 2 — Analyser et corriger les données orphelines + +Dans le cas où plusieurs données orphelines ont été détecté dans l'étape précédente, consultez le fichier `tmp/habref/orphans_habref.csv`. Pour chaque ligne de ce fichier, la valeur `fk_value` est un code habitat qui sera supprimé lors de la mise à jour. + +Le CSV généré contient les colonnes suivantes : + +| Colonne | Description | +| --------------------- | -------------------------------------------------------- | +| `ref_table` | Table du référentiel concernée (ex: `habref`, `typoref`) | +| `table_name` | Table applicative contenant la référence orpheline | +| `schema` | Schéma de cette table | +| `fk_column` | Colonne portant la clé étrangère | +| `fk_value` | Valeur orpheline | +| `nb_lignes_affectees` | Nombre de lignes concernées | + +Vous devez décider, pour chaque cas : + +- **Mettre à jour** les données concernées avec le `cd_hab` de remplacement dans la nouvelle version +- **Supprimer** les observations si elles ne peuvent plus être rattachées + +--- + +## 3 — Appliquer la mise à jour + +Une fois les données orphelines corrigées, lancez la mise à jour effective du référentiel à l'aide de la commande suivante : + +```bash +geonature habref apply-v07 +``` + +Cette commande : + +1. Désactive temporairement les contraintes de clés étrangères +2. Vide toutes les tables du schéma `ref_habitats` (dans l'ordre inverse des dépendances) +3. Recopie le contenu des tables temporaires `tmp_*` dans les tables de production +4. Reconstruit la table `ref_habitats.autocomplete_habitat` +5. Réactive les contraintes de clés étrangères +6. Supprime les tables temporaires `tmp_*` + +> [!WARNING] +> ⚠️ Si des données orphelines subsistent au moment du `apply-v07`, des erreurs de contraintes FK pourraient apparaître et la migration ne se feras pas. Assurez-vous que toutes les corrections ont bien été appliquées avant de lancer cette commande. diff --git a/pyproject.toml b/pyproject.toml index 3845a99..fe257bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,9 @@ Homepage = "https://github.com/PnX-SI/Habref-api-module" [project.entry-points."alembic"] migrations = "pypn_habref_api.migrations:versions" +[project.entry-points."flask.commands"] +habref = "pypn_habref_api.commands.habref:habref" + [tool.setuptools] package-dir = { "" = "src" } packages = ["pypn_habref_api"] diff --git a/src/pypn_habref_api/commands/habref.py b/src/pypn_habref_api/commands/habref.py new file mode 100644 index 0000000..19f5626 --- /dev/null +++ b/src/pypn_habref_api/commands/habref.py @@ -0,0 +1,18 @@ +import click + +from flask.cli import with_appcontext + +from .habref_v7 import import_v07, apply_v07 + +import logging + +logger = logging.getLogger("habref_commands") + + +@click.group(help="Manager HabRef referentials.") +def habref(): + pass + + +habref.add_command(import_v07) +habref.add_command(apply_v07) diff --git a/src/pypn_habref_api/commands/habref_v7.py b/src/pypn_habref_api/commands/habref_v7.py new file mode 100644 index 0000000..8899ed7 --- /dev/null +++ b/src/pypn_habref_api/commands/habref_v7.py @@ -0,0 +1,234 @@ +import logging + +from pypn_habref_api.env import db + +import click +from flask.cli import with_appcontext + +from utils_flask_sqla.referential import collect_orphan_rows, export_orphans_to_csv +from .utils import import_habref, apply_habref + +base_url = "https://geonature.fr/data/inpn/habitats/" +table_files = { + "typoref": { + "filename": "TYPOREF_70.csv", + "unique_column": "cd_typo", + "table_fields": { + "cd_typo": "CD_TYPO", + "cd_table": "CD_TABLE", + "lb_nom_typo": "LB_NOM_TYPO", + "nom_jeu_donnees": "NOM_JEU_DONNEES", + "date_creation": "DATE_CREATION", + "auteur_typo": "AUTEUR_TYPO", + "auteur_table": "AUTEUR_TABLE", + "territoire": "TERRITOIRE", + "organisme": "ORGANISME", + "langue": "LANGUE", + "presentation": "PRESENTATION", + "description": "DESCRIPTION", + "origine": "ORIGINE", + "ref_biblio": "REF_BIBLIO", + "mots_cles": "MOTS_CLES", + "referencement": "REFERENCEMENT", + "diffusion": "DIFFUSION", + "derniere_modif": "DERNIERE_MODIF", + "type_table": "TYPE_TABLE", + "cd_typo_entre": "CD_TYPO_ENTRE", + "cd_typo_sortie": "CD_TYPO_SORTIE", + }, + }, + "bib_habref_typo_rel": { + "filename": "HABREF_TYPE_REL_70.csv", + "unique_column": "cd_type_rel", + "table_fields": { + "cd_type_rel": "CD_TYPE_REL", + "lb_type_rel": "LB_TYPE_REL", + "lb_rel": "LB_REL", + "corresp_hab": "CORRESP_HAB", + "corresp_esp": "CORRESP_ESP", + "corresp_syn": "CORRESP_SYN", + "date_crea": "DATE_CREA", + "date_modif": "DATE_MODIF", + }, + }, + "bib_habref_statuts": { + "filename": "HABREF_STATUTS.csv", + "unique_column": "statut", + "table_fields": { + "statut": "STATUT", + "description": "DESCRIPTION", + "definition": "DEFINITION", + "ordre": "ORDRE", + }, + }, + "habref_sources": { + "filename": "HABREF_SOURCES_70.csv", + "unique_column": "cd_source", + "table_fields": { + "cd_source": "CD_SOURCE", + "cd_doc": "CD_DOC", + "type_source": "TYPE_SOURCE", + "auteur_source": "AUTEUR_SOURCE", + "date_source": "DATE_SOURCE", + "lb_source": "LB_SOURCE", + "lb_source_complet": "LB_SOURCE_COMPLET", + "titre": "TITRE", + "link": "LINK", + "date_crea": "DATE_CREA", + "date_modif": "DATE_MODIF", + }, + }, + "habref": { + "filename": "HABREF_70.csv", + "unique_column": "cd_hab", + "table_fields": { + "cd_hab": "CD_HAB", + "fg_validite": "FG_VALIDITE", + "cd_typo": "CD_TYPO", + "lb_code": "LB_CODE", + "lb_hab_fr": "LB_HAB_FR", + "lb_hab_fr_complet": "LB_HAB_FR_COMPLET", + "lb_hab_en": "LB_HAB_EN", + "lb_auteur": "LB_AUTEUR", + "niveau": "NIVEAU", + "lb_niveau": "LB_NIVEAU", + "cd_hab_sup": "CD_HAB_SUP", + "path_cd_hab": "PATH_CD_HAB", + "france": "FRANCE", + "lb_description": "LB_DESCRIPTION", + }, + }, + "habref_corresp_hab": { + "filename": "HABREF_CORRESP_HAB_70.csv", + "unique_column": "cd_corresp_hab", + "table_fields": { + "cd_corresp_hab": "CD_CORRESP_HAB", + "cd_hab_entre": "CD_HAB_ENTRE", + "cd_hab_sortie": "CD_HAB_SORTIE", + "cd_type_relation": "CD_TYPE_RELATION", + "lb_condition": "LB_CONDITION", + "lb_remarques": "LB_REMARQUES", + "validite": "VALIDITE", + "cd_typo_entre": "CD_TYPO_ENTRE", + "cd_typo_sortie": "CD_TYPO_SORTIE", + }, + }, + "habref_corresp_taxon": { + "filename": "HABREF_CORRESP_TAXON_70.csv", + "unique_column": "cd_corresp_tax", + "table_fields": { + "cd_corresp_tax": "CD_CORRESP_TAX", + "cd_hab_entre": "CD_HAB_ENTRE", + "cd_nom": "CD_NOM", + "cd_type_relation": "CD_TYPE_RELATION", + "lb_condition": "LB_CONDITION", + "lb_remarques": "LB_REMARQUES", + "nom_cite": "NOM_CITE", + "validite": "VALIDITE", + "date_crea": "DATE_CREA", + "date_modif": "DATE_MODIF", + }, + }, + "cor_habref_terr_statut": { + "filename": "HABREF_TERR_70.csv", + "unique_column": "cd_hab_ter", + "table_fields": { + "cd_hab_ter": "CD_HAB_TERR", + "cd_hab": "CD_HAB", + "cd_sig_terr": "CD_SIG_TERR", + "cd_statut_presence": "CD_STATUT_PRESENCE", + "date_crea": "DATE_CREA", + "date_modif": "DATE_MODIF", + }, + }, + "typoref_fields": { + "filename": "TYPOREF_FIELDS_70.csv", + "unique_column": "cd_hab_field", + "table_fields": { + "cd_hab_field": "CD_HAB_FIELD", + "cd_typo": "CD_TYPO", + "lb_hab_field": "LB_HAB_FIELD", + "format_hab_field": "FORMAT_HAB_FIELD", + "descript_hab_field": "DESCRIPT_HAB_FIELD", + "ordre_hab_field": "ORDRE_HAB_FIELD", + "length_hab_field": "LENGTH_HAB_FIELD", + "lb_label": "LB_LABEL", + "date_crea": "DATE_CREA", + "date_modif": "DATE_MODIF", + }, + }, + "cor_habref_description": { + "filename": "HABREF_DESCRIPTION_70.csv", + "unique_column": "cd_hab_description", + "table_fields": { + "cd_hab_description": "CD_HAB_DESCRIPTION", + "cd_hab": "CD_HAB", + "cd_hab_field": "CD_HAB_FIELD", + "valeurs": "VALEURS", + }, + }, + "cor_hab_source": { + "filename": "HABREF_LIEN_SOURCES_70.csv", + "unique_column": "cd_hab_lien_source", + "table_fields": { + "cd_hab_lien_source": "CD_HAB_LIEN_SOURCE", + "cd": "CD", + "type_lien": "TYPE_LIEN", + "cd_source": "CD_SOURCE", + "origine": "ORIGINE", + "date_crea": "DATE_CREA", + "date_modif": "DATE_MODIF", + }, + }, +} + + +@click.command() +@with_appcontext +def import_v07(): + logger = logging.getLogger() + + import_habref( + logger, + table_files=table_files, + schema="ref_habitats", + base_url=base_url, + num_version="07", + archive_name="HABREF_70.zip", + ) + + logger.info("Détection des données orphelines…") + all_orphans = [] + for table, config in table_files.items(): + logger.info(f" Vérification de {table}…") + all_orphans.extend( + collect_orphan_rows( + ref_table=table, + new_ref_table=f"tmp_{table}", + pk_col=config["unique_column"], + db=db, + schema="ref_habitats", + exclude_tables=list(table_files.keys()), + ) + ) + nb = export_orphans_to_csv(all_orphans, "tmp/habref/orphans_habref.csv") + if nb: + logger.warning( + f"{nb} valeur(s) orpheline(s) détectée(s), voir tmp/habref/orphans_habref.csv" + ) + else: + logger.info("Aucune donnée orpheline détectée.") + + logger.info("Committing…") + db.session.commit() + + +@click.command() +@with_appcontext +def apply_v07(): + logger = logging.getLogger() + + apply_habref(logger, table_files=table_files, schema="ref_habitats") + + logger.info("Committing…") + db.session.commit() diff --git a/src/pypn_habref_api/commands/utils.py b/src/pypn_habref_api/commands/utils.py new file mode 100644 index 0000000..3dc8fe5 --- /dev/null +++ b/src/pypn_habref_api/commands/utils.py @@ -0,0 +1,151 @@ +from csv import DictReader +from io import TextIOWrapper +from zipfile import ZipFile + +import sqlalchemy as sa +from sqlalchemy import inspect as sa_inspect, text as sa_text +from sqlalchemy.schema import ( + Table, + MetaData, + PrimaryKeyConstraint, + ForeignKeyConstraint, +) + +from utils_flask_sqla.migrations.utils import open_remote_file + +from pypn_habref_api.env import db + + +def get_csv_field_names(f, encoding, delimiter): + if encoding == "WIN1252": # postgresql encoding + encoding = "cp1252" # python encoding + t = TextIOWrapper(f, encoding=encoding) + reader = DictReader(t, delimiter=delimiter) + field_names = reader.fieldnames + t.detach() # avoid f to be closed on t garbage collection + f.seek(0) + return field_names + + +""" +Insert CSV file into specified table. +If source columns are specified, CSV file in copied in a temporary table, +then data restricted to specified source columns are copied in final table. +""" + + +def copy_from_csv( + f, + table_name, + table_fields, + schema="", + header=True, + encoding=None, + delimiter=None, + db=None, +): + bind = db.session.get_bind() + metadata = MetaData(bind=bind) + engine = db.engine + + table_fields_list = list(table_fields.values()) + table_fields_key_list = list(table_fields.keys()) + + final_table_name = table_name + table_name = f"import_{table_name}" + field_names = get_csv_field_names(f, encoding=encoding, delimiter=delimiter) + field_names = list(map(lambda field_name: field_name.replace("\ufeff", ""), field_names)) + table = Table( + table_name, + metadata, + *[sa.Column(c, sa.String) for c in map(str.lower, field_names)], + schema=schema, + ) + table.create(bind=db.session.connection()) + + options = ["FORMAT CSV"] + if header: + options.append("HEADER") + if encoding: + options.append(f"ENCODING '{encoding}'") + if delimiter: + options.append(f"DELIMITER E'{delimiter}'") + options = ", ".join(options) + cursor = db.session.connection().connection.cursor() + cursor.copy_expert( + f""" + COPY {schema}.{table_name} + FROM STDIN WITH ({options}) + """, + f, + ) + + testTable = Table( + final_table_name, MetaData(), schema=schema, autoload_with=db.session.connection() + ) + + for col in testTable.columns: + if col.name in table_fields: + table_fields[col.name] = f"{table_fields[col.name]}::{col.type}" + table_fields_list = list(table_fields.values()) + + db.session.execute(f""" + INSERT INTO {schema}.{final_table_name} ({", ".join(table_fields_key_list)}) + SELECT {", ".join(table_fields_list)} + FROM {schema}.{table_name}; + """) + table.drop(bind=db.session.connection()) + + +def import_habref(logger, table_files, schema, base_url, num_version, archive_name): + with open_remote_file(base_url, archive_name, open_fct=ZipFile) as archive: + for table, value in table_files.items(): + logger.info(f"Insert HABREF v{num_version} {table}…") + with archive.open(value["filename"]) as f: + db.session.execute(sa_text(f"DROP TABLE IF EXISTS {schema}.tmp_{table}")) + db.session.execute( + sa_text(f"CREATE TABLE {schema}.tmp_{table} AS TABLE {schema}.{table} WITH NO DATA") + ) + copy_from_csv( + f, + f"tmp_{table}", + value["table_fields"], + encoding="UTF-8", + delimiter=";", + schema=schema, + db=db, + ) + + +def apply_habref(logger, table_files, schema): + db.session.execute(sa_text("SET session_replication_role = 'replica'")) + + for table in reversed(list(table_files.keys())): + logger.info(f"Vidage de {table}…") + db.session.execute(sa_text(f"DELETE FROM {schema}.{table}")) + + for table in table_files.keys(): + logger.info(f"Remplissage de {table} depuis tmp_{table}…") + db.session.execute( + sa_text(f"INSERT INTO {schema}.{table} SELECT * FROM {schema}.tmp_{table}") + ) + + logger.info("Remplissage de autocomplete_habitat…") + db.session.execute(sa_text(f"DELETE FROM {schema}.autocomplete_habitat")) + db.session.execute(sa_text(f""" + INSERT INTO {schema}.autocomplete_habitat + SELECT + cd_hab, + h.cd_typo, + lb_code, + lb_nom_typo, + concat(lb_code, ' - ', lb_hab_fr, ' ', lb_hab_fr_complet) + FROM {schema}.habref h + JOIN {schema}.typoref t ON t.cd_typo = h.cd_typo + """)) + + db.session.execute(sa_text("SET session_replication_role = 'origin'")) + + for table in reversed(list(table_files.keys())): + logger.info(f"Suppression de tmp_{table}…") + db.session.execute(sa_text(f"DROP TABLE IF EXISTS {schema}.tmp_{table}")) diff --git a/src/pypn_habref_api/migrations/versions/46e91e738845_insert_inpn_data_in_ref_habitats_schema.py b/src/pypn_habref_api/migrations/versions/46e91e738845_insert_inpn_data_in_ref_habitats_schema.py index e62134c..6f9c308 100644 --- a/src/pypn_habref_api/migrations/versions/46e91e738845_insert_inpn_data_in_ref_habitats_schema.py +++ b/src/pypn_habref_api/migrations/versions/46e91e738845_insert_inpn_data_in_ref_habitats_schema.py @@ -52,7 +52,8 @@ def upgrade(): ) logger.info("Populate table autocomplete_habitat…") - op.execute(""" + op.execute( + """ INSERT INTO ref_habitats.autocomplete_habitat SELECT cd_hab, @@ -62,7 +63,8 @@ def upgrade(): concat(lb_code, ' - ', lb_hab_fr, ' ', lb_hab_fr_complet) FROM ref_habitats.habref h JOIN ref_habitats.typoref t ON t.cd_typo = h.cd_typo - """) + """ + ) def downgrade(): diff --git a/src/pypn_habref_api/migrations/versions/805442837a68_correction_on_habref_data.py b/src/pypn_habref_api/migrations/versions/805442837a68_correction_on_habref_data.py index fe83ced..23951f8 100644 --- a/src/pypn_habref_api/migrations/versions/805442837a68_correction_on_habref_data.py +++ b/src/pypn_habref_api/migrations/versions/805442837a68_correction_on_habref_data.py @@ -18,7 +18,8 @@ def upgrade(): - op.execute(""" + op.execute( + """ ALTER TABLE ref_habitats.habref ADD column lb_hab_en_save character varying(500); UPDATE ref_habitats.habref SET lb_hab_en_save = lb_auteur; @@ -27,8 +28,10 @@ def upgrade(): UPDATE ref_habitats.habref SET lb_hab_en = lb_hab_en_save; ALTER TABLE ref_habitats.habref DROP column lb_hab_en_save - """) - op.execute(""" + """ + ) + op.execute( + """ DELETE FROM ref_habitats.autocomplete_habitat; INSERT INTO ref_habitats.autocomplete_habitat SELECT @@ -39,7 +42,8 @@ def upgrade(): concat(lb_code, ' - ', lb_hab_fr, ' ', lb_hab_fr_complet) FROM ref_habitats.habref h JOIN ref_habitats.typoref t ON t.cd_typo = h.cd_typo; - """) + """ + ) def downgrade(): diff --git a/src/pypn_habref_api/models.py b/src/pypn_habref_api/models.py index 165ca10..febd8c5 100644 --- a/src/pypn_habref_api/models.py +++ b/src/pypn_habref_api/models.py @@ -20,6 +20,31 @@ class BibHabrefTypoRel(DB.Model): corresp_syn: Mapped[Optional[bool]] = mapped_column(Boolean) +@serializable +class BibHabrefStatuts(DB.Model): + __tablename__ = "bib_habref_statuts" + __table_args__ = {"schema": "ref_habitats"} + statut = DB.Column(DB.Unicode, primary_key=True) + description = DB.Column(DB.Unicode) + definition = DB.Column(DB.Unicode) + ordre = DB.Column(DB.Integer) + + +@serializable +class HabrefSources(DB.Model): + __tablename__ = "habref_sources" + __table_args__ = {"schema": "ref_habitats"} + cd_source = DB.Column(DB.Integer, primary_key=True) + cd_doc = DB.Column(DB.Integer) + type_source = DB.Column(DB.Unicode) + auteur_source = DB.Column(DB.Unicode) + date_source = DB.Column(DB.Integer) + lb_source = DB.Column(DB.Unicode) + lb_source_complet = DB.Column(DB.Unicode) + titre = DB.Column(DB.Unicode) + link = DB.Column(DB.Unicode) + + @serializable class CorespHab(DB.Model): __tablename__ = "habref_corresp_hab" @@ -41,6 +66,68 @@ class CorespHab(DB.Model): type_rel = DB.relationship("BibHabrefTypoRel", lazy="select") +@serializable +class HabrefCorrespTaxon(DB.Model): + __tablename__ = "habref_corresp_taxon" + __table_args__ = {"schema": "ref_habitats"} + cd_corresp_tax = DB.Column(DB.Integer, primary_key=True) + cd_hab_entre = DB.Column(DB.Integer) + cd_nom = DB.Column(DB.Integer) + cd_type_relation = DB.Column(DB.Integer) + lb_condition = DB.Column(DB.Unicode) + lb_remarques = DB.Column(DB.Unicode) + nom_cite = DB.Column(DB.Unicode) + validite = DB.Column(DB.Boolean) + + +@serializable +class CorHabrefTerrStatut(DB.Model): + __tablename__ = "cor_habref_terr_statut" + __table_args__ = {"schema": "ref_habitats"} + cd_hab_ter = DB.Column(DB.Integer, primary_key=True) + cd_hab = DB.Column(DB.Integer) + cd_sig_terr = DB.Column(DB.Unicode) + cd_statut_presence = DB.Column(DB.Unicode) + + +@serializable +class TyporefFields(DB.Model): + __tablename__ = "typoref_fields" + __table_args__ = {"schema": "ref_habitats"} + cd_hab_field = DB.Column(DB.Integer, primary_key=True) + cd_typo = DB.Column(DB.Integer) + lb_hab_field = DB.Column(DB.Unicode) + format_hab_field = DB.Column(DB.Unicode) + descript_hab_field = DB.Column(DB.Unicode) + ordre_hab_field = DB.Column(DB.Integer) + length_hab_field = DB.Column(DB.Integer) + lb_label = DB.Column(DB.Unicode) + + +@serializable +class CorHabrefDescription(DB.Model): + __tablename__ = "cor_habref_description" + __table_args__ = {"schema": "ref_habitats"} + cd_hab_description = DB.Column(DB.Integer, primary_key=True) + cd_hab = DB.Column(DB.Integer) + cd_hab_field = DB.Column(DB.Integer) + cd_typo = DB.Column(DB.Integer) + lb_code = DB.Column(DB.Unicode) + lb_hab_field = DB.Column(DB.Unicode) + valeurs = DB.Column(DB.Unicode) + + +@serializable +class CorHabSource(DB.Model): + __tablename__ = "cor_hab_source" + __table_args__ = {"schema": "ref_habitats"} + cd_hab_lien_source = DB.Column(DB.Integer, primary_key=True) + cd = DB.Column(DB.Integer) + type_lien = DB.Column(DB.Unicode) + cd_source = DB.Column(DB.Integer) + origine = DB.Column(DB.Unicode) + + @serializable class TypoRef(DB.Model): __tablename__ = "typoref" diff --git a/src/pypn_habref_api/tests/test_habref_update.py b/src/pypn_habref_api/tests/test_habref_update.py new file mode 100644 index 0000000..d3afa97 --- /dev/null +++ b/src/pypn_habref_api/tests/test_habref_update.py @@ -0,0 +1,173 @@ +import csv +import io +import logging +import os +import tempfile +import zipfile +from contextlib import contextmanager +from unittest.mock import patch + +import pytest +from sqlalchemy import inspect as sa_inspect, text + +from pypn_habref_api.commands.habref_v7 import table_files, base_url +from pypn_habref_api.commands.utils import apply_habref, import_habref +from utils_flask_sqla.referential import collect_orphan_rows, export_orphans_to_csv +from pypn_habref_api.env import db + + +def make_habref_zip(): + """ZIP minimal en mémoire — uniquement les en-têtes CSV, aucune ligne de données.""" + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + for config in table_files.values(): + csv_content = io.StringIO() + writer = csv.writer(csv_content, delimiter=";") + writer.writerow(config["table_fields"].values()) + zf.writestr(config["filename"], csv_content.getvalue().encode("utf-8")) + buf.seek(0) + return buf + + +@contextmanager +def tmp_tables(): + """Crée les tables tmp_* en copiant la structure des tables habref, et les supprime à la fin.""" + try: + for table in table_files: + db.session.execute(text(f"DROP TABLE IF EXISTS ref_habitats.tmp_{table}")) + db.session.execute( + text( + f"CREATE TABLE ref_habitats.tmp_{table} " + f"AS TABLE ref_habitats.{table} WITH NO DATA" + ) + ) + db.session.flush() + yield + finally: + for table in reversed(list(table_files.keys())): + db.session.execute(text(f"DROP TABLE IF EXISTS ref_habitats.tmp_{table}")) + db.session.flush() + + +@pytest.mark.usefixtures("app") +class TestImportHabref: + def test_tmp_tables_created(self): + """import_habref crée une table tmp_* pour chaque table de table_files.""" + logger = logging.getLogger() + zip_data = make_habref_zip() + + @contextmanager + def fake_open_remote(url, filename, open_fct=None): + zip_data.seek(0) + with zipfile.ZipFile(zip_data) as zf: + yield zf + + try: + with patch("pypn_habref_api.commands.utils.open_remote_file", fake_open_remote): + import_habref( + logger, + table_files=table_files, + schema="ref_habitats", + base_url=base_url, + num_version="07", + archive_name="HABREF_70.zip", + ) + + inspector = sa_inspect(db.engine) + existing = inspector.get_table_names(schema="ref_habitats") + for table in table_files: + assert f"tmp_{table}" in existing + finally: + for table in reversed(list(table_files.keys())): + db.session.execute(text(f"DROP TABLE IF EXISTS ref_habitats.tmp_{table}")) + db.session.flush() + + +@pytest.mark.usefixtures("app") +class TestExportOrphans: + def test_orphan_detected_in_csv(self): + """Un cd_hab présent dans cor_list_habitat mais absent de tmp_habref apparaît dans le CSV.""" + orphan_cd_hab = db.session.execute( + text("SELECT cd_hab FROM ref_habitats.habref LIMIT 1") + ).scalar() + + with tmp_tables(): + db.session.execute( + text( + "INSERT INTO ref_habitats.tmp_habref " + "SELECT * FROM ref_habitats.habref WHERE cd_hab != :cd" + ), + {"cd": orphan_cd_hab}, + ) + + db.session.execute(text("SET session_replication_role = 'replica'")) + db.session.execute( + text( + "INSERT INTO ref_habitats.cor_list_habitat (id_cor_list, cd_hab) " + "SELECT COALESCE(MAX(id_cor_list), 0) + 1, :cd " + "FROM ref_habitats.cor_list_habitat" + ), + {"cd": orphan_cd_hab}, + ) + db.session.execute(text("SET session_replication_role = 'origin'")) + db.session.flush() + + with tempfile.NamedTemporaryFile(suffix=".csv", delete=False) as f: + output_path = f.name + try: + orphan_rows = collect_orphan_rows( + ref_table="habref", + new_ref_table="tmp_habref", + pk_col="cd_hab", + db=db, + schema="ref_habitats", + exclude_tables=list(table_files.keys()), + ) + nb = export_orphans_to_csv(orphan_rows, output_path) + assert nb > 0 + with open(output_path) as f: + rows = list(csv.DictReader(f)) + assert any(int(r["fk_value"]) == orphan_cd_hab for r in rows) + finally: + os.unlink(output_path) + db.session.execute(text("SET session_replication_role = 'replica'")) + db.session.execute( + text("DELETE FROM ref_habitats.cor_list_habitat WHERE cd_hab = :cd"), + {"cd": orphan_cd_hab}, + ) + db.session.execute(text("SET session_replication_role = 'origin'")) + db.session.flush() + + +@pytest.mark.usefixtures("app") +class TestApplyHabref: + def test_tables_updated_and_tmp_dropped(self): + """apply_habref remplace le contenu des tables réelles et supprime les tmp_*.""" + logger = logging.getLogger() + + count_before = db.session.execute( + text("SELECT COUNT(*) FROM ref_habitats.habref") + ).scalar() + + with tmp_tables(): + for table in table_files: + db.session.execute( + text( + f"INSERT INTO ref_habitats.tmp_{table} " + f"SELECT * FROM ref_habitats.{table}" + ) + ) + db.session.flush() + + apply_habref(logger, table_files=table_files, schema="ref_habitats") + db.session.flush() + + count_after = db.session.execute( + text("SELECT COUNT(*) FROM ref_habitats.habref") + ).scalar() + assert count_after == count_before + + inspector = sa_inspect(db.engine) + existing = inspector.get_table_names(schema="ref_habitats") + for table in table_files: + assert f"tmp_{table}" not in existing