From 393fb978477d8fa0810261ae1dc941cea9f7aec5 Mon Sep 17 00:00:00 2001 From: Pierre-Narcisi Date: Wed, 3 Dec 2025 11:04:58 +0100 Subject: [PATCH 01/11] Update habref --- setup.py | 50 +++++ src/pypn_habref_api/commands/habref.py | 17 ++ src/pypn_habref_api/commands/habref_v7.py | 226 ++++++++++++++++++++++ src/pypn_habref_api/commands/utils.py | 126 ++++++++++++ src/pypn_habref_api/models.py | 87 +++++++++ 5 files changed, 506 insertions(+) create mode 100644 setup.py create mode 100644 src/pypn_habref_api/commands/habref.py create mode 100644 src/pypn_habref_api/commands/habref_v7.py create mode 100644 src/pypn_habref_api/commands/utils.py diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..07d2dcb --- /dev/null +++ b/setup.py @@ -0,0 +1,50 @@ +import setuptools +from pathlib import Path + +root_dir = Path(__file__).absolute().parent +with (root_dir / "VERSION").open() as f: + version = f.read() +with (root_dir / "README.md").open() as f: + long_description = f.read() +with (root_dir / "requirements.in").open() as f: + requirements = f.read().splitlines() + + +setuptools.setup( + name="pypn_habref_api", + version=version, + description="Python lib related to Habref referential (INPN)", + long_description=long_description, + long_description_content_type="text/markdown", + maintainer="Parcs nationaux des Écrins et des Cévennes", + maintainer_email="geonature@ecrins-parcnational.fr", + url="https://github.com/PnX-SI/Habref-api-module", + packages=setuptools.find_packages("src"), + package_dir={"": "src"}, + install_requires=requirements, + extras_require={ + "tests": [ + "pytest", + "pytest-flask", + ], + }, + include_package_data=True, + entry_points={ + "alembic": [ + "migrations = pypn_habref_api.migrations:versions", + ], + "flask.commands": [ + "habref = pypn_habref_api.commands.habref:habref", + ], + }, + zip_safe=False, + classifiers=[ + "Development Status :: 1 - Planning", + "Intended Audience :: Developers", + "Natural Language :: English", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.11", + "License :: OSI Approved :: GNU Affero General Public License v3", + "Operating System :: OS Independent", + ], +) diff --git a/src/pypn_habref_api/commands/habref.py b/src/pypn_habref_api/commands/habref.py new file mode 100644 index 0000000..2b13a25 --- /dev/null +++ b/src/pypn_habref_api/commands/habref.py @@ -0,0 +1,17 @@ +import click + +from flask.cli import with_appcontext + +from .habref_v7 import import_v07 + +import logging + +logger = logging.getLogger("habref_commands") + + +@click.group(help="Manager HabRef referentials.") +def habref(): + pass + + +habref.add_command(import_v07) diff --git a/src/pypn_habref_api/commands/habref_v7.py b/src/pypn_habref_api/commands/habref_v7.py new file mode 100644 index 0000000..4a4def2 --- /dev/null +++ b/src/pypn_habref_api/commands/habref_v7.py @@ -0,0 +1,226 @@ +from zipfile import ZipFile +from collections import OrderedDict +import logging + +from pypn_habref_api.models import BibListHabitat, Habref +from pypn_habref_api.env import db + +from sqlalchemy.schema import Table, MetaData, PrimaryKeyConstraint + +import click +from flask.cli import with_appcontext + +from alembic import op + +from utils_flask_sqla.migrations.utils import open_remote_file +from .utils import copy_from_csv, empty_table, restore_constraints + +base_url = "https://geonature.fr/data/inpn/habitats/" +table_files = { + "typoref": { + "filename": "TYPOREF_70.csv", + "table_fields": { + "cd_typo": "CD_TYPO", + "cd_table": "CD_TABLE", + "lb_nom_typo": "LB_NOM_TYPO", + "nom_jeu_donnees": "NOM_JEU_DONNEES", + "date_creation": "DATE_CREATION", + "auteur_typo": "AUTEUR_TYPO", + "auteur_table": "AUTEUR_TABLE", + "territoire": "TERRITOIRE", + "organisme": "ORGANISME", + "langue": "LANGUE", + "presentation": "PRESENTATION", + "description": "DESCRIPTION", + "origine": "ORIGINE", + "ref_biblio": "REF_BIBLIO", + "mots_cles": "MOTS_CLES", + "referencement": "REFERENCEMENT", + "diffusion": "DIFFUSION", + "derniere_modif": "DERNIERE_MODIF", + "type_table": "TYPE_TABLE", + "cd_typo_entre": "CD_TYPO_ENTRE", + "cd_typo_sortie": "CD_TYPO_SORTIE", + }, + }, + "bib_habref_typo_rel": { + "filename": "HABREF_TYPE_REL_70.csv", + "table_fields": { + "cd_type_rel": "CD_TYPE_REL", + "lb_type_rel": "LB_TYPE_REL", + "lb_rel": "LB_REL", + "corresp_hab": "CORRESP_HAB", + "corresp_esp": "CORRESP_ESP", + "corresp_syn": "CORRESP_SYN", + "date_crea": "DATE_CREA", + "date_modif": "DATE_MODIF", + }, + }, + "bib_habref_statuts": { + "filename": "HABREF_STATUTS.csv", + "table_fields": { + "statut": "STATUT", + "description": "DESCRIPTION", + "definition": "DEFINITION", + "ordre": "ORDRE", + }, + }, + "habref_sources": { + "filename": "HABREF_SOURCES_70.csv", + "table_fields": { + "cd_source": "CD_SOURCE", + "cd_doc": "CD_DOC", + "type_source": "TYPE_SOURCE", + "auteur_source": "AUTEUR_SOURCE", + "date_source": "DATE_SOURCE", + "lb_source": "LB_SOURCE", + "lb_source_complet": "LB_SOURCE_COMPLET", + "titre": "TITRE", + "link": "LINK", + "date_crea": "DATE_CREA", + "date_modif": "DATE_MODIF", + }, + }, + "habref": { + "filename": "HABREF_70.csv", + "table_fields": { + "cd_hab": "CD_HAB", + "fg_validite": "FG_VALIDITE", + "cd_typo": "CD_TYPO", + "lb_code": "LB_CODE", + "lb_hab_fr": "LB_HAB_FR", + "lb_hab_fr_complet": "LB_HAB_FR_COMPLET", + "lb_hab_en": "LB_HAB_EN", + "lb_auteur": "LB_AUTEUR", + "niveau": "NIVEAU", + "lb_niveau": "LB_NIVEAU", + "cd_hab_sup": "CD_HAB_SUP", + "path_cd_hab": "PATH_CD_HAB", + "france": "FRANCE", + "lb_description": "LB_DESCRIPTION", + }, + }, + "habref_corresp_hab": { + "filename": "HABREF_CORRESP_HAB_70.csv", + "table_fields": { + "cd_corresp_hab": "CD_CORRESP_HAB", + "cd_hab_entre": "CD_HAB_ENTRE", + "cd_hab_sortie": "CD_HAB_SORTIE", + "cd_type_relation": "CD_TYPE_RELATION", + "lb_condition": "LB_CONDITION", + "lb_remarques": "LB_REMARQUES", + "validite": "VALIDITE", + "cd_typo_entre": "CD_TYPO_ENTRE", + "cd_typo_sortie": "CD_TYPO_SORTIE", + }, + }, + "habref_corresp_taxon": { + "filename": "HABREF_CORRESP_TAXON_70.csv", + "table_fields": { + "cd_corresp_tax": "CD_CORRESP_TAX", + "cd_hab_entre": "CD_HAB_ENTRE", + "cd_nom": "CD_NOM", + "cd_type_relation": "CD_TYPE_RELATION", + "lb_condition": "LB_CONDITION", + "lb_remarques": "LB_REMARQUES", + "nom_cite": "NOM_CITE", + "validite": "VALIDITE", + "date_crea": "DATE_CREA", + "date_modif": "DATE_MODIF", + }, + }, + "cor_habref_terr_statut": { + "filename": "HABREF_TERR_70.csv", + "table_fields": { + "cd_hab_ter": "CD_HAB_TERR", + "cd_hab": "CD_HAB", + "cd_sig_terr": "CD_SIG_TERR", + "cd_statut_presence": "CD_STATUT_PRESENCE", + "date_crea": "DATE_CREA", + "date_modif": "DATE_MODIF", + }, + }, + "typoref_fields": { + "filename": "TYPOREF_FIELDS_70.csv", + "table_fields": { + "cd_hab_field": "CD_HAB_FIELD", + "cd_typo": "CD_TYPO", + "lb_hab_field": "LB_HAB_FIELD", + "format_hab_field": "FORMAT_HAB_FIELD", + "descript_hab_field": "DESCRIPT_HAB_FIELD", + "ordre_hab_field": "ORDRE_HAB_FIELD", + "length_hab_field": "LENGTH_HAB_FIELD", + "lb_label": "LB_LABEL", + "date_crea": "DATE_CREA", + "date_modif": "DATE_MODIF", + }, + }, + "cor_habref_description": { + "filename": "HABREF_DESCRIPTION_70.csv", + "table_fields": { + "cd_hab_description": "CD_HAB_DESCRIPTION", + "cd_hab": "CD_HAB", + "cd_hab_field": "CD_HAB_FIELD", + "cd_typo": "CD_TYPO", + "lb_code": "LB_CODE", + "lb_hab_field": "LB_HAB_FIELD", + "valeurs": "VALEURS", + }, + }, + "cor_hab_source": { + "filename": "HABREF_LIEN_SOURCES_70.csv", + "table_fields": { + "cd_hab_lien_source": "CD_HAB_LIEN_SOURCE", + "cd": "CD", + "type_lien": "TYPE_LIEN", + "cd_source": "CD_SOURCE", + "origine": "ORIGINE", + "date_crea": "DATE_CREA", + "date_modif": "DATE_MODIF", + }, + }, +} + + +def import_habref(logger, num_version, habref_archive_name): + with open_remote_file(base_url, habref_archive_name, open_fct=ZipFile) as archive: + for table, value in table_files.items(): + + constraints = empty_table( + table, + db, + schema="ref_habitats", + ) + logger.info(f"Insert HABREF v{num_version} {table}…") + with archive.open(value["filename"]) as f: + + copy_from_csv( + f, + table, + value["table_fields"], + encoding="UTF-8", + delimiter=";", + schema="ref_habitats", + db=db, + ) + restore_constraints( + table, + db, + constraints, + schema="ref_habitats", + ) + + +@click.command() +@with_appcontext +def import_v07(): + logger = logging.getLogger() + + import_habref( + logger, + num_version="07", + habref_archive_name="HABREF_70.zip", + ) + + logger.info("Committing…") + db.session.commit() diff --git a/src/pypn_habref_api/commands/utils.py b/src/pypn_habref_api/commands/utils.py new file mode 100644 index 0000000..d0e7195 --- /dev/null +++ b/src/pypn_habref_api/commands/utils.py @@ -0,0 +1,126 @@ +from csv import DictReader +from io import TextIOWrapper + +import sqlalchemy as sa +from sqlalchemy import inspect as sa_inspect +from sqlalchemy.schema import ( + Table, + MetaData, + PrimaryKeyConstraint, + ForeignKeyConstraint, +) + + +def get_csv_field_names(f, encoding, delimiter): + if encoding == "WIN1252": # postgresql encoding + encoding = "cp1252" # python encoding + t = TextIOWrapper(f, encoding=encoding) + reader = DictReader(t, delimiter=delimiter) + field_names = reader.fieldnames + t.detach() # avoid f to be closed on t garbage collection + f.seek(0) + return field_names + + +""" +Insert CSV file into specified table. +If source columns are specified, CSV file in copied in a temporary table, +then data restricted to specified source columns are copied in final table. +""" + + +def copy_from_csv( + f, + table_name, + table_fields, + schema="", + header=True, + encoding=None, + delimiter=None, + db=None, +): + bind = db.session.get_bind() + metadata = MetaData(bind=bind) + engine = db.engine + + table_fields_list = list(table_fields.values()) + table_fields_key_list = list(table_fields.keys()) + + final_table_name = table_name + table_name = f"import_{table_name}" + field_names = get_csv_field_names(f, encoding=encoding, delimiter=delimiter) + field_names = list(map(lambda field_name: field_name.replace("\ufeff", ""), field_names)) + table = Table( + table_name, + metadata, + *[sa.Column(c, sa.String) for c in map(str.lower, field_names)], + schema=schema, + ) + table.create(bind=db.session.connection()) + + options = ["FORMAT CSV"] + if header: + options.append("HEADER") + if encoding: + options.append(f"ENCODING '{encoding}'") + if delimiter: + options.append(f"DELIMITER E'{delimiter}'") + options = ", ".join(options) + cursor = db.session.connection().connection.cursor() + cursor.copy_expert( + f""" + COPY {schema}.{table_name} + FROM STDIN WITH ({options}) + """, + f, + ) + + testTable = Table(final_table_name, db.metadata, schema=schema, autoload_with=engine) + + for col in testTable.columns: + if col.name in table_fields: + table_fields[col.name] = f"{table_fields[col.name]}::{col.type}" + table_fields_list = list(table_fields.values()) + + db.session.execute( + f""" + INSERT INTO {schema}.{final_table_name} ({", ".join(table_fields_key_list)}) + SELECT {", ".join(table_fields_list)} + FROM {schema}.{table_name}; + """ + ) + table.drop(bind=db.session.connection()) + + +def empty_table(table_name, db, schema=""): + inspector = sa_inspect(db.engine) + + # Récupère les foreign keys + foreign_keys = inspector.get_foreign_keys(table_name, schema=schema) + for fk in foreign_keys: + constraint_name = fk["name"] + if constraint_name: # Vérifie que le nom existe + db.session.execute( + f"ALTER TABLE {schema}.{table_name} DROP CONSTRAINT {constraint_name} CASCADE" + ) + db.session.execute(f"TRUNCATE TABLE {schema}.{table_name} CASCADE") + return foreign_keys + + +def restore_constraints(table_name, db, constraints, schema=""): + if constraints is None: + return + for constraint in constraints: + if isinstance(constraint, PrimaryKeyConstraint): + cols = ", ".join(str(col).split(".")[1] for col in constraint.columns) + db.session.execute( + f"ALTER TABLE {schema}.{table_name} ADD CONSTRAINT {constraint.name} UNIQUE ({cols})" + ) + elif isinstance(constraint, ForeignKeyConstraint): + cols = ", ".join(str(col).split(".")[1] for col in constraint.columns) + ref_cols = ", ".join( + str(col).split(".")[1] for col in constraint.references[0].columns + ) + db.session.execute( + f"ALTER TABLE {schema}.{table_name} ADD CONSTRAINT {constraint.name} FOREIGN KEY {cols} REFERENCES {constraint.references[0].table.name}({ref_cols})" + ) diff --git a/src/pypn_habref_api/models.py b/src/pypn_habref_api/models.py index 165ca10..febd8c5 100644 --- a/src/pypn_habref_api/models.py +++ b/src/pypn_habref_api/models.py @@ -20,6 +20,31 @@ class BibHabrefTypoRel(DB.Model): corresp_syn: Mapped[Optional[bool]] = mapped_column(Boolean) +@serializable +class BibHabrefStatuts(DB.Model): + __tablename__ = "bib_habref_statuts" + __table_args__ = {"schema": "ref_habitats"} + statut = DB.Column(DB.Unicode, primary_key=True) + description = DB.Column(DB.Unicode) + definition = DB.Column(DB.Unicode) + ordre = DB.Column(DB.Integer) + + +@serializable +class HabrefSources(DB.Model): + __tablename__ = "habref_sources" + __table_args__ = {"schema": "ref_habitats"} + cd_source = DB.Column(DB.Integer, primary_key=True) + cd_doc = DB.Column(DB.Integer) + type_source = DB.Column(DB.Unicode) + auteur_source = DB.Column(DB.Unicode) + date_source = DB.Column(DB.Integer) + lb_source = DB.Column(DB.Unicode) + lb_source_complet = DB.Column(DB.Unicode) + titre = DB.Column(DB.Unicode) + link = DB.Column(DB.Unicode) + + @serializable class CorespHab(DB.Model): __tablename__ = "habref_corresp_hab" @@ -41,6 +66,68 @@ class CorespHab(DB.Model): type_rel = DB.relationship("BibHabrefTypoRel", lazy="select") +@serializable +class HabrefCorrespTaxon(DB.Model): + __tablename__ = "habref_corresp_taxon" + __table_args__ = {"schema": "ref_habitats"} + cd_corresp_tax = DB.Column(DB.Integer, primary_key=True) + cd_hab_entre = DB.Column(DB.Integer) + cd_nom = DB.Column(DB.Integer) + cd_type_relation = DB.Column(DB.Integer) + lb_condition = DB.Column(DB.Unicode) + lb_remarques = DB.Column(DB.Unicode) + nom_cite = DB.Column(DB.Unicode) + validite = DB.Column(DB.Boolean) + + +@serializable +class CorHabrefTerrStatut(DB.Model): + __tablename__ = "cor_habref_terr_statut" + __table_args__ = {"schema": "ref_habitats"} + cd_hab_ter = DB.Column(DB.Integer, primary_key=True) + cd_hab = DB.Column(DB.Integer) + cd_sig_terr = DB.Column(DB.Unicode) + cd_statut_presence = DB.Column(DB.Unicode) + + +@serializable +class TyporefFields(DB.Model): + __tablename__ = "typoref_fields" + __table_args__ = {"schema": "ref_habitats"} + cd_hab_field = DB.Column(DB.Integer, primary_key=True) + cd_typo = DB.Column(DB.Integer) + lb_hab_field = DB.Column(DB.Unicode) + format_hab_field = DB.Column(DB.Unicode) + descript_hab_field = DB.Column(DB.Unicode) + ordre_hab_field = DB.Column(DB.Integer) + length_hab_field = DB.Column(DB.Integer) + lb_label = DB.Column(DB.Unicode) + + +@serializable +class CorHabrefDescription(DB.Model): + __tablename__ = "cor_habref_description" + __table_args__ = {"schema": "ref_habitats"} + cd_hab_description = DB.Column(DB.Integer, primary_key=True) + cd_hab = DB.Column(DB.Integer) + cd_hab_field = DB.Column(DB.Integer) + cd_typo = DB.Column(DB.Integer) + lb_code = DB.Column(DB.Unicode) + lb_hab_field = DB.Column(DB.Unicode) + valeurs = DB.Column(DB.Unicode) + + +@serializable +class CorHabSource(DB.Model): + __tablename__ = "cor_hab_source" + __table_args__ = {"schema": "ref_habitats"} + cd_hab_lien_source = DB.Column(DB.Integer, primary_key=True) + cd = DB.Column(DB.Integer) + type_lien = DB.Column(DB.Unicode) + cd_source = DB.Column(DB.Integer) + origine = DB.Column(DB.Unicode) + + @serializable class TypoRef(DB.Model): __tablename__ = "typoref" From bfad6787d05a2484881092faddcde2af0f08e0c9 Mon Sep 17 00:00:00 2001 From: Pierre-Narcisi Date: Wed, 17 Dec 2025 11:47:49 +0100 Subject: [PATCH 02/11] change constrainte handling --- src/pypn_habref_api/commands/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pypn_habref_api/commands/utils.py b/src/pypn_habref_api/commands/utils.py index d0e7195..e052e87 100644 --- a/src/pypn_habref_api/commands/utils.py +++ b/src/pypn_habref_api/commands/utils.py @@ -97,13 +97,14 @@ def empty_table(table_name, db, schema=""): # Récupère les foreign keys foreign_keys = inspector.get_foreign_keys(table_name, schema=schema) + db.session.execute("SET session_replication_role = 'replica'") for fk in foreign_keys: constraint_name = fk["name"] if constraint_name: # Vérifie que le nom existe db.session.execute( f"ALTER TABLE {schema}.{table_name} DROP CONSTRAINT {constraint_name} CASCADE" ) - db.session.execute(f"TRUNCATE TABLE {schema}.{table_name} CASCADE") + db.session.execute(f"DELETE FROM {schema}.{table_name}") return foreign_keys From ed6b615f47c8bf0809bbeb34a4190c55ab903f71 Mon Sep 17 00:00:00 2001 From: Pierre-Narcisi Date: Wed, 14 Jan 2026 15:43:39 +0100 Subject: [PATCH 03/11] wip --- src/pypn_habref_api/commands/habref_v7.py | 18 ++++-------------- src/pypn_habref_api/commands/utils.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/pypn_habref_api/commands/habref_v7.py b/src/pypn_habref_api/commands/habref_v7.py index 4a4def2..effdd28 100644 --- a/src/pypn_habref_api/commands/habref_v7.py +++ b/src/pypn_habref_api/commands/habref_v7.py @@ -185,30 +185,20 @@ def import_habref(logger, num_version, habref_archive_name): with open_remote_file(base_url, habref_archive_name, open_fct=ZipFile) as archive: for table, value in table_files.items(): - - constraints = empty_table( - table, - db, - schema="ref_habitats", - ) logger.info(f"Insert HABREF v{num_version} {table}…") with archive.open(value["filename"]) as f: - + db.execute( + f"CREATE TABLE ref_habitats.tmp_{table} AS TABLE ref_habitats.{table} WITH NO DATA;" + ) copy_from_csv( f, - table, + f"tmp_{table}", value["table_fields"], encoding="UTF-8", delimiter=";", schema="ref_habitats", db=db, ) - restore_constraints( - table, - db, - constraints, - schema="ref_habitats", - ) @click.command() diff --git a/src/pypn_habref_api/commands/utils.py b/src/pypn_habref_api/commands/utils.py index e052e87..2c29cc7 100644 --- a/src/pypn_habref_api/commands/utils.py +++ b/src/pypn_habref_api/commands/utils.py @@ -125,3 +125,16 @@ def restore_constraints(table_name, db, constraints, schema=""): db.session.execute( f"ALTER TABLE {schema}.{table_name} ADD CONSTRAINT {constraint.name} FOREIGN KEY {cols} REFERENCES {constraint.references[0].table.name}({ref_cols})" ) + + +def detect_changes(): + pass + + +def detect_missing_cd_hab(): + op.create_table( + "tmp_missing_cd_hab", + Column("cd_hab", Integer, primary_key=True), + schema="ref_habitats", + ) + pass From 62443af8a0deca4aef8ddd77583750c2dc6e92ab Mon Sep 17 00:00:00 2001 From: Pierre-Narcisi Date: Fri, 23 Jan 2026 10:20:08 +0100 Subject: [PATCH 04/11] merge --- src/pypn_habref_api/commands/utils.py | 121 ++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) diff --git a/src/pypn_habref_api/commands/utils.py b/src/pypn_habref_api/commands/utils.py index 2c29cc7..4208ffa 100644 --- a/src/pypn_habref_api/commands/utils.py +++ b/src/pypn_habref_api/commands/utils.py @@ -138,3 +138,124 @@ def detect_missing_cd_hab(): schema="ref_habitats", ) pass + + +def test(): + inspector = sa_inspect(db.engine) + + # Trouve toutes les tables qui référencent cette table + broken_refs = [] + + foreign_keys = inspector.get_foreign_keys(table_name, schema=schema) + + # Désactive les contraintes FK + db.session.execute(text("SET session_replication_role = 'replica'")) + try: + # Vide la table + db.session.execute(text(f"TRUNCATE TABLE {table_full_name}")) + db.session.commit() + + # Charge les nouvelles données si une fonction est fournie + if data_loader_func: + data_loader_func(db) + + # Réactive les contraintes + db.session.execute(text("SET session_replication_role = 'origin'")) + db.session.commit() + + # Vérifie les références cassées + broken = check_broken_references(table_name, db, schema) + if broken: + print("⚠️ ATTENTION : Références cassées détectées !") + for ref in broken: + print( + f" - Table {ref['table']}.{ref['column']} : {ref['broken_count']} lignes orphelines" + ) + return False, broken + + print("✓ Aucune référence cassée détectée") + return True, foreign_keys + + except Exception as e: + db.session.rollback() + db.session.execute(text("SET session_replication_role = 'origin'")) + db.session.commit() + raise e + + +def get_referencing_tables(table_name, db, schema=""): + """Trouve toutes les tables (tous schémas) qui ont des FK pointant vers table_name""" + inspector = sa_inspect(db.engine) + referencing_tables = [] + + all_schemas = inspector.get_schema_names() + + for other_schema in all_schemas: + if other_schema in ("pg_catalog", "information_schema", "pg_toast"): + continue + + try: + for other_table in inspector.get_table_names(schema=other_schema): + fks = inspector.get_foreign_keys(other_table, schema=other_schema) + + for fk in fks: + referred_schema = fk.get("referred_schema", other_schema) + + if fk["referred_table"] == table_name and ( + not schema or referred_schema == schema + ): + referencing_tables.append( + { + "schema": other_schema, + "table": other_table, + "constraint_name": fk["name"], + "fk_column": fk["constrained_columns"][0], + "ref_column": fk["referred_columns"][0], + "referred_schema": referred_schema, + } + ) + except Exception as e: + print(f"Impossible d'inspecter le schéma {other_schema}: {e}") + continue + + return referencing_tables + + +def check_broken_references(): + inspector = sa_inspect(db.engine) + + # Trouve toutes les tables qui référencent cette table + broken_refs = [] + + for other_table in inspector.get_table_names(schema=schema): + fks = inspector.get_foreign_keys(other_table, schema=schema) + + for fk in fks: + # Si cette FK pointe vers notre table + if fk["referred_table"] == table_name: + fk_column = fk["constrained_columns"][0] + ref_column = fk["referred_columns"][0] + + table_full = f"{schema}.{other_table}" if schema else other_table + ref_full = f"{schema}.{table_name}" if schema else table_name + + # Requête pour trouver les orphelins + query = text( + f""" + SELECT COUNT(*) as broken_count + FROM {table_full} t + WHERE t.{fk_column} IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM {ref_full} r + WHERE r.{ref_column} = t.{fk_column} + ) + """ + ) + + result = db.session.execute(query).fetchone() + if result[0] > 0: + broken_refs.append( + {"table": other_table, "column": fk_column, "broken_count": result[0]} + ) + + return broken_refs From 8c6ca5c05a8cbbd149dee52063616ac00cdb15a5 Mon Sep 17 00:00:00 2001 From: Pierre-Narcisi Date: Tue, 31 Mar 2026 17:26:41 +0200 Subject: [PATCH 05/11] wip --- src/pypn_habref_api/commands/habref_v7.py | 31 +++++++++++++- src/pypn_habref_api/commands/utils.py | 51 ++++++++++++++++++++++- 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/src/pypn_habref_api/commands/habref_v7.py b/src/pypn_habref_api/commands/habref_v7.py index effdd28..c44ddf7 100644 --- a/src/pypn_habref_api/commands/habref_v7.py +++ b/src/pypn_habref_api/commands/habref_v7.py @@ -13,12 +13,20 @@ from alembic import op from utils_flask_sqla.migrations.utils import open_remote_file -from .utils import copy_from_csv, empty_table, restore_constraints +from .utils import ( + copy_from_csv, + empty_table, + restore_constraints, + delete_tmp_tables, + compare_tables_on_column, + get_referencing_tables, +) base_url = "https://geonature.fr/data/inpn/habitats/" table_files = { "typoref": { "filename": "TYPOREF_70.csv", + "unique_column": "cd_typo", "table_fields": { "cd_typo": "CD_TYPO", "cd_table": "CD_TABLE", @@ -45,6 +53,7 @@ }, "bib_habref_typo_rel": { "filename": "HABREF_TYPE_REL_70.csv", + "unique_column": "cd_type_rel", "table_fields": { "cd_type_rel": "CD_TYPE_REL", "lb_type_rel": "LB_TYPE_REL", @@ -58,6 +67,7 @@ }, "bib_habref_statuts": { "filename": "HABREF_STATUTS.csv", + "unique_column": "statut", "table_fields": { "statut": "STATUT", "description": "DESCRIPTION", @@ -67,6 +77,7 @@ }, "habref_sources": { "filename": "HABREF_SOURCES_70.csv", + "unique_column": "cd_source", "table_fields": { "cd_source": "CD_SOURCE", "cd_doc": "CD_DOC", @@ -83,6 +94,7 @@ }, "habref": { "filename": "HABREF_70.csv", + "unique_column": "cd_hab", "table_fields": { "cd_hab": "CD_HAB", "fg_validite": "FG_VALIDITE", @@ -102,6 +114,7 @@ }, "habref_corresp_hab": { "filename": "HABREF_CORRESP_HAB_70.csv", + "unique_column": "cd_corresp_hab", "table_fields": { "cd_corresp_hab": "CD_CORRESP_HAB", "cd_hab_entre": "CD_HAB_ENTRE", @@ -116,6 +129,7 @@ }, "habref_corresp_taxon": { "filename": "HABREF_CORRESP_TAXON_70.csv", + "unique_column": "cd_corresp_tax", "table_fields": { "cd_corresp_tax": "CD_CORRESP_TAX", "cd_hab_entre": "CD_HAB_ENTRE", @@ -131,6 +145,7 @@ }, "cor_habref_terr_statut": { "filename": "HABREF_TERR_70.csv", + "unique_column": "cd_hab_ter", "table_fields": { "cd_hab_ter": "CD_HAB_TERR", "cd_hab": "CD_HAB", @@ -142,6 +157,7 @@ }, "typoref_fields": { "filename": "TYPOREF_FIELDS_70.csv", + "unique_column": "cd_hab_field", "table_fields": { "cd_hab_field": "CD_HAB_FIELD", "cd_typo": "CD_TYPO", @@ -157,6 +173,7 @@ }, "cor_habref_description": { "filename": "HABREF_DESCRIPTION_70.csv", + "unique_column": "cd_hab_description", "table_fields": { "cd_hab_description": "CD_HAB_DESCRIPTION", "cd_hab": "CD_HAB", @@ -169,6 +186,7 @@ }, "cor_hab_source": { "filename": "HABREF_LIEN_SOURCES_70.csv", + "unique_column": "cd_hab_lien_source", "table_fields": { "cd_hab_lien_source": "CD_HAB_LIEN_SOURCE", "cd": "CD", @@ -183,11 +201,12 @@ def import_habref(logger, num_version, habref_archive_name): + delete_tmp_tables(table_files) with open_remote_file(base_url, habref_archive_name, open_fct=ZipFile) as archive: for table, value in table_files.items(): logger.info(f"Insert HABREF v{num_version} {table}…") with archive.open(value["filename"]) as f: - db.execute( + db.session.execute( f"CREATE TABLE ref_habitats.tmp_{table} AS TABLE ref_habitats.{table} WITH NO DATA;" ) copy_from_csv( @@ -199,6 +218,14 @@ def import_habref(logger, num_version, habref_archive_name): schema="ref_habitats", db=db, ) + diff = compare_tables_on_column( + "ref_habitats", table, f"tmp_{table}", value["unique_column"] + ) + print("TEEEEEEEEEESSSSSTTT") + print("table : ", table) + print("référence : ", get_referencing_tables(table, "ref_habitats")) + # print(diff[f"only_in_{table}"]) + # print(diff[f"only_in_tmp_{table}"]) @click.command() diff --git a/src/pypn_habref_api/commands/utils.py b/src/pypn_habref_api/commands/utils.py index 4208ffa..f0bdd53 100644 --- a/src/pypn_habref_api/commands/utils.py +++ b/src/pypn_habref_api/commands/utils.py @@ -10,6 +10,8 @@ ForeignKeyConstraint, ) +from pypn_habref_api.env import db + def get_csv_field_names(f, encoding, delimiter): if encoding == "WIN1252": # postgresql encoding @@ -75,7 +77,12 @@ def copy_from_csv( f, ) - testTable = Table(final_table_name, db.metadata, schema=schema, autoload_with=engine) + print("final_table_name : ", final_table_name) + print("schema : ", schema) + + testTable = Table( + final_table_name, db.metadata, schema=schema, autoload_with=db.session.connection() + ) for col in testTable.columns: if col.name in table_fields: @@ -183,7 +190,7 @@ def test(): raise e -def get_referencing_tables(table_name, db, schema=""): +def get_referencing_tables(table_name, schema=""): """Trouve toutes les tables (tous schémas) qui ont des FK pointant vers table_name""" inspector = sa_inspect(db.engine) referencing_tables = [] @@ -259,3 +266,43 @@ def check_broken_references(): ) return broken_refs + + +def delete_tmp_tables(table_files): + for table, value in table_files.items(): + db.session.execute(f"DROP TABLE ref_habitats.tmp_{table}") + + +def compare_tables_on_column(schema, table1, table2, column="cd_hab"): + + query = f""" + SELECT '{table1}' as source, {column} + FROM {schema}.{table1} + EXCEPT + SELECT '{table1}' as source, {column} + FROM {schema}.{table2} + + UNION ALL + + SELECT '{table2}' as source, {column} + FROM {schema}.{table2} + EXCEPT + SELECT '{table2}' as source, {column} + FROM {schema}.{table1} + """ + + result = db.session.execute(query) + + only_in_table1 = [] + only_in_table2 = [] + + for row in result: + if row.source == table1: + only_in_table1.append(row[column]) + else: + only_in_table2.append(row[column]) + + return { + f"only_in_{table1}": only_in_table1, + f"only_in_{table2}": only_in_table2, + } From d39b7d82427e7f4f82b7f5881a67b42187d56273 Mon Sep 17 00:00:00 2001 From: Pierre-Narcisi Date: Mon, 29 Jun 2026 17:48:40 +0200 Subject: [PATCH 06/11] Finish habref update --- src/pypn_habref_api/commands/habref.py | 3 +- src/pypn_habref_api/commands/habref_v7.py | 71 ++++++- src/pypn_habref_api/commands/utils.py | 218 +++++++--------------- 3 files changed, 131 insertions(+), 161 deletions(-) diff --git a/src/pypn_habref_api/commands/habref.py b/src/pypn_habref_api/commands/habref.py index 2b13a25..19f5626 100644 --- a/src/pypn_habref_api/commands/habref.py +++ b/src/pypn_habref_api/commands/habref.py @@ -2,7 +2,7 @@ from flask.cli import with_appcontext -from .habref_v7 import import_v07 +from .habref_v7 import import_v07, apply_v07 import logging @@ -15,3 +15,4 @@ def habref(): habref.add_command(import_v07) +habref.add_command(apply_v07) diff --git a/src/pypn_habref_api/commands/habref_v7.py b/src/pypn_habref_api/commands/habref_v7.py index c44ddf7..94e2e59 100644 --- a/src/pypn_habref_api/commands/habref_v7.py +++ b/src/pypn_habref_api/commands/habref_v7.py @@ -5,6 +5,7 @@ from pypn_habref_api.models import BibListHabitat, Habref from pypn_habref_api.env import db +from sqlalchemy import text as sa_text from sqlalchemy.schema import Table, MetaData, PrimaryKeyConstraint import click @@ -13,14 +14,7 @@ from alembic import op from utils_flask_sqla.migrations.utils import open_remote_file -from .utils import ( - copy_from_csv, - empty_table, - restore_constraints, - delete_tmp_tables, - compare_tables_on_column, - get_referencing_tables, -) +from .utils import copy_from_csv, empty_table, restore_constraints, export_orphans_to_csv base_url = "https://geonature.fr/data/inpn/habitats/" table_files = { @@ -206,6 +200,7 @@ def import_habref(logger, num_version, habref_archive_name): for table, value in table_files.items(): logger.info(f"Insert HABREF v{num_version} {table}…") with archive.open(value["filename"]) as f: + db.session.execute(f"DROP TABLE IF EXISTS ref_habitats.tmp_{table};") db.session.execute( f"CREATE TABLE ref_habitats.tmp_{table} AS TABLE ref_habitats.{table} WITH NO DATA;" ) @@ -239,5 +234,65 @@ def import_v07(): habref_archive_name="HABREF_70.zip", ) + logger.info("Détection des données orphelines…") + nb = export_orphans_to_csv( + ref_table="habref", + new_ref_table="tmp_habref", + pk_col="cd_hab", + output_path="tmp/habref/orphans_habref.csv", + db=db, + schema="ref_habitats", + exclude_tables=list(table_files.keys()), + ) + if nb: + logger.warning(f"{nb} valeur(s) orpheline(s) détectée(s), voir {orphans_output}") + else: + logger.info("Aucune donnée orpheline détectée.") + + logger.info("Committing…") + db.session.commit() + + +def apply_habref(logger): + db.session.execute(sa_text("SET session_replication_role = 'replica'")) + + for table in reversed(list(table_files.keys())): + logger.info(f"Vidage de {table}…") + db.session.execute(sa_text(f"DELETE FROM ref_habitats.{table}")) + + for table in table_files.keys(): + logger.info(f"Remplissage de {table} depuis tmp_{table}…") + db.session.execute( + sa_text(f"INSERT INTO ref_habitats.{table} SELECT * FROM ref_habitats.tmp_{table}") + ) + + logger.info("Remplissage de autocomplete_habitat…") + db.session.execute(sa_text("DELETE FROM ref_habitats.autocomplete_habitat")) + db.session.execute(sa_text(""" + INSERT INTO ref_habitats.autocomplete_habitat + SELECT + cd_hab, + h.cd_typo, + lb_code, + lb_nom_typo, + concat(lb_code, ' - ', lb_hab_fr, ' ', lb_hab_fr_complet) + FROM ref_habitats.habref h + JOIN ref_habitats.typoref t ON t.cd_typo = h.cd_typo + """)) + + db.session.execute(sa_text("SET session_replication_role = 'origin'")) + + for table in reversed(list(table_files.keys())): + logger.info(f"Suppression de tmp_{table}…") + db.session.execute(sa_text(f"DROP TABLE IF EXISTS ref_habitats.tmp_{table}")) + + +@click.command() +@with_appcontext +def apply_v07(): + logger = logging.getLogger() + + apply_habref(logger) + logger.info("Committing…") db.session.commit() diff --git a/src/pypn_habref_api/commands/utils.py b/src/pypn_habref_api/commands/utils.py index f0bdd53..d8add82 100644 --- a/src/pypn_habref_api/commands/utils.py +++ b/src/pypn_habref_api/commands/utils.py @@ -1,8 +1,10 @@ +import csv +import os from csv import DictReader from io import TextIOWrapper import sqlalchemy as sa -from sqlalchemy import inspect as sa_inspect +from sqlalchemy import inspect as sa_inspect, text from sqlalchemy.schema import ( Table, MetaData, @@ -77,12 +79,7 @@ def copy_from_csv( f, ) - print("final_table_name : ", final_table_name) - print("schema : ", schema) - - testTable = Table( - final_table_name, db.metadata, schema=schema, autoload_with=db.session.connection() - ) + testTable = Table(final_table_name, MetaData(), schema=schema, autoload_with=db.session.connection()) for col in testTable.columns: if col.name in table_fields: @@ -134,80 +131,22 @@ def restore_constraints(table_name, db, constraints, schema=""): ) -def detect_changes(): - pass - - -def detect_missing_cd_hab(): - op.create_table( - "tmp_missing_cd_hab", - Column("cd_hab", Integer, primary_key=True), - schema="ref_habitats", - ) - pass - - -def test(): - inspector = sa_inspect(db.engine) - - # Trouve toutes les tables qui référencent cette table - broken_refs = [] - - foreign_keys = inspector.get_foreign_keys(table_name, schema=schema) - - # Désactive les contraintes FK - db.session.execute(text("SET session_replication_role = 'replica'")) - try: - # Vide la table - db.session.execute(text(f"TRUNCATE TABLE {table_full_name}")) - db.session.commit() - - # Charge les nouvelles données si une fonction est fournie - if data_loader_func: - data_loader_func(db) - - # Réactive les contraintes - db.session.execute(text("SET session_replication_role = 'origin'")) - db.session.commit() - - # Vérifie les références cassées - broken = check_broken_references(table_name, db, schema) - if broken: - print("⚠️ ATTENTION : Références cassées détectées !") - for ref in broken: - print( - f" - Table {ref['table']}.{ref['column']} : {ref['broken_count']} lignes orphelines" - ) - return False, broken - - print("✓ Aucune référence cassée détectée") - return True, foreign_keys - - except Exception as e: - db.session.rollback() - db.session.execute(text("SET session_replication_role = 'origin'")) - db.session.commit() - raise e - - -def get_referencing_tables(table_name, schema=""): - """Trouve toutes les tables (tous schémas) qui ont des FK pointant vers table_name""" +def get_referencing_tables(table_name, db, schema="", exclude_tables=None): + """Trouve toutes les tables qui ont des FK pointant vers table_name, + en excluant les tables listées dans exclude_tables.""" + exclude_tables = set(exclude_tables or []) inspector = sa_inspect(db.engine) referencing_tables = [] - all_schemas = inspector.get_schema_names() - - for other_schema in all_schemas: + for other_schema in inspector.get_schema_names(): if other_schema in ("pg_catalog", "information_schema", "pg_toast"): continue - try: for other_table in inspector.get_table_names(schema=other_schema): - fks = inspector.get_foreign_keys(other_table, schema=other_schema) - - for fk in fks: - referred_schema = fk.get("referred_schema", other_schema) - + if other_schema == schema and other_table in exclude_tables: + continue + for fk in inspector.get_foreign_keys(other_table, schema=other_schema): + referred_schema = fk.get("referred_schema") or other_schema if fk["referred_table"] == table_name and ( not schema or referred_schema == schema ): @@ -215,94 +154,69 @@ def get_referencing_tables(table_name, schema=""): { "schema": other_schema, "table": other_table, - "constraint_name": fk["name"], "fk_column": fk["constrained_columns"][0], "ref_column": fk["referred_columns"][0], - "referred_schema": referred_schema, } ) except Exception as e: print(f"Impossible d'inspecter le schéma {other_schema}: {e}") - continue return referencing_tables -def check_broken_references(): - inspector = sa_inspect(db.engine) - - # Trouve toutes les tables qui référencent cette table - broken_refs = [] - - for other_table in inspector.get_table_names(schema=schema): - fks = inspector.get_foreign_keys(other_table, schema=schema) - - for fk in fks: - # Si cette FK pointe vers notre table - if fk["referred_table"] == table_name: - fk_column = fk["constrained_columns"][0] - ref_column = fk["referred_columns"][0] - - table_full = f"{schema}.{other_table}" if schema else other_table - ref_full = f"{schema}.{table_name}" if schema else table_name - - # Requête pour trouver les orphelins - query = text( - f""" - SELECT COUNT(*) as broken_count - FROM {table_full} t - WHERE t.{fk_column} IS NOT NULL - AND NOT EXISTS ( - SELECT 1 FROM {ref_full} r - WHERE r.{ref_column} = t.{fk_column} - ) - """ - ) - - result = db.session.execute(query).fetchone() - if result[0] > 0: - broken_refs.append( - {"table": other_table, "column": fk_column, "broken_count": result[0]} - ) - - return broken_refs - - -def delete_tmp_tables(table_files): - for table, value in table_files.items(): - db.session.execute(f"DROP TABLE ref_habitats.tmp_{table}") - - -def compare_tables_on_column(schema, table1, table2, column="cd_hab"): - - query = f""" - SELECT '{table1}' as source, {column} - FROM {schema}.{table1} - EXCEPT - SELECT '{table1}' as source, {column} - FROM {schema}.{table2} - - UNION ALL - - SELECT '{table2}' as source, {column} - FROM {schema}.{table2} - EXCEPT - SELECT '{table2}' as source, {column} - FROM {schema}.{table1} +def export_orphans_to_csv(ref_table, new_ref_table, pk_col, output_path, db, schema="", exclude_tables=None): """ + Compare ref_table (ancienne version) et new_ref_table (nouvelle version importée) + et exporte dans un CSV unique toutes les valeurs de pk_col présentes dans les tables + référençantes qui n'existent plus dans new_ref_table. - result = db.session.execute(query) - - only_in_table1 = [] - only_in_table2 = [] - - for row in result: - if row.source == table1: - only_in_table1.append(row[column]) - else: - only_in_table2.append(row[column]) + Colonnes CSV : table_name, schema, fk_column, fk_value, nb_lignes_affectees + """ + ref_full = f"{schema}.{new_ref_table}" if schema else new_ref_table + referencing = get_referencing_tables(ref_table, db, schema=schema, exclude_tables=exclude_tables) + + rows = [] + for ref in referencing: + src_full = f"{ref['schema']}.{ref['table']}" + fk_col = ref["fk_column"] + result = db.session.execute( + text( + f""" + SELECT t.{fk_col}, COUNT(*) AS nb_lignes + FROM {src_full} t + WHERE t.{fk_col} IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM {ref_full} r WHERE r.{pk_col} = t.{fk_col} + ) + GROUP BY t.{fk_col} + ORDER BY t.{fk_col} + """ + ) + ).fetchall() + + for fk_value, nb_lignes in result: + rows.append( + { + "table_name": ref["table"], + "schema": ref["schema"], + "fk_column": fk_col, + "fk_value": fk_value, + "nb_lignes_affectees": nb_lignes, + } + ) - return { - f"only_in_{table1}": only_in_table1, - f"only_in_{table2}": only_in_table2, - } + if not rows: + return 0 + + dirname = os.path.dirname(output_path) + if dirname: + os.makedirs(dirname, exist_ok=True) + with open(output_path, "w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter( + f, + fieldnames=["table_name", "schema", "fk_column", "fk_value", "nb_lignes_affectees"], + ) + writer.writeheader() + writer.writerows(rows) + + return len(rows) From 6c13aa4a98e1a1264ec61ecedabb2a79fa5fa828 Mon Sep 17 00:00:00 2001 From: Pierre-Narcisi Date: Wed, 1 Jul 2026 08:54:58 +0200 Subject: [PATCH 07/11] extend to other tables than habref --- src/pypn_habref_api/commands/habref_v7.py | 36 +++++++-------- src/pypn_habref_api/commands/utils.py | 56 ++++++++++------------- 2 files changed, 40 insertions(+), 52 deletions(-) diff --git a/src/pypn_habref_api/commands/habref_v7.py b/src/pypn_habref_api/commands/habref_v7.py index 94e2e59..a361539 100644 --- a/src/pypn_habref_api/commands/habref_v7.py +++ b/src/pypn_habref_api/commands/habref_v7.py @@ -14,7 +14,7 @@ from alembic import op from utils_flask_sqla.migrations.utils import open_remote_file -from .utils import copy_from_csv, empty_table, restore_constraints, export_orphans_to_csv +from .utils import copy_from_csv, empty_table, restore_constraints, collect_orphan_rows, export_orphans_to_csv base_url = "https://geonature.fr/data/inpn/habitats/" table_files = { @@ -195,7 +195,6 @@ def import_habref(logger, num_version, habref_archive_name): - delete_tmp_tables(table_files) with open_remote_file(base_url, habref_archive_name, open_fct=ZipFile) as archive: for table, value in table_files.items(): logger.info(f"Insert HABREF v{num_version} {table}…") @@ -213,14 +212,6 @@ def import_habref(logger, num_version, habref_archive_name): schema="ref_habitats", db=db, ) - diff = compare_tables_on_column( - "ref_habitats", table, f"tmp_{table}", value["unique_column"] - ) - print("TEEEEEEEEEESSSSSTTT") - print("table : ", table) - print("référence : ", get_referencing_tables(table, "ref_habitats")) - # print(diff[f"only_in_{table}"]) - # print(diff[f"only_in_tmp_{table}"]) @click.command() @@ -235,17 +226,22 @@ def import_v07(): ) logger.info("Détection des données orphelines…") - nb = export_orphans_to_csv( - ref_table="habref", - new_ref_table="tmp_habref", - pk_col="cd_hab", - output_path="tmp/habref/orphans_habref.csv", - db=db, - schema="ref_habitats", - exclude_tables=list(table_files.keys()), - ) + all_orphans = [] + for table, config in table_files.items(): + logger.info(f" Vérification de {table}…") + all_orphans.extend( + collect_orphan_rows( + ref_table=table, + new_ref_table=f"tmp_{table}", + pk_col=config["unique_column"], + db=db, + schema="ref_habitats", + exclude_tables=list(table_files.keys()), + ) + ) + nb = export_orphans_to_csv(all_orphans, "tmp/habref/orphans_habref.csv") if nb: - logger.warning(f"{nb} valeur(s) orpheline(s) détectée(s), voir {orphans_output}") + logger.warning(f"{nb} valeur(s) orpheline(s) détectée(s), voir tmp/habref/orphans_habref.csv") else: logger.info("Aucune donnée orpheline détectée.") diff --git a/src/pypn_habref_api/commands/utils.py b/src/pypn_habref_api/commands/utils.py index d8add82..7b029d6 100644 --- a/src/pypn_habref_api/commands/utils.py +++ b/src/pypn_habref_api/commands/utils.py @@ -4,7 +4,7 @@ from io import TextIOWrapper import sqlalchemy as sa -from sqlalchemy import inspect as sa_inspect, text +from sqlalchemy import inspect as sa_inspect, func, exists, select, table as sa_table, column as sa_column from sqlalchemy.schema import ( Table, MetaData, @@ -164,39 +164,32 @@ def get_referencing_tables(table_name, db, schema="", exclude_tables=None): return referencing_tables -def export_orphans_to_csv(ref_table, new_ref_table, pk_col, output_path, db, schema="", exclude_tables=None): - """ - Compare ref_table (ancienne version) et new_ref_table (nouvelle version importée) - et exporte dans un CSV unique toutes les valeurs de pk_col présentes dans les tables - référençantes qui n'existent plus dans new_ref_table. +CSV_FIELDNAMES = ["ref_table", "table_name", "schema", "fk_column", "fk_value", "nb_lignes_affectees"] + - Colonnes CSV : table_name, schema, fk_column, fk_value, nb_lignes_affectees +def collect_orphan_rows(ref_table, new_ref_table, pk_col, db, schema="", exclude_tables=None): + """ + Retourne la liste des lignes orphelines pour une table du référentiel : + valeurs de pk_col présentes dans les tables référençantes mais absentes de new_ref_table. """ - ref_full = f"{schema}.{new_ref_table}" if schema else new_ref_table referencing = get_referencing_tables(ref_table, db, schema=schema, exclude_tables=exclude_tables) - rows = [] for ref in referencing: - src_full = f"{ref['schema']}.{ref['table']}" fk_col = ref["fk_column"] - result = db.session.execute( - text( - f""" - SELECT t.{fk_col}, COUNT(*) AS nb_lignes - FROM {src_full} t - WHERE t.{fk_col} IS NOT NULL - AND NOT EXISTS ( - SELECT 1 FROM {ref_full} r WHERE r.{pk_col} = t.{fk_col} - ) - GROUP BY t.{fk_col} - ORDER BY t.{fk_col} - """ - ) - ).fetchall() - - for fk_value, nb_lignes in result: + src = sa_table(ref["table"], sa_column(fk_col), schema=ref["schema"]) + ref_t = sa_table(new_ref_table, sa_column(pk_col), schema=schema) + subq = select(1).select_from(ref_t).where(ref_t.c[pk_col] == src.c[fk_col]) + stmt = ( + select(src.c[fk_col], func.count().label("nb_lignes")) + .where(src.c[fk_col].isnot(None)) + .where(~exists(subq)) + .group_by(src.c[fk_col]) + .order_by(src.c[fk_col]) + ) + for fk_value, nb_lignes in db.session.execute(stmt).fetchall(): rows.append( { + "ref_table": ref_table, "table_name": ref["table"], "schema": ref["schema"], "fk_column": fk_col, @@ -204,19 +197,18 @@ def export_orphans_to_csv(ref_table, new_ref_table, pk_col, output_path, db, sch "nb_lignes_affectees": nb_lignes, } ) + return rows + +def export_orphans_to_csv(rows, output_path): + """Écrit la liste de lignes orphelines dans un CSV.""" if not rows: return 0 - dirname = os.path.dirname(output_path) if dirname: os.makedirs(dirname, exist_ok=True) with open(output_path, "w", newline="", encoding="utf-8") as f: - writer = csv.DictWriter( - f, - fieldnames=["table_name", "schema", "fk_column", "fk_value", "nb_lignes_affectees"], - ) + writer = csv.DictWriter(f, fieldnames=CSV_FIELDNAMES) writer.writeheader() writer.writerows(rows) - return len(rows) From 4b9b8ce0552efeb81f785c600a206b2c9c5e574c Mon Sep 17 00:00:00 2001 From: Pierre-Narcisi Date: Thu, 2 Jul 2026 13:42:11 +0200 Subject: [PATCH 08/11] add doc for migration --- docs/habref_update.md | 81 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 docs/habref_update.md diff --git a/docs/habref_update.md b/docs/habref_update.md new file mode 100644 index 0000000..2558936 --- /dev/null +++ b/docs/habref_update.md @@ -0,0 +1,81 @@ +# Mise à jour du référentiel HABREF + +Scripts permettant de mettre à jour le référentiel des habitats (HABREF) vers une nouvelle version. + +## Avant de commencer + +La mise à jour du référentiel HABREF remplace l'intégralité des données de référence. + +Il est fortement recommandé de **faire une sauvegarde de la base de données** avant de commencer. + +## Commandes disponibles + +Les commandes sont accessibles depuis le virtualenv de GeoNature : + +```bash +cd ~/geonature +source backend/venv/bin/activate +``` + +--- + +## Étape 1 — Importer la nouvelle version et détecter les orphelins + +```bash +geonature habref import-v07 +``` + +Cette commande : + +1. Télécharge l'archive `HABREF_70.zip` +2. Importe les données dans des tables temporaires `ref_habitats.tmp_*` (sans toucher aux données en production) +3. Parcourt toutes les tables de la base (tous modules confondus) qui référencent les tables HABREF et identifie les valeurs qui deviendront orphelines +4. Exporte le résultat dans le fichier `tmp/habref/orphans_habref.csv` + +Le CSV généré contient les colonnes suivantes : + +| Colonne | Description | +| --------------------- | -------------------------------------------------------- | +| `ref_table` | Table du référentiel concernée (ex: `habref`, `typoref`) | +| `table_name` | Table applicative contenant la référence orpheline | +| `schema` | Schéma de cette table | +| `fk_column` | Colonne portant la clé étrangère | +| `fk_value` | Valeur orpheline | +| `nb_lignes_affectees` | Nombre de lignes concernées | + +Si la commande est relancée, les tables temporaires existantes sont automatiquement supprimées et recréées. + +--- + +## Étape 2 — Analyser et corriger les données orphelines + +Consultez le fichier `tmp/habref/orphans_habref.csv`. Pour chaque ligne, la valeur `fk_value` est un code habitat qui sera supprimé lors de la mise à jour. + +Vous devez décider, pour chaque cas : + +- **Mettre à jour** les observations concernées avec le `cd_hab` de remplacement dans la nouvelle version +- **Supprimer** les observations si elles ne peuvent plus être rattachées +- **Ignorer** si les données ne sont plus actives + +Ces corrections peuvent être regroupées dans un fichier SQL et exécutées manuellement avant de passer à l'étape suivante. + +--- + +## Étape 3 — Appliquer la mise à jour + +Une fois les données orphelines corrigées : + +```bash +geonature habref apply-v07 +``` + +Cette commande : + +1. Désactive temporairement les contraintes de clés étrangères +2. Vide toutes les tables du schéma `ref_habitats` (dans l'ordre inverse des dépendances) +3. Recopie le contenu des tables temporaires `tmp_*` dans les tables de production +4. Reconstruit la table `ref_habitats.autocomplete_habitat` +5. Réactive les contraintes de clés étrangères +6. Supprime les tables temporaires `tmp_*` + +> ⚠️ Si des données orphelines subsistent au moment du `apply-v07`, des erreurs de contraintes FK pourraient apparaître et la migration ne se feras pas. Assurez-vous que toutes les corrections ont bien été appliquées avant de lancer cette commande. From fa2686be898ee8e2838a7340475f3dd0c24c2c7d Mon Sep 17 00:00:00 2001 From: Pierre-Narcisi Date: Thu, 2 Jul 2026 13:44:57 +0200 Subject: [PATCH 09/11] add test and apply black --- src/pypn_habref_api/commands/habref_v7.py | 20 ++- src/pypn_habref_api/commands/utils.py | 26 ++- ...insert_inpn_data_in_ref_habitats_schema.py | 6 +- .../805442837a68_correction_on_habref_data.py | 12 +- .../tests/test_habref_update.py | 165 ++++++++++++++++++ 5 files changed, 215 insertions(+), 14 deletions(-) create mode 100644 src/pypn_habref_api/tests/test_habref_update.py diff --git a/src/pypn_habref_api/commands/habref_v7.py b/src/pypn_habref_api/commands/habref_v7.py index a361539..774854b 100644 --- a/src/pypn_habref_api/commands/habref_v7.py +++ b/src/pypn_habref_api/commands/habref_v7.py @@ -14,7 +14,13 @@ from alembic import op from utils_flask_sqla.migrations.utils import open_remote_file -from .utils import copy_from_csv, empty_table, restore_constraints, collect_orphan_rows, export_orphans_to_csv +from .utils import ( + copy_from_csv, + empty_table, + restore_constraints, + collect_orphan_rows, + export_orphans_to_csv, +) base_url = "https://geonature.fr/data/inpn/habitats/" table_files = { @@ -241,7 +247,9 @@ def import_v07(): ) nb = export_orphans_to_csv(all_orphans, "tmp/habref/orphans_habref.csv") if nb: - logger.warning(f"{nb} valeur(s) orpheline(s) détectée(s), voir tmp/habref/orphans_habref.csv") + logger.warning( + f"{nb} valeur(s) orpheline(s) détectée(s), voir tmp/habref/orphans_habref.csv" + ) else: logger.info("Aucune donnée orpheline détectée.") @@ -264,7 +272,9 @@ def apply_habref(logger): logger.info("Remplissage de autocomplete_habitat…") db.session.execute(sa_text("DELETE FROM ref_habitats.autocomplete_habitat")) - db.session.execute(sa_text(""" + db.session.execute( + sa_text( + """ INSERT INTO ref_habitats.autocomplete_habitat SELECT cd_hab, @@ -274,7 +284,9 @@ def apply_habref(logger): concat(lb_code, ' - ', lb_hab_fr, ' ', lb_hab_fr_complet) FROM ref_habitats.habref h JOIN ref_habitats.typoref t ON t.cd_typo = h.cd_typo - """)) + """ + ) + ) db.session.execute(sa_text("SET session_replication_role = 'origin'")) diff --git a/src/pypn_habref_api/commands/utils.py b/src/pypn_habref_api/commands/utils.py index 7b029d6..9014e74 100644 --- a/src/pypn_habref_api/commands/utils.py +++ b/src/pypn_habref_api/commands/utils.py @@ -4,7 +4,14 @@ from io import TextIOWrapper import sqlalchemy as sa -from sqlalchemy import inspect as sa_inspect, func, exists, select, table as sa_table, column as sa_column +from sqlalchemy import ( + inspect as sa_inspect, + func, + exists, + select, + table as sa_table, + column as sa_column, +) from sqlalchemy.schema import ( Table, MetaData, @@ -79,7 +86,9 @@ def copy_from_csv( f, ) - testTable = Table(final_table_name, MetaData(), schema=schema, autoload_with=db.session.connection()) + testTable = Table( + final_table_name, MetaData(), schema=schema, autoload_with=db.session.connection() + ) for col in testTable.columns: if col.name in table_fields: @@ -164,7 +173,14 @@ def get_referencing_tables(table_name, db, schema="", exclude_tables=None): return referencing_tables -CSV_FIELDNAMES = ["ref_table", "table_name", "schema", "fk_column", "fk_value", "nb_lignes_affectees"] +CSV_FIELDNAMES = [ + "ref_table", + "table_name", + "schema", + "fk_column", + "fk_value", + "nb_lignes_affectees", +] def collect_orphan_rows(ref_table, new_ref_table, pk_col, db, schema="", exclude_tables=None): @@ -172,7 +188,9 @@ def collect_orphan_rows(ref_table, new_ref_table, pk_col, db, schema="", exclude Retourne la liste des lignes orphelines pour une table du référentiel : valeurs de pk_col présentes dans les tables référençantes mais absentes de new_ref_table. """ - referencing = get_referencing_tables(ref_table, db, schema=schema, exclude_tables=exclude_tables) + referencing = get_referencing_tables( + ref_table, db, schema=schema, exclude_tables=exclude_tables + ) rows = [] for ref in referencing: fk_col = ref["fk_column"] diff --git a/src/pypn_habref_api/migrations/versions/46e91e738845_insert_inpn_data_in_ref_habitats_schema.py b/src/pypn_habref_api/migrations/versions/46e91e738845_insert_inpn_data_in_ref_habitats_schema.py index e62134c..6f9c308 100644 --- a/src/pypn_habref_api/migrations/versions/46e91e738845_insert_inpn_data_in_ref_habitats_schema.py +++ b/src/pypn_habref_api/migrations/versions/46e91e738845_insert_inpn_data_in_ref_habitats_schema.py @@ -52,7 +52,8 @@ def upgrade(): ) logger.info("Populate table autocomplete_habitat…") - op.execute(""" + op.execute( + """ INSERT INTO ref_habitats.autocomplete_habitat SELECT cd_hab, @@ -62,7 +63,8 @@ def upgrade(): concat(lb_code, ' - ', lb_hab_fr, ' ', lb_hab_fr_complet) FROM ref_habitats.habref h JOIN ref_habitats.typoref t ON t.cd_typo = h.cd_typo - """) + """ + ) def downgrade(): diff --git a/src/pypn_habref_api/migrations/versions/805442837a68_correction_on_habref_data.py b/src/pypn_habref_api/migrations/versions/805442837a68_correction_on_habref_data.py index fe83ced..23951f8 100644 --- a/src/pypn_habref_api/migrations/versions/805442837a68_correction_on_habref_data.py +++ b/src/pypn_habref_api/migrations/versions/805442837a68_correction_on_habref_data.py @@ -18,7 +18,8 @@ def upgrade(): - op.execute(""" + op.execute( + """ ALTER TABLE ref_habitats.habref ADD column lb_hab_en_save character varying(500); UPDATE ref_habitats.habref SET lb_hab_en_save = lb_auteur; @@ -27,8 +28,10 @@ def upgrade(): UPDATE ref_habitats.habref SET lb_hab_en = lb_hab_en_save; ALTER TABLE ref_habitats.habref DROP column lb_hab_en_save - """) - op.execute(""" + """ + ) + op.execute( + """ DELETE FROM ref_habitats.autocomplete_habitat; INSERT INTO ref_habitats.autocomplete_habitat SELECT @@ -39,7 +42,8 @@ def upgrade(): concat(lb_code, ' - ', lb_hab_fr, ' ', lb_hab_fr_complet) FROM ref_habitats.habref h JOIN ref_habitats.typoref t ON t.cd_typo = h.cd_typo; - """) + """ + ) def downgrade(): diff --git a/src/pypn_habref_api/tests/test_habref_update.py b/src/pypn_habref_api/tests/test_habref_update.py new file mode 100644 index 0000000..ef45d32 --- /dev/null +++ b/src/pypn_habref_api/tests/test_habref_update.py @@ -0,0 +1,165 @@ +import csv +import io +import logging +import os +import tempfile +import zipfile +from contextlib import contextmanager +from unittest.mock import patch + +import pytest +from sqlalchemy import inspect as sa_inspect, text + +from pypn_habref_api.commands.habref_v7 import apply_habref, import_habref, table_files +from pypn_habref_api.commands.utils import collect_orphan_rows, export_orphans_to_csv +from pypn_habref_api.env import db + + +def make_habref_zip(): + """ZIP minimal en mémoire — uniquement les en-têtes CSV, aucune ligne de données.""" + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + for config in table_files.values(): + csv_content = io.StringIO() + writer = csv.writer(csv_content, delimiter=";") + writer.writerow(config["table_fields"].values()) + zf.writestr(config["filename"], csv_content.getvalue().encode("utf-8")) + buf.seek(0) + return buf + + +@contextmanager +def tmp_tables(): + """Crée les tables tmp_* en copiant la structure des tables habref, et les supprime à la fin.""" + try: + for table in table_files: + db.session.execute(text(f"DROP TABLE IF EXISTS ref_habitats.tmp_{table}")) + db.session.execute( + text( + f"CREATE TABLE ref_habitats.tmp_{table} " + f"AS TABLE ref_habitats.{table} WITH NO DATA" + ) + ) + db.session.flush() + yield + finally: + for table in reversed(list(table_files.keys())): + db.session.execute(text(f"DROP TABLE IF EXISTS ref_habitats.tmp_{table}")) + db.session.flush() + + +@pytest.mark.usefixtures("app") +class TestImportHabref: + def test_tmp_tables_created(self): + """import_habref crée une table tmp_* pour chaque table de table_files.""" + logger = logging.getLogger() + zip_data = make_habref_zip() + + @contextmanager + def fake_open_remote(url, filename, open_fct=None): + zip_data.seek(0) + with zipfile.ZipFile(zip_data) as zf: + yield zf + + try: + with patch("pypn_habref_api.commands.habref_v7.open_remote_file", fake_open_remote): + import_habref(logger, num_version="07", habref_archive_name="HABREF_70.zip") + + inspector = sa_inspect(db.engine) + existing = inspector.get_table_names(schema="ref_habitats") + for table in table_files: + assert f"tmp_{table}" in existing + finally: + for table in reversed(list(table_files.keys())): + db.session.execute(text(f"DROP TABLE IF EXISTS ref_habitats.tmp_{table}")) + db.session.flush() + + +@pytest.mark.usefixtures("app") +class TestExportOrphans: + def test_orphan_detected_in_csv(self): + """Un cd_hab présent dans cor_list_habitat mais absent de tmp_habref apparaît dans le CSV.""" + orphan_cd_hab = db.session.execute( + text("SELECT cd_hab FROM ref_habitats.habref LIMIT 1") + ).scalar() + + with tmp_tables(): + db.session.execute( + text( + "INSERT INTO ref_habitats.tmp_habref " + "SELECT * FROM ref_habitats.habref WHERE cd_hab != :cd" + ), + {"cd": orphan_cd_hab}, + ) + + db.session.execute(text("SET session_replication_role = 'replica'")) + db.session.execute( + text( + "INSERT INTO ref_habitats.cor_list_habitat (id_cor_list, cd_hab) " + "SELECT COALESCE(MAX(id_cor_list), 0) + 1, :cd " + "FROM ref_habitats.cor_list_habitat" + ), + {"cd": orphan_cd_hab}, + ) + db.session.execute(text("SET session_replication_role = 'origin'")) + db.session.flush() + + with tempfile.NamedTemporaryFile(suffix=".csv", delete=False) as f: + output_path = f.name + try: + orphan_rows = collect_orphan_rows( + ref_table="habref", + new_ref_table="tmp_habref", + pk_col="cd_hab", + db=db, + schema="ref_habitats", + exclude_tables=list(table_files.keys()), + ) + nb = export_orphans_to_csv(orphan_rows, output_path) + assert nb > 0 + with open(output_path) as f: + rows = list(csv.DictReader(f)) + assert any(int(r["fk_value"]) == orphan_cd_hab for r in rows) + finally: + os.unlink(output_path) + db.session.execute(text("SET session_replication_role = 'replica'")) + db.session.execute( + text("DELETE FROM ref_habitats.cor_list_habitat WHERE cd_hab = :cd"), + {"cd": orphan_cd_hab}, + ) + db.session.execute(text("SET session_replication_role = 'origin'")) + db.session.flush() + + +@pytest.mark.usefixtures("app") +class TestApplyHabref: + def test_tables_updated_and_tmp_dropped(self): + """apply_habref remplace le contenu des tables réelles et supprime les tmp_*.""" + logger = logging.getLogger() + + count_before = db.session.execute( + text("SELECT COUNT(*) FROM ref_habitats.habref") + ).scalar() + + with tmp_tables(): + for table in table_files: + db.session.execute( + text( + f"INSERT INTO ref_habitats.tmp_{table} " + f"SELECT * FROM ref_habitats.{table}" + ) + ) + db.session.flush() + + apply_habref(logger) + db.session.flush() + + count_after = db.session.execute( + text("SELECT COUNT(*) FROM ref_habitats.habref") + ).scalar() + assert count_after == count_before + + inspector = sa_inspect(db.engine) + existing = inspector.get_table_names(schema="ref_habitats") + for table in table_files: + assert f"tmp_{table}" not in existing From bae71fcbd284e324220a97f6952fada5d45a9a80 Mon Sep 17 00:00:00 2001 From: Pierre-Narcisi Date: Thu, 2 Jul 2026 15:31:18 +0200 Subject: [PATCH 10/11] add command to pyproj --- pyproject.toml | 3 +++ setup.py | 50 -------------------------------------------------- 2 files changed, 3 insertions(+), 50 deletions(-) delete mode 100644 setup.py diff --git a/pyproject.toml b/pyproject.toml index 3845a99..fe257bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,9 @@ Homepage = "https://github.com/PnX-SI/Habref-api-module" [project.entry-points."alembic"] migrations = "pypn_habref_api.migrations:versions" +[project.entry-points."flask.commands"] +habref = "pypn_habref_api.commands.habref:habref" + [tool.setuptools] package-dir = { "" = "src" } packages = ["pypn_habref_api"] diff --git a/setup.py b/setup.py deleted file mode 100644 index 07d2dcb..0000000 --- a/setup.py +++ /dev/null @@ -1,50 +0,0 @@ -import setuptools -from pathlib import Path - -root_dir = Path(__file__).absolute().parent -with (root_dir / "VERSION").open() as f: - version = f.read() -with (root_dir / "README.md").open() as f: - long_description = f.read() -with (root_dir / "requirements.in").open() as f: - requirements = f.read().splitlines() - - -setuptools.setup( - name="pypn_habref_api", - version=version, - description="Python lib related to Habref referential (INPN)", - long_description=long_description, - long_description_content_type="text/markdown", - maintainer="Parcs nationaux des Écrins et des Cévennes", - maintainer_email="geonature@ecrins-parcnational.fr", - url="https://github.com/PnX-SI/Habref-api-module", - packages=setuptools.find_packages("src"), - package_dir={"": "src"}, - install_requires=requirements, - extras_require={ - "tests": [ - "pytest", - "pytest-flask", - ], - }, - include_package_data=True, - entry_points={ - "alembic": [ - "migrations = pypn_habref_api.migrations:versions", - ], - "flask.commands": [ - "habref = pypn_habref_api.commands.habref:habref", - ], - }, - zip_safe=False, - classifiers=[ - "Development Status :: 1 - Planning", - "Intended Audience :: Developers", - "Natural Language :: English", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.11", - "License :: OSI Approved :: GNU Affero General Public License v3", - "Operating System :: OS Independent", - ], -) From 7370784b3f02e2bd29da7f4e94cee6a47025667a Mon Sep 17 00:00:00 2001 From: Pierre-Narcisi Date: Thu, 16 Jul 2026 16:07:12 +0200 Subject: [PATCH 11/11] apply fixes --- docs/habref_update.md | 47 ++--- src/pypn_habref_api/commands/habref_v7.py | 86 +------- src/pypn_habref_api/commands/utils.py | 195 +++++------------- .../tests/test_habref_update.py | 18 +- 4 files changed, 94 insertions(+), 252 deletions(-) diff --git a/docs/habref_update.md b/docs/habref_update.md index 2558936..d7d35fd 100644 --- a/docs/habref_update.md +++ b/docs/habref_update.md @@ -4,33 +4,30 @@ Scripts permettant de mettre à jour le référentiel des habitats (HABREF) vers ## Avant de commencer -La mise à jour du référentiel HABREF remplace l'intégralité des données de référence. +> [!WARNING] +> La mise à jour du référentiel HABREF efface les données de la version du référentiel précédemment installée. -Il est fortement recommandé de **faire une sauvegarde de la base de données** avant de commencer. +> [!WARNING] +> Il est fortement recommandé de **faire une sauvegarde de la base de données** avant de commencer. -## Commandes disponibles +## 1. Importer la nouvelle version et détecter les orphelins -Les commandes sont accessibles depuis le virtualenv de GeoNature : +Dans la première étape, il faut télécharger les données du référentiels, stocker ces dernières dans une table temporaire (ref*habitats.tmp*). Pour cela, on lance la commande : ```bash -cd ~/geonature -source backend/venv/bin/activate +geonature habref import-v07 ``` ---- +> [!DANGER] +> Il se peut que certaines entrées du référentiels soient supprimées lors d'une mise à jours. Dans ce cas, le fichier `tmp/habref/orphans_habref.csv` liste l'ensemble des données dans votre base utilisant ces entrées Habref. -## Étape 1 — Importer la nouvelle version et détecter les orphelins +Si la commande est relancée, les tables temporaires existantes sont automatiquement supprimées et recréées. -```bash -geonature habref import-v07 -``` +--- -Cette commande : +## 2 — Analyser et corriger les données orphelines -1. Télécharge l'archive `HABREF_70.zip` -2. Importe les données dans des tables temporaires `ref_habitats.tmp_*` (sans toucher aux données en production) -3. Parcourt toutes les tables de la base (tous modules confondus) qui référencent les tables HABREF et identifie les valeurs qui deviendront orphelines -4. Exporte le résultat dans le fichier `tmp/habref/orphans_habref.csv` +Dans le cas où plusieurs données orphelines ont été détecté dans l'étape précédente, consultez le fichier `tmp/habref/orphans_habref.csv`. Pour chaque ligne de ce fichier, la valeur `fk_value` est un code habitat qui sera supprimé lors de la mise à jour. Le CSV généré contient les colonnes suivantes : @@ -43,27 +40,16 @@ Le CSV généré contient les colonnes suivantes : | `fk_value` | Valeur orpheline | | `nb_lignes_affectees` | Nombre de lignes concernées | -Si la commande est relancée, les tables temporaires existantes sont automatiquement supprimées et recréées. - ---- - -## Étape 2 — Analyser et corriger les données orphelines - -Consultez le fichier `tmp/habref/orphans_habref.csv`. Pour chaque ligne, la valeur `fk_value` est un code habitat qui sera supprimé lors de la mise à jour. - Vous devez décider, pour chaque cas : -- **Mettre à jour** les observations concernées avec le `cd_hab` de remplacement dans la nouvelle version +- **Mettre à jour** les données concernées avec le `cd_hab` de remplacement dans la nouvelle version - **Supprimer** les observations si elles ne peuvent plus être rattachées -- **Ignorer** si les données ne sont plus actives - -Ces corrections peuvent être regroupées dans un fichier SQL et exécutées manuellement avant de passer à l'étape suivante. --- -## Étape 3 — Appliquer la mise à jour +## 3 — Appliquer la mise à jour -Une fois les données orphelines corrigées : +Une fois les données orphelines corrigées, lancez la mise à jour effective du référentiel à l'aide de la commande suivante : ```bash geonature habref apply-v07 @@ -78,4 +64,5 @@ Cette commande : 5. Réactive les contraintes de clés étrangères 6. Supprime les tables temporaires `tmp_*` +> [!WARNING] > ⚠️ Si des données orphelines subsistent au moment du `apply-v07`, des erreurs de contraintes FK pourraient apparaître et la migration ne se feras pas. Assurez-vous que toutes les corrections ont bien été appliquées avant de lancer cette commande. diff --git a/src/pypn_habref_api/commands/habref_v7.py b/src/pypn_habref_api/commands/habref_v7.py index 774854b..8899ed7 100644 --- a/src/pypn_habref_api/commands/habref_v7.py +++ b/src/pypn_habref_api/commands/habref_v7.py @@ -1,26 +1,12 @@ -from zipfile import ZipFile -from collections import OrderedDict import logging -from pypn_habref_api.models import BibListHabitat, Habref from pypn_habref_api.env import db -from sqlalchemy import text as sa_text -from sqlalchemy.schema import Table, MetaData, PrimaryKeyConstraint - import click from flask.cli import with_appcontext -from alembic import op - -from utils_flask_sqla.migrations.utils import open_remote_file -from .utils import ( - copy_from_csv, - empty_table, - restore_constraints, - collect_orphan_rows, - export_orphans_to_csv, -) +from utils_flask_sqla.referential import collect_orphan_rows, export_orphans_to_csv +from .utils import import_habref, apply_habref base_url = "https://geonature.fr/data/inpn/habitats/" table_files = { @@ -178,9 +164,6 @@ "cd_hab_description": "CD_HAB_DESCRIPTION", "cd_hab": "CD_HAB", "cd_hab_field": "CD_HAB_FIELD", - "cd_typo": "CD_TYPO", - "lb_code": "LB_CODE", - "lb_hab_field": "LB_HAB_FIELD", "valeurs": "VALEURS", }, }, @@ -200,26 +183,6 @@ } -def import_habref(logger, num_version, habref_archive_name): - with open_remote_file(base_url, habref_archive_name, open_fct=ZipFile) as archive: - for table, value in table_files.items(): - logger.info(f"Insert HABREF v{num_version} {table}…") - with archive.open(value["filename"]) as f: - db.session.execute(f"DROP TABLE IF EXISTS ref_habitats.tmp_{table};") - db.session.execute( - f"CREATE TABLE ref_habitats.tmp_{table} AS TABLE ref_habitats.{table} WITH NO DATA;" - ) - copy_from_csv( - f, - f"tmp_{table}", - value["table_fields"], - encoding="UTF-8", - delimiter=";", - schema="ref_habitats", - db=db, - ) - - @click.command() @with_appcontext def import_v07(): @@ -227,8 +190,11 @@ def import_v07(): import_habref( logger, + table_files=table_files, + schema="ref_habitats", + base_url=base_url, num_version="07", - habref_archive_name="HABREF_70.zip", + archive_name="HABREF_70.zip", ) logger.info("Détection des données orphelines…") @@ -257,50 +223,12 @@ def import_v07(): db.session.commit() -def apply_habref(logger): - db.session.execute(sa_text("SET session_replication_role = 'replica'")) - - for table in reversed(list(table_files.keys())): - logger.info(f"Vidage de {table}…") - db.session.execute(sa_text(f"DELETE FROM ref_habitats.{table}")) - - for table in table_files.keys(): - logger.info(f"Remplissage de {table} depuis tmp_{table}…") - db.session.execute( - sa_text(f"INSERT INTO ref_habitats.{table} SELECT * FROM ref_habitats.tmp_{table}") - ) - - logger.info("Remplissage de autocomplete_habitat…") - db.session.execute(sa_text("DELETE FROM ref_habitats.autocomplete_habitat")) - db.session.execute( - sa_text( - """ - INSERT INTO ref_habitats.autocomplete_habitat - SELECT - cd_hab, - h.cd_typo, - lb_code, - lb_nom_typo, - concat(lb_code, ' - ', lb_hab_fr, ' ', lb_hab_fr_complet) - FROM ref_habitats.habref h - JOIN ref_habitats.typoref t ON t.cd_typo = h.cd_typo - """ - ) - ) - - db.session.execute(sa_text("SET session_replication_role = 'origin'")) - - for table in reversed(list(table_files.keys())): - logger.info(f"Suppression de tmp_{table}…") - db.session.execute(sa_text(f"DROP TABLE IF EXISTS ref_habitats.tmp_{table}")) - - @click.command() @with_appcontext def apply_v07(): logger = logging.getLogger() - apply_habref(logger) + apply_habref(logger, table_files=table_files, schema="ref_habitats") logger.info("Committing…") db.session.commit() diff --git a/src/pypn_habref_api/commands/utils.py b/src/pypn_habref_api/commands/utils.py index 9014e74..3dc8fe5 100644 --- a/src/pypn_habref_api/commands/utils.py +++ b/src/pypn_habref_api/commands/utils.py @@ -1,17 +1,9 @@ -import csv -import os from csv import DictReader from io import TextIOWrapper +from zipfile import ZipFile import sqlalchemy as sa -from sqlalchemy import ( - inspect as sa_inspect, - func, - exists, - select, - table as sa_table, - column as sa_column, -) +from sqlalchemy import inspect as sa_inspect, text as sa_text from sqlalchemy.schema import ( Table, MetaData, @@ -19,6 +11,8 @@ ForeignKeyConstraint, ) +from utils_flask_sqla.migrations.utils import open_remote_file + from pypn_habref_api.env import db @@ -95,138 +89,63 @@ def copy_from_csv( table_fields[col.name] = f"{table_fields[col.name]}::{col.type}" table_fields_list = list(table_fields.values()) - db.session.execute( - f""" + db.session.execute(f""" INSERT INTO {schema}.{final_table_name} ({", ".join(table_fields_key_list)}) SELECT {", ".join(table_fields_list)} FROM {schema}.{table_name}; - """ - ) + """) table.drop(bind=db.session.connection()) -def empty_table(table_name, db, schema=""): - inspector = sa_inspect(db.engine) - - # Récupère les foreign keys - foreign_keys = inspector.get_foreign_keys(table_name, schema=schema) - db.session.execute("SET session_replication_role = 'replica'") - for fk in foreign_keys: - constraint_name = fk["name"] - if constraint_name: # Vérifie que le nom existe - db.session.execute( - f"ALTER TABLE {schema}.{table_name} DROP CONSTRAINT {constraint_name} CASCADE" - ) - db.session.execute(f"DELETE FROM {schema}.{table_name}") - return foreign_keys - - -def restore_constraints(table_name, db, constraints, schema=""): - if constraints is None: - return - for constraint in constraints: - if isinstance(constraint, PrimaryKeyConstraint): - cols = ", ".join(str(col).split(".")[1] for col in constraint.columns) - db.session.execute( - f"ALTER TABLE {schema}.{table_name} ADD CONSTRAINT {constraint.name} UNIQUE ({cols})" - ) - elif isinstance(constraint, ForeignKeyConstraint): - cols = ", ".join(str(col).split(".")[1] for col in constraint.columns) - ref_cols = ", ".join( - str(col).split(".")[1] for col in constraint.references[0].columns - ) - db.session.execute( - f"ALTER TABLE {schema}.{table_name} ADD CONSTRAINT {constraint.name} FOREIGN KEY {cols} REFERENCES {constraint.references[0].table.name}({ref_cols})" - ) - - -def get_referencing_tables(table_name, db, schema="", exclude_tables=None): - """Trouve toutes les tables qui ont des FK pointant vers table_name, - en excluant les tables listées dans exclude_tables.""" - exclude_tables = set(exclude_tables or []) - inspector = sa_inspect(db.engine) - referencing_tables = [] - - for other_schema in inspector.get_schema_names(): - if other_schema in ("pg_catalog", "information_schema", "pg_toast"): - continue - try: - for other_table in inspector.get_table_names(schema=other_schema): - if other_schema == schema and other_table in exclude_tables: - continue - for fk in inspector.get_foreign_keys(other_table, schema=other_schema): - referred_schema = fk.get("referred_schema") or other_schema - if fk["referred_table"] == table_name and ( - not schema or referred_schema == schema - ): - referencing_tables.append( - { - "schema": other_schema, - "table": other_table, - "fk_column": fk["constrained_columns"][0], - "ref_column": fk["referred_columns"][0], - } - ) - except Exception as e: - print(f"Impossible d'inspecter le schéma {other_schema}: {e}") - - return referencing_tables - - -CSV_FIELDNAMES = [ - "ref_table", - "table_name", - "schema", - "fk_column", - "fk_value", - "nb_lignes_affectees", -] - - -def collect_orphan_rows(ref_table, new_ref_table, pk_col, db, schema="", exclude_tables=None): - """ - Retourne la liste des lignes orphelines pour une table du référentiel : - valeurs de pk_col présentes dans les tables référençantes mais absentes de new_ref_table. - """ - referencing = get_referencing_tables( - ref_table, db, schema=schema, exclude_tables=exclude_tables - ) - rows = [] - for ref in referencing: - fk_col = ref["fk_column"] - src = sa_table(ref["table"], sa_column(fk_col), schema=ref["schema"]) - ref_t = sa_table(new_ref_table, sa_column(pk_col), schema=schema) - subq = select(1).select_from(ref_t).where(ref_t.c[pk_col] == src.c[fk_col]) - stmt = ( - select(src.c[fk_col], func.count().label("nb_lignes")) - .where(src.c[fk_col].isnot(None)) - .where(~exists(subq)) - .group_by(src.c[fk_col]) - .order_by(src.c[fk_col]) +def import_habref(logger, table_files, schema, base_url, num_version, archive_name): + with open_remote_file(base_url, archive_name, open_fct=ZipFile) as archive: + for table, value in table_files.items(): + logger.info(f"Insert HABREF v{num_version} {table}…") + with archive.open(value["filename"]) as f: + db.session.execute(sa_text(f"DROP TABLE IF EXISTS {schema}.tmp_{table}")) + db.session.execute( + sa_text(f"CREATE TABLE {schema}.tmp_{table} AS TABLE {schema}.{table} WITH NO DATA") + ) + copy_from_csv( + f, + f"tmp_{table}", + value["table_fields"], + encoding="UTF-8", + delimiter=";", + schema=schema, + db=db, + ) + + +def apply_habref(logger, table_files, schema): + db.session.execute(sa_text("SET session_replication_role = 'replica'")) + + for table in reversed(list(table_files.keys())): + logger.info(f"Vidage de {table}…") + db.session.execute(sa_text(f"DELETE FROM {schema}.{table}")) + + for table in table_files.keys(): + logger.info(f"Remplissage de {table} depuis tmp_{table}…") + db.session.execute( + sa_text(f"INSERT INTO {schema}.{table} SELECT * FROM {schema}.tmp_{table}") ) - for fk_value, nb_lignes in db.session.execute(stmt).fetchall(): - rows.append( - { - "ref_table": ref_table, - "table_name": ref["table"], - "schema": ref["schema"], - "fk_column": fk_col, - "fk_value": fk_value, - "nb_lignes_affectees": nb_lignes, - } - ) - return rows - - -def export_orphans_to_csv(rows, output_path): - """Écrit la liste de lignes orphelines dans un CSV.""" - if not rows: - return 0 - dirname = os.path.dirname(output_path) - if dirname: - os.makedirs(dirname, exist_ok=True) - with open(output_path, "w", newline="", encoding="utf-8") as f: - writer = csv.DictWriter(f, fieldnames=CSV_FIELDNAMES) - writer.writeheader() - writer.writerows(rows) - return len(rows) + + logger.info("Remplissage de autocomplete_habitat…") + db.session.execute(sa_text(f"DELETE FROM {schema}.autocomplete_habitat")) + db.session.execute(sa_text(f""" + INSERT INTO {schema}.autocomplete_habitat + SELECT + cd_hab, + h.cd_typo, + lb_code, + lb_nom_typo, + concat(lb_code, ' - ', lb_hab_fr, ' ', lb_hab_fr_complet) + FROM {schema}.habref h + JOIN {schema}.typoref t ON t.cd_typo = h.cd_typo + """)) + + db.session.execute(sa_text("SET session_replication_role = 'origin'")) + + for table in reversed(list(table_files.keys())): + logger.info(f"Suppression de tmp_{table}…") + db.session.execute(sa_text(f"DROP TABLE IF EXISTS {schema}.tmp_{table}")) diff --git a/src/pypn_habref_api/tests/test_habref_update.py b/src/pypn_habref_api/tests/test_habref_update.py index ef45d32..d3afa97 100644 --- a/src/pypn_habref_api/tests/test_habref_update.py +++ b/src/pypn_habref_api/tests/test_habref_update.py @@ -10,8 +10,9 @@ import pytest from sqlalchemy import inspect as sa_inspect, text -from pypn_habref_api.commands.habref_v7 import apply_habref, import_habref, table_files -from pypn_habref_api.commands.utils import collect_orphan_rows, export_orphans_to_csv +from pypn_habref_api.commands.habref_v7 import table_files, base_url +from pypn_habref_api.commands.utils import apply_habref, import_habref +from utils_flask_sqla.referential import collect_orphan_rows, export_orphans_to_csv from pypn_habref_api.env import db @@ -62,8 +63,15 @@ def fake_open_remote(url, filename, open_fct=None): yield zf try: - with patch("pypn_habref_api.commands.habref_v7.open_remote_file", fake_open_remote): - import_habref(logger, num_version="07", habref_archive_name="HABREF_70.zip") + with patch("pypn_habref_api.commands.utils.open_remote_file", fake_open_remote): + import_habref( + logger, + table_files=table_files, + schema="ref_habitats", + base_url=base_url, + num_version="07", + archive_name="HABREF_70.zip", + ) inspector = sa_inspect(db.engine) existing = inspector.get_table_names(schema="ref_habitats") @@ -151,7 +159,7 @@ def test_tables_updated_and_tmp_dropped(self): ) db.session.flush() - apply_habref(logger) + apply_habref(logger, table_files=table_files, schema="ref_habitats") db.session.flush() count_after = db.session.execute(