diff --git a/core/widgets.py b/core/widgets.py new file mode 100644 index 000000000..1ea6893b8 --- /dev/null +++ b/core/widgets.py @@ -0,0 +1,28 @@ +import json +from django import forms + +class ReadOnlyPrettyJSONWidget(forms.Textarea): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.attrs.update({ + 'readonly': True, + 'rows': 20, + 'style': ( + 'font-family: monospace;' + ), + }) + + def format_value(self, value): + if not value: + return '' + try: + parsed = json.loads(value) if isinstance(value, str) else value + return json.dumps( + parsed, + indent=2, + ensure_ascii=False, + sort_keys=True, + default=str, # fallback: converte qualquer coisa para str + ) + except (json.JSONDecodeError, TypeError): + return value \ No newline at end of file diff --git a/pid_provider/choices.py b/pid_provider/choices.py index 9415dab9b..2d3127e76 100644 --- a/pid_provider/choices.py +++ b/pid_provider/choices.py @@ -2,6 +2,15 @@ ENDPOINTS = (("fix-pid-v2", "fix-pid-v2"),) +XMLURL_STATUS_SUCCESS = "success" +XMLURL_STATUS_XML_FETCH_FAILED = "xml_fetch_failed" +XMLURL_STATUS_PID_PROVIDER_XML_FAILED = "pid_provider_xml_failed" +XMLURL_STATUS = ( + (XMLURL_STATUS_SUCCESS, _("Success")), + (XMLURL_STATUS_XML_FETCH_FAILED, _("XML fetch failed")), + (XMLURL_STATUS_PID_PROVIDER_XML_FAILED, _("PID provider XML failed")), +) + PPXML_STATUS_WAIT = "WAIT" PPXML_STATUS_IGNORED = "IGNORE" PPXML_STATUS_TODO = "TODO" diff --git a/pid_provider/exceptions.py b/pid_provider/exceptions.py index d91e40792..a8ff9624e 100644 --- a/pid_provider/exceptions.py +++ b/pid_provider/exceptions.py @@ -1,3 +1,6 @@ +class SkipSavePidProviderXML(Exception): ... + + class APIPidProviderPostError(Exception): ... diff --git a/pid_provider/migrations/0017_pidproviderxmlregistration_and_more.py b/pid_provider/migrations/0017_pidproviderxmlregistration_and_more.py new file mode 100644 index 000000000..df583db64 --- /dev/null +++ b/pid_provider/migrations/0017_pidproviderxmlregistration_and_more.py @@ -0,0 +1,175 @@ +# Generated by Django 5.2.7 on 2026-07-21 14:25 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("pid_provider", "0016_alter_pidproviderxml_proc_status_xmlevent"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="PidProviderXMLRegistration", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + models.DateTimeField( + auto_now_add=True, verbose_name="Creation date" + ), + ), + ( + "updated", + models.DateTimeField( + auto_now=True, verbose_name="Last update date" + ), + ), + ( + "pkg_name", + models.CharField( + blank=True, + max_length=100, + null=True, + verbose_name="Package name", + ), + ), + ( + "event_status", + models.CharField( + blank=True, + choices=[ + ("created", "created"), + ("updated", "updated"), + ("skipped", "skipped"), + ("forbidden", "forbidden"), + ("conflict", "conflict"), + ("unmatched", "unmatched"), + ("bad_request", "bad_request"), + ("error", "error"), + ], + max_length=15, + null=True, + verbose_name="Event status", + ), + ), + ( + "detail", + models.JSONField(blank=True, null=True, verbose_name="Detail"), + ), + ], + options={ + "verbose_name": "PidProviderXML Registration", + "verbose_name_plural": "PidProviderXML Registrations", + "ordering": ["-created"], + }, + ), + migrations.AddField( + model_name="pidproviderxml", + name="readable_data", + field=models.JSONField(blank=True, null=True, verbose_name="Readable data"), + ), + migrations.AddField( + model_name="xmlurl", + name="detail", + field=models.JSONField(blank=True, null=True, verbose_name="Detail"), + ), + migrations.AddField( + model_name="xmlurl", + name="is_public", + field=models.BooleanField( + blank=True, default=None, null=True, verbose_name="Is Public" + ), + ), + migrations.AlterField( + model_name="xmlurl", + name="status", + field=models.CharField( + blank=True, + choices=[ + ("success", "Success"), + ("xml_fetch_failed", "XML fetch failed"), + ("pid_provider_xml_failed", "PID provider XML failed"), + ], + max_length=50, + null=True, + verbose_name="Status", + ), + ), + migrations.AddIndex( + model_name="xmlurl", + index=models.Index(fields=["is_public"], name="pid_provide_is_public_idx"), + ), + migrations.AddField( + model_name="pidproviderxmlregistration", + name="creator", + field=models.ForeignKey( + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="%(class)s_creator", + to=settings.AUTH_USER_MODEL, + verbose_name="Creator", + ), + ), + migrations.AddField( + model_name="pidproviderxmlregistration", + name="pid_provider_xml", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="registration_events", + to="pid_provider.pidproviderxml", + ), + ), + migrations.AddField( + model_name="pidproviderxmlregistration", + name="updated_by", + field=models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="%(class)s_last_mod_user", + to=settings.AUTH_USER_MODEL, + verbose_name="Updater", + ), + ), + migrations.AddIndex( + model_name="pidproviderxmlregistration", + index=models.Index( + fields=["pkg_name"], name="pid_provide_pkg_nam_2db0b2_idx" + ), + ), + migrations.AddIndex( + model_name="pidproviderxmlregistration", + index=models.Index( + fields=["event_status"], name="pid_provide_event_s_3c9ae7_idx" + ), + ), + migrations.AddIndex( + model_name="pidproviderxmlregistration", + index=models.Index( + fields=["-created"], name="pid_provide_created_94fb08_idx" + ), + ), + migrations.AddIndex( + model_name="pidproviderxmlregistration", + index=models.Index( + fields=["pid_provider_xml"], name="pid_provide_pid_pro_c9fb0e_idx" + ), + ), + ] diff --git a/pid_provider/models.py b/pid_provider/models.py index cb67abda7..0fea7e48a 100644 --- a/pid_provider/models.py +++ b/pid_provider/models.py @@ -1,28 +1,27 @@ import io -import json import logging import os import sys import traceback import zipfile from datetime import datetime -from functools import lru_cache, cached_property +from functools import cached_property from zlib import crc32 from django.core.files.base import ContentFile +from django.core.exceptions import FieldError from django.db import IntegrityError, models -from django.db.models import Q, Count, Min +from django.db.models import Prefetch, Q, Count from django.utils.translation import gettext_lazy as _ from modelcluster.fields import ParentalKey from modelcluster.models import ClusterableModel from packtools.sps.pid_provider import v3_gen, xml_sps_adapter from packtools.sps.pid_provider.xml_sps_lib import XMLWithPre from wagtail.admin.panels import FieldPanel, InlinePanel, ObjectList, TabbedInterface -from wagtail.fields import RichTextField -from wagtail.models import Orderable from wagtailautocomplete.edit_handlers import AutocompletePanel from collection.models import Collection +from core.widgets import ReadOnlyPrettyJSONWidget from core.forms import CoreAdminModelForm from core.models import CommonControlField from core.utils.profiling_tools import ( # ajuste o import conforme sua estrutura @@ -31,15 +30,16 @@ profile_property, profile_staticmethod, ) -from core.utils.similarity import how_similar from pid_provider import choices, exceptions from pid_provider.query_params import ( - get_score, zero_to_none, + compare, QueryBuilderPidProviderXML, ) from tracker.models import BaseEvent, UnexpectedEvent +PARTIAL_BODY_MAX = 300 + try: from django_prometheus.models import ExportModelOperationsMixin @@ -136,14 +136,10 @@ def create( obj.pid_provider_xml = pid_provider_xml obj.finger_print = xml_with_pre.finger_print obj.creator = user - # Salvar primeiro sem arquivo para obter o PK obj.save() - # save_file já faz self.file.save() que persiste o campo file, - # mas precisamos persistir o registro completo com o path do arquivo obj.save_file( f"{pid_provider_xml.v3}.xml", xml_with_pre.tostring(pretty_print=True) ) - # Único save final após salvar o arquivo obj.save() return obj except IntegrityError: @@ -348,6 +344,24 @@ def created_updated(self): return self.updated or self.created +class PidProviderXMLManager(models.Manager): + """ + Manager customizado: aplica select_related("current_version") em toda + consulta de PidProviderXML.objects, evitando repetir esse select_related + manualmente em cada classmethod (get_xml_with_pre, get_record_by_pid_v3, + select_records, public_items, mark_items_as_invalid, get_by_pid_v3, etc). + + Nota: prefetch_related("collections") NÃO entra aqui de propósito — + prefetch_related sempre dispara uma query extra, mesmo quando + "collections" não é usado (ex.: em _is_registered_pid, que só faz + .exists()). Por isso ele é aplicado pontualmente em get_queryset(), + que é o método de listagem que de fato usa collection_list. + """ + + def get_queryset(self): + return super().get_queryset().select_related("current_version") + + class PidProviderXML(BasePidProviderXML, CommonControlField, ClusterableModel): """ Tem responsabilidade de garantir a atribuição do PID da versão 3, @@ -414,6 +428,13 @@ class PidProviderXML(BasePidProviderXML, CommonControlField, ClusterableModel): registered_in_core = models.BooleanField(default=False) collections = models.ManyToManyField(Collection, blank=True) + # dados legíveis para facilitar a análise + readable_data = models.JSONField( + _("Readable data"), null=True, blank=True + ) + + objects = PidProviderXMLManager() + base_form_class = CoreAdminModelForm panel_a = [ @@ -436,13 +457,11 @@ class PidProviderXML(BasePidProviderXML, CommonControlField, ClusterableModel): InlinePanel("other_pid", label=_("Other PID")), ] panel_c = [ - FieldPanel("z_surnames"), - FieldPanel("z_collab"), - FieldPanel("z_links"), - FieldPanel("z_partial_body"), - ] - panels_event = [ - InlinePanel("events", label=_("Events")), + FieldPanel("z_surnames", read_only=True), + FieldPanel("z_collab", read_only=True), + FieldPanel("z_links", read_only=True), + FieldPanel("z_partial_body", read_only=True), + FieldPanel("readable_data", widget=ReadOnlyPrettyJSONWidget(), read_only=True), ] edit_handler = TabbedInterface( @@ -450,7 +469,6 @@ class PidProviderXML(BasePidProviderXML, CommonControlField, ClusterableModel): ObjectList(panel_a, heading=_("Identification")), ObjectList(panel_b, heading=_("Other PIDs")), ObjectList(panel_c, heading=_("Data")), - ObjectList(panels_event, heading=_("Events")), ] ) @@ -527,19 +545,6 @@ class Meta: def __str__(self): return f"{self.pkg_name} {self.v3}" - - @property - def article_pid_suffix_source(self): - try: - return self.xml_with_pre.get_article_pid_suffix_source() - except AttributeError: - return self.elocation_id or self.fpage or self.xml_with_pre.order - - def get_article_pid_suffix(self): - data = self.article_pid_suffix_source - if not data: - data = self.pkg_name.split("-")[-1] - return string_to_5_digits(data) @property def collection_list(self): @@ -571,13 +576,22 @@ def get_queryset( params["pub_year__lte"] = until_pub_year if proc_status_list: params["proc_status__in"] = proc_status_list - return cls.objects.filter(q, **params) + # select_related("current_version") já vem do manager; + # prefetch_related("collections") é aplicado aqui pois este método + # é usado em listagens que iteram collection_list. + return cls.objects.prefetch_related("collections").filter(q, **params) + + @classmethod + def delete_queryset(cls, qs): + OtherPid.objects.filter(pid_provider_xml__in=qs).delete() + qs.delete() @classmethod @profile_classmethod def public_items(cls, from_date): now = datetime.utcnow().isoformat()[:10] - return cls.objects.select_related("current_version").filter( + # select_related("current_version") já vem do manager + return cls.objects.filter( (Q(available_since__isnull=True) | Q(available_since__lte=now)) & (Q(created__gte=from_date) | Q(updated__gte=from_date)), current_version__pid_provider_xml__v3__isnull=False, @@ -607,8 +621,8 @@ def data(self): @profile_classmethod def get_xml_with_pre(cls, v3): try: - # Usar select_related para evitar query extra ao acessar current_version - return cls.objects.select_related("current_version").get(v3=v3).xml_with_pre + # select_related("current_version") já vem do manager + return cls.objects.get(v3=v3).xml_with_pre except cls.DoesNotExist: return None except Exception: @@ -635,6 +649,20 @@ def is_aop(self): return False return True + @property + def data_to_compare(self): + readable = self.readable_data or {} + titles = readable.get("article_titles") + body_fragment = readable.get("body_fragment") + return { + "article_titles": titles or self.xml_with_pre.article_titles_texts, + "z_surnames": self.z_surnames, + "z_collab": self.z_collab, + "z_links": self.z_links, + "z_partial_body": self.z_partial_body, + "body_fragment": body_fragment or self.xml_with_pre.get_body_fragment(PARTIAL_BODY_MAX), + } + @classmethod @profile_classmethod def register( @@ -695,80 +723,116 @@ def register( Parâmetros insuficientes para identificar documento """ try: + # outputs + response = {} + registered = None + event_status = None + error_type = None + select_record_response = None + + # inputs + pkg_name = filename input_data = None xml_adapter_data = None - response = {} - response["input_data"] = xml_with_pre.data - response["input_data"].update({"origin": origin}) + input_data = {} + input_data.update(xml_with_pre.data) + input_data.update(xml_with_pre.get_article_data()) + input_data["origin"] = origin + response["input_data"] = input_data # adaptador do xml with pre xml_adapter = xml_sps_adapter.PidProviderXMLAdapter(xml_with_pre) - response["xml_adapter_data"] = xml_adapter.data + xml_adapter_data = xml_adapter.data + response["xml_adapter_data"] = xml_adapter_data # consulta se documento já está registrado try: - records = cls.get_records(xml_adapter) - registered = cls.get_record(xml_adapter, records=records) + records = cls.select_records(xml_adapter) + select_record_response = cls.select_record(xml_adapter, records) + try: + registered = select_record_response.pop("registered") + except KeyError: + unmatched_items = select_record_response.get("unmatched_items") + if unmatched_items: + raise exceptions.UnmatchedPidProviderXMLError + raise cls.DoesNotExist + event_status = "updated" + if select_record_response.get("matched_items"): + response["select_record_response"] = select_record_response except cls.DoesNotExist as exc: registered = None + event_status = "created" except (cls.MultipleObjectsReturned, exceptions.UnmatchedPidProviderXMLError) as exc: - response["records"] = [item.data for item in records] + event_status = "unmatched" + response["select_record_response"] = select_record_response raise exceptions.QueryDocumentMultipleObjectsReturnedError(exc) except ( - exceptions.RequiredPublicationYearErrorToGetPidProviderXMLError + exceptions.RequiredPublicationYearErrorToGetPidProviderXMLError, + exceptions.RequiredISSNErrorToGetPidProviderXMLError, + exceptions.NotEnoughParametersToGetPidProviderXMLError, ) as exc: - raise exc - except exceptions.RequiredISSNErrorToGetPidProviderXMLError as exc: - raise exc - except exceptions.NotEnoughParametersToGetPidProviderXMLError as exc: + event_status = "bad_request" raise exc # valida os PIDs do XML # - não podem ter conflito com outros registros # - identifica mudança - response["xml_changed"] = cls.complete_missing_xml_pids( - xml_adapter, registered, auto_solve_pid_conflict - ) + try: + response["xml_changed"] = cls.complete_missing_xml_pids( + xml_adapter, registered, auto_solve_pid_conflict + ) + except PidProviderXMLPidV3ConflictError as exc: + event_status = "conflict" + raise exc # analisa se continua o registro - updated_data = cls.is_updated( - xml_with_pre, - registered, - force_update, - origin_date, - registered_in_core, - ) - if updated_data: - response["skip_update"] = True - response.update(updated_data) - return response - - # cria ou atualiza registro - registered = cls._save( - registered, - xml_adapter, - user, - origin_date, - available_since, - registered_in_core, - ) - - # data to return - response.update(registered.data) - return response - - except Exception as e: + try: + PidProviderXML.is_updated( + xml_with_pre, + registered, + force_update, + origin_date, + registered_in_core, + ) + registered = cls._save( + registered, + xml_adapter, + user, + origin_date, + available_since, + registered_in_core, + ) + # data to return + response.update(registered.data) + except exceptions.ForbiddenPidProviderXMLRegistrationError: + event_status = "forbidden" + raise + except exceptions.SkipSavePidProviderXML: + event_status = "skipped" + response["skipped"] = True + response.update(registered.data) + # do not raise + except Exception as exc: + event_status = event_status or "error" exc_type, exc_value, exc_traceback = sys.exc_info() - UnexpectedEvent.create( - item=xml_with_pre.sps_pkg_name, - action="PidProviderXML.register", - exception=e, - exc_traceback=exc_traceback, - detail=response, - ) - response.update({"error_msg": str(e), "error_type": str(type(e))}) - return response + error_type = str(type(exc)) + response.update({ + "error_msg": str(exc), + "error_type": error_type, + "traceback": traceback.format_exc() + }) + finally: + response["event_status"] = event_status + if error_type or (select_record_response or {}).get("matched_items"): + PidProviderXMLRegistration.record( + user=user, + pid_provider_xml=registered, + pkg_name=pkg_name, + event_status=event_status, + detail=response, + ) + return response @classmethod @profile_classmethod @@ -814,15 +878,19 @@ def get_valid_pid_v3( if xml_pid and xml_pid != registered_pid: # Verifica se o XML PID já está em uso por outro documento. try: - # garantir que xml_adapter.v3 não tenha conflito + # verificar se xml_adapter.v3 pertence a outro xml cls.get_record_by_pid_v3(xml_adapter) + # pertence a xml_adapter return xml_pid except cls.DoesNotExist: + # não pertence a nenhum xml return xml_pid except PidProviderXMLPidV3ConflictError: + # pertence a um xml diferente de xml_adapter if not auto_solve_pid_conflict: + # rejeita o uso deste pid raise - + # ignora # XML PID não fornecido, ou igual ao registrado # ou em conflito sem exceção # retorna o PID registrado ou gera um novo. @@ -840,7 +908,6 @@ def _save( registered_in_core=None, ): if registered: - # obtém os dados de substituição para registrar em other_pid registered_changed = registered.check_registered_pids_changed( xml_adapter.xml_with_pre ) @@ -849,51 +916,44 @@ def _save( registered = cls() registered.creator = user registered_changed = None - + registered.proc_status = choices.PPXML_STATUS_TODO registered._add_dates(xml_adapter, origin_date, available_since) registered._add_data(xml_adapter, registered_in_core) registered._add_journal(xml_adapter) registered._add_issue(xml_adapter) - - # Primeiro save: necessário para obter PK (se novo) antes de criar XMLVersion / OtherPid - if registered.pk is None: - registered.save() - + + registered.save() + if registered_changed: registered._add_other_pid(registered_changed, user) registered._add_current_version(xml_adapter.xml_with_pre, user) + + registered.add_collections(xml_adapter) + return registered - # Save final consolidado: persiste current_version e other_pid_count - registered.save() + def add_collections(self, xml_adapter): q = Q() - if COLLECTION_PREFIX == "scielojournal": - if xml_adapter.journal_issn_print: - q |= Q( - scielojournal__journal__official__issn_print=xml_adapter.journal_issn_print - ) - if xml_adapter.journal_issn_electronic: - q |= Q( - scielojournal__journal__official__issn_electronic=xml_adapter.journal_issn_electronic - ) - else: - if xml_adapter.journal_issn_print: - q |= Q( - journalproc__journal__official_journal__issn_print=xml_adapter.journal_issn_print - ) - if xml_adapter.journal_issn_electronic: - q |= Q( - journalproc__journal__official_journal__issn_electronic=xml_adapter.journal_issn_electronic - ) + issn_print = xml_adapter.journal_issn_print + issn_electronic = xml_adapter.journal_issn_electronic + + try: + Collection.objects.filter(scielojournal__isnull=True).exists() + issn_path = "scielojournal__journal__official" + except FieldError: + issn_path = "journalproc__journal__official_journal" + + if issn_print: + q |= Q(**{f"{issn_path}__issn_print": issn_print}) + if issn_electronic: + q |= Q(**{f"{issn_path}__issn_electronic": issn_electronic}) for collection in Collection.objects.filter(q): - registered.collections.add(collection) - return registered + self.collections.add(collection) - @classmethod - @profile_classmethod + @staticmethod def is_updated( - cls, xml_with_pre, registered, force_update, origin_date, registered_in_core + xml_with_pre, registered, force_update, origin_date, registered_in_core ): """ XML é versão AOP, mas @@ -919,7 +979,7 @@ def is_updated( if registered.is_equal_to(xml_with_pre): # XML fornecido é igual ao registrado, não precisa continuar logging.info(f"Skip update: equal") - return registered.data + raise exceptions.SkipSavePidProviderXML if xml_with_pre.is_aop and registered and not registered.is_aop: logging.info(f"Skip update: forbidden") @@ -935,9 +995,7 @@ def is_updated( and registered.origin_date and registered.origin_date > origin_date ): - # retorna item registrado que está mais atualizado - logging.info(f"Skip update: is already up-to-date") - return registered.data + raise exceptions.SkipSavePidProviderXML @profile_method def is_equal_to(self, xml_with_pre): @@ -947,23 +1005,78 @@ def is_equal_to(self, xml_with_pre): @classmethod @profile_classmethod - def get_records(cls, xml_adapter): + def select_records(cls, xml_adapter): + """ + Gera pares (label, lista_de_candidatos) para cada estratégia de + correspondência, do mais específico ao mais genérico. + + Cada branch é materializada (list(...)) uma única vez aqui, para + que o consumidor (select_record) nunca precise avaliar a queryset + mais de uma vez (evita repetir .exists() + .count() + iteração, + que geram queries separadas no banco). Por ser um generator, uma + branch só é construída e avaliada quando o consumidor de fato + solicita o próximo item — se a primeira branch já resolver, as + demais nunca chegam a rodar no banco. + """ qbuilder = QueryBuilderPidProviderXML(xml_adapter) - q_ids = qbuilder.identifier_queries - q_journal = qbuilder.issn_query - q_issue = Q(**qbuilder.issue_params) - return cls.objects.filter(q_ids | (q_journal & q_issue)).distinct() + qbuilder.validate_input_data() - @classmethod - @profile_classmethod - def get_record(cls, xml_adapter, records): - results = records - if not results.exists(): - raise cls.DoesNotExist - matched = cls.best_matches(results, xml_adapter) - if not matched: - raise cls.DoesNotExist - return cls.objects.get(id=sorted(matched)[-1][-1]) + # select_related("current_version") já vem do manager + objects = cls.objects.all() + + # 1) correspondência direta por identificadores + yield "ids", list(objects.filter(qbuilder.identifier_queries)) + + selected_journal = objects.filter(qbuilder.issn_query) + + # 2) journal + issue + dados do artigo + yield ( + "journal-issue-article", + list( + selected_journal.filter( + Q(**qbuilder.issue_params) & qbuilder.article_data_query + ) + ), + ) + + # 3) journal + dados do artigo + yield "journal-article", list(selected_journal.filter(qbuilder.article_data_query)) + + @staticmethod + def select_record(xml_adapter, selection_results): + """ + Consome os pares (label, lista_de_candidatos) produzidos por + select_records. As listas já vêm materializadas, então aqui só + checamos truthiness (nunca .exists()/.count() sobre queryset). + """ + unmatched_items = {} + xml_adapter_data_to_compare = xml_adapter.get_data_to_compare() + for label, results in selection_results: + if not results: + continue + + result = PidProviderXML.get_best_match(results, xml_adapter_data_to_compare) + + matched = result.get("matched") + unmatched = result.get("unmatched") + registered = result.get("registered") + if registered: + response = { + "total_results": len(results), + "registered": registered, + } + if matched: + response["matched_items"] = {label: matched} + if unmatched: + response["unmatched_items"] = {label: unmatched} + return response + + if unmatched: + unmatched_items[label] = unmatched + + if unmatched_items: + return {"unmatched_items": unmatched_items} + return {} @classmethod @profile_classmethod @@ -972,106 +1085,84 @@ def get_record_by_pid_v3(cls, xml_adapter): if not xml_adapter.v3: raise ValueError("get_record_by_pid_v3: XML has not pid v3") xml_pid_v3 = xml_adapter.v3 - results = ( - cls.objects.filter(Q(v3=xml_pid_v3) | Q(other_pid__pid_in_xml=xml_pid_v3)) + # select_related("current_version") já vem do manager + results = cls.objects.filter( + Q(v3=xml_pid_v3) | Q(other_pid__pid_in_xml=xml_pid_v3) ) if not results.exists(): + # pid v3 é inédito raise cls.DoesNotExist - matched = cls.best_matches(results, xml_adapter) - if not matched: - UnexpectedEvent.create( - item=xml_adapter.sps_pkg_name, - action="PidProviderXML.get_record_by_pid_v3", - exception=PidProviderXMLPidV3ConflictError, - detail={"xml_adapter": xml_adapter.data, "results": [i.data for i in results]}, - ) + + xml_adapter_data_to_compare = xml_adapter.get_data_to_compare() + result = PidProviderXML.get_best_match(results, xml_adapter_data_to_compare) + registered = result.get("registered") + if not registered: + xml_data = xml_adapter.xml_with_pre.get_article_data(PARTIAL_BODY_MAX) + items = [item.data for item in results] raise PidProviderXMLPidV3ConflictError( - _("No matching record found for the provided XML data.") + _(f"{xml_pid_v3} belongs to {items}, not to {xml_data}") ) - return cls.objects.get(id=sorted(matched)[-1][-1]) + return registered - @profile_method - def match(self, xml_adapter): - """ + @staticmethod + def get_best_match(results, xml_adapter_data): """ - labels = [] - score = self.title_similarity(xml_adapter) * 100 - if score > 50: - labels.append("title") - if score_item := get_score(self.z_surnames, xml_adapter.z_surnames, 10, 100): - labels.append("z_surnames") - score += score_item - if score_item := get_score(self.z_collab, xml_adapter.z_collab, 10, 100): - labels.append("z_collab") - score += score_item - if score_item := get_score(self.z_links, xml_adapter.z_links, 10, 100): - labels.append("z_links") - score += score_item - if score_item := get_score(self.z_partial_body, xml_adapter.z_partial_body, 10, 100): - labels.append("z_partial_body") - score += score_item - return {"score": score, "labels": labels} - - def title_similarity(self, xml_adapter): - try: - registered = self.xml_with_pre.article_titles_texts - except Exception: - registered = [] - xml_adapter_titles = xml_adapter.xml_with_pre.article_titles_texts - if xml_adapter_titles == registered: - return 1 - if not xml_adapter_titles: - return 0 - if not registered: - return 0 - words1 = set() - for item in xml_adapter_titles: - words1.update(item.split()) - words2 = set() - for item in registered: - words2.update(item.split()) - return how_similar(" ".join(sorted(words1)), " ".join(sorted(words2))) + Compara uma lista de candidatos (PidProviderXML) com os dados do XML + recebido e classifica os candidatos por similaridade. - @classmethod - def best_matches(cls, results, xml_adapter): - data = [] + Parameters + ---------- + results : list[PidProviderXML] + Lista JÁ MATERIALIZADA (não queryset) de candidatos a comparar. + xml_adapter_data : dict + Dados de comparação do XML de entrada, ou seja, o retorno de + ``xml_adapter.get_data_to_compare()``. + + Returns + ------- + dict + Todas as chaves abaixo são OPCIONAIS — só aparecem quando há + conteúdo para elas. Use ``.get(...)`` ou ``"chave" in result`` + ao consumir o retorno, nunca acesso direto. + + - ``"unmatched"``: presente apenas se houver ao menos 1 + candidato com ``percentual_score`` <= 0.6. Lista de + ``item.data`` desses candidatos. + - ``"registered"``: presente apenas se houver ao menos 1 + candidato aprovado (score > 0.6). Contém o OBJETO + ``PidProviderXML`` (não o dict ``.data``) do candidato com + maior score — em caso de empate, o critério de desempate é + ``updated`` mais recente e, em seguida, maior ``id``. + - ``"matched"``: presente apenas se houver 2 OU MAIS candidatos + aprovados. Contém ``item.data`` dos candidatos aprovados + EXCLUINDO o que já está em ``"registered"`` (ou seja, é a + lista de aprovados a partir do 2º colocado), na mesma ordem + de score decrescente. + """ + detail = {} + found = [] + items = {} + for item in results: + item_data = item.data_to_compare + response = compare(item_data, xml_adapter_data) + items[item.id] = item + found.append((response["percentual_score"], item.updated.isoformat(), item.id)) + + found = sorted(found, reverse=True) matched = [] - for item in results.select_related("current_version").iterator(): - response = item.match(xml_adapter) - score = response["score"] - - if xml_adapter.v2: - if item.v2 == xml_adapter.v2: - score += 100 - elif xml_adapter.order and item.v2 and item.v2.endswith(xml_adapter.order): - score += 100 - if item.v3 == xml_adapter.v3: - score += 100 - if item.pkg_name == xml_adapter.pkg_name: - score += 100 - if item.main_doi == xml_adapter.main_doi: - score += 100 - - _data = response - _data.update(item.data) - data.append(_data) - - if score > 50: - matched.append((score, item.updated.isoformat(), item.id)) - - if results.count() > 1 or not matched: - detail = { - "xml_adapter_data": xml_adapter.data, - "data": data, - "matched": matched, - } - UnexpectedEvent.create( - item=xml_adapter.sps_pkg_name, - action="PidProviderXML.best_matches", - exception=cls.MultipleObjectsReturned, - detail=detail, - ) - return matched + unmatched = [] + for percentual_score, updated, item_id in found: + if percentual_score > 0.6: + matched.append(items[item_id].data) + else: + unmatched.append(items[item_id].data) + if matched: + detail["registered"] = items[found[0][-1]] + if len(matched) > 1: + detail["matched"] = matched[1:] + if unmatched: + detail["unmatched"] = unmatched + return detail @profile_method def _add_data(self, xml_adapter, registered_in_core): @@ -1095,12 +1186,18 @@ def _add_data(self, xml_adapter, registered_in_core): self.z_links = xml_adapter.z_links self.z_partial_body = xml_adapter.z_partial_body + self.readable_data = xml_adapter.xml_with_pre.get_article_data() + @profile_method def _add_dates(self, xml_adapter, origin_date, available_since): # evita que artigos WIP fique disponíveis antes de estarem públicos try: + # Usa get_complete_publication_date para evitar logs de erro do + # packtools quando a data de publicação no XML é incompleta + # (ex.: apenas com e , sem mes/dia). + # Mesmo padrão adotado em proc/models.py e package/models.py. self.available_since = available_since or ( - xml_adapter.xml_with_pre.article_publication_date + xml_adapter.xml_with_pre.get_complete_publication_date() ) except Exception as e: # packtools error @@ -1128,7 +1225,7 @@ def _add_current_version(self, xml_with_pre, user, delete=False): pass self.current_version = XMLVersion.get_or_create(user, self, xml_with_pre) - # Não faz save() aqui; chamador é responsável por consolidar o save + self.save() @profile_method def check_registered_pids_changed(self, xml_with_pre): @@ -1164,20 +1261,15 @@ def check_registered_pids_changed(self, xml_with_pre): @profile_method def _add_other_pid(self, registered_changed, user): - # registrados passam a ser other pid - # os pids do XML passam a ser os vigentes if not registered_changed: return for change_args in registered_changed: - change_args["pid_in_xml"] = change_args.pop("registered") - change_args["user"] = user change_args["pid_provider_xml"] = self - OtherPid.get_or_create(**change_args) self.other_pid_count = self.other_pid.count() - # Não é necessário save() aqui; será consolidado no _save() pai + self.save(update_fields=["other_pid_count"]) @classmethod @profile_classmethod @@ -1229,58 +1321,52 @@ def is_registered( """ try: + select_record_response = None response = {} response["input_data"] = xml_with_pre.data - xml_adapter_data = None - xml_adapter = xml_sps_adapter.PidProviderXMLAdapter(xml_with_pre) - xml_adapter_data = xml_adapter.data - response["xml_adapter_data"] = xml_adapter_data + xml_adapter = xml_sps_adapter.PidProviderXMLAdapter(xml_with_pre) + response["xml_adapter_data"] = xml_adapter.data try: - records = cls.get_records(xml_adapter) - registered = cls.get_record(xml_adapter, records=records) + records = cls.select_records(xml_adapter) + select_record_response = cls.select_record(xml_adapter, records) + try: + registered = select_record_response.pop("registered") + except KeyError: + unmatched_items = select_record_response.get("unmatched_items") + if unmatched_items: + raise exceptions.UnmatchedPidProviderXMLError + raise cls.DoesNotExist + matched_items = select_record_response.get("matched_items") + if matched_items: + response["select_record_response"] = select_record_response except cls.DoesNotExist as exc: response.update( {"filename": xml_with_pre.filename, "registered": False} ) return response except (cls.MultipleObjectsReturned, exceptions.UnmatchedPidProviderXMLError) as exc: - exc_type, exc_value, exc_traceback = sys.exc_info() - response["records"] = [item.data for item in records] - UnexpectedEvent.create( - item=xml_with_pre.sps_pkg_name, - action="PidProviderXML.is_registered", - exception=exc, - exc_traceback=exc_traceback, - detail=response, - ) - response.update({"error_msg": str(exc), "error_type": str(type(exc))}) - return response + response["select_record_response"] = select_record_response + raise except ( - exceptions.RequiredPublicationYearErrorToGetPidProviderXMLError + exceptions.RequiredPublicationYearErrorToGetPidProviderXMLError, + exceptions.RequiredISSNErrorToGetPidProviderXMLError, + exceptions.NotEnoughParametersToGetPidProviderXMLError, ) as exc: raise exc - except exceptions.RequiredISSNErrorToGetPidProviderXMLError as exc: - raise exc - except exceptions.NotEnoughParametersToGetPidProviderXMLError as exc: - raise exc response["registered"] = True response.update(registered.data) response["is_equal"] = registered.is_equal_to(xml_with_pre) return response except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() - UnexpectedEvent.create( - item=xml_with_pre.sps_pkg_name, - action="PidProviderXML.is_registered", - exception=e, - exc_traceback=exc_traceback, - detail=response, - ) - response.update({"error_msg": str(e), "error_type": str(type(e))}) + response.update({ + "error_msg": str(e), + "error_type": str(type(e)), + "traceback": traceback.format_exc() + }) return response - return {} @classmethod def get_by_pid_v3(cls, pid_v3, partial_pid_v2=None, pid_v2=None): @@ -1291,6 +1377,7 @@ def get_by_pid_v3(cls, pid_v3, partial_pid_v2=None, pid_v2=None): params["v2"] = pid_v2 if partial_pid_v2: params["v2__contains"] = partial_pid_v2 + # select_related("current_version") já vem do manager try: return cls.objects.get(**params) except cls.MultipleObjectsReturned as e: @@ -1333,13 +1420,22 @@ def mark_as_done(self): @classmethod @profile_classmethod def mark_items_as_invalid(cls, issns): - for item in cls.objects.filter( + # select_related("current_version") já vem do manager + # (necessário aqui pois o loop acessa item.xml_with_pre, que usa + # self.current_version) + items = cls.objects.filter( Q(issn_print__in=issns) | Q(issn_electronic__in=issns), - ).iterator(): + ) + items_to_update = [] + for item in items.iterator(): try: - invalid = bool(item.xml_with_pre) + valid = bool(item.xml_with_pre) except Exception as e: - invalid = True + valid = False + if not valid: + item.proc_status = choices.PPXML_STATUS_INVALID + items_to_update.append(item) + cls.objects.bulk_update(items_to_update, ["proc_status"], batch_size=100) @classmethod @profile_classmethod @@ -1403,7 +1499,19 @@ def fix_duplicated_pkg_name(cls, pkg_name, user): int: Número de items atualizados. """ try: - items = cls.objects.filter(pkg_name=pkg_name) + # select_related("current_version") já vem do manager. + # prefetch_related com Prefetch + to_attr é necessário aqui + # porque o loop chama item.other_pid.filter(pid_type="pid_v3"), + # e um .filter() sobre manager relacionado ignora o cache do + # prefetch_related simples (só .all() usa o cache) — por isso + # a filtragem precisa estar dentro do próprio Prefetch. + items = cls.objects.prefetch_related( + Prefetch( + "other_pid", + queryset=OtherPid.objects.filter(pid_type="pid_v3"), + to_attr="pid_v3_others", + ) + ).filter(pkg_name=pkg_name) if items.count() <= 1: return 0 @@ -1419,12 +1527,16 @@ def fix_duplicated_pkg_name(cls, pkg_name, user): most_recent_item.save() for item in items.exclude(id=most_recent_item.id): - for other_pid in item.other_pid.filter(pid_type="pid_v3"): + for other_pid in item.pid_v3_others: OtherPid.get_or_create( user=user, pid_type=other_pid.pid_type, pid_in_xml=other_pid.pid_in_xml, - version=other_pid.current_version, + # Nota: OtherPid não tem campo current_version, e + # sim `version` — corrigido aqui (era + # other_pid.current_version, que não existe no + # modelo e lançaria AttributeError). + version=other_pid.version, pid_provider_xml=most_recent_item, ) OtherPid.get_or_create( @@ -1451,7 +1563,7 @@ def fix_pkg_name(self, pkg_name): self.save() return True return False - + def add_event(self, name, proc_status, detail=None, errors=None, exceptions=None): self.proc_status = proc_status self.save() @@ -1618,23 +1730,25 @@ def xml_url_zipfile_path(instance, filename): class XMLURL(CommonControlField): """ Model to store URLs that experienced failures and should be retried in the future. - + This model tracks URLs that failed during processing, along with their status and associated article PID, enabling retry mechanisms to reprocess them later. - + Fields: url: URLField - The URL that needs to be retried status: CharField - To control the request status (e.g., "pending", "failed", "retrying") pid: CharField - Article PID associated with this URL zipfile: FileField - Compressed XML content retrieved from the URL - exceptions: CharField - Exception traceback information (truncated to 255 chars if needed) + detail: JSONField + is_public: BooleanField - Whether the document is public (derived from item status) """ url = models.URLField( _("URL"), max_length=500, null=False, blank=False ) status = models.CharField( - _("Status"), max_length=50, null=True, blank=True + _("Status"), max_length=50, null=True, blank=True, + choices=choices.XMLURL_STATUS, ) pid = models.CharField( _("Article PID"), max_length=23, null=True, blank=True @@ -1642,8 +1756,12 @@ class XMLURL(CommonControlField): zipfile = models.FileField( _("ZIP File"), upload_to=xml_url_zipfile_path, null=True, blank=True, max_length=300, ) - exceptions = models.CharField( - _("Exceptions"), max_length=255, null=True, blank=True + exceptions = models.CharField(_("Exceptions"), max_length=255, null=True, blank=True) + detail = models.JSONField( + _("Detail"), null=True, blank=True + ) + is_public = models.BooleanField( + _("Is Public"), null=True, blank=True, default=None ) base_form_class = CoreAdminModelForm @@ -1653,7 +1771,9 @@ class XMLURL(CommonControlField): FieldPanel("status"), FieldPanel("pid"), FieldPanel("zipfile"), + FieldPanel("detail", widget=ReadOnlyPrettyJSONWidget()), FieldPanel("exceptions"), + FieldPanel("is_public"), ] class Meta: @@ -1665,6 +1785,7 @@ class Meta: models.Index(fields=["url"]), models.Index(fields=["status"]), models.Index(fields=["pid"]), + models.Index(fields=["is_public"], name="pid_provide_is_public_idx"), ] def __str__(self): @@ -1683,14 +1804,16 @@ def create( url=None, status=None, pid=None, - exceptions=None, + detail=None, + is_public=None, ): try: obj = cls() obj.url = url obj.status = status obj.pid = pid - obj.exceptions = exceptions + obj.detail = detail + obj.is_public = is_public obj.creator = user obj.save() return obj @@ -1704,7 +1827,8 @@ def create_or_update( url=None, status=None, pid=None, - exceptions=None, + detail=None, + is_public=None, ): try: obj = cls.get(url=url) @@ -1713,8 +1837,10 @@ def create_or_update( obj.status = status if pid is not None: obj.pid = pid - if exceptions is not None: - obj.exceptions = exceptions + if detail is not None: + obj.detail = detail + if is_public is not None: + obj.is_public = is_public obj.save() return obj except cls.DoesNotExist: @@ -1723,17 +1849,18 @@ def create_or_update( url, status, pid, - exceptions, + detail, + is_public=is_public, ) def save_file(self, xml_content, filename=None): """ Create a zip file from XML content and save it to the zipfile field. - + Args: xml_content: str or bytes - The XML content to compress filename: str - Optional filename for the XML inside the zip (defaults to 'content.xml') - + Returns: bool - True if file was saved successfully, False otherwise """ @@ -1741,22 +1868,46 @@ def save_file(self, xml_content, filename=None): # Convert string to bytes if needed if isinstance(xml_content, str): xml_content = xml_content.encode('utf-8') - + # Create in-memory zip file zip_buffer = io.BytesIO() with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zip_file: # Use provided filename or default xml_filename = filename or 'content.xml' zip_file.writestr(xml_filename, xml_content) - + # Save the zip file to the model zip_filename = f"{self.pid or 'unknown'}_{self.pk or 'new'}.zip" self.zipfile.save(zip_filename, ContentFile(zip_buffer.getvalue()), save=True) - + return True except Exception as e: logging.error(f"Error saving zip file for XMLURL {self.url}: {e}") return False + + @classmethod + def record(cls, user, url, status, document_item, *, exception=None, response=None, xml_with_pre=None, name=None): + detail = {"document_item": document_item} + if exception is not None: + detail["exceptions"] = traceback.format_exc() + if response is not None: + detail["response"] = response + + pid = response.get("v3") if response else None + + is_public = None + if document_item: + doc_status = document_item.get("status") + if doc_status is not None: + is_public = doc_status != "false" + + xmlurl_obj = cls.create_or_update(user=user, url=url, status=status, pid=pid, detail=detail, is_public=is_public) + + if xml_with_pre is not None: + filename = name or pid or "content.xml" + xmlurl_obj.save_file(xml_with_pre.tostring(), filename=filename) + + return xmlurl_obj class XMLEvent(BaseEvent, CommonControlField): @@ -1790,4 +1941,105 @@ def register(cls, ppxml, name, detail=None, errors=None, exceptions=None): obj.name = name completed = bool(not errors and not exceptions) obj.finish(completed=completed, detail=detail, errors=errors, exceptions=exceptions) - return obj \ No newline at end of file + return obj + +# ----------------------------------------------------------------------------- +# [models.py] MODELO NOVO — PidProviderXMLRegistration +# Auditoria por documento. Grava SEMPRE (created/updated/skipped/forbidden/ +# conflict/unmatched/error). FK nullable (unmatched/error podem não ter PPX). +# ----------------------------------------------------------------------------- +class PidProviderXMLRegistration(CommonControlField): + LIGHTWEIGHT_STATUSES = {"created", "updated", "skip_update"} + + EVENT_CREATED = "created" + EVENT_UPDATED = "updated" + EVENT_SKIPPED = "skipped" + EVENT_FORBIDDEN = "forbidden" + EVENT_CONFLICT = "conflict" + EVENT_UNMATCHED = "unmatched" + EVENT_ERROR = "error" + EVENT_BAD_REQUEST = "bad_request" + + EVENT_STATUS_CHOICES = ( + (EVENT_CREATED, "created"), + (EVENT_UPDATED, "updated"), + (EVENT_SKIPPED, "skipped"), + (EVENT_FORBIDDEN, "forbidden"), + (EVENT_CONFLICT, "conflict"), + (EVENT_UNMATCHED, "unmatched"), + (EVENT_BAD_REQUEST, "bad_request"), + (EVENT_ERROR, "error"), + ) + + pid_provider_xml = models.ForeignKey( + PidProviderXML, + null=True, + blank=True, + on_delete=models.SET_NULL, + related_name="registration_events", + ) + pkg_name = models.CharField( + _("Package name"), max_length=100, null=True, blank=True + ) + event_status = models.CharField( + _("Event status"), + max_length=15, + null=True, + blank=True, + choices=EVENT_STATUS_CHOICES, + ) + detail = models.JSONField(_("Detail"), null=True, blank=True) + + base_form_class = CoreAdminModelForm + + panels = [ + FieldPanel("event_status", read_only=True), + FieldPanel("pkg_name", read_only=True), + AutocompletePanel("pid_provider_xml", read_only=True), + FieldPanel("detail", widget=ReadOnlyPrettyJSONWidget(), read_only=True), + ] + + class Meta: + ordering = ["-created"] + verbose_name = _("PidProviderXML Registration") + verbose_name_plural = _("PidProviderXML Registrations") + indexes = [ + models.Index(fields=["pkg_name"]), + models.Index(fields=["event_status"]), + models.Index(fields=["-created"]), + models.Index(fields=["pid_provider_xml"]), + ] + + def __str__(self): + return f"{self.pkg_name} {self.event_status} {self.created}" + + @staticmethod + def _serialize_detail(detail): + """ + O detail do detail contém o objeto PidProviderXML em + detail['registered']. Para gravar em JSON, troca pelo v3/id. + """ + if not detail: + return None + data = dict(detail) + registered = data.get("registered") + if registered is not None and hasattr(registered, "v3"): + data["registered"] = {"id": registered.id, "v3": registered.v3} + return data + + @classmethod + def record(cls, user, event_status, pid_provider_xml=None, pkg_name=None, + detail=None): + try: + obj = cls() + obj.creator = user + obj.pid_provider_xml = pid_provider_xml + obj.pkg_name = pkg_name or (pid_provider_xml and pid_provider_xml.pkg_name) + obj.event_status = event_status + if event_status not in cls.LIGHTWEIGHT_STATUSES: + obj.detail = cls._serialize_detail(detail) + obj.save() + return obj + except Exception as e: + logging.exception(f"Unable to record PidProviderXMLRegistration: {e}") + return None diff --git a/pid_provider/query_params.py b/pid_provider/query_params.py index bfb6901bd..b4dbc1f9c 100644 --- a/pid_provider/query_params.py +++ b/pid_provider/query_params.py @@ -3,10 +3,55 @@ from django.db.models import Q from django.utils.translation import gettext_lazy as _ -from core.utils.profiling_tools import profile_function +from core.utils.similarity import how_similar from pid_provider import exceptions +def compare(registered_items, input_data): + """ + """ + total_score = 0 + items = [] + for label, registered_item in registered_items.items(): + result = compare_items(label, registered_item, input_data.get(label)) + items.append(result) + total_score += result["score"] + return { + "items": items, + "total_score": total_score, + "percentual_score": total_score / len(items) + } + + +def compare_lists(registered, xml_adapter_titles): + if xml_adapter_titles == registered: + return 1 + if not xml_adapter_titles: + return 0 + if not registered: + return 0 + words1 = set() + for item in xml_adapter_titles: + words1.update(item.split()) + words2 = set() + for item in registered: + words2.update(item.split()) + return how_similar(" ".join(sorted(words1)), " ".join(sorted(words2))) + + +def compare_items(label, registered, input_data): + if isinstance(registered, list): + score = compare_lists(registered, input_data) + elif (input_data or None) == (registered or None): + score = 1 + else: + score = how_similar(input_data, registered) + response = {"label": label, "score": score} + if score != 1: + response["registered"] = registered + return response + + def get_score(registered, xml_data, min_value, max_value): if registered == xml_data: if registered: @@ -35,7 +80,7 @@ class QueryBuilderPidProviderXML: def __init__(self, xml_adapter): """ - Inicializa o construtor de queries. + Inicializa o construtor de queries obtendo os dicionários de dados do adaptador. Parameters ---------- @@ -43,269 +88,165 @@ def __init__(self, xml_adapter): Adaptador com dados do XML para busca """ self.xml_adapter = xml_adapter - - # ========== Cached Properties para Atributos do XML Adapter ========== - - @cached_property - def v3(self): - """PID v3 do documento.""" - return self.xml_adapter.v3 - - @cached_property - def v2(self): - """PID v2 do documento.""" - return self.xml_adapter.v2 - - @cached_property - def aop_pid(self): - """PID AOP (Ahead of Print) do documento.""" - return self.xml_adapter.aop_pid - - @cached_property - def pkg_name(self): - """Nome do pacote do documento, parâmtro usado ao instanciar XMLAdapter""" - return self.xml_adapter.pkg_name - - @cached_property - def sps_pkg_name(self): - """Nome do pacote do documento (deprecated).""" - return self.xml_adapter.sps_pkg_name + # Centraliza o acesso aos dados brutos e normalizados (hashes de 64 chars) + self.adapter_data = xml_adapter.data + self.compare_data = xml_adapter.get_data_to_compare() + self.xml_with_pre_data = xml_adapter.xml_with_pre.get_article_data(300) - @cached_property - def deprecated_sps_pkg_name(self): - """Nome do pacote do documento (deprecated).""" - return self.xml_adapter.sps_pkg_name - - @cached_property - def main_doi(self): - """DOI principal do documento.""" - return self.xml_adapter.main_doi - - @cached_property - def journal_issn_electronic(self): - """ISSN eletrônico do periódico.""" - return self.xml_adapter.journal_issn_electronic - - @cached_property - def journal_issn_print(self): - """ISSN impresso do periódico.""" - return self.xml_adapter.journal_issn_print - - @cached_property - def elocation_id(self): - """Identificador de localização eletrônica.""" - return self.xml_adapter.elocation_id - - @cached_property - def fpage(self): - """Primeira página do artigo.""" - return self.xml_adapter.fpage - - @cached_property - def fpage_seq(self): - """Sequência da primeira página.""" - return self.xml_adapter.fpage_seq - - @cached_property - def lpage(self): - """Última página do artigo.""" - return self.xml_adapter.lpage - - @cached_property - def pub_year(self): - """Ano de publicação.""" - return self.xml_adapter.pub_year - - @cached_property - def volume(self): - """Volume da publicação.""" - return self.xml_adapter.volume - - @cached_property - def number(self): - """Número/fascículo da publicação.""" - return self.xml_adapter.number - - @cached_property - def suppl(self): - """Suplemento da publicação.""" - return self.xml_adapter.suppl - - @cached_property - def z_surnames(self): - """Sobrenomes dos autores concatenados.""" - return self.xml_adapter.z_surnames - - @cached_property - def z_collab(self): - """Colaborações do artigo.""" - return self.xml_adapter.z_collab - - @cached_property - def z_links(self): - """Links relacionados ao artigo.""" - return self.xml_adapter.z_links - - @cached_property - def z_partial_body(self): - """Conteúdo parcial do corpo do artigo.""" - return self.xml_adapter.z_partial_body + @property + def pkg_name_list(self): + # --- Resolução Consolidada de Package Names --- + pkg_names = set() + # 1. Nome enviado originalmente via parâmetro no construtor + if self.xml_adapter.pkg_name: + pkg_names.add(self.xml_adapter.pkg_name) + # 2. Nome oficial atual gerado pelo motor de cálculo do XML + if self.xml_adapter.sps_pkg_name: + pkg_names.add(self.xml_adapter.sps_pkg_name) + # 3. Consolida todas as listas de nomes depreciados/alternativos + pkg_names.update(self.xml_adapter.xml_with_pre.deprecated_sps_pkg_name_list) + return set(item for item in pkg_names if item) + + def validate_input_data(self): + if not self.adapter_data.get("pub_year"): + raise exceptions.RequiredPublicationYearErrorToGetPidProviderXMLError() + issn_electronic = self.adapter_data.get("issn_electronic") + issn_print = self.adapter_data.get("issn_print") + if not issn_electronic and not issn_print: + raise exceptions.RequiredISSNErrorToGetPidProviderXMLError() + items = list(self.article_location_params.values()) + if any(items): + return + article_titles = (self.xml_with_pre_data.get("article_titles") or []) + article_titles = [x for x in article_titles if x] + items = [ + article_titles, + self.xml_with_pre_data.get("surnames"), + self.xml_with_pre_data.get("collab"), + self.xml_with_pre_data.get("links"), + self.xml_with_pre_data.get("partial_body"), + ] + if any(items): + return + raise exceptions.NotEnoughParametersToGetPidProviderXMLError() - @cached_property - def order(self): - """Conteúdo parcial do corpo do artigo.""" - return self.xml_adapter.order - # ========== Queries Construídas ========== - @cached_property + @property def identifier_queries(self): """ Constrói queries para busca por identificadores (v3, v2, aop_pid, pkg_name, DOI). - - Busca em múltiplos campos incluindo other_pid para garantir - compatibilidade com diferentes formatos de PIDs. - - Returns - ------- - Q - Query object combinando buscas por v3, v2, aop_pid, pkg_name e main_doi """ q = Q() + other_pids = set() + # PIDs diretos do xml_adapter (não envelopados no data dict) + v3 = self.xml_adapter.v3 + v2 = self.xml_adapter.v2 + aop_pid = self.xml_adapter.aop_pid + # PID v3 - máxima prioridade - if self.v3: - q |= Q(v3=self.v3) + if v3: + q |= Q(v3=v3) # PID v2 - if self.v2: - q |= Q(v2=self.v2) + if v2: + q |= Q(v2=v2) # AOP PID - if self.aop_pid: - q |= Q(v2=self.aop_pid) | Q(aop_pid=self.aop_pid) + if aop_pid: + q |= Q(v2=aop_pid) | Q(aop_pid=aop_pid) - # Package name - pkg_names = set() - if self.pkg_name: - pkg_names.add(self.pkg_name) - if self.sps_pkg_name: - pkg_names.add(self.sps_pkg_name) - if self.deprecated_sps_pkg_name: - pkg_names.add(self.deprecated_sps_pkg_name) + # Package names históricos e atuais + pkg_names = self.pkg_name_list if pkg_names: q |= Q(pkg_name__in=pkg_names) - # # DOI principal - # if self.main_doi: - # q |= Q(main_doi=self.main_doi) - + main_doi = self.adapter_data.get("main_doi") + if main_doi: + q |= Q(main_doi=main_doi) + return q - @cached_property + @property def issn_query(self): """ Constrói query base para busca por ISSN (eletrônico ou impresso). - - Returns - ------- - Q - Query object combinando ISSN eletrônico e impresso com operador OR - - Raises - ------ - RequiredISSNErrorToGetPidProviderXMLError - Se nenhum ISSN (eletrônico ou impresso) estiver disponível """ q = Q() + issn_electronic = self.adapter_data.get("issn_electronic") + issn_print = self.adapter_data.get("issn_print") - if not self.journal_issn_electronic and not self.journal_issn_print: + if not issn_electronic and not issn_print: raise exceptions.RequiredISSNErrorToGetPidProviderXMLError( _("Required Print or Electronic ISSN to identify XML {}").format( - self.pkg_name, + self.xml_adapter.pkg_name, ) ) - if self.journal_issn_electronic: - q |= Q(issn_electronic=self.journal_issn_electronic) + if issn_electronic: + q |= Q(issn_electronic=issn_electronic) - if self.journal_issn_print: - q |= Q(issn_print=self.journal_issn_print) + if issn_print: + q |= Q(issn_print=issn_print) return q - @cached_property + @property def issue_params(self): """ Constrói dicionário com metadados do fascículo e paginação do artigo. - - Retorna todos os campos sem verificar presença, permitindo - que o ORM do Django filtre automaticamente valores None. - - Returns - ------- - dict - Dicionário com elocation_id, fpage, fpage_seq, lpage, - pub_year, volume, number e suppl + """ + return { + "pub_year": self.adapter_data.get("pub_year"), + "volume": self.adapter_data.get("volume"), + "number": self.adapter_data.get("number"), + "suppl": self.adapter_data.get("suppl"), + } + + @property + def article_location_params(self): + """ + Constrói dicionário com metadados de localização do artigo. """ data = { - "elocation_id": self.elocation_id, - "fpage": self.fpage, - "fpage_seq": self.fpage_seq, - "lpage": self.lpage, - "pub_year": self.pub_year, - "volume": self.volume, - "number": self.number, - "suppl": self.suppl, + "elocation_id": self.adapter_data.get("elocation_id"), + "fpage": self.adapter_data.get("fpage"), + "fpage_seq": self.adapter_data.get("fpage_seq"), + "lpage": self.adapter_data.get("lpage"), } - if self.order: - data["v2__endswith"] = self.order - elif not self.elocation_id and not self.fpage and self.main_doi: - data["main_doi__iexact"] = self.main_doi + order = self.xml_adapter.order + if order: + data["v2__endswith"] = order return data - @cached_property + @property def article_data_query(self): """ - Constrói query para busca por dados textuais do artigo. - - Combina buscas por sobrenomes de autores, colaborações, - links e conteúdo parcial do corpo do artigo. - - Returns - ------- - Q or None - Query object combinando z_surnames, z_collab, z_links e z_partial_body, - ou None se nenhum dado textual estiver disponível + Constrói query para busca por dados textuais codificados (hashes sha256). """ - # Verifica se há algum dado textual disponível - if not any([ - self.z_surnames, - self.z_collab, - self.z_links, - self.z_partial_body, - ]): - return Q( - z_surnames=self.z_surnames, - z_collab=self.z_collab, - z_links=self.z_links, - z_partial_body=self.z_partial_body, - ) - - q = Q() - - # Adiciona query para sobrenomes se disponível - if self.z_surnames: - q |= Q(z_surnames=self.z_surnames) - - # Adiciona queries para outros campos textuais - if self.z_collab: - q |= Q(z_collab=self.z_collab) - - if self.z_links: - q |= Q(z_links=self.z_links) - - if self.z_partial_body: - q |= Q(z_partial_body=self.z_partial_body) - - return q \ No newline at end of file + z_surnames = self.adapter_data.get("z_surnames") + z_collab = self.adapter_data.get("z_collab") + z_links = self.adapter_data.get("z_links") + z_partial_body = self.adapter_data.get("z_partial_body") + + # Se houver qualquer dado textual disponível, constrói query com OR (|) + if z_surnames or z_partial_body or z_collab or z_links: + q = Q() + if z_surnames: + q |= Q(z_surnames=z_surnames) + if z_collab: + q |= Q(z_collab=z_collab) + if z_links: + q |= Q(z_links=z_links) + if z_partial_body: + q |= Q(z_partial_body=z_partial_body) + return q + + # Caso contrário, retorna os campos (geralmente None neste ponto) com AND + return Q( + z_surnames=z_surnames, + z_collab=z_collab, + z_links=z_links, + z_partial_body=z_partial_body, + ) & Q(**self.article_location_params) \ No newline at end of file diff --git a/pid_provider/test_controller.py b/pid_provider/test_controller.py deleted file mode 100644 index 3ff177825..000000000 --- a/pid_provider/test_controller.py +++ /dev/null @@ -1,87 +0,0 @@ -from unittest.mock import ANY, Mock, patch - -from django.contrib.auth import get_user_model -from django.test import TestCase - -from pid_provider.controller import PidProvider -from pid_provider.models import PidProviderXML - -User = get_user_model() - - -# def get_mock_config(): -# config = object() -# config.host = '' -# config.access_key = '' -# config.secret_key = '' -# config.bucket_root = '' -# config.bucket_app_subdir = 'bucket-app-subdir' -# config.secure = '' -# return config - - -class PidProviderTest(TestCase): - @patch("pid_provider.models.XMLSPS.save") - @patch("pid_provider.models.XMLVersion.save") - @patch("pid_provider.models.XMLIssue.save") - @patch("pid_provider.models.XMLJournal.save") - @patch("pid_provider.models.PidProviderXML.save") - @patch( - "pid_provider.models.PidProviderXML._get_unique_v3", - return_value="SJLD63mRxz9nTXtyMj7SLwk", - ) - @patch( - "pid_provider.models.PidProviderXML._get_unique_v2", - return_value="S2236-89062022061645340", - ) - @patch("pid_provider.controller.PidProviderConfig.get_or_create") - def test_provide_pid_for_xml_zip( - self, - mock_pid_provider_config, - mock_get_unique_v2, - mock_get_unique_v3, - mock_pid_provider_xml_save, - mock_xml_journal_save, - mock_xml_issue_save, - mock_xml_version_save, - mock_xmlsps_save, - ): - pid_provider = PidProvider() - result = pid_provider.provide_pid_for_xml_zip( - zip_xml_file_path="./pid_provider/fixtures/sub-article/2236-8906-hoehnea-49-e1082020.xml.zip", - user=User.objects.first(), - ) - result = list(result) - self.assertEqual("SJLD63mRxz9nTXtyMj7SLwk", result[0]["v3"]) - self.assertEqual("S2236-89062022061645340", result[0]["v2"]) - self.assertIsNone(result[0]["aop_pid"]) - self.assertIsNotNone(result[0]["created"]) - self.assertIsNone(result[0]["updated"]) - self.assertEqual("2236-8906-hoehnea-49-e1082020.xml", result[0]["filename"]) - self.assertEqual("created", result[0]["record_status"]) - self.assertEqual(True, result[0]["xml_changed"]) - - @patch("pid_provider.models.PidProviderXML._query_document") - @patch("pid_provider.models.PidProviderXML.is_equal_to", return_value=True) - def test_provide_pid_for_xml_with_pre_do_nothing_because_it_is_already_updated( - self, - mock_is_equal, - mock_query_document, - ): - # dubla o registro encontrado - pid_provider_xml = Mock(PidProviderXML) - pid_provider_xml.data = {"v3": ""} - mock_query_document.return_value = pid_provider_xml - - pid_provider_ = PidProvider() - result = pid_provider_.provide_pid_for_xml_zip( - zip_xml_file_path="./pid_provider/fixtures/sub-article/2236-8906-hoehnea-49-e1082020.xml.zip", - user=User.objects.first(), - ) - result = list(result) - expected = { - "filename": "2236-8906-hoehnea-49-e1082020.xml", - "v3": "", - "xml_with_pre": ANY, - } - self.assertDictEqual(result[0], expected) diff --git a/pid_provider/test_models.py b/pid_provider/test_models.py deleted file mode 100644 index 09626540c..000000000 --- a/pid_provider/test_models.py +++ /dev/null @@ -1,1029 +0,0 @@ -import logging -from datetime import datetime -from unittest import mock -from unittest.mock import ANY, MagicMock, Mock, call, patch - -from django.contrib.auth import get_user_model -from django.test import TestCase -from lxml import etree -from xmlsps.xml_sps_lib import XMLWithPre - -from pid_provider import exceptions, models -from pid_provider.xml_sps_adapter import PidProviderXMLAdapter - -User = get_user_model() - - -def _get_xml_adapter_from_file(path): - for xml_with_pre in XMLWithPre.create(path=path): - obj = PidProviderXMLAdapter(xml_with_pre) - return obj - - -def _get_xml_with_pre(xml=None): - xml = xml or "
" - return XMLWithPre("", etree.fromstring(xml)) - - -def _get_xml_adapter(xml=None): - xml = xml or "
" - xml_with_pre = XMLWithPre("", etree.fromstring(xml)) - obj = PidProviderXMLAdapter(xml_with_pre) - return obj - - -def _get_xml_adapter_with_issue_data(): - xml_adapter = _get_xml_adapter() - xml_adapter.journal_issn_electronic = "data-issn-e" - xml_adapter.journal_issn_print = "data-issn-p" - xml_adapter.volume = "data-vol" - xml_adapter.number = "data-num" - xml_adapter.suppl = "data-suppl" - xml_adapter.pub_year = "data-year" - xml_adapter.issue = models.XMLIssue.get_or_create( - models.XMLJournal.get_or_create("data-issn-e", "data-issn-p"), - "data-vol", - "data-num", - "data-suppl", - "data-year", - ) - xml_adapter.fpage = "data-fpage" - xml_adapter.fpage_seq = "data-fpage-seq" - xml_adapter.lpage = "data-lpage" - - xml_adapter.article_pub_year = "data-pub-year" - xml_adapter.v3 = "123456789012345678901v3" - xml_adapter.v2 = "123456789012345678901v2" - xml_adapter.aop_pid = "12345678901234567890aop" - - xml_adapter.main_doi = "data-main_doi" - xml_adapter.elocation_id = "data-elocation_id" - return xml_adapter - - -class PidProviderXMLValidateQueryParamsTest(TestCase): - def setUp(self): - self.article_params = { - "z_collab": "VALUE", - "z_links": "Links", - "z_partial_body": "Body", - "z_surnames": "Z_SURNAMES", - "article_pub_year": "2020", - "elocation_id": "e19347", - "journal__issn_electronic": "issn electronic", - "journal__issn_print": "issn print", - "main_doi": "DOI", - "pkg_name": "pkgName", - } - - self.issue_params = { - "issue__pub_year": "year", - "issue__volume": "vol", - "issue__number": "num", - "issue__suppl": "suppl", - "fpage": "1", - "fpage_seq": "a", - "lpage": "11", - } - - def test_validate_query_params_all_present(self): - params = self.article_params - result = models.PidProviderXML.validate_query_params(params) - self.assertTrue(result) - - def test_validate_query_params_all_present_plus_issue_params(self): - params = self.article_params - params.update(self.issue_params) - result = models.PidProviderXML.validate_query_params(params) - self.assertTrue(result) - - def test_validate_query_params_issue_params_only(self): - params = {} - params.update(self.issue_params) - with self.assertRaises(exceptions.NotEnoughParametersToGetDocumentRecordError): - result = models.PidProviderXML.validate_query_params(params) - - def test_validate_query_params_journal_issns_absence(self): - params = self.article_params - params.update(self.issue_params) - del params["journal__issn_print"] - del params["journal__issn_electronic"] - with self.assertRaises(exceptions.NotEnoughParametersToGetDocumentRecordError): - result = models.PidProviderXML.validate_query_params(params) - - def test_validate_query_params_pub_year_absence(self): - params = self.article_params - params.update(self.issue_params) - del params["article_pub_year"] - del params["issue__pub_year"] - with self.assertRaises(exceptions.NotEnoughParametersToGetDocumentRecordError): - result = models.PidProviderXML.validate_query_params(params) - - def test_validate_query_params_main_doi_absence(self): - params = self.article_params - params.update(self.issue_params) - del params["main_doi"] - result = models.PidProviderXML.validate_query_params(params) - self.assertTrue(result) - - def test_validate_query_params_fpage_absence(self): - params = self.article_params - params.update(self.issue_params) - del params["fpage"] - result = models.PidProviderXML.validate_query_params(params) - self.assertTrue(result) - - def test_validate_query_params_elocation_id_absence(self): - params = self.article_params - params.update(self.issue_params) - del params["elocation_id"] - result = models.PidProviderXML.validate_query_params(params) - self.assertTrue(result) - - def test_validate_query_params_main_doi_fpage_elocation_id_absence(self): - params = self.article_params - params.update(self.issue_params) - del params["main_doi"] - del params["fpage"] - del params["elocation_id"] - result = models.PidProviderXML.validate_query_params(params) - self.assertTrue(result) - - def test_validate_query_params_z_surnames_id_absence(self): - params = self.article_params - params.update(self.issue_params) - del params["main_doi"] - del params["fpage"] - del params["elocation_id"] - del params["z_surnames"] - result = models.PidProviderXML.validate_query_params(params) - self.assertTrue(result) - - def test_validate_query_params_z_collab_id_absence(self): - params = self.article_params - params.update(self.issue_params) - del params["main_doi"] - del params["fpage"] - del params["elocation_id"] - del params["z_collab"] - result = models.PidProviderXML.validate_query_params(params) - self.assertTrue(result) - - def test_validate_query_params_z_collab_id_absence(self): - params = self.article_params - params.update(self.issue_params) - del params["main_doi"] - del params["fpage"] - del params["elocation_id"] - del params["z_links"] - result = models.PidProviderXML.validate_query_params(params) - self.assertTrue(result) - - def test_validate_query_params_z_collab_id_absence(self): - params = self.article_params - params.update(self.issue_params) - del params["main_doi"] - del params["fpage"] - del params["elocation_id"] - del params["pkg_name"] - result = models.PidProviderXML.validate_query_params(params) - self.assertTrue(result) - - def test_validate_query_params_z_collab_id_absence(self): - params = self.article_params - params.update(self.issue_params) - del params["main_doi"] - del params["fpage"] - del params["elocation_id"] - del params["pkg_name"] - del params["z_surnames"] - del params["z_collab"] - del params["z_links"] - - with self.assertRaises(exceptions.NotEnoughParametersToGetDocumentRecordError): - result = models.PidProviderXML.validate_query_params(params) - - -@patch( - "pid_provider.xml_sps_adapter.PidProviderXMLAdapter.query_list", - new_callable=mock.PropertyMock, -) -@patch( - "pid_provider.models.PidProviderXML.validate_query_params", - return_value=True, -) -@patch("pid_provider.models.PidProviderXML.objects.get") -class PidProviderXMLQueryDocumentTest(TestCase): - def test_query_document_is_called_with_query_params( - self, - mock_get, - mock_validate_params, - mock_query_list, - ): - """ - PidProviderXML._query_document is called with parameters returned by - PidProviderXML.query_list - """ - params_list = [ - {"key": "value"}, - ] - mock_query_list.return_value = params_list - mock_get.side_effect = models.PidProviderXML.DoesNotExist - xml_adapter = _get_xml_adapter() - result = models.PidProviderXML._query_document(xml_adapter) - mock_get.assert_called_once_with(**{"key": "value"}) - - def test_query_document_returns_none_if_document_does_not_exist( - self, - mock_get, - mock_validate_params, - mock_query_list, - ): - params_list = [ - {"key": "value"}, - ] - mock_query_list.return_value = params_list - mock_get.side_effect = models.PidProviderXML.DoesNotExist - xml_adapter = _get_xml_adapter() - result = models.PidProviderXML._query_document(xml_adapter) - self.assertIsNone(result) - - def test_query_document_returns_found_document( - self, - mock_get, - mock_validate_params, - mock_query_list, - ): - params_list = [ - {"key": "value"}, - ] - mock_query_list.return_value = params_list - mock_get.return_value = models.PidProviderXML() - xml_adapter = _get_xml_adapter() - result = models.PidProviderXML._query_document(xml_adapter) - self.assertEqual(models.PidProviderXML, type(result)) - - def test_query_document_returns_found_item_at_the_second_round( - self, - mock_get, - mock_validate_params, - mock_query_list, - ): - params_list = [ - {"key": "value"}, - {"key": "value2"}, - ] - mock_query_list.return_value = params_list - mock_get.side_effect = [ - models.PidProviderXML.DoesNotExist, - models.PidProviderXML(), - ] - xml_adapter = _get_xml_adapter() - result = models.PidProviderXML._query_document(xml_adapter) - self.assertEqual(models.PidProviderXML, type(result)) - - def test_query_document_raises_query_document_error_because_multiple_objects_returned( - self, - mock_get, - mock_validate_params, - mock_query_list, - ): - params_list = [ - {"key": "value"}, - ] - mock_query_list.return_value = params_list - mock_get.side_effect = models.PidProviderXML.MultipleObjectsReturned - with self.assertRaises( - exceptions.QueryDocumentMultipleObjectsReturnedError - ) as exc: - xml_adapter = _get_xml_adapter() - result = models.PidProviderXML._query_document(xml_adapter) - - def test_query_document_raises_error( - self, - mock_get, - mock_validate_params, - mock_query_list, - ): - """ - PidProviderXML._query_document is called with parameters returned by - PidProviderXML.query_list - """ - params_list = [ - {"key": "value"}, - ] - mock_query_list.return_value = params_list - mock_validate_params.side_effect = ( - exceptions.NotEnoughParametersToGetDocumentRecordError - ) - - with self.assertRaises(exceptions.NotEnoughParametersToGetDocumentRecordError): - xml_adapter = _get_xml_adapter() - result = models.PidProviderXML._query_document(xml_adapter) - - -@patch("pid_provider.models.PidProviderXML._query_document") -class PidProviderXMLGetRegisteredTest(TestCase): - def setUp(self): - self.xml_with_pre = _get_xml_with_pre() - - def test_get_registered_returns_dict_with_registered_data( - self, - mock_query_document, - ): - pid_req_xml = models.PidProviderXML() - pid_req_xml.pkg_name = "registered_pkg_name" - pid_req_xml.v2 = "registered_v2" - pid_req_xml.v3 = "registered_v3" - pid_req_xml.aop_pid = "registered_aop_pid" - pid_req_xml.created = datetime(2023, 2, 20) - pid_req_xml.updated = datetime(2023, 2, 20) - - mock_query_document.return_value = pid_req_xml - - result = models.PidProviderXML.get_registered(self.xml_with_pre) - expected = { - "v3": "registered_v3", - "v2": "registered_v2", - "aop_pid": "registered_aop_pid", - "pkg_name": "registered_pkg_name", - "created": "2023-02-20T00:00:00", - "updated": "2023-02-20T00:00:00", - "record_status": "updated", - } - self.assertDictEqual(expected, result) - - def test_get_registered_returns_none( - self, - mock_query_document, - ): - mock_query_document.return_value = None - - result = models.PidProviderXML.get_registered(self.xml_with_pre) - self.assertIsNone(result) - - def test_get_registered_returns_error_multiple_return( - self, - mock_query_document, - ): - mock_query_document.side_effect = ( - exceptions.QueryDocumentMultipleObjectsReturnedError - ) - - result = models.PidProviderXML.get_registered(self.xml_with_pre) - self.assertIn("error_type", result.keys()) - self.assertIn("error_msg", result.keys()) - - def test_get_registered_returns_error_not_enough_params( - self, - mock_query_document, - ): - mock_query_document.side_effect = ( - exceptions.NotEnoughParametersToGetDocumentRecordError - ) - - result = models.PidProviderXML.get_registered(self.xml_with_pre) - self.assertIn("error_type", result.keys()) - self.assertIn("error_msg", result.keys()) - - -class PidProviderXMLEvaluateRegistrationTest(TestCase): - def setUp(self): - self.xml_adapter = _get_xml_adapter() - - def test_evaluate_registration_accepts_xml_is_aop_and_registered_is_aop(self): - registered = Mock(spec=models.PidProviderXML) - registered.is_aop = True - - self.xml_adapter.is_aop = True - - result = models.PidProviderXML.evaluate_registration( - self.xml_adapter, registered - ) - self.assertTrue(result) - - def test_evaluate_registration_accepts_xml_is_not_aop_and_registered_is_aop(self): - registered = Mock(spec=models.PidProviderXML) - registered.is_aop = True - - self.xml_adapter.is_aop = False - - result = models.PidProviderXML.evaluate_registration( - self.xml_adapter, registered - ) - self.assertTrue(result) - - def test_evaluate_registration_raises_error(self): - registered = Mock(spec=models.PidProviderXML) - registered.is_aop = False - - self.xml_adapter.is_aop = True - - with self.assertRaises(exceptions.ForbiddenPidProviderXMLRegistrationError): - result = models.PidProviderXML.evaluate_registration( - self.xml_adapter, registered - ) - - -@patch("pid_provider.models.PidProviderXML._get_unique_v2") -class PidProviderXMLAddV2Test(TestCase): - def _get_xml_adapter(self, v2=None, v3=None, aop_pid=None): - v2 = ( - v2 - and f'{v2}' - or "" - ) - v3 = ( - v3 - and f'{v3}' - or "" - ) - aop_pid = ( - aop_pid - and f'{aop_pid}' - or "" - ) - - return _get_xml_adapter( - f"""
- - {v2} - {v3} - {aop_pid} - 10.36416/1806-3756/e20220072 - 01100 - -
""" - ) - - # TODO - # def test_add_pid_v2_uses_registered_pid_v2( - # self, - # mock_get_unique_v2, - # ): - # found = models.PidProviderXML() - # found.v2 = "registered_v2" - - # xml_adapter = self._get_xml_adapter(v2='xml_v2') - - # mock_get_unique_v2.return_value = "generated_v2" - - # models.PidProviderXML._add_pid_v2(xml_adapter, found) - # self.assertEqual("registered_v2", xml_adapter.v2) - - def test_add_pid_v2_replace_xml_v2_because_its_value_is_invalid_length_is_not_23( - self, - mock_get_unique_v2, - ): - found = models.PidProviderXML() - found.v2 = None - - xml_adapter = self._get_xml_adapter(v2="bad_size_not_23") - - mock_get_unique_v2.return_value = "S1806-37132022000201100" - - models.PidProviderXML._add_pid_v2(xml_adapter, found) - self.assertEqual("S1806-37132022000201100", xml_adapter.v2) - - def test_add_pid_v2_keeps_xml_v2( - self, - mock_get_unique_v2, - ): - found = models.PidProviderXML() - found.v2 = None - - xml_adapter = self._get_xml_adapter(v2="S1806-37132022000199999") - - mock_get_unique_v2.return_value = "S1806-37132022000300001" - - models.PidProviderXML._add_pid_v2(xml_adapter, found) - self.assertEqual("S1806-37132022000199999", xml_adapter.v2) - - def test_add_pid_v2_uses_unique_v2( - self, - mock_get_unique_v2, - ): - found = models.PidProviderXML() - found.v2 = None - - xml_adapter = self._get_xml_adapter() - - mock_get_unique_v2.return_value = "S1806-37132022000201100" - - models.PidProviderXML._add_pid_v2(xml_adapter, found) - self.assertEqual("S1806-37132022000201100", xml_adapter.v2) - - -class PidProviderXMLAddAopPidTest(TestCase): - def _get_xml_adapter(self, v2=None, v3=None, aop_pid=None): - v2 = ( - v2 - and f'{v2}' - or "" - ) - v3 = ( - v3 - and f'{v3}' - or "" - ) - aop_pid = ( - aop_pid - and f'{aop_pid}' - or "" - ) - - return _get_xml_adapter( - f"""
- - {v2} - {v3} - {aop_pid} - 10.36416/1806-3756/e20220072 - 01100 - -
""" - ) - - def test_add_aop_pid_uses_registered_aop_pid( - self, - ): - found = models.PidProviderXML() - found.aop_pid = "12345678901234567890aop" - - xml_adapter = self._get_xml_adapter(aop_pid="xml_aop_pid") - - models.PidProviderXML._add_aop_pid(xml_adapter, found) - self.assertEqual("12345678901234567890aop", xml_adapter.aop_pid) - - def test_add_aop_pid_does_not_replace_by_none( - self, - ): - found = models.PidProviderXML() - found.aop_pid = None - - xml_adapter = self._get_xml_adapter(aop_pid="xml_aop_pid") - - models.PidProviderXML._add_aop_pid(xml_adapter, found) - self.assertEqual("xml_aop_pid", xml_adapter.aop_pid) - - -@patch("pid_provider.models.PidProviderXML._is_registered_pid") -@patch("pid_provider.models.PidProviderXML._get_unique_v3") -class PidProviderXMLAddPidV3Test(TestCase): - def _get_xml_adapter(self, v2=None, v3=None, aop_pid=None): - v2 = ( - v2 - and f'{v2}' - or "" - ) - v3 = ( - v3 - and f'{v3}' - or "" - ) - aop_pid = ( - aop_pid - and f'{aop_pid}' - or "" - ) - - return _get_xml_adapter( - f"""
- - {v2} - {v3} - {aop_pid} - 10.36416/1806-3756/e20220072 - 01100 - -
""" - ) - - def test_add_pid_v3_uses_registered_v3( - self, - mock__get_unique_v3, - mock__is_registered_pid, - ): - found = models.PidProviderXML() - found.v3 = "123456789012345678901v3" - - xml_adapter = self._get_xml_adapter(v3="xml_v3") - - models.PidProviderXML._add_pid_v3(xml_adapter, found) - self.assertEqual("123456789012345678901v3", xml_adapter.v3) - - def test_add_pid_v3_replaced_by_generated( - self, - mock__get_unique_v3, - mock__is_registered_pid, - ): - mock__is_registered_pid.return_value = True - mock__get_unique_v3.return_value = "gen456789012345678901v3" - - found = None - - xml_adapter = self._get_xml_adapter(v3="xml_v3") - - models.PidProviderXML._add_pid_v3(xml_adapter, found) - self.assertEqual("gen456789012345678901v3", xml_adapter.v3) - - def test_add_pid_v3_keeps_xml_v3( - self, - mock__get_unique_v3, - mock__is_registered_pid, - ): - mock__is_registered_pid.return_value = False - mock__get_unique_v3.return_value = "gen456789012345678901v3" - - found = None - - xml_adapter = self._get_xml_adapter(v3="xml456789012345678901v3") - - models.PidProviderXML._add_pid_v3(xml_adapter, found) - self.assertEqual("xml456789012345678901v3", xml_adapter.v3) - - -class PidProviderXMLIsEqualToTest(TestCase): - def test_is_equal_to_returns_false(self): - xml_adapter = _get_xml_adapter_from_file( - "./pid_provider/fixtures/article/ex-aop.xml" - ) - registered = models.PidProviderXML() - - result = registered.is_equal_to(xml_adapter) - self.assertFalse(result) - - -class PidProviderXMLAddDataForRegularArticleTest(TestCase): - def setUp(self): - user = User() - xml_adapter = _get_xml_adapter_from_file( - "./pid_provider/fixtures/article/ex-aop.xml" - ) - self.registered = models.PidProviderXML() - self.registered._add_data(xml_adapter, user) - - def test_v3(self): - self.assertEqual("yH6CLqxFJsQKrHj7zXkwL3G", self.registered.v3) - - def test_v2(self): - self.assertEqual("S1413-41522020000400627", self.registered.v2) - - def test_aop_pid(self): - self.assertEqual("S1413-41522020005000111", self.registered.aop_pid) - - def test_main_doi(self): - self.assertEqual("10.1590/S1413-4152202020180029", self.registered.main_doi) - - def test_fpage(self): - self.assertEqual("627", self.registered.fpage) - - def test_fpage_seq(self): - self.assertEqual(None, self.registered.fpage_seq) - - def test_lpage(self): - self.assertEqual("634", self.registered.lpage) - - def test_elocation_id(self): - self.assertEqual(None, self.registered.elocation_id) - - def test_article_pub_year(self): - self.assertEqual("2020", self.registered.article_pub_year) - - def test_z_surnames(self): - self.assertEqual( - "544700df348a47fdd7c55713054e12663a0c530e60e7a166395a496f77de9d36", - self.registered.z_surnames, - ) - - def test_z_collab(self): - self.assertIsNone(self.registered.z_collab) - - def test_z_links(self): - self.assertIsNone(self.registered.z_links) - - def test_z_partial_body(self): - self.assertEqual( - "2e07675bfe91c65e1544ada450ff2e956fef9b492d30e997ebd47687e0f7afa2", - self.registered.z_partial_body, - ) - - -@patch( - "pid_provider.models.utcnow", - side_effect=[datetime(2020, 2, 2, 0, 0), datetime(2020, 2, 3, 0, 0)], -) -@patch("pid_provider.models.XMLVersion.save_file") -@patch("pid_provider.models.XMLVersion.save") -@patch("pid_provider.models.XMLIssue.save") -@patch("pid_provider.models.XMLJournal.save") -@patch("pid_provider.models.PidProviderXML.save") -@patch("pid_provider.models.PidRequest.save") -class PidProviderXMLRegisterTest(TestCase): - def test_register_returns_error( - self, - mock_pid_request_save, - mock_pid_provider_xml_save, - mock_xml_journal_save, - mock_xml_issue_save, - mock_xml_version_save, - mock_xml_version_save_file, - mock_now, - ): - expected = { - "result_type": "", - "result_message": "No attribute enough for disambiguations {'z_surnames': None, 'z_collab': None, 'main_doi': None, 'z_links': None, 'z_partial_body': None, 'pkg_name': None, 'elocation_id': None, 'journal__issn_print': None, 'journal__issn_electronic': None, 'article_pub_year': None}", - "origin": "filename.xml", - "xml": "
", - } - - user = User() - xml_with_pre = _get_xml_with_pre() - result = models.PidProviderXML.register( - xml_with_pre=xml_with_pre, - filename="filename.xml", - user=user, - ) - print(result) - self.assertEqual(expected["result_type"], result["result_type"]) - self.assertIsNotNone(result["result_msg"]) - # self.assertEqual(expected["result_message"], result["result_msg"]) - self.assertEqual(expected["origin"], result["origin"]) - self.assertEqual(expected["xml"], result["detail"]["xml"]) - mock_pid_provider_xml_save.assert_not_called() - mock_pid_request_save.assert_called_once_with() - - -@patch( - "pid_provider.models.utcnow", - side_effect=[datetime(2020, 2, 2, 0, 0), datetime(2020, 2, 3, 0, 0)], -) -@patch("pid_provider.models.XMLSPS.save") -@patch("pid_provider.models.XMLVersion.save_file") -@patch("pid_provider.models.XMLVersion.save") -@patch("pid_provider.models.XMLIssue.save") -@patch("pid_provider.models.XMLJournal.save") -@patch("pid_provider.models.PidProviderXML.save") -@patch("pid_provider.models.PidRequest.save") -class PidProviderXMLRegisterTest(TestCase): - def test_register_with_success( - self, - mock_pid_request_save, - mock_pid_provider_xml_save, - mock_xml_journal_save, - mock_xml_issue_save, - mock_xml_version_save, - mock_xml_version_save_file, - mock_xml_sps_save, - mock_now, - ): - expected = { - "result_type": "", - "result_message": "No attribute enough for disambiguations {'z_surnames': None, 'z_collab': None, 'main_doi': None, 'z_links': None, 'z_partial_body': None, 'pkg_name': None, 'elocation_id': None, 'journal__issn_print': None, 'journal__issn_electronic': None, 'article_pub_year': None}", - "origin": "filename.xml", - "xml": "
", - } - - user = User() - xml_adapter = _get_xml_adapter_from_file( - "./pid_provider/fixtures/article/ex-aop.xml" - ) - result = models.PidProviderXML.register( - xml_with_pre=xml_adapter.xml_with_pre, - filename="ex-aop.xml", - user=user, - ) - self.assertEqual("yH6CLqxFJsQKrHj7zXkwL3G", result["v3"]) - self.assertEqual("S1413-41522020000400627", result["v2"]) - self.assertEqual("S1413-41522020005000111", result["aop_pid"]) - self.assertEqual("1809-4457-esa-25-04-627", result["pkg_name"]) - self.assertEqual(False, result["xml_changed"]) - self.assertEqual("created", result["record_status"]) - self.assertIsNone(result["updated"]) - self.assertIsNotNone(result["created"]) - mock_pid_request_save.assert_not_called() - - -class XMLURLTest(TestCase): - """Tests for XMLURL model""" - - def setUp(self): - self.user = User.objects.create_user(username="testuser", password="testpass") - self.test_url = "http://example.com/article.xml" - self.test_pid = "ABC123XYZ456" - - def test_create_xmlurl(self): - """Test creating a new XMLURL instance""" - xmlurl = models.XMLURL.create( - user=self.user, - url=self.test_url, - status="pending", - pid=self.test_pid, - ) - - self.assertIsNotNone(xmlurl) - self.assertEqual(xmlurl.url, self.test_url) - self.assertEqual(xmlurl.status, "pending") - self.assertEqual(xmlurl.pid, self.test_pid) - self.assertEqual(xmlurl.creator, self.user) - - def test_get_xmlurl(self): - """Test getting an XMLURL by URL""" - models.XMLURL.create( - user=self.user, - url=self.test_url, - status="pending", - pid=self.test_pid, - ) - - xmlurl = models.XMLURL.get(url=self.test_url) - self.assertIsNotNone(xmlurl) - self.assertEqual(xmlurl.url, self.test_url) - - def test_create_or_update_existing(self): - """Test updating an existing XMLURL""" - # Create initial record - xmlurl = models.XMLURL.create( - user=self.user, - url=self.test_url, - status="pending", - pid=None, - ) - - # Update it - updated_xmlurl = models.XMLURL.create_or_update( - user=self.user, - url=self.test_url, - status="success", - pid=self.test_pid, - ) - - self.assertEqual(updated_xmlurl.id, xmlurl.id) - self.assertEqual(updated_xmlurl.status, "success") - self.assertEqual(updated_xmlurl.pid, self.test_pid) - self.assertEqual(updated_xmlurl.updated_by, self.user) - - def test_create_or_update_new(self): - """Test creating a new XMLURL when it doesn't exist""" - xmlurl = models.XMLURL.create_or_update( - user=self.user, - url=self.test_url, - status="pending", - pid=self.test_pid, - ) - - self.assertIsNotNone(xmlurl) - self.assertEqual(xmlurl.url, self.test_url) - self.assertEqual(xmlurl.status, "pending") - - def test_save_file_with_string_content(self): - """Test save_file method with string XML content""" - xmlurl = models.XMLURL.create( - user=self.user, - url=self.test_url, - status="pending", - pid=self.test_pid, - ) - - xml_content = "
Test Article
" - result = xmlurl.save_file(xml_content, filename="test.xml") - - self.assertTrue(result) - self.assertTrue(xmlurl.zipfile.name) - - def test_save_file_with_bytes_content(self): - """Test save_file method with bytes XML content""" - xmlurl = models.XMLURL.create( - user=self.user, - url=self.test_url, - status="pending", - pid=self.test_pid, - ) - - xml_content = b"
Test Article
" - result = xmlurl.save_file(xml_content, filename="test.xml") - - self.assertTrue(result) - self.assertTrue(xmlurl.zipfile.name) - - def test_save_file_default_filename(self): - """Test save_file method with default filename""" - xmlurl = models.XMLURL.create( - user=self.user, - url=self.test_url, - status="pending", - pid=self.test_pid, - ) - - xml_content = "
Test Article
" - result = xmlurl.save_file(xml_content) - - self.assertTrue(result) - self.assertTrue(xmlurl.zipfile.name) - - def test_str_method(self): - """Test __str__ method""" - xmlurl = models.XMLURL.create( - user=self.user, - url=self.test_url, - status="pending", - pid=self.test_pid, - ) - - expected_str = f"{self.test_url} - pending" - self.assertEqual(str(xmlurl), expected_str) - - -class BasePidProviderXMLURITest(TestCase): - """Tests for BasePidProvider.provide_pid_for_xml_uri method""" - - def setUp(self): - self.user = User.objects.create_user(username="testuser", password="testpass") - - @patch("pid_provider.base_pid_provider.XMLWithPre.create") - def test_provide_pid_for_xml_uri_fetch_failure(self, mock_create): - """Test exception type a) - Failure to obtain XML""" - from pid_provider.base_pid_provider import BasePidProvider - - # Mock XMLWithPre.create to raise an exception - mock_create.side_effect = Exception("Connection timeout") - - provider = BasePidProvider() - result = provider.provide_pid_for_xml_uri( - xml_uri="http://example.com/article.xml", - name="test.xml", - user=self.user, - ) - - # Should return error details - self.assertIn("error_msg", result) - self.assertIn("error_type", result) - - # Should create XMLURL with failed status - xmlurl = models.XMLURL.get(url="http://example.com/article.xml") - self.assertEqual(xmlurl.status, "xml_fetch_failed") - self.assertIsNone(xmlurl.pid) - - @patch("pid_provider.base_pid_provider.XMLWithPre.create") - @patch.object(models.PidProviderXML, "register") - def test_provide_pid_for_xml_uri_success(self, mock_register, mock_create): - """Test successful processing with XMLURL creation""" - from pid_provider.base_pid_provider import BasePidProvider - - # Mock XMLWithPre.create - xml_with_pre = _get_xml_with_pre("
Test
") - mock_create.return_value = [xml_with_pre] - - # Mock successful registration - mock_register.return_value = { - "v3": "test_v3_pid", - "v2": "test_v2_pid", - "created": datetime.now(), - } - - provider = BasePidProvider() - result = provider.provide_pid_for_xml_uri( - xml_uri="http://example.com/article.xml", - name="test.xml", - user=self.user, - ) - - # Should return success response - self.assertEqual(result.get("v3"), "test_v3_pid") - - # Should create XMLURL with success status - xmlurl = models.XMLURL.get(url="http://example.com/article.xml") - self.assertEqual(xmlurl.status, "success") - self.assertEqual(xmlurl.pid, "test_v3_pid") - - @patch("pid_provider.base_pid_provider.XMLWithPre.create") - @patch.object(models.PidProviderXML, "register") - def test_provide_pid_for_xml_uri_registration_failure(self, mock_register, mock_create): - """Test exception type b) - XML obtained but registration failed""" - from pid_provider.base_pid_provider import BasePidProvider - - # Mock XMLWithPre.create - xml_with_pre = _get_xml_with_pre("
Test
") - mock_create.return_value = [xml_with_pre] - - # Mock failed registration - mock_register.return_value = { - "error_type": "ValidationError", - "error_msg": "Invalid XML structure", - "v3": "test_v3_pid", - } - - provider = BasePidProvider() - result = provider.provide_pid_for_xml_uri( - xml_uri="http://example.com/article2.xml", - name="test2.xml", - user=self.user, - ) - - # Should return error response - self.assertIn("error_type", result) - - # Should create XMLURL with failed status and save zipfile - xmlurl = models.XMLURL.get(url="http://example.com/article2.xml") - self.assertEqual(xmlurl.status, "pid_provider_xml_failed") - self.assertEqual(xmlurl.pid, "test_v3_pid") diff --git a/pid_provider/tests.py b/pid_provider/tests.py deleted file mode 100644 index 7ce503c2d..000000000 --- a/pid_provider/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/pid_provider/wagtail_hooks.py b/pid_provider/wagtail_hooks.py index 228742585..2c0f586f8 100644 --- a/pid_provider/wagtail_hooks.py +++ b/pid_provider/wagtail_hooks.py @@ -1,12 +1,10 @@ -from django.http import HttpResponseRedirect from django.utils.translation import gettext_lazy as _ -from wagtail import hooks from wagtail.snippets.models import register_snippet from wagtail.snippets.views.snippets import SnippetViewSetGroup from config.menu import get_menu_order from core.views import CommonControlFieldViewSet -from pid_provider.models import XMLVersion, FixPidV2, OtherPid, PidProviderConfig, PidProviderXML +from pid_provider.models import XMLURL, XMLVersion, FixPidV2, OtherPid, PidProviderConfig, PidProviderXML, PidProviderXMLRegistration class PidProviderXMLViewSet(CommonControlFieldViewSet): @@ -178,6 +176,64 @@ class XMLVersionViewSet(CommonControlFieldViewSet): "available_since", ) +class XMLURLViewSet(CommonControlFieldViewSet): + model = XMLURL + menu_label = _("XML URLs") + menu_icon = "folder" + menu_order = 300 + add_to_settings_menu = False + list_per_page = 10 + + # Configuração de listagem + list_display = [ + "url", + "status", + "pid", + ] + list_filter = { + "status": ["exact"], + "is_public": ["exact"], + } + search_fields = ( + "url", + "status", + "pid", + ) + + +class PidProviderXMLRegistrationViewSet(CommonControlFieldViewSet): + model = PidProviderXMLRegistration + icon = "doc-empty-inverse" + menu_label = _("PID Registration Events") + menu_name = "pid_provider_xml_registration" + + # ordenação na listagem + ordering = ["-created"] + + # colunas da listagem + list_display = ( + "pkg_name", + "event_status", + "pid_provider_xml", + "created", + ) + + # filtros laterais + list_filter = ("event_status", "created") + + # busca + search_fields = ("pkg_name", "pid_provider_xml__v2", "pid_provider_xml__v3") + + # paginação (tabela cresce em volume) + list_per_page = 50 + + def get_queryset(self, request): + # super = CommonControlFieldViewSet + queryset = super().get_queryset(request) + if queryset is None: + queryset = self.model._default_manager.all() + return queryset.select_related("pid_provider_xml") + # Grupo de ViewSets class PidProviderViewSetGroup(SnippetViewSetGroup): @@ -190,6 +246,8 @@ class PidProviderViewSetGroup(SnippetViewSetGroup): FixPidV2ViewSet, PidProviderConfigViewSet, XMLVersionViewSet, + XMLURLViewSet, + PidProviderXMLRegistrationViewSet, ) diff --git a/requirements/base.txt b/requirements/base.txt index afc93ff4e..0849c45de 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -91,7 +91,7 @@ opensearch-py==3.2.0 # https://github.com/opensearch-project/opensearch-py # ------------------------------------------------------------------------------ tornado>=6.5.2 # not directly required, pinned by Snyk to avoid a vulnerability lxml==6.0.2 # https://github.com/lxml/lxml -git+https://git@github.com/scieloorg/packtools@4.15.0#egg=packtools +git+https://git@github.com/scieloorg/packtools@4.16.8#egg=packtools # pymongo # ------------------------------------------------------------------------------