From f1639ab13cb68a8d9fad7735f2855fe198e22bcc Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Mon, 6 Apr 2026 20:56:11 -0400 Subject: [PATCH 01/13] Migration to peprs --- pepdbagent/const.py | 1 - pepdbagent/models.py | 4 +- pepdbagent/modules/project.py | 73 +++++---- pepdbagent/modules/sample.py | 33 ++--- pepdbagent/modules/view.py | 27 ++-- pepdbagent/utils.py | 2 +- requirements/requirements-all.txt | 2 +- .../amendments1/project_config.yaml | 1 - .../namespace2/derive/project_config.yaml | 2 +- .../namespace3/piface/project_config.yaml | 2 +- .../namespace3/remove/project_config.yaml | 1 - .../amendments1/project_config.yaml | 1 - .../private_test/derive/project_config.yaml | 1 - .../private_test/remove/project_config.yaml | 1 - tests/test_project.py | 58 +++++--- tests/test_project_history.py | 62 ++++---- tests/test_samples.py | 12 +- tests/test_updates.py | 140 +++++++++--------- tests/utils.py | 8 +- 19 files changed, 218 insertions(+), 213 deletions(-) diff --git a/pepdbagent/const.py b/pepdbagent/const.py index cf0577b6..2bf7c931 100644 --- a/pepdbagent/const.py +++ b/pepdbagent/const.py @@ -6,7 +6,6 @@ DESCRIPTION_KEY = "description" NAME_KEY = "name" -# from peppy.const import SAMPLE_RAW_DICT_KEY, SUBSAMPLE_RAW_LIST_KEY DEFAULT_OFFSET = 0 DEFAULT_LIMIT = 100 diff --git a/pepdbagent/models.py b/pepdbagent/models.py index 95aa783f..6291dc78 100644 --- a/pepdbagent/models.py +++ b/pepdbagent/models.py @@ -2,7 +2,7 @@ import datetime from typing import Dict, List, Optional, Union -from peppy.const import CONFIG_KEY, SAMPLE_RAW_DICT_KEY, SUBSAMPLE_RAW_LIST_KEY +from peprs.const import CONFIG_KEY, SAMPLE_RAW_DICT_KEY, SUBSAMPLE_RAW_DICT_KEY from pydantic import BaseModel, ConfigDict, Field, field_validator from pepdbagent.const import DEFAULT_TAG @@ -14,7 +14,7 @@ class ProjectDict(BaseModel): """ config: dict = Field(alias=CONFIG_KEY) - subsample_list: Optional[Union[list, None]] = Field(alias=SUBSAMPLE_RAW_LIST_KEY) + subsample_list: Optional[Union[list, None]] = Field(alias=SUBSAMPLE_RAW_DICT_KEY) sample_dict: list = Field(alias=SAMPLE_RAW_DICT_KEY) model_config = ConfigDict(populate_by_name=True, extra="forbid") diff --git a/pepdbagent/modules/project.py b/pepdbagent/modules/project.py index bf87e789..62590411 100644 --- a/pepdbagent/modules/project.py +++ b/pepdbagent/modules/project.py @@ -4,14 +4,15 @@ from typing import Dict, List, NoReturn, Union import numpy as np -import peppy -from peppy.const import ( +import peprs +from peprs.const import ( CONFIG_KEY, - SAMPLE_NAME_ATTR, SAMPLE_RAW_DICT_KEY, - SAMPLE_TABLE_INDEX_KEY, - SUBSAMPLE_RAW_LIST_KEY, + SUBSAMPLE_RAW_DICT_KEY, ) + +SAMPLE_NAME_ATTR = "sample_name" +SAMPLE_TABLE_INDEX_KEY = "sample_table_index" from sqlalchemy import Select, and_, delete, select from sqlalchemy.exc import IntegrityError, NoResultFound from sqlalchemy.orm import Session @@ -86,7 +87,7 @@ def get( tag: str = DEFAULT_TAG, raw: bool = True, with_id: bool = False, - ) -> Union[peppy.Project, dict, None]: + ) -> Union[peprs.Project, dict, None]: """ Retrieve project from database by specifying namespace, name and tag @@ -95,7 +96,7 @@ def get( :param tag: tag (or version) of the project. :param raw: retrieve unprocessed (raw) PEP dict. :param with_id: retrieve project with id [default: False] - :return: peppy.Project object with found project or dict with unprocessed + :return: peprs.Project object with found project or dict with unprocessed PEP elements: { name: str description: str @@ -133,13 +134,13 @@ def get( project_value = { CONFIG_KEY: found_prj.config, SAMPLE_RAW_DICT_KEY: sample_list, - SUBSAMPLE_RAW_LIST_KEY: subsample_list, + SUBSAMPLE_RAW_DICT_KEY: subsample_list, } if raw: return project_value else: - project_obj = peppy.Project().from_dict(project_value) + project_obj = peprs.Project.from_dict(project_value) return project_obj else: @@ -224,13 +225,13 @@ def get_by_rp( self, registry_path: str, raw: bool = False, - ) -> Union[peppy.Project, dict, None]: + ) -> Union[peprs.Project, dict, None]: """ Retrieve project from database by specifying project registry_path :param registry_path: project registry_path [e.g. namespace/name:tag] :param raw: retrieve unprocessed (raw) PEP dict. - :return: peppy.Project object with found project or dict with unprocessed + :return: peprs.Project object with found project or dict with unprocessed PEP elements: { name: str description: str @@ -296,7 +297,7 @@ def delete_by_rp( def create( self, - project: Union[peppy.Project, dict], + project: Union[peprs.Project, dict], namespace: str, name: str = None, tag: str = DEFAULT_TAG, @@ -312,14 +313,8 @@ def create( Project with the key, that already exists won't be uploaded(but case, when argument update is set True) - :param peppy.Project project: Project object that has to be uploaded to the DB - danger zone: - optionally, project can be a dictionary with PEP elements - ({ - _config: dict, - _sample_dict: Union[list, dict], - _subsample_list: list - }) + :param project: peprs.Project object or dict with PEP elements + ({config: dict, samples: list, subsamples: list}) :param namespace: namespace of the project (Default: 'other') :param name: name of the project (Default: name is taken from the project object) :param tag: tag (or version) of the project. @@ -332,8 +327,8 @@ def create( :param description: description of the project :return: None """ - if isinstance(project, peppy.Project): - proj_dict = project.to_dict(extended=True, orient="records") + if isinstance(project, peprs.Project): + proj_dict = project.to_dict(raw=True, by_sample=True) elif isinstance(project, dict): # verify if the dictionary has all necessary elements. # samples should be always presented as list of dicts (orient="records")) @@ -343,11 +338,14 @@ def create( proj_dict = ProjectDict(**project).model_dump(by_alias=True) else: raise PEPDatabaseAgentError( - "Project has to be peppy.Project object or dictionary with PEP elements" + "Project has to be peprs.Project object or dictionary with PEP elements" ) if not description: - description = project.get(description, "") + if isinstance(project, peprs.Project): + description = project.description or "" + else: + description = proj_dict.get(CONFIG_KEY, {}).get(DESCRIPTION_KEY, "") proj_dict[CONFIG_KEY][DESCRIPTION_KEY] = description namespace = namespace.lower() @@ -442,8 +440,8 @@ def create( ), ) - if proj_dict[SUBSAMPLE_RAW_LIST_KEY]: - subsamples = proj_dict[SUBSAMPLE_RAW_LIST_KEY] + if proj_dict.get(SUBSAMPLE_RAW_DICT_KEY): + subsamples = proj_dict[SUBSAMPLE_RAW_DICT_KEY] self._add_subsamples_to_project(new_prj, subsamples) with Session(self._sa_engine) as session: @@ -553,9 +551,9 @@ def _overwrite( sample_table_index=project_dict[CONFIG_KEY].get(SAMPLE_TABLE_INDEX_KEY), ) - if project_dict[SUBSAMPLE_RAW_LIST_KEY]: + if project_dict.get(SUBSAMPLE_RAW_DICT_KEY): self._add_subsamples_to_project( - found_prj, project_dict[SUBSAMPLE_RAW_LIST_KEY] + found_prj, project_dict[SUBSAMPLE_RAW_DICT_KEY] ) session.commit() @@ -579,7 +577,7 @@ def update( :param update_dict: dict with update key->values. Dict structure: { - project: Optional[peppy.Project] + project: Optional[peprs.Project] is_private: Optional[bool] tag: Optional[str] name: Optional[str] @@ -603,11 +601,11 @@ def update( else: if "project" in update_dict: project_dict = update_dict.pop("project").to_dict( - extended=True, orient="records" + raw=True, by_sample=True ) update_dict["config"] = project_dict[CONFIG_KEY] update_dict["samples"] = project_dict[SAMPLE_RAW_DICT_KEY] - update_dict["subsamples"] = project_dict[SUBSAMPLE_RAW_LIST_KEY] + update_dict["subsamples"] = project_dict.get(SUBSAMPLE_RAW_DICT_KEY, []) update_values = UpdateItems(**update_dict) @@ -1174,7 +1172,8 @@ def get_samples( ).get(SAMPLE_RAW_DICT_KEY) return ( self.get(namespace=namespace, name=name, tag=tag, raw=False, with_id=with_ids) - .sample_table.replace({np.nan: None}) + .to_pandas() + .replace({np.nan: None}) .to_dict(orient="records") ) @@ -1233,7 +1232,7 @@ def get_project_from_history( history_id: int, raw: bool = True, with_id: bool = False, - ) -> Union[dict, peppy.Project]: + ) -> Union[dict, peprs.Project]: """ Get project sample history annotation by providing namespace, name, and tag @@ -1319,13 +1318,13 @@ def get_project_from_history( return { CONFIG_KEY: project_config or project_mapping.config, SAMPLE_RAW_DICT_KEY: ordered_samples_list, - SUBSAMPLE_RAW_LIST_KEY: self.get_subsamples(namespace, name, tag), + SUBSAMPLE_RAW_DICT_KEY: self.get_subsamples(namespace, name, tag), } - return peppy.Project.from_dict( + return peprs.Project.from_dict( pep_dictionary={ CONFIG_KEY: project_config or project_mapping.config, SAMPLE_RAW_DICT_KEY: ordered_samples_list, - SUBSAMPLE_RAW_LIST_KEY: self.get_subsamples(namespace, name, tag), + SUBSAMPLE_RAW_DICT_KEY: self.get_subsamples(namespace, name, tag), } ) @@ -1440,7 +1439,7 @@ def restore( with_id=True, ) self.update( - update_dict={"project": peppy.Project.from_dict(restore_project)}, + update_dict={"project": peprs.Project.from_dict(restore_project)}, namespace=namespace, name=name, tag=tag, diff --git a/pepdbagent/modules/sample.py b/pepdbagent/modules/sample.py index 90e216bf..c9d12bf9 100644 --- a/pepdbagent/modules/sample.py +++ b/pepdbagent/modules/sample.py @@ -2,8 +2,7 @@ import logging from typing import Union -import peppy -from peppy.const import SAMPLE_TABLE_INDEX_KEY +import peprs from sqlalchemy import and_, select from sqlalchemy.orm import Session from sqlalchemy.orm.attributes import flag_modified @@ -37,7 +36,7 @@ def get( sample_name: str, tag: str = DEFAULT_TAG, raw: bool = True, - ) -> Union[peppy.Sample, dict, None]: + ) -> Union[peprs.Sample, dict, None]: """ Retrieve sample from the database using namespace, name, tag, and sample_name @@ -45,15 +44,8 @@ def get( :param name: name of the project (Default: name is taken from the project object) :param tag: tag (or version) of the project. :param sample_name: sample_name of the sample - :param raw: return raw dict or peppy.Sample object [Default: True] - :return: peppy.Project object with found project or dict with unprocessed - PEP elements: { - name: str - description: str - _config: dict - _sample_dict: dict - _subsample_dict: dict - } + :param raw: return raw dict or peprs.Sample object [Default: True] + :return: peprs.Sample object or raw dict """ statement_sample = select(Samples).where( and_( @@ -83,13 +75,10 @@ def get( if result: if not raw: config = session.execute(project_config_statement).one_or_none()[0] - project = peppy.Project().from_dict( + project = peprs.Project.from_dict( pep_dictionary={ - "name": name, - "description": config.get("description"), - "_config": config, - "_sample_dict": [result.sample], - "_subsample_dict": None, + "config": config, + "samples": [result.sample], } ) return project.samples[0] @@ -155,11 +144,11 @@ def update( sample_mapping.sample.update(update_dict) try: sample_mapping.sample_name = sample_mapping.sample[ - project_mapping.config.get(SAMPLE_TABLE_INDEX_KEY, "sample_name") + project_mapping.config.get("sample_table_index", "sample_name") ] except KeyError: raise KeyError( - f"Sample index key {project_mapping.config.get(SAMPLE_TABLE_INDEX_KEY, 'sample_name')} not found in sample dict" + f"Sample index key {project_mapping.config.get('sample_table_index', 'sample_name')} not found in sample dict" ) # This line needed due to: https://github.com/sqlalchemy/sqlalchemy/issues/5218 @@ -206,11 +195,11 @@ def add( project_mapping = session.scalar(project_statement) try: sample_name = sample_dict[ - project_mapping.config.get(SAMPLE_TABLE_INDEX_KEY, "sample_name") + project_mapping.config.get("sample_table_index", "sample_name") ] except KeyError: raise KeyError( - f"Sample index key {project_mapping.config.get(SAMPLE_TABLE_INDEX_KEY, 'sample_name')} not found in sample dict" + f"Sample index key {project_mapping.config.get('sample_table_index', 'sample_name')} not found in sample dict" ) statement = select(Samples).where( and_(Samples.project_id == project_mapping.id, Samples.sample_name == sample_name) diff --git a/pepdbagent/modules/view.py b/pepdbagent/modules/view.py index 8704c97a..5dd79ea8 100644 --- a/pepdbagent/modules/view.py +++ b/pepdbagent/modules/view.py @@ -3,7 +3,7 @@ import logging from typing import List, Union -import peppy +import peprs from sqlalchemy import and_, delete, select from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session @@ -44,7 +44,7 @@ def get( tag: str = DEFAULT_TAG, view_name: str = None, raw: bool = True, - ) -> Union[peppy.Project, dict, None]: + ) -> Union[peprs.Project, dict, None]: """ Retrieve view of the project from the database. View is a subset of the samples in the project. e.g. bed-db project has all the samples in bedbase, @@ -55,14 +55,7 @@ def get( :param tag: tag of the project (Default: tag is taken from the project object) :param view_name: name of the view :param raw: retrieve unprocessed (raw) PEP dict. [Default: True] - :return: peppy.Project object with found project or dict with unprocessed - PEP elements: { - name: str - description: str - _config: dict - _sample_dict: dict - _subsample_dict: dict - } + :return: peprs.Project object or raw dict """ _LOGGER.debug(f"Get view {view_name} from {namespace}/{name}:{tag}") view_statement = select(Views).where( @@ -80,11 +73,11 @@ def get( ) samples = [sample.sample.sample for sample in view.samples] config = view.project_mapping.config - sub_project_dict = {"_config": config, "_sample_dict": samples, "_subsample_dict": None} + sub_project_dict = {"config": config, "samples": samples} if raw: return sub_project_dict else: - return peppy.Project.from_dict(sub_project_dict) + return peprs.Project.from_dict(sub_project_dict) def get_annotation( self, namespace: str, name: str, tag: str = DEFAULT_TAG, view_name: str = None @@ -349,7 +342,7 @@ def remove_sample( def get_snap_view( self, namespace: str, name: str, tag: str, sample_name_list: List[str], raw: bool = False - ) -> Union[peppy.Project, dict]: + ) -> Union[peprs.Project, dict]: """ Get a snap view of the project. Snap view is a view of the project with only the samples in the list. This view won't be saved in the database. @@ -359,7 +352,7 @@ def get_snap_view( :param tag: tag of the project :param sample_name_list: list of sample names e.g. ["sample1", "sample2"] :param raw: retrieve unprocessed (raw) PEP dict. - :return: peppy.Project object + :return: peprs.Project object """ _LOGGER.debug(f"Creating snap view for {namespace}/{name}:{tag}") project_statement = select(Projects).where( @@ -390,10 +383,10 @@ def get_snap_view( config = project.config if raw: - return {"_config": config, "_sample_dict": samples, "_subsample_dict": None} + return {"config": config, "samples": samples} else: - return peppy.Project.from_dict( - {"_config": config, "_sample_dict": samples, "_subsample_dict": None} + return peprs.Project.from_dict( + {"config": config, "samples": samples} ) def get_views_annotation( diff --git a/pepdbagent/utils.py b/pepdbagent/utils.py index fc966842..a15809ad 100644 --- a/pepdbagent/utils.py +++ b/pepdbagent/utils.py @@ -6,7 +6,7 @@ from typing import List, Tuple, Union import ubiquerg -from peppy.const import SAMPLE_RAW_DICT_KEY +from peprs.const import SAMPLE_RAW_DICT_KEY from pepdbagent.exceptions import RegistryPathError diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt index 786a29d8..e6a85bc0 100644 --- a/requirements/requirements-all.txt +++ b/requirements/requirements-all.txt @@ -1,6 +1,6 @@ sqlalchemy>=2.0.0 logmuse>=0.2.7 -peppy>=0.40.6 +peprs>=0.1.2 ubiquerg>=0.6.2 coloredlogs>=15.0.1 pytest-mock diff --git a/tests/data/namespace1/amendments1/project_config.yaml b/tests/data/namespace1/amendments1/project_config.yaml index 79da9ab1..86c8fd83 100644 --- a/tests/data/namespace1/amendments1/project_config.yaml +++ b/tests/data/namespace1/amendments1/project_config.yaml @@ -7,7 +7,6 @@ sample_modifiers: attributes: [file_path] sources: source1: /data/lab/project/{organism}_{time}h.fastq - source2: /path/from/collaborator/weirdNamingScheme_{external_id}.fastq project_modifiers: amend: newLib: diff --git a/tests/data/namespace2/derive/project_config.yaml b/tests/data/namespace2/derive/project_config.yaml index 445929de..2e23feb7 100644 --- a/tests/data/namespace2/derive/project_config.yaml +++ b/tests/data/namespace2/derive/project_config.yaml @@ -7,4 +7,4 @@ sample_modifiers: attributes: [file_path] sources: source1: $HOME/data/lab/project/{organism}_{time}h.fastq - source2: /path/from/collaborator/weirdNamingScheme_{external_id}.fastq + diff --git a/tests/data/namespace3/piface/project_config.yaml b/tests/data/namespace3/piface/project_config.yaml index f808189d..7326eaa2 100644 --- a/tests/data/namespace3/piface/project_config.yaml +++ b/tests/data/namespace3/piface/project_config.yaml @@ -11,7 +11,7 @@ looper: sample_modifiers: append: attr: "val" - pipeline_interfaces: ["pipeline_interface1_sample.yaml", "pipeline_interface2_sample.yaml"] + pipeline_interfaces: "pipeline_interface1_sample.yaml" derive: attributes: [read1, read2] sources: diff --git a/tests/data/namespace3/remove/project_config.yaml b/tests/data/namespace3/remove/project_config.yaml index 7821eba6..8a789eb5 100644 --- a/tests/data/namespace3/remove/project_config.yaml +++ b/tests/data/namespace3/remove/project_config.yaml @@ -7,6 +7,5 @@ sample_modifiers: attributes: [file_path] sources: source1: /data/lab/project/{organism}_{time}h.fastq - source2: /path/from/collaborator/weirdNamingScheme_{external_id}.fastq remove: - protocol diff --git a/tests/data/private_test/amendments1/project_config.yaml b/tests/data/private_test/amendments1/project_config.yaml index 79da9ab1..86c8fd83 100644 --- a/tests/data/private_test/amendments1/project_config.yaml +++ b/tests/data/private_test/amendments1/project_config.yaml @@ -7,7 +7,6 @@ sample_modifiers: attributes: [file_path] sources: source1: /data/lab/project/{organism}_{time}h.fastq - source2: /path/from/collaborator/weirdNamingScheme_{external_id}.fastq project_modifiers: amend: newLib: diff --git a/tests/data/private_test/derive/project_config.yaml b/tests/data/private_test/derive/project_config.yaml index 445929de..6f24c702 100644 --- a/tests/data/private_test/derive/project_config.yaml +++ b/tests/data/private_test/derive/project_config.yaml @@ -7,4 +7,3 @@ sample_modifiers: attributes: [file_path] sources: source1: $HOME/data/lab/project/{organism}_{time}h.fastq - source2: /path/from/collaborator/weirdNamingScheme_{external_id}.fastq diff --git a/tests/data/private_test/remove/project_config.yaml b/tests/data/private_test/remove/project_config.yaml index 7821eba6..8a789eb5 100644 --- a/tests/data/private_test/remove/project_config.yaml +++ b/tests/data/private_test/remove/project_config.yaml @@ -7,6 +7,5 @@ sample_modifiers: attributes: [file_path] sources: source1: /data/lab/project/{organism}_{time}h.fastq - source2: /path/from/collaborator/weirdNamingScheme_{external_id}.fastq remove: - protocol diff --git a/tests/test_project.py b/tests/test_project.py index 426f0260..c79198a1 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -1,5 +1,5 @@ import numpy as np -import peppy +import peprs import pytest from pepdbagent.exceptions import ProjectNotFoundError @@ -18,15 +18,15 @@ class TestProject: def test_create_project(self): with PEPDBAgentContextManager(add_data=False) as agent: - prj = peppy.Project(list_of_available_peps()["namespace3"]["subtables"]) + prj = peprs.Project(list_of_available_peps()["namespace3"]["subtables"]) agent.project.create(prj, namespace="test", name="imply", overwrite=False) assert True def test_create_project_from_dict(self): with PEPDBAgentContextManager(add_data=False) as agent: - prj = peppy.Project(list_of_available_peps()["namespace3"]["subtables"]) + prj = peprs.Project(list_of_available_peps()["namespace3"]["subtables"]) agent.project.create( - prj.to_dict(extended=True, orient="records"), + prj.to_dict(raw=True, by_sample=True), namespace="test", name="imply", overwrite=True, @@ -47,9 +47,16 @@ def test_create_project_from_dict(self): ) def test_get_project(self, namespace, name): with PEPDBAgentContextManager(add_data=True) as agent: - kk = agent.project.get(namespace=namespace, name=name, tag="default", raw=False) - ff = peppy.Project(get_path_to_example_file(namespace, name)) - assert kk == ff + kk = agent.project.get(namespace=namespace, name=name, tag="default", raw=True) + ff = peprs.Project(get_path_to_example_file(namespace, name)).to_dict( + raw=True, by_sample=True + ) + # pepdbagent always sets the registry name on the stored config, + # overriding any name in the file (which may be empty or different). + ff["config"]["name"] = name + assert kk["config"] == ff["config"] + assert kk["samples"] == ff["samples"] + assert kk.get("subsamples", []) == ff.get("subsamples", []) @pytest.mark.parametrize( "namespace, name", @@ -65,10 +72,11 @@ def test_get_config(self, namespace, name): name=name, tag="default", ) - ff = peppy.Project(get_path_to_example_file(namespace, name)) - ff["_original_config"]["description"] = description - ff["_original_config"]["name"] = name - assert kk == ff["_original_config"] + ff = peprs.Project(get_path_to_example_file(namespace, name)) + expected_config = ff.config.copy() + expected_config["description"] = description + expected_config["name"] = name + assert kk == expected_config @pytest.mark.parametrize( "namespace, name", @@ -83,11 +91,11 @@ def test_get_subsamples(self, namespace, name): name=name, tag="default", ) - orgiginal_prj = peppy.Project(get_path_to_example_file(namespace, name)) + orgiginal_prj = peprs.Project(get_path_to_example_file(namespace, name)) assert ( prj_subtables - == orgiginal_prj.to_dict(extended=True, orient="records")["_subsample_list"] + == orgiginal_prj.to_dict(raw=True, by_sample=True)["subsamples"] ) @pytest.mark.parametrize( @@ -101,11 +109,11 @@ def test_get_samples_raw(self, namespace, name): prj_samples = agent.project.get_samples( namespace=namespace, name=name, tag="default", raw=True ) - orgiginal_prj = peppy.Project(get_path_to_example_file(namespace, name)) + orgiginal_prj = peprs.Project(get_path_to_example_file(namespace, name)) assert ( prj_samples - == orgiginal_prj.to_dict(extended=True, orient="records")["_sample_dict"] + == orgiginal_prj.to_dict(raw=True, by_sample=True)["samples"] ) @pytest.mark.parametrize( @@ -122,12 +130,24 @@ def test_get_samples_processed(self, namespace, name): tag="default", raw=False, ) - orgiginal_prj = peppy.Project(get_path_to_example_file(namespace, name)) - - assert prj_samples == orgiginal_prj.sample_table.replace({np.nan: None}).to_dict( - orient="records" + orgiginal_prj = peprs.Project(get_path_to_example_file(namespace, name)) + expected = ( + orgiginal_prj.to_pandas().replace({np.nan: None}).to_dict(orient="records") ) + # Normalize numpy arrays (used for subsample list columns) to plain lists + # so dict equality works without raising "truth value is ambiguous". + def _normalize(samples): + return [ + { + k: (v.tolist() if isinstance(v, np.ndarray) else v) + for k, v in s.items() + } + for s in samples + ] + + assert _normalize(prj_samples) == _normalize(expected) + @pytest.mark.parametrize( "namespace, name,tag", [ diff --git a/tests/test_project_history.py b/tests/test_project_history.py index 332e1ca4..20d00aeb 100644 --- a/tests/test_project_history.py +++ b/tests/test_project_history.py @@ -1,4 +1,4 @@ -import peppy +import peprs import pytest from pepdbagent.const import PEPHUB_SAMPLE_ID_KEY @@ -26,10 +26,10 @@ def test_get_add_history_all_annotation(self, namespace, name, sample_name): with PEPDBAgentContextManager(add_data=True) as agent: prj = agent.project.get(namespace, name, tag="default", with_id=True) - prj["_sample_dict"][0]["sample_name"] = "new_sample_name" + prj["samples"][0]["sample_name"] = "new_sample_name" - del prj["_sample_dict"][1] - del prj["_sample_dict"][2] + del prj["samples"][1] + del prj["samples"][2] new_sample1 = { "sample_name": "new_sample", "protocol": "new_protocol", @@ -41,14 +41,14 @@ def test_get_add_history_all_annotation(self, namespace, name, sample_name): PEPHUB_SAMPLE_ID_KEY: None, } - prj["_sample_dict"].append(new_sample1.copy()) - prj["_sample_dict"].append(new_sample2.copy()) + prj["samples"].append(new_sample1.copy()) + prj["samples"].append(new_sample2.copy()) agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) project_history = agent.project.get_history(namespace, name, tag="default") @@ -66,10 +66,10 @@ def test_get_add_history_all_project(self, namespace, name, sample_name): prj_init = agent.project.get(namespace, name, tag="default", raw=False) prj = agent.project.get(namespace, name, tag="default", with_id=True) - # prj["_sample_dict"][0]["sample_name"] = "new_sample_name" + # prj["samples"][0]["sample_name"] = "new_sample_name" - del prj["_sample_dict"][1] - del prj["_sample_dict"][2] + del prj["samples"][1] + del prj["samples"][2] new_sample1 = { "sample_name": "new_sample", "protocol": "new_protocol", @@ -81,14 +81,14 @@ def test_get_add_history_all_project(self, namespace, name, sample_name): PEPHUB_SAMPLE_ID_KEY: None, } - prj["_sample_dict"].append(new_sample1.copy()) - prj["_sample_dict"].append(new_sample2.copy()) + prj["samples"].append(new_sample1.copy()) + prj["samples"].append(new_sample2.copy()) agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) history_prj = agent.project.get_project_from_history( @@ -106,13 +106,13 @@ def test_get_history_multiple_changes(self, namespace, name, sample_name): with PEPDBAgentContextManager(add_data=True) as agent: prj = agent.project.get(namespace, name, tag="default", with_id=True) - del prj["_sample_dict"][1] + del prj["samples"][1] agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) prj = agent.project.get(namespace, name, tag="default", with_id=True) @@ -122,13 +122,13 @@ def test_get_history_multiple_changes(self, namespace, name, sample_name): "protocol": "new_protocol", PEPHUB_SAMPLE_ID_KEY: None, } - prj["_sample_dict"].append(new_sample1.copy()) + prj["samples"].append(new_sample1.copy()) agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) history = agent.project.get_history(namespace, name, tag="default") @@ -144,13 +144,13 @@ def test_get_history_multiple_changes(self, namespace, name, sample_name): def test_get_project_incorrect_history_id(self, namespace, name, sample_name): with PEPDBAgentContextManager(add_data=True) as agent: prj = agent.project.get(namespace, name, tag="default", with_id=True) - del prj["_sample_dict"][1] + del prj["samples"][1] agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) with pytest.raises(HistoryNotFoundError): @@ -179,13 +179,13 @@ def test_delete_all_history(self, namespace, name, sample_name): with PEPDBAgentContextManager(add_data=True) as agent: prj = agent.project.get(namespace, name, tag="default", with_id=True) - del prj["_sample_dict"][1] + del prj["samples"][1] agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) prj = agent.project.get(namespace, name, tag="default", with_id=True) @@ -195,13 +195,13 @@ def test_delete_all_history(self, namespace, name, sample_name): "protocol": "new_protocol", PEPHUB_SAMPLE_ID_KEY: None, } - prj["_sample_dict"].append(new_sample1.copy()) + prj["samples"].append(new_sample1.copy()) agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) history = agent.project.get_history(namespace, name, tag="default") @@ -226,13 +226,13 @@ def test_delete_one_history(self, namespace, name, sample_name): with PEPDBAgentContextManager(add_data=True) as agent: prj = agent.project.get(namespace, name, tag="default", with_id=True) - del prj["_sample_dict"][1] + del prj["samples"][1] agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) prj = agent.project.get(namespace, name, tag="default", with_id=True) @@ -242,13 +242,13 @@ def test_delete_one_history(self, namespace, name, sample_name): "protocol": "new_protocol", PEPHUB_SAMPLE_ID_KEY: None, } - prj["_sample_dict"].append(new_sample1.copy()) + prj["samples"].append(new_sample1.copy()) agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) history = agent.project.get_history(namespace, name, tag="default") @@ -273,13 +273,13 @@ def test_restore_project(self, namespace, name, sample_name): prj_org = agent.project.get(namespace, name, tag="default", with_id=False) prj = agent.project.get(namespace, name, tag="default", with_id=True) - del prj["_sample_dict"][1] + del prj["samples"][1] agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) prj = agent.project.get(namespace, name, tag="default", with_id=True) @@ -289,13 +289,13 @@ def test_restore_project(self, namespace, name, sample_name): "protocol": "new_protocol", PEPHUB_SAMPLE_ID_KEY: None, } - prj["_sample_dict"].append(new_sample1.copy()) + prj["samples"].append(new_sample1.copy()) agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) agent.project.restore(namespace, name, tag="default", history_id=1) diff --git a/tests/test_samples.py b/tests/test_samples.py index e8a68620..15acc73d 100644 --- a/tests/test_samples.py +++ b/tests/test_samples.py @@ -1,4 +1,4 @@ -import peppy +import peprs import pytest from pepdbagent.exceptions import SampleNotFoundError @@ -20,7 +20,7 @@ class TestSamples: def test_retrieve_one_sample(self, namespace, name, sample_name): with PEPDBAgentContextManager(add_data=True) as agent: one_sample = agent.sample.get(namespace, name, sample_name, raw=False) - assert isinstance(one_sample, peppy.Sample) + assert isinstance(one_sample, peprs.Sample) assert one_sample.sample_name == sample_name @pytest.mark.parametrize( @@ -44,7 +44,7 @@ def test_retrieve_raw_sample(self, namespace, name, sample_name): def test_retrieve_sample_with_modified_sample_id(self, namespace, name, sample_name): with PEPDBAgentContextManager(add_data=True) as agent: one_sample = agent.sample.get(namespace, name, sample_name, raw=False) - assert isinstance(one_sample, peppy.Sample) + assert isinstance(one_sample, peprs.Sample) assert one_sample.sample_id == "frog_1" @pytest.mark.parametrize( @@ -154,7 +154,7 @@ def test_project_timestamp_was_changed(self, namespace, name, sample_name): def test_delete_sample(self, namespace, name, sample_name): with PEPDBAgentContextManager(add_data=True) as agent: one_sample = agent.sample.get(namespace, name, sample_name, raw=False) - assert isinstance(one_sample, peppy.Sample) + assert isinstance(one_sample, peprs.Sample) agent.sample.delete(namespace, name, tag="default", sample_name=sample_name) @@ -203,7 +203,9 @@ def test_add_sample(self, namespace, name, tag, sample_dict): ) def test_overwrite_sample(self, namespace, name, tag, sample_dict): with PEPDBAgentContextManager(add_data=True) as agent: - assert agent.project.get(namespace, name, raw=False).get_sample("pig_0h").time == "0" + # peprs/polars infers numeric values from the sample table, so the + # original time is loaded as int (not string). + assert agent.project.get(namespace, name, raw=False).get_sample("pig_0h").time == 0 agent.sample.add(namespace, name, tag, sample_dict, overwrite=True) assert ( diff --git a/tests/test_updates.py b/tests/test_updates.py index 628772e1..3c2a15f5 100644 --- a/tests/test_updates.py +++ b/tests/test_updates.py @@ -1,6 +1,5 @@ -import peppy +import peprs import pytest -from peppy.exceptions import IllegalStateException from pepdbagent.const import PEPHUB_SAMPLE_ID_KEY from pepdbagent.exceptions import ProjectDuplicatedSampleGUIDsError, SampleTableUpdateError @@ -248,16 +247,16 @@ def test_project_can_have_2_sample_names(self, namespace, name): """ with PEPDBAgentContextManager(add_data=True) as agent: new_prj = agent.project.get(namespace=namespace, name=name, raw=False, with_id=True) - prj_dict = new_prj.to_dict(extended=True, orient="records") + prj_dict = new_prj.to_dict(raw=True, by_sample=True) - prj_dict["_sample_dict"].append( + prj_dict["samples"].append( { "file": "data/frog23_data.txt", "protocol": "anySample3Type", "sample_name": "frog_2", } ) - prj_dict["_sample_dict"].append( + prj_dict["samples"].append( { "file": "data/frog23_data.txt4", "protocol": "anySample3Type4", @@ -270,12 +269,12 @@ def test_project_can_have_2_sample_names(self, namespace, name): namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj_dict)}, + update_dict={"project": peprs.Project.from_dict(prj_dict)}, ) prj = agent.project.get(namespace=namespace, name=name, raw=True) - assert len(prj["_sample_dict"]) == 4 + assert len(prj["samples"]) == 4 @pytest.mark.skipif( @@ -304,23 +303,23 @@ def test_update_whole_project_with_id(self, namespace, name): PEPHUB_SAMPLE_ID_KEY: None, } - prj["_sample_dict"].append(new_sample.copy()) - prj["_sample_dict"][0]["sample_name"] = "new_sample_name2" - del prj["_sample_dict"][1] + prj["samples"].append(new_sample.copy()) + prj["samples"][0]["sample_name"] = "new_sample_name2" + del prj["samples"][1] agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) del new_sample[PEPHUB_SAMPLE_ID_KEY] - peppy_prj["_sample_dict"].append(new_sample.copy()) # add sample without id - peppy_prj["_sample_dict"][0]["sample_name"] = "new_sample_name2" # modify sample - del peppy_prj["_sample_dict"][1] # delete sample + peppy_prj["samples"].append(new_sample.copy()) # add sample without id + peppy_prj["samples"][0]["sample_name"] = "new_sample_name2" # modify sample + del peppy_prj["samples"][1] # delete sample - assert peppy.Project.from_dict(peppy_prj) == agent.project.get( + assert peprs.Project.from_dict(peppy_prj) == agent.project.get( namespace=namespace, name=name, raw=False ) @@ -342,19 +341,19 @@ def test_insert_new_row(self, namespace, name): PEPHUB_SAMPLE_ID_KEY: None, } - prj["_sample_dict"].append(new_sample.copy()) + prj["samples"].append(new_sample.copy()) agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) del new_sample[PEPHUB_SAMPLE_ID_KEY] - peppy_prj["_sample_dict"].append(new_sample.copy()) # add sample without id + peppy_prj["samples"].append(new_sample.copy()) # add sample without id - assert peppy.Project.from_dict(peppy_prj) == agent.project.get( + assert peprs.Project.from_dict(peppy_prj) == agent.project.get( namespace=namespace, name=name, raw=False ) @@ -381,22 +380,22 @@ def test_insert_new_multiple_rows(self, namespace, name): PEPHUB_SAMPLE_ID_KEY: None, } - prj["_sample_dict"].append(new_sample1.copy()) - prj["_sample_dict"].append(new_sample2.copy()) + prj["samples"].append(new_sample1.copy()) + prj["samples"].append(new_sample2.copy()) agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) del new_sample1[PEPHUB_SAMPLE_ID_KEY] del new_sample2[PEPHUB_SAMPLE_ID_KEY] - peppy_prj["_sample_dict"].append(new_sample1.copy()) # add sample without id - peppy_prj["_sample_dict"].append(new_sample2.copy()) # add sample without id + peppy_prj["samples"].append(new_sample1.copy()) # add sample without id + peppy_prj["samples"].append(new_sample2.copy()) # add sample without id - assert peppy.Project.from_dict(peppy_prj) == agent.project.get( + assert peprs.Project.from_dict(peppy_prj) == agent.project.get( namespace=namespace, name=name, raw=False ) @@ -407,7 +406,11 @@ def test_insert_new_multiple_rows(self, namespace, name): ], ) def test_insert_new_multiple_rows_duplicated_samples(self, namespace, name): + """PEP 2.1.0 allows duplicate sample names, so this should succeed.""" with PEPDBAgentContextManager(add_data=True) as agent: + original_count = len( + agent.project.get(namespace=namespace, name=name, raw=True)["samples"] + ) prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) new_sample1 = { @@ -421,16 +424,17 @@ def test_insert_new_multiple_rows_duplicated_samples(self, namespace, name): PEPHUB_SAMPLE_ID_KEY: None, } - prj["_sample_dict"].append(new_sample1.copy()) - prj["_sample_dict"].append(new_sample2.copy()) + prj["samples"].append(new_sample1.copy()) + prj["samples"].append(new_sample2.copy()) - with pytest.raises(IllegalStateException): - agent.project.update( - namespace=namespace, - name=name, - tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, - ) + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"project": peprs.Project.from_dict(prj)}, + ) + updated = agent.project.get(namespace=namespace, name=name, raw=True) + assert len(updated["samples"]) == original_count + 2 @pytest.mark.parametrize( "namespace, name", @@ -444,20 +448,20 @@ def test_delete_multiple_rows(self, namespace, name): peppy_prj = agent.project.get(namespace=namespace, name=name, raw=True) prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) - del prj["_sample_dict"][1] - del prj["_sample_dict"][2] + del prj["samples"][1] + del prj["samples"][2] agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) - del peppy_prj["_sample_dict"][1] # delete sample - del peppy_prj["_sample_dict"][2] # delete sample + del peppy_prj["samples"][1] # delete sample + del peppy_prj["samples"][2] # delete sample - assert peppy.Project.from_dict(peppy_prj) == agent.project.get( + assert peprs.Project.from_dict(peppy_prj) == agent.project.get( namespace=namespace, name=name, raw=False ) @@ -473,18 +477,18 @@ def test_modify_one_row(self, namespace, name): peppy_prj = agent.project.get(namespace=namespace, name=name, raw=True) prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) - prj["_sample_dict"][0]["sample_name"] = "new_sample_name2" + prj["samples"][0]["sample_name"] = "new_sample_name2" agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) - peppy_prj["_sample_dict"][0]["sample_name"] = "new_sample_name2" # modify sample + peppy_prj["samples"][0]["sample_name"] = "new_sample_name2" # modify sample - assert peppy.Project.from_dict(peppy_prj) == agent.project.get( + assert peprs.Project.from_dict(peppy_prj) == agent.project.get( namespace=namespace, name=name, raw=False ) @@ -500,20 +504,20 @@ def test_modify_multiple_rows(self, namespace, name): peppy_prj = agent.project.get(namespace=namespace, name=name, raw=True) prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) - prj["_sample_dict"][0]["sample_name"] = "new_sample_name2" - prj["_sample_dict"][1]["sample_name"] = "new_sample_name3" + prj["samples"][0]["sample_name"] = "new_sample_name2" + prj["samples"][1]["sample_name"] = "new_sample_name3" agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) - peppy_prj["_sample_dict"][0]["sample_name"] = "new_sample_name2" # modify sample - peppy_prj["_sample_dict"][1]["sample_name"] = "new_sample_name3" # modify sample + peppy_prj["samples"][0]["sample_name"] = "new_sample_name2" # modify sample + peppy_prj["samples"][1]["sample_name"] = "new_sample_name3" # modify sample - assert peppy.Project.from_dict(peppy_prj) == agent.project.get( + assert peprs.Project.from_dict(peppy_prj) == agent.project.get( namespace=namespace, name=name, raw=False ) @@ -535,19 +539,19 @@ def test_add_new_first_sample(self, namespace, name): PEPHUB_SAMPLE_ID_KEY: None, } - prj["_sample_dict"].insert(0, new_sample.copy()) + prj["samples"].insert(0, new_sample.copy()) agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) del new_sample[PEPHUB_SAMPLE_ID_KEY] - peppy_prj["_sample_dict"].insert(0, new_sample.copy()) # add sample without id + peppy_prj["samples"].insert(0, new_sample.copy()) # add sample without id - assert peppy.Project.from_dict(peppy_prj) == agent.project.get( + assert peprs.Project.from_dict(peppy_prj) == agent.project.get( namespace=namespace, name=name, raw=False ) @@ -563,26 +567,26 @@ def test_change_sample_order(self, namespace, name): peppy_prj = agent.project.get(namespace=namespace, name=name, raw=True) prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) - sample1 = prj["_sample_dict"][0].copy() - sample2 = prj["_sample_dict"][1].copy() + sample1 = prj["samples"][0].copy() + sample2 = prj["samples"][1].copy() - prj["_sample_dict"][0] = sample2 - prj["_sample_dict"][1] = sample1 + prj["samples"][0] = sample2 + prj["samples"][1] = sample1 agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) - peppy_prj["_sample_dict"][0] = sample2 - peppy_prj["_sample_dict"][1] = sample1 + peppy_prj["samples"][0] = sample2 + peppy_prj["samples"][1] = sample1 - del peppy_prj["_sample_dict"][0][PEPHUB_SAMPLE_ID_KEY] - del peppy_prj["_sample_dict"][1][PEPHUB_SAMPLE_ID_KEY] + del peppy_prj["samples"][0][PEPHUB_SAMPLE_ID_KEY] + del peppy_prj["samples"][1][PEPHUB_SAMPLE_ID_KEY] - assert peppy.Project.from_dict(peppy_prj) == agent.project.get( + assert peprs.Project.from_dict(peppy_prj) == agent.project.get( namespace=namespace, name=name, raw=False ) @@ -597,7 +601,7 @@ def test_update_porject_without_ids(self, namespace, name): with PEPDBAgentContextManager(add_data=True) as agent: prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=False) - prj["_sample_dict"][0]["sample_name"] = "new_sample_name2" + prj["samples"][0]["sample_name"] = "new_sample_name2" with pytest.raises(SampleTableUpdateError): @@ -605,7 +609,7 @@ def test_update_porject_without_ids(self, namespace, name): namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) @pytest.mark.parametrize( @@ -617,12 +621,12 @@ def test_update_porject_without_ids(self, namespace, name): def test_update_project_with_duplicated_sample_guids(self, namespace, name): with PEPDBAgentContextManager(add_data=True) as agent: new_prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) - new_prj["_sample_dict"].append(new_prj["_sample_dict"][0]) + new_prj["samples"].append(new_prj["samples"][0]) with pytest.raises(ProjectDuplicatedSampleGUIDsError): agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(new_prj)}, + update_dict={"project": peprs.Project.from_dict(new_prj)}, ) diff --git a/tests/utils.py b/tests/utils.py index fc9cc897..56f878a9 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,7 +1,7 @@ import os import warnings -import peppy +import peprs import yaml from sqlalchemy.exc import OperationalError @@ -106,7 +106,11 @@ def _insert_data(self): else: private = False for name, path in item.items(): - prj = peppy.Project(path) + try: + prj = peprs.Project(path) + except Exception as e: + warnings.warn(f"Skipping {namespace}/{name}: {e}") + continue pepdb_con.project.create( namespace=namespace, name=name, From 6f4bae0d76d7fe4d1f51fca79fc7eef165b25daf Mon Sep 17 00:00:00 2001 From: Oleksandr <41573628+khoroshevskyi@users.noreply.github.com> Date: Mon, 29 Jun 2026 14:05:58 -0400 Subject: [PATCH 02/13] Update tests/utils.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- tests/utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/utils.py b/tests/utils.py index 56f878a9..f440124f 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -109,8 +109,9 @@ def _insert_data(self): try: prj = peprs.Project(path) except Exception as e: - warnings.warn(f"Skipping {namespace}/{name}: {e}") - continue + raise RuntimeError( + f"Failed to load test project {namespace}/{name} from {path}" + ) from e pepdb_con.project.create( namespace=namespace, name=name, From 8f4972dffb52340dd00b5338e5e3adaf82965d17 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Mon, 29 Jun 2026 14:16:39 -0400 Subject: [PATCH 03/13] PR fixes and lint --- docs/changelog.md | 5 +++++ pepdbagent/_version.py | 2 +- pepdbagent/const.py | 3 +++ pepdbagent/modules/project.py | 9 ++++----- pepdbagent/modules/sample.py | 6 +++--- pepdbagent/modules/view.py | 4 +--- requirements/requirements-all.txt | 2 +- tests/test_project.py | 19 ++++--------------- 8 files changed, 22 insertions(+), 28 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 96b6a035..f97b6952 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,11 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format. + +## [0.13.0] -- 2026-07-01 +- Removed peppy from dependencies, using peprs instead +- Bug fixes and speed improvements in project update, validate and upload methods + ## [0.12.4] -- 2026-01-26 - Added project search by tag in annotation module - Updated github actions workflows diff --git a/pepdbagent/_version.py b/pepdbagent/_version.py index 6dd4954d..f23a6b39 100644 --- a/pepdbagent/_version.py +++ b/pepdbagent/_version.py @@ -1 +1 @@ -__version__ = "0.12.4" +__version__ = "0.13.0" diff --git a/pepdbagent/const.py b/pepdbagent/const.py index 2bf7c931..72b6170a 100644 --- a/pepdbagent/const.py +++ b/pepdbagent/const.py @@ -24,3 +24,6 @@ DEFAULT_TAG_VERSION = "1.0.0" LATEST_SCHEMA_VERSION = "latest" + +SAMPLE_NAME_ATTR = "sample_name" +SAMPLE_TABLE_INDEX_KEY = "sample_table_index" diff --git a/pepdbagent/modules/project.py b/pepdbagent/modules/project.py index 62590411..8fc75dc7 100644 --- a/pepdbagent/modules/project.py +++ b/pepdbagent/modules/project.py @@ -11,8 +11,7 @@ SUBSAMPLE_RAW_DICT_KEY, ) -SAMPLE_NAME_ATTR = "sample_name" -SAMPLE_TABLE_INDEX_KEY = "sample_table_index" + from sqlalchemy import Select, and_, delete, select from sqlalchemy.exc import IntegrityError, NoResultFound from sqlalchemy.orm import Session @@ -26,6 +25,8 @@ PEPHUB_SAMPLE_ID_KEY, PKG_NAME, LATEST_SCHEMA_VERSION, + SAMPLE_NAME_ATTR, + SAMPLE_TABLE_INDEX_KEY, ) from pepdbagent.db_utils import ( BaseEngine, @@ -600,9 +601,7 @@ def update( update_values = update_dict else: if "project" in update_dict: - project_dict = update_dict.pop("project").to_dict( - raw=True, by_sample=True - ) + project_dict = update_dict.pop("project").to_dict(raw=True, by_sample=True) update_dict["config"] = project_dict[CONFIG_KEY] update_dict["samples"] = project_dict[SAMPLE_RAW_DICT_KEY] update_dict["subsamples"] = project_dict.get(SUBSAMPLE_RAW_DICT_KEY, []) diff --git a/pepdbagent/modules/sample.py b/pepdbagent/modules/sample.py index c9d12bf9..56c1339f 100644 --- a/pepdbagent/modules/sample.py +++ b/pepdbagent/modules/sample.py @@ -7,7 +7,7 @@ from sqlalchemy.orm import Session from sqlalchemy.orm.attributes import flag_modified -from pepdbagent.const import DEFAULT_TAG, PKG_NAME +from pepdbagent.const import DEFAULT_TAG, PKG_NAME, SAMPLE_TABLE_INDEX_KEY from pepdbagent.db_utils import BaseEngine, Projects, Samples from pepdbagent.exceptions import SampleAlreadyExistsError, SampleNotFoundError from pepdbagent.utils import generate_guid, order_samples @@ -144,7 +144,7 @@ def update( sample_mapping.sample.update(update_dict) try: sample_mapping.sample_name = sample_mapping.sample[ - project_mapping.config.get("sample_table_index", "sample_name") + project_mapping.config.get(SAMPLE_TABLE_INDEX_KEY, "sample_name") ] except KeyError: raise KeyError( @@ -195,7 +195,7 @@ def add( project_mapping = session.scalar(project_statement) try: sample_name = sample_dict[ - project_mapping.config.get("sample_table_index", "sample_name") + project_mapping.config.get(SAMPLE_TABLE_INDEX_KEY, "sample_name") ] except KeyError: raise KeyError( diff --git a/pepdbagent/modules/view.py b/pepdbagent/modules/view.py index 5dd79ea8..673e1722 100644 --- a/pepdbagent/modules/view.py +++ b/pepdbagent/modules/view.py @@ -385,9 +385,7 @@ def get_snap_view( if raw: return {"config": config, "samples": samples} else: - return peprs.Project.from_dict( - {"config": config, "samples": samples} - ) + return peprs.Project.from_dict({"config": config, "samples": samples}) def get_views_annotation( self, namespace: str, name: str, tag: str = DEFAULT_TAG diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt index e6a85bc0..6f9874e3 100644 --- a/requirements/requirements-all.txt +++ b/requirements/requirements-all.txt @@ -1,6 +1,6 @@ sqlalchemy>=2.0.0 logmuse>=0.2.7 -peprs>=0.1.2 +peprs>=0.2.0 ubiquerg>=0.6.2 coloredlogs>=15.0.1 pytest-mock diff --git a/tests/test_project.py b/tests/test_project.py index c79198a1..86ea816d 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -93,10 +93,7 @@ def test_get_subsamples(self, namespace, name): ) orgiginal_prj = peprs.Project(get_path_to_example_file(namespace, name)) - assert ( - prj_subtables - == orgiginal_prj.to_dict(raw=True, by_sample=True)["subsamples"] - ) + assert prj_subtables == orgiginal_prj.to_dict(raw=True, by_sample=True)["subsamples"] @pytest.mark.parametrize( "namespace, name", @@ -111,10 +108,7 @@ def test_get_samples_raw(self, namespace, name): ) orgiginal_prj = peprs.Project(get_path_to_example_file(namespace, name)) - assert ( - prj_samples - == orgiginal_prj.to_dict(raw=True, by_sample=True)["samples"] - ) + assert prj_samples == orgiginal_prj.to_dict(raw=True, by_sample=True)["samples"] @pytest.mark.parametrize( "namespace, name", @@ -131,18 +125,13 @@ def test_get_samples_processed(self, namespace, name): raw=False, ) orgiginal_prj = peprs.Project(get_path_to_example_file(namespace, name)) - expected = ( - orgiginal_prj.to_pandas().replace({np.nan: None}).to_dict(orient="records") - ) + expected = orgiginal_prj.to_pandas().replace({np.nan: None}).to_dict(orient="records") # Normalize numpy arrays (used for subsample list columns) to plain lists # so dict equality works without raising "truth value is ambiguous". def _normalize(samples): return [ - { - k: (v.tolist() if isinstance(v, np.ndarray) else v) - for k, v in s.items() - } + {k: (v.tolist() if isinstance(v, np.ndarray) else v) for k, v in s.items()} for s in samples ] From d01bfec73f77752c2f7509c9c8c3f9c772cd8b7a Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Mon, 29 Jun 2026 16:58:55 -0400 Subject: [PATCH 04/13] Modernization steps 1-3 --- .github/workflows/black.yml | 8 ++- .github/workflows/cli-coverage.yml | 10 ++- .github/workflows/pytest.yml | 9 +-- .github/workflows/python-publish.yml | 15 ++--- MANIFEST.in | 5 -- Makefile | 14 ---- pepdbagent/__init__.py | 8 ++- pepdbagent/_version.py | 1 - pepdbagent/alembic/env.py | 4 +- .../44cb1e7a80de_initial_migration.py | 2 +- .../8a037f13b4e5_upgrading_schemas.py | 2 +- pepdbagent/db_utils.py | 3 +- pepdbagent/modules/annotation.py | 6 +- pepdbagent/modules/namespace.py | 6 +- pepdbagent/modules/project.py | 4 +- pepdbagent/modules/schema.py | 19 ++++-- pepdbagent/modules/view.py | 8 ++- pyproject.toml | 67 +++++++++++++++++-- requirements/requirements-all.txt | 10 --- requirements/requirements-dev.txt | 5 -- setup.py | 64 ------------------ tests/test_project.py | 6 +- tests/test_schema.py | 4 +- tests/test_updates.py | 5 +- 24 files changed, 131 insertions(+), 154 deletions(-) delete mode 100644 MANIFEST.in delete mode 100644 Makefile delete mode 100644 pepdbagent/_version.py delete mode 100644 requirements/requirements-all.txt delete mode 100644 requirements/requirements-dev.txt delete mode 100644 setup.py diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index 79ec0bb8..093667e9 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -1,6 +1,6 @@ name: Lint -on: [pull_request] +on: [push, pull_request] jobs: lint: @@ -8,4 +8,8 @@ jobs: steps: - uses: actions/checkout@v6 - uses: actions/setup-python@v6 - - uses: psf/black@stable + with: + python-version: "3.12" + - run: pip install ruff + - run: ruff check . + - run: ruff format --check . diff --git a/.github/workflows/cli-coverage.yml b/.github/workflows/cli-coverage.yml index cbd7cfe5..dd6b92c4 100644 --- a/.github/workflows/cli-coverage.yml +++ b/.github/workflows/cli-coverage.yml @@ -8,7 +8,7 @@ jobs: cli-coverage-report: strategy: matrix: - python-version: [ "3.11" ] + python-version: [ "3.10" ] os: [ ubuntu-latest ] # can't use macOS when using service containers or container jobs r: [ release ] runs-on: ${{ matrix.os }} @@ -31,10 +31,8 @@ jobs: with: python-version: '3.10' - - name: Install dev dependancies - run: if [ -f requirements/requirements-dev.txt ]; then pip install -r requirements/requirements-dev.txt; fi - - - run: pip install . + - name: Install package with test dependencies + run: pip install ".[test]" - name: Run tests run: coverage run -m pytest @@ -49,4 +47,4 @@ jobs: SMOKESHOW_GITHUB_CONTEXT: coverage SMOKESHOW_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SMOKESHOW_GITHUB_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} - SMOKESHOW_AUTH_KEY: ${{ secrets.SMOKESHOW_AUTH_KEY }} \ No newline at end of file + SMOKESHOW_AUTH_KEY: ${{ secrets.SMOKESHOW_AUTH_KEY }} diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 7771b349..cfd1c91f 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -10,7 +10,7 @@ jobs: pytest: strategy: matrix: - python-version: ["3.9", "3.13"] + python-version: ["3.10", "3.13"] os: [ubuntu-latest] # can't use macOS when using service containers or container jobs r: [release] runs-on: ${{ matrix.os }} @@ -33,11 +33,8 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install dev dependencies - run: if [ -f requirements/requirements-dev.txt ]; then pip install -r requirements/requirements-dev.txt; fi - - - name: Install package - run: python -m pip install . + - name: Install package with test dependencies + run: python -m pip install ".[test]" - name: Run pytest tests run: pytest tests -x -vv diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 1aa6b6aa..6368928a 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -9,19 +9,18 @@ jobs: name: upload release to PyPI runs-on: ubuntu-latest permissions: + contents: read id-token: write + steps: - uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v6 with: python-version: '3.x' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - - name: Build and publish - run: | - python setup.py sdist bdist_wheel + - name: Install build dependencies + run: python -m pip install --upgrade pip build + - name: Build package + run: python -m build - name: Publish package distributions to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 \ No newline at end of file + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 814aee1e..00000000 --- a/MANIFEST.in +++ /dev/null @@ -1,5 +0,0 @@ -include LICENSE.txt -include requirements/* -include README.md -include pepdbagent/alembic/* -include pepdbagent/alembic/versions/* \ No newline at end of file diff --git a/Makefile b/Makefile deleted file mode 100644 index 067ac8af..00000000 --- a/Makefile +++ /dev/null @@ -1,14 +0,0 @@ -lint: - # black should be last in the list, as it lint the code. Tests can fail if order will be different - flake8 && isort . && black . - -run-coverage: - coverage run -m pytest - -html-report: - coverage html - -open-coverage: - cd htmlcov && google-chrome index.html - -coverage: run-coverage html-report open-coverage \ No newline at end of file diff --git a/pepdbagent/__init__.py b/pepdbagent/__init__.py index a8b67f24..e332236a 100644 --- a/pepdbagent/__init__.py +++ b/pepdbagent/__init__.py @@ -1,13 +1,15 @@ -"""Package-level data""" +"""Package-level data.""" + +from importlib.metadata import version import coloredlogs import logmuse -from pepdbagent._version import __version__ from pepdbagent.pepdbagent import PEPDatabaseAgent -__all__ = ["__version__", "PEPDatabaseAgent"] +__version__ = version("pepdbagent") +__all__ = ["__version__", "PEPDatabaseAgent"] _LOGGER = logmuse.init_logger("pepdbagent") coloredlogs.install( diff --git a/pepdbagent/_version.py b/pepdbagent/_version.py deleted file mode 100644 index f23a6b39..00000000 --- a/pepdbagent/_version.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "0.13.0" diff --git a/pepdbagent/alembic/env.py b/pepdbagent/alembic/env.py index 26101269..1e979b03 100644 --- a/pepdbagent/alembic/env.py +++ b/pepdbagent/alembic/env.py @@ -1,9 +1,7 @@ from logging.config import fileConfig -from sqlalchemy import engine_from_config -from sqlalchemy import pool - from alembic import context +from sqlalchemy import engine_from_config, pool # this is the Alembic Config object, which provides # access to the values within the .ini file in use. diff --git a/pepdbagent/alembic/versions/44cb1e7a80de_initial_migration.py b/pepdbagent/alembic/versions/44cb1e7a80de_initial_migration.py index 2ec118cd..64e61acb 100644 --- a/pepdbagent/alembic/versions/44cb1e7a80de_initial_migration.py +++ b/pepdbagent/alembic/versions/44cb1e7a80de_initial_migration.py @@ -8,8 +8,8 @@ from typing import Sequence, Union -from alembic import op import sqlalchemy as sa +from alembic import op from sqlalchemy.dialects import postgresql from sqlalchemy.schema import FetchedValue diff --git a/pepdbagent/alembic/versions/8a037f13b4e5_upgrading_schemas.py b/pepdbagent/alembic/versions/8a037f13b4e5_upgrading_schemas.py index 5e3e19c7..752d86c1 100644 --- a/pepdbagent/alembic/versions/8a037f13b4e5_upgrading_schemas.py +++ b/pepdbagent/alembic/versions/8a037f13b4e5_upgrading_schemas.py @@ -8,8 +8,8 @@ from typing import Sequence, Union -from alembic import op import sqlalchemy as sa +from alembic import op from sqlalchemy.dialects import postgresql from sqlalchemy.schema import FetchedValue diff --git a/pepdbagent/db_utils.py b/pepdbagent/db_utils.py index 610c574f..c7a42e37 100644 --- a/pepdbagent/db_utils.py +++ b/pepdbagent/db_utils.py @@ -1,12 +1,11 @@ import datetime import enum import logging -from typing import List, Optional import os +from typing import List, Optional from alembic import command from alembic.config import Config - from sqlalchemy import ( TIMESTAMP, BigInteger, diff --git a/pepdbagent/modules/annotation.py b/pepdbagent/modules/annotation.py index 7159ece7..99f1e40f 100644 --- a/pepdbagent/modules/annotation.py +++ b/pepdbagent/modules/annotation.py @@ -17,7 +17,11 @@ from pepdbagent.db_utils import BaseEngine, Projects from pepdbagent.exceptions import FilterError, ProjectNotFoundError, RegistryPathError from pepdbagent.models import AnnotationList, AnnotationModel, RegistryPath -from pepdbagent.utils import convert_date_string_to_date, registry_path_converter, tuple_converter +from pepdbagent.utils import ( + convert_date_string_to_date, + registry_path_converter, + tuple_converter, +) _LOGGER = logging.getLogger(PKG_NAME) diff --git a/pepdbagent/modules/namespace.py b/pepdbagent/modules/namespace.py index 7766fb61..44dbc441 100644 --- a/pepdbagent/modules/namespace.py +++ b/pepdbagent/modules/namespace.py @@ -3,12 +3,12 @@ from datetime import datetime, timedelta from typing import List, Tuple, Union -from sqlalchemy import distinct, func, or_, select, delete +from sqlalchemy import delete, distinct, func, or_, select from sqlalchemy.orm import Session from sqlalchemy.sql.selectable import Select from pepdbagent.const import DEFAULT_LIMIT, DEFAULT_LIMIT_INFO, DEFAULT_OFFSET, PKG_NAME -from pepdbagent.db_utils import BaseEngine, Projects, User, TarNamespace +from pepdbagent.db_utils import BaseEngine, Projects, TarNamespace, User from pepdbagent.exceptions import NamespaceNotFoundError from pepdbagent.models import ( ListOfNamespaceInfo, @@ -16,9 +16,9 @@ NamespaceInfo, NamespaceList, NamespaceStats, + PaginationResult, TarNamespaceModel, TarNamespaceModelReturn, - PaginationResult, ) from pepdbagent.utils import tuple_converter diff --git a/pepdbagent/modules/project.py b/pepdbagent/modules/project.py index 8fc75dc7..0f92344f 100644 --- a/pepdbagent/modules/project.py +++ b/pepdbagent/modules/project.py @@ -10,8 +10,6 @@ SAMPLE_RAW_DICT_KEY, SUBSAMPLE_RAW_DICT_KEY, ) - - from sqlalchemy import Select, and_, delete, select from sqlalchemy.exc import IntegrityError, NoResultFound from sqlalchemy.orm import Session @@ -20,11 +18,11 @@ from pepdbagent.const import ( DEFAULT_TAG, DESCRIPTION_KEY, + LATEST_SCHEMA_VERSION, MAX_HISTORY_SAMPLES_NUMBER, NAME_KEY, PEPHUB_SAMPLE_ID_KEY, PKG_NAME, - LATEST_SCHEMA_VERSION, SAMPLE_NAME_ATTR, SAMPLE_TABLE_INDEX_KEY, ) diff --git a/pepdbagent/modules/schema.py b/pepdbagent/modules/schema.py index fe216ad9..4362ed67 100644 --- a/pepdbagent/modules/schema.py +++ b/pepdbagent/modules/schema.py @@ -1,27 +1,32 @@ import logging - -from typing import List, Optional, Union, Dict +from typing import Dict, List, Optional, Union from sqlalchemy import Select, and_, func, or_, select from sqlalchemy.orm import Session from sqlalchemy.orm.attributes import flag_modified -from pepdbagent.const import PKG_NAME, DEFAULT_TAG_VERSION, LATEST_SCHEMA_VERSION -from pepdbagent.db_utils import BaseEngine, SchemaRecords, SchemaTags, SchemaVersions, User +from pepdbagent.const import DEFAULT_TAG_VERSION, LATEST_SCHEMA_VERSION, PKG_NAME +from pepdbagent.db_utils import ( + BaseEngine, + SchemaRecords, + SchemaTags, + SchemaVersions, + User, +) from pepdbagent.exceptions import ( SchemaAlreadyExistsError, - SchemaVersionDoesNotExistError, SchemaDoesNotExistError, SchemaTagAlreadyExistsError, SchemaTagDoesNotExistError, SchemaVersionAlreadyExistsError, + SchemaVersionDoesNotExistError, ) from pepdbagent.models import ( + PaginationResult, SchemaRecordAnnotation, + SchemaSearchResult, SchemaVersionAnnotation, - PaginationResult, SchemaVersionSearchResult, - SchemaSearchResult, UpdateSchemaRecordFields, UpdateSchemaVersionFields, ) diff --git a/pepdbagent/modules/view.py b/pepdbagent/modules/view.py index 673e1722..3c6a07da 100644 --- a/pepdbagent/modules/view.py +++ b/pepdbagent/modules/view.py @@ -9,7 +9,13 @@ from sqlalchemy.orm import Session from pepdbagent.const import DEFAULT_TAG, PKG_NAME -from pepdbagent.db_utils import BaseEngine, Projects, Samples, Views, ViewSampleAssociation +from pepdbagent.db_utils import ( + BaseEngine, + Projects, + Samples, + Views, + ViewSampleAssociation, +) from pepdbagent.exceptions import ( ProjectNotFoundError, SampleAlreadyInView, diff --git a/pyproject.toml b/pyproject.toml index 9348158b..27ac75cf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,4 +1,63 @@ -[tool.black] -line-length = 99 -target-version = ['py38', 'py311'] -include = '\.pyi?$' +[project] +name = "pepdbagent" +version = "0.13.0" +description = "A python-based database manager for portable encapsulated projects" +readme = "README.md" +license = "BSD-2-Clause" +requires-python = ">=3.10" +authors = [ + { name = "Oleksandr Khoroshevskyi" }, +] +keywords = ["project", "metadata", "bioinformatics", "database"] +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Scientific/Engineering :: Bio-Informatics", +] +dependencies = [ + "sqlalchemy>=2.0.0", + "logmuse>=0.2.7", + "peprs>=0.2.0", + "ubiquerg>=0.6.2", + "coloredlogs>=15.0.1", + "pydantic>=2.0", + "psycopg>=3.1.15", + "numpy>=1.24.4", + "alembic>=1.15.1", +] + +[project.urls] +Homepage = "https://github.com/pepkit/pepdbagent/" + +[project.optional-dependencies] +test = [ + "pytest", + "pytest-mock", + "python-dotenv", + "coverage", + "smokeshow", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.pytest.ini_options] +addopts = "-rfE" +testpaths = ["tests"] + +[tool.ruff] +line-length = 88 + +[tool.ruff.lint] +select = ["E", "F", "I"] +ignore = ["F403", "F405", "E501"] + +[tool.ruff.lint.isort] +known-first-party = ["pepdbagent"] + +[tool.ruff.lint.per-file-ignores] +"pepdbagent/alembic/env.py" = ["E402"] diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt deleted file mode 100644 index 6f9874e3..00000000 --- a/requirements/requirements-all.txt +++ /dev/null @@ -1,10 +0,0 @@ -sqlalchemy>=2.0.0 -logmuse>=0.2.7 -peprs>=0.2.0 -ubiquerg>=0.6.2 -coloredlogs>=15.0.1 -pytest-mock -pydantic>=2.0 -psycopg>=3.1.15 -numpy>=1.24.4 -alembic>=1.15.1 diff --git a/requirements/requirements-dev.txt b/requirements/requirements-dev.txt deleted file mode 100644 index 9b1f849a..00000000 --- a/requirements/requirements-dev.txt +++ /dev/null @@ -1,5 +0,0 @@ -black -pytest -python-dotenv -coverage -smokeshow \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index f53601e0..00000000 --- a/setup.py +++ /dev/null @@ -1,64 +0,0 @@ -import os -import sys - -from setuptools import find_packages, setup - -PACKAGE_NAME = "pepdbagent" - -# Ordinary dependencies -DEPENDENCIES = [] -with open("requirements/requirements-all.txt", "r") as reqs_file: - for line in reqs_file: - if not line.strip(): - continue - # DEPENDENCIES.append(line.split("=")[0].rstrip("<>")) - DEPENDENCIES.append(line) - -# Additional keyword arguments for setup(). -extra = {"install_requires": DEPENDENCIES} - - -# Additional files to include with package -def get_static(name, condition=None): - static = [ - os.path.join(name, f) - for f in os.listdir(os.path.join(os.path.dirname(os.path.realpath(__file__)), name)) - ] - if condition is None: - return static - else: - return [i for i in filter(lambda x: eval(condition), static)] - - -with open(f"{PACKAGE_NAME}/_version.py", "r") as versionfile: - version = versionfile.readline().split()[-1].strip("\"'\n") - -with open("README.md") as f: - long_description = f.read() - -setup( - name=PACKAGE_NAME, - packages=find_packages(), - version=version, - description="A python-based database manager for portable encapsulated projects", - long_description=long_description, - long_description_content_type="text/markdown", - classifiers=[ - "Development Status :: 4 - Beta", - "License :: OSI Approved :: BSD License", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Topic :: Scientific/Engineering :: Bio-Informatics", - ], - keywords="project, metadata, bioinformatics, database", - url="https://github.com/pepkit/pepdbagent/", - author="Oleksandr Khoroshevskyi ", - license="BSD2", - include_package_data=True, - # tests_require=(["pytest"]), - setup_requires=(["pytest-runner"] if {"test", "pytest", "ptr"} & set(sys.argv) else []), - **extra, -) diff --git a/tests/test_project.py b/tests/test_project.py index 86ea816d..8df174fd 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -4,7 +4,11 @@ from pepdbagent.exceptions import ProjectNotFoundError -from .utils import PEPDBAgentContextManager, get_path_to_example_file, list_of_available_peps +from .utils import ( + PEPDBAgentContextManager, + get_path_to_example_file, + list_of_available_peps, +) @pytest.mark.skipif( diff --git a/tests/test_schema.py b/tests/test_schema.py index d4b8ecb6..0202fc9e 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -1,8 +1,8 @@ import pytest -from .utils import PEPDBAgentContextManager +from pepdbagent.models import UpdateSchemaRecordFields, UpdateSchemaVersionFields -from pepdbagent.models import UpdateSchemaVersionFields, UpdateSchemaRecordFields +from .utils import PEPDBAgentContextManager DEFAULT_SCHEMA_VERSION = "1.0.0" diff --git a/tests/test_updates.py b/tests/test_updates.py index 3c2a15f5..a3fe0dd4 100644 --- a/tests/test_updates.py +++ b/tests/test_updates.py @@ -2,7 +2,10 @@ import pytest from pepdbagent.const import PEPHUB_SAMPLE_ID_KEY -from pepdbagent.exceptions import ProjectDuplicatedSampleGUIDsError, SampleTableUpdateError +from pepdbagent.exceptions import ( + ProjectDuplicatedSampleGUIDsError, + SampleTableUpdateError, +) from .utils import PEPDBAgentContextManager From 3d26179bb94c7d0887f728c2a6eff9d1a31da34e Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Mon, 29 Jun 2026 17:37:56 -0400 Subject: [PATCH 05/13] Modernization step 4 --- pepdbagent/const.py | 34 ++++---- pepdbagent/db_utils.py | 97 +++++++++++---------- pepdbagent/exceptions.py | 52 ++++++------ pepdbagent/models.py | 139 +++++++++++++++---------------- pepdbagent/modules/annotation.py | 70 ++++++++-------- pepdbagent/modules/namespace.py | 7 +- pepdbagent/modules/project.py | 42 +++++----- pepdbagent/modules/sample.py | 3 +- pepdbagent/modules/schema.py | 15 ++-- pepdbagent/modules/user.py | 3 +- pepdbagent/modules/view.py | 15 ++-- pepdbagent/pepdbagent.py | 20 ++--- pepdbagent/utils.py | 9 +- 13 files changed, 248 insertions(+), 258 deletions(-) diff --git a/pepdbagent/const.py b/pepdbagent/const.py index 72b6170a..52ef7f40 100644 --- a/pepdbagent/const.py +++ b/pepdbagent/const.py @@ -1,29 +1,29 @@ -PKG_NAME = "pepdbagent" +PKG_NAME: str = "pepdbagent" -DEFAULT_NAMESPACE = "_" -DEFAULT_TAG = "default" +DEFAULT_NAMESPACE: str = "_" +DEFAULT_TAG: str = "default" -DESCRIPTION_KEY = "description" -NAME_KEY = "name" +DESCRIPTION_KEY: str = "description" +NAME_KEY: str = "name" -DEFAULT_OFFSET = 0 -DEFAULT_LIMIT = 100 +DEFAULT_OFFSET: int = 0 +DEFAULT_LIMIT: int = 100 # db_dialects -POSTGRES_DIALECT = "postgresql+psycopg" +POSTGRES_DIALECT: str = "postgresql+psycopg" -DEFAULT_LIMIT_INFO = 5 +DEFAULT_LIMIT_INFO: int = 5 -SUBMISSION_DATE_KEY = "submission_date" -LAST_UPDATE_DATE_KEY = "last_update_date" +SUBMISSION_DATE_KEY: str = "submission_date" +LAST_UPDATE_DATE_KEY: str = "last_update_date" -PEPHUB_SAMPLE_ID_KEY = "ph_id" +PEPHUB_SAMPLE_ID_KEY: str = "ph_id" -MAX_HISTORY_SAMPLES_NUMBER = 2000 +MAX_HISTORY_SAMPLES_NUMBER: int = 2000 -DEFAULT_TAG_VERSION = "1.0.0" -LATEST_SCHEMA_VERSION = "latest" +DEFAULT_TAG_VERSION: str = "1.0.0" +LATEST_SCHEMA_VERSION: str = "latest" -SAMPLE_NAME_ATTR = "sample_name" -SAMPLE_TABLE_INDEX_KEY = "sample_table_index" +SAMPLE_NAME_ATTR: str = "sample_name" +SAMPLE_TABLE_INDEX_KEY: str = "sample_table_index" diff --git a/pepdbagent/db_utils.py b/pepdbagent/db_utils.py index c7a42e37..07a55fe5 100644 --- a/pepdbagent/db_utils.py +++ b/pepdbagent/db_utils.py @@ -2,7 +2,6 @@ import enum import logging import os -from typing import List, Optional from alembic import command from alembic.config import Config @@ -20,7 +19,7 @@ select, ) from sqlalchemy.dialects.postgresql import JSON -from sqlalchemy.engine import URL, create_engine +from sqlalchemy.engine import URL, Engine, create_engine from sqlalchemy.exc import ProgrammingError from sqlalchemy.ext.compiler import compiles from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column, relationship @@ -36,12 +35,12 @@ class BIGSERIAL(BigInteger): @compiles(BIGSERIAL, POSTGRES_DIALECT) -def compile_bigserial_pg(type_, compiler, **kw): +def compile_bigserial_pg(type_, compiler, **kw) -> str: return "BIGSERIAL" @compiles(JSON, POSTGRES_DIALECT) -def compile_jsonb_pg(type_, compiler, **kw): +def compile_jsonb_pg(type_, compiler, **kw) -> str: return "JSON" @@ -50,7 +49,7 @@ class Base(DeclarativeBase): @event.listens_for(Base.metadata, "after_create") -def receive_after_create(target, connection, tables, **kw): +def receive_after_create(target, connection, tables, **kw) -> None: """ listen for the 'after_create' event """ @@ -64,7 +63,7 @@ def receive_after_create(target, connection, tables, **kw): # return context.get_current_parameters()["config"]["description"] -def deliver_update_date(context): +def deliver_update_date(context) -> datetime.datetime: return datetime.datetime.now(datetime.timezone.utc) @@ -80,37 +79,37 @@ class Projects(Base): name: Mapped[str] = mapped_column() tag: Mapped[str] = mapped_column() digest: Mapped[str] = mapped_column(String(32)) - description: Mapped[Optional[str]] + description: Mapped[str | None] config: Mapped[dict] = mapped_column(JSON, server_default=FetchedValue()) private: Mapped[bool] number_of_samples: Mapped[int] number_of_stars: Mapped[int] = mapped_column(default=0) submission_date: Mapped[datetime.datetime] - last_update_date: Mapped[Optional[datetime.datetime]] = mapped_column( + last_update_date: Mapped[datetime.datetime | None] = mapped_column( default=deliver_update_date, # onupdate=deliver_update_date, # This field should not be updated, while we are adding project to favorites ) - schema_id: Mapped[Optional[int]] = mapped_column( + schema_id: Mapped[int | None] = mapped_column( ForeignKey("schema_versions.id", ondelete="SET NULL"), nullable=True ) schema_mapping: Mapped["SchemaVersions"] = relationship("SchemaVersions", lazy="joined") - pop: Mapped[Optional[bool]] = mapped_column(default=False) - samples_mapping: Mapped[List["Samples"]] = relationship( + pop: Mapped[bool | None] = mapped_column(default=False) + samples_mapping: Mapped[list["Samples"]] = relationship( back_populates="project_mapping", cascade="all, delete-orphan" ) - subsamples_mapping: Mapped[List["Subsamples"]] = relationship( + subsamples_mapping: Mapped[list["Subsamples"]] = relationship( back_populates="subsample_mapping", cascade="all, delete-orphan" ) - stars_mapping: Mapped[List["Stars"]] = relationship( + stars_mapping: Mapped[list["Stars"]] = relationship( back_populates="project_mapping", cascade="all, delete-orphan" ) - views_mapping: Mapped[List["Views"]] = relationship( + views_mapping: Mapped[list["Views"]] = relationship( back_populates="project_mapping", cascade="all, delete-orphan" ) # Self-referential relationship. The parent project is the one that was forked to create this one. - forked_from_id: Mapped[Optional[int]] = mapped_column( + forked_from_id: Mapped[int | None] = mapped_column( ForeignKey("projects.id", ondelete="SET NULL"), nullable=True ) forked_from_mapping = relationship( @@ -129,7 +128,7 @@ class Projects(Base): namespace_mapping: Mapped["User"] = relationship("User", back_populates="projects_mapping") - history_mapping: Mapped[List["HistoryProjects"]] = relationship( + history_mapping: Mapped[list["HistoryProjects"]] = relationship( back_populates="project_mapping", cascade="all, delete-orphan" ) @@ -147,16 +146,16 @@ class Samples(Base): sample: Mapped[dict] = mapped_column(JSON, server_default=FetchedValue()) project_id = mapped_column(ForeignKey("projects.id", ondelete="CASCADE")) project_mapping: Mapped["Projects"] = relationship(back_populates="samples_mapping") - sample_name: Mapped[Optional[str]] = mapped_column() - guid: Mapped[Optional[str]] = mapped_column(nullable=False, unique=True) + sample_name: Mapped[str | None] = mapped_column() + guid: Mapped[str | None] = mapped_column(nullable=False, unique=True) submission_date: Mapped[datetime.datetime] = mapped_column(default=deliver_update_date) - last_update_date: Mapped[Optional[datetime.datetime]] = mapped_column( + last_update_date: Mapped[datetime.datetime | None] = mapped_column( default=deliver_update_date, onupdate=deliver_update_date, ) - parent_guid: Mapped[Optional[str]] = mapped_column( + parent_guid: Mapped[str | None] = mapped_column( ForeignKey("samples.guid", ondelete="CASCADE"), nullable=True, doc="Parent sample id. Used to create a hierarchy of samples.", @@ -167,7 +166,7 @@ class Samples(Base): ) child_mapping: Mapped["Samples"] = relationship("Samples", back_populates="parent_mapping") - views: Mapped[Optional[List["ViewSampleAssociation"]]] = relationship( + views: Mapped[list["ViewSampleAssociation"] | None] = relationship( back_populates="sample", cascade="all, delete-orphan" ) @@ -196,7 +195,7 @@ class User(Base): id: Mapped[int] = mapped_column(primary_key=True) namespace: Mapped[str] = mapped_column(nullable=False, unique=True) - stars_mapping: Mapped[List["Stars"]] = relationship( + stars_mapping: Mapped[list["Stars"]] = relationship( back_populates="user_mapping", cascade="all, delete-orphan", order_by="Stars.star_date.desc()", @@ -204,10 +203,10 @@ class User(Base): number_of_projects: Mapped[int] = mapped_column(default=0) number_of_schemas: Mapped[int] = mapped_column(default=0) - projects_mapping: Mapped[List["Projects"]] = relationship( + projects_mapping: Mapped[list["Projects"]] = relationship( "Projects", back_populates="namespace_mapping" ) - schemas_mapping: Mapped[List["SchemaRecords"]] = relationship( + schemas_mapping: Mapped[list["SchemaRecords"]] = relationship( "SchemaRecords", back_populates="user_mapping" ) @@ -221,7 +220,7 @@ class Stars(Base): user_id = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), primary_key=True) project_id = mapped_column(ForeignKey("projects.id", ondelete="CASCADE"), primary_key=True) - user_mapping: Mapped[List["User"]] = relationship(back_populates="stars_mapping") + user_mapping: Mapped[list["User"]] = relationship(back_populates="stars_mapping") project_mapping: Mapped["Projects"] = relationship(back_populates="stars_mapping") star_date: Mapped[datetime.datetime] = mapped_column( onupdate=deliver_update_date, default=deliver_update_date @@ -237,12 +236,12 @@ class Views(Base): id: Mapped[int] = mapped_column(primary_key=True) name: Mapped[str] = mapped_column() - description: Mapped[Optional[str]] + description: Mapped[str | None] project_id = mapped_column(ForeignKey("projects.id", ondelete="CASCADE")) project_mapping = relationship("Projects", back_populates="views_mapping") - samples: Mapped[List["ViewSampleAssociation"]] = relationship( + samples: Mapped[list["ViewSampleAssociation"]] = relationship( back_populates="view", cascade="all, delete-orphan" ) @@ -277,7 +276,7 @@ class HistoryProjects(Base): project_mapping: Mapped["Projects"] = relationship( "Projects", back_populates="history_mapping" ) - sample_changes_mapping: Mapped[List["HistorySamples"]] = relationship( + sample_changes_mapping: Mapped[list["HistorySamples"]] = relationship( back_populates="history_project_mapping", cascade="all, delete-orphan" ) @@ -299,7 +298,7 @@ class HistorySamples(Base): id: Mapped[int] = mapped_column(primary_key=True) history_id: Mapped[int] = mapped_column(ForeignKey("project_history.id", ondelete="CASCADE")) guid: Mapped[str] = mapped_column(nullable=False) - parent_guid: Mapped[Optional[str]] = mapped_column(nullable=True) + parent_guid: Mapped[str | None] = mapped_column(nullable=True) sample_json: Mapped[dict] = mapped_column(JSON, server_default=FetchedValue()) change_type: Mapped[UpdateTypes] = mapped_column(Enum(UpdateTypes), nullable=False) @@ -316,15 +315,15 @@ class SchemaRecords(Base): name: Mapped[str] = mapped_column(nullable=False) maintainers: Mapped[str] = mapped_column(nullable=True) lifecycle_stage: Mapped[str] = mapped_column(nullable=True) - description: Mapped[Optional[str]] = mapped_column(nullable=True) + description: Mapped[str | None] = mapped_column(nullable=True) private: Mapped[bool] = mapped_column(default=False) - last_update_date: Mapped[Optional[datetime.datetime]] = mapped_column( + last_update_date: Mapped[datetime.datetime | None] = mapped_column( default=deliver_update_date, onupdate=deliver_update_date ) __table_args__ = (UniqueConstraint("namespace", "name"),) - versions_mapping: Mapped[List["SchemaVersions"]] = relationship( + versions_mapping: Mapped[list["SchemaVersions"]] = relationship( "SchemaVersions", back_populates="schema_mapping", cascade="all, delete-orphan", @@ -341,11 +340,11 @@ class SchemaVersions(Base): version: Mapped[str] = mapped_column(nullable=False) schema_value: Mapped[dict] = mapped_column(JSON, server_default=FetchedValue()) release_date: Mapped[datetime.datetime] = mapped_column(default=deliver_update_date) - last_update_date: Mapped[Optional[datetime.datetime]] = mapped_column( + last_update_date: Mapped[datetime.datetime | None] = mapped_column( default=deliver_update_date, onupdate=deliver_update_date ) - contributors: Mapped[Optional[str]] = mapped_column(nullable=True) - release_notes: Mapped[Optional[str]] = mapped_column(nullable=True) + contributors: Mapped[str | None] = mapped_column(nullable=True) + release_notes: Mapped[str | None] = mapped_column(nullable=True) __table_args__ = (UniqueConstraint("schema_id", "version"),) @@ -353,7 +352,7 @@ class SchemaVersions(Base): "SchemaRecords", back_populates="versions_mapping" ) - tags_mapping: Mapped[List["SchemaTags"]] = relationship( + tags_mapping: Mapped[list["SchemaTags"]] = relationship( "SchemaTags", back_populates="schema_mapping", lazy="joined", cascade="all, delete-orphan" ) @@ -392,9 +391,9 @@ class BedBaseStats(Base): gse: Mapped[str] = mapped_column() gsm: Mapped[str] = mapped_column() sample_name: Mapped[str] = mapped_column(nullable=True) - genome: Mapped[Optional[str]] = mapped_column(nullable=True, default="") - last_update_date: Mapped[Optional[str]] = mapped_column() - submission_date: Mapped[Optional[str]] = mapped_column() + genome: Mapped[str | None] = mapped_column(nullable=True, default="") + last_update_date: Mapped[str | None] = mapped_column() + submission_date: Mapped[str | None] = mapped_column() class BaseEngine: @@ -408,13 +407,13 @@ def __init__( host: str = "localhost", port: int = 5432, database: str = "pep-db", - user: str = None, - password: str = None, + user: str | None = None, + password: str | None = None, drivername: str = POSTGRES_DIALECT, - dsn: str = None, + dsn: str | None = None, echo: bool = False, run_migrations: bool = False, - ): + ) -> None: """ Initialize connection to the pep_db database. You can use The basic connection parameters or libpq connection string. @@ -447,7 +446,7 @@ def __init__( self.create_schema(self._engine) self.check_db_connection() - def create_schema(self, engine=None): + def create_schema(self, engine=None) -> None: """ Create sql schema in the database. @@ -474,17 +473,17 @@ def session_execute(self, statement: Select) -> Result: return query_result @property - def session(self): + def session(self) -> Session: """ :return: started sqlalchemy session """ return self._start_session() @property - def engine(self): + def engine(self) -> Engine: return self._engine - def _start_session(self): + def _start_session(self) -> Session: session = Session(self.engine) try: session.execute(select(Projects).limit(1)) @@ -493,7 +492,7 @@ def _start_session(self): return session - def check_db_connection(self): + def check_db_connection(self) -> None: try: self.session_execute(select(Projects).limit(1)) except ProgrammingError: @@ -511,7 +510,7 @@ def delete_schema(self, engine=None) -> None: Base.metadata.drop_all(engine) return None - def run_db_migration(self, database_url: str): + def run_db_migration(self, database_url: str) -> None: """ Migrate the database to the required version. """ diff --git a/pepdbagent/exceptions.py b/pepdbagent/exceptions.py index 17be6976..5ef92f66 100644 --- a/pepdbagent/exceptions.py +++ b/pepdbagent/exceptions.py @@ -4,37 +4,37 @@ class PEPDatabaseAgentError(Exception): """Base error type for pepdbagent custom errors.""" - def __init__(self, msg): + def __init__(self, msg: str) -> None: super(PEPDatabaseAgentError, self).__init__(msg) class SchemaError(PEPDatabaseAgentError): - def __init__(self): + def __init__(self) -> None: super().__init__("""PEP_db connection error! The schema of connected db is incorrect""") class RegistryPathError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Provided registry path is incorrect. {msg}""") class ProjectNotFoundError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Project does not exist. {msg}""") class ProjectUniqueNameError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""{msg}""") class IncorrectDateFormat(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Incorrect date format was provided. {msg}""") class FilterError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""pepdbagent filter error. {msg}""") @@ -43,7 +43,7 @@ class ProjectNotInFavorites(PEPDatabaseAgentError): Project doesn't exist in favorites """ - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Project is not in favorites list. {msg}""") @@ -52,37 +52,37 @@ class ProjectAlreadyInFavorites(PEPDatabaseAgentError): Project doesn't exist in favorites """ - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Project is already in favorites list. {msg}""") class SampleNotFoundError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Sample does not exist. {msg}""") class SampleTableUpdateError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Sample table update error. {msg}""") class ProjectDuplicatedSampleGUIDsError(SampleTableUpdateError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Project has duplicated sample GUIDs. {msg}""") class SampleAlreadyExistsError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Sample already exists. {msg}""") class ViewNotFoundError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""View does not exist. {msg}""") class SampleNotInViewError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Sample is not in the view. {msg}""") @@ -91,7 +91,7 @@ class SampleAlreadyInView(PEPDatabaseAgentError): Sample is already in the view exception """ - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Sample is already in the view. {msg}""") @@ -100,50 +100,50 @@ class ViewAlreadyExistsError(PEPDatabaseAgentError): View is already in the project exception """ - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""View already in the project. {msg}""") class NamespaceNotFoundError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Project does not exist. {msg}""") class HistoryNotFoundError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""History does not exist. {msg}""") class UserNotFoundError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""User does not exist. {msg}""") class SchemaDoesNotExistError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Schema does not exist. {msg}""") class SchemaAlreadyExistsError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Schema already exists. {msg}""") class SchemaVersionDoesNotExistError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Schema version does not exist. {msg}""") class SchemaVersionAlreadyExistsError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Schema version already exists. {msg}""") class SchemaTagDoesNotExistError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Schema tag does not exist. {msg}""") class SchemaTagAlreadyExistsError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Schema tag already exists. {msg}""") diff --git a/pepdbagent/models.py b/pepdbagent/models.py index 6291dc78..a78542c1 100644 --- a/pepdbagent/models.py +++ b/pepdbagent/models.py @@ -1,6 +1,5 @@ # file with pydantic models import datetime -from typing import Dict, List, Optional, Union from peprs.const import CONFIG_KEY, SAMPLE_RAW_DICT_KEY, SUBSAMPLE_RAW_DICT_KEY from pydantic import BaseModel, ConfigDict, Field, field_validator @@ -14,7 +13,7 @@ class ProjectDict(BaseModel): """ config: dict = Field(alias=CONFIG_KEY) - subsample_list: Optional[Union[list, None]] = Field(alias=SUBSAMPLE_RAW_DICT_KEY) + subsample_list: list | None = Field(alias=SUBSAMPLE_RAW_DICT_KEY) sample_dict: list = Field(alias=SAMPLE_RAW_DICT_KEY) model_config = ConfigDict(populate_by_name=True, extra="forbid") @@ -25,19 +24,19 @@ class AnnotationModel(BaseModel): Project Annotation model. All meta metadata """ - namespace: Optional[str] - name: Optional[str] - tag: Optional[str] - is_private: Optional[bool] - number_of_samples: Optional[int] - description: Optional[str] - last_update_date: Optional[str] - submission_date: Optional[str] - digest: Optional[str] - pep_schema: Optional[str] - pop: Optional[bool] = False - stars_number: Optional[int] = 0 - forked_from: Optional[Union[str, None]] = None + namespace: str | None = None + name: str | None = None + tag: str | None = None + is_private: bool | None = None + number_of_samples: int | None = None + description: str | None = None + last_update_date: str | None = None + submission_date: str | None = None + digest: str | None = None + pep_schema: str | None = None + pop: bool | None = False + stars_number: int | None = 0 + forked_from: str | None = None model_config = ConfigDict( validate_assignment=True, @@ -66,7 +65,7 @@ class AnnotationList(BaseModel): count: int limit: int offset: int - results: List[Union[AnnotationModel, None]] + results: list[AnnotationModel | None] class Namespace(BaseModel): @@ -87,7 +86,7 @@ class NamespaceList(BaseModel): count: int limit: int offset: int - results: List[Namespace] + results: list[Namespace] class UpdateItems(BaseModel): @@ -95,17 +94,17 @@ class UpdateItems(BaseModel): Model used for updating individual items in db """ - name: Optional[str] = None - description: Optional[str] = None - tag: Optional[str] = None - is_private: Optional[bool] = None - pep_schema: Optional[str] = None - digest: Optional[str] = None - config: Optional[dict] = None - samples: Optional[List[dict]] = None - subsamples: Optional[List[List[dict]]] = None - pop: Optional[bool] = None - schema_id: Optional[int] = None + name: str | None = None + description: str | None = None + tag: str | None = None + is_private: bool | None = None + pep_schema: str | None = None + digest: str | None = None + config: dict | None = None + samples: list[dict] | None = None + subsamples: list[list[dict]] | None = None + pop: bool | None = None + schema_id: int | None = None model_config = ConfigDict( arbitrary_types_allowed=True, @@ -113,7 +112,7 @@ class UpdateItems(BaseModel): ) @property - def number_of_samples(self) -> Union[int, None]: + def number_of_samples(self) -> int | None: if self.samples: return len(self.samples) return None @@ -125,16 +124,16 @@ class UpdateModel(BaseModel): Model used for updating individual items and creating sql string in the code """ - config: Optional[dict] = None - name: Optional[str] = None - tag: Optional[str] = None - private: Optional[bool] = Field(alias="is_private", default=None) - digest: Optional[str] = None - number_of_samples: Optional[int] = None - pep_schema: Optional[str] = None - description: Optional[str] = "" - # last_update_date: Optional[datetime.datetime] = datetime.datetime.now(datetime.timezone.utc) - pop: Optional[bool] = False + config: dict | None = None + name: str | None = None + tag: str | None = None + private: bool | None = Field(alias="is_private", default=None) + digest: str | None = None + number_of_samples: int | None = None + pep_schema: str | None = None + description: str | None = "" + # last_update_date: datetime.datetime | None = datetime.datetime.now(datetime.timezone.utc) + pop: bool | None = False @field_validator("tag", "name") def value_must_not_be_empty(cls, v): @@ -163,7 +162,7 @@ class NamespaceInfo(BaseModel): """ namespace_name: str - contact_url: Optional[str] = None + contact_url: str | None = None number_of_projects: int number_of_schemas: int @@ -174,7 +173,7 @@ class ListOfNamespaceInfo(BaseModel): """ pagination: PaginationResult - results: List[NamespaceInfo] + results: list[NamespaceInfo] class ProjectRegistryPath(BaseModel): @@ -193,7 +192,7 @@ class ViewAnnotation(BaseModel): """ name: str - description: Optional[str] = None + description: str | None = None number_of_samples: int = 0 @@ -205,7 +204,7 @@ class ProjectViews(BaseModel): namespace: str name: str tag: str = DEFAULT_TAG - views: List[ViewAnnotation] = [] + views: list[ViewAnnotation] = [] class CreateViewDictModel(BaseModel): @@ -216,13 +215,13 @@ class CreateViewDictModel(BaseModel): project_namespace: str project_name: str project_tag: str - sample_list: List[str] + sample_list: list[str] class RegistryPath(BaseModel): namespace: str name: str - tag: Optional[str] = "default" + tag: str | None = "default" class NamespaceStats(BaseModel): @@ -230,9 +229,9 @@ class NamespaceStats(BaseModel): Namespace stats model """ - namespace: Union[str, None] = None - projects_updated: Dict[str, int] = None - projects_created: Dict[str, int] = None + namespace: str | None = None + projects_updated: dict[str, int] | None = None + projects_created: dict[str, int] | None = None class HistoryChangeModel(BaseModel): @@ -253,7 +252,7 @@ class HistoryAnnotationModel(BaseModel): namespace: str name: str tag: str = DEFAULT_TAG - history: List[HistoryChangeModel] + history: list[HistoryChangeModel] class SchemaVersionAnnotation(BaseModel): @@ -264,9 +263,9 @@ class SchemaVersionAnnotation(BaseModel): namespace: str schema_name: str version: str - contributors: Optional[Union[str, None]] = "" - release_notes: Optional[Union[str, None]] = "" - tags: Dict[str, Union[str, None]] = {} + contributors: str | None = "" + release_notes: str | None = "" + tags: dict[str, str | None] = {} release_date: datetime.datetime last_update_date: datetime.datetime @@ -278,11 +277,11 @@ class SchemaRecordAnnotation(BaseModel): namespace: str schema_name: str - description: Optional[Union[str, None]] = "" - maintainers: Optional[Union[str, None]] = "" - lifecycle_stage: Optional[Union[str, None]] = "" - latest_released_version: Optional[Union[str, None]] - private: Optional[bool] = False + description: str | None = "" + maintainers: str | None = "" + lifecycle_stage: str | None = "" + latest_released_version: str | None = None + private: bool | None = False last_update_date: datetime.datetime @@ -292,7 +291,7 @@ class SchemaSearchResult(BaseModel): """ pagination: PaginationResult - results: List[SchemaRecordAnnotation] + results: list[SchemaRecordAnnotation] class SchemaVersionSearchResult(BaseModel): @@ -301,21 +300,21 @@ class SchemaVersionSearchResult(BaseModel): """ pagination: PaginationResult - results: List[SchemaVersionAnnotation] + results: list[SchemaVersionAnnotation] class UpdateSchemaRecordFields(BaseModel): - maintainers: Optional[Union[str, None]] = None - lifecycle_stage: Optional[Union[str, None]] = None - private: Optional[bool] = False - name: Optional[Union[str, None]] = None - description: Optional[Union[str, None]] = None + maintainers: str | None = None + lifecycle_stage: str | None = None + private: bool | None = False + name: str | None = None + description: str | None = None class UpdateSchemaVersionFields(BaseModel): - contributors: Optional[Union[str, None]] = None - schema_value: Optional[dict] = None - release_notes: Optional[Union[str, None]] = None + contributors: str | None = None + schema_value: dict | None = None + release_notes: str | None = None class TarNamespaceModel(BaseModel): @@ -323,10 +322,10 @@ class TarNamespaceModel(BaseModel): Namespace archive model """ - identifier: int = None + identifier: int | None = None namespace: str file_path: str - creation_date: datetime.datetime = None + creation_date: datetime.datetime | None = None number_of_projects: int = 0 file_size: int = 0 @@ -337,4 +336,4 @@ class TarNamespaceModelReturn(BaseModel): """ count: int - results: List[TarNamespaceModel] + results: list[TarNamespaceModel] diff --git a/pepdbagent/modules/annotation.py b/pepdbagent/modules/annotation.py index 99f1e40f..b53ead78 100644 --- a/pepdbagent/modules/annotation.py +++ b/pepdbagent/modules/annotation.py @@ -1,6 +1,6 @@ import logging from datetime import datetime -from typing import List, Literal, Optional, Union +from typing import Literal from sqlalchemy import and_, func, or_, select from sqlalchemy.orm import Session @@ -46,15 +46,15 @@ def get( name: str = None, tag: str = None, query: str = None, - admin: Union[List[str], str] = None, + admin: list[str] | str | None = None, limit: int = DEFAULT_LIMIT, offset: int = DEFAULT_OFFSET, order_by: str = "update_date", order_desc: bool = False, - filter_by: Optional[Literal["submission_date", "last_update_date"]] = None, - filter_start_date: Optional[str] = None, - filter_end_date: Optional[str] = None, - pep_type: Optional[Literal["pep", "pop"]] = None, + filter_by: Literal["submission_date", "last_update_date"] | None = None, + filter_start_date: str | None = None, + filter_end_date: str | None = None, + pep_type: Literal["pep", "pop"] | None = None, ) -> AnnotationList: """ Get project annotations. @@ -135,8 +135,8 @@ def get( def get_by_rp( self, - registry_paths: Union[List[str], str], - admin: Union[List[str], str] = None, + registry_paths: list[str] | str, + admin: list[str] | str | None = None, ) -> AnnotationList: """ Get project annotations by providing registry_path or list of registry paths. @@ -179,8 +179,8 @@ def _get_single_annotation( namespace: str, name: str, tag: str = DEFAULT_TAG, - admin: Union[List[str], str] = None, - ) -> Union[AnnotationModel, None]: + admin: list[str] | str | None = None, + ) -> AnnotationModel | None: """ Retrieving project annotation dict by specifying project name :param namespace: project registry_path - will return dict of project annotations @@ -242,11 +242,11 @@ def _count_projects( namespace: str = None, search_str: str = None, tag: str = None, - admin: Union[str, List[str]] = None, - filter_by: Optional[Literal["submission_date", "last_update_date"]] = None, - filter_start_date: Optional[str] = None, - filter_end_date: Optional[str] = None, - pep_type: Optional[Literal["pep", "pop"]] = None, + admin: str | list[str] | None = None, + filter_by: Literal["submission_date", "last_update_date"] | None = None, + filter_start_date: str | None = None, + filter_end_date: str | None = None, + pep_type: Literal["pep", "pop"] | None = None, ) -> int: """ Count projects. [This function is related to _find_projects] @@ -292,16 +292,16 @@ def _get_projects( namespace: str = None, tag: str = None, search_str: str = None, - admin: Union[str, List[str]] = None, + admin: str | list[str] | None = None, limit: int = DEFAULT_LIMIT, offset: int = DEFAULT_OFFSET, order_by: str = "update_date", order_desc: bool = False, - filter_by: Optional[Literal["submission_date", "last_update_date"]] = None, - filter_start_date: Optional[str] = None, - filter_end_date: Optional[str] = None, - pep_type: Optional[Literal["pep", "pop"]] = None, - ) -> List[AnnotationModel]: + filter_by: Literal["submission_date", "last_update_date"] | None = None, + filter_start_date: str | None = None, + filter_end_date: str | None = None, + pep_type: Literal["pep", "pop"] | None = None, + ) -> list[AnnotationModel]: """ Get projects by providing search string. @@ -418,7 +418,7 @@ def _add_condition( statement: Select, namespace: str = None, search_str: str = None, - admin_list: Union[str, List[str]] = None, + admin_list: str | list[str] | None = None, tag: str = None, ) -> Select: """ @@ -455,10 +455,10 @@ def _add_condition( @staticmethod def _add_date_filter_if_provided( statement: Select, - filter_by: Optional[Literal["submission_date", "last_update_date"]], - filter_start_date: Optional[str], - filter_end_date: Optional[str] = None, - ): + filter_by: Literal["submission_date", "last_update_date"] | None, + filter_start_date: str | None, + filter_end_date: str | None = None, + ) -> Select: """ Add filter to where clause to sqlalchemy statement (in project search) @@ -494,7 +494,7 @@ def _add_date_filter_if_provided( def get_project_number_in_namespace( self, namespace: str, - admin: Union[str, List[str]] = None, + admin: str | list[str] | None = None, ) -> int: """ Get number of found projects by providing search string. @@ -521,8 +521,8 @@ def get_project_number_in_namespace( def get_by_rp_list( self, - registry_paths: List[str], - admin: Union[str, List[str]] = None, + registry_paths: list[str], + admin: str | list[str] | None = None, ) -> AnnotationList: """ Get project annotations by providing list of registry paths. @@ -614,16 +614,16 @@ def get_projects_list( self, namespace: str = None, search_str: str = None, - admin: Union[str, List[str]] = None, + admin: str | list[str] | None = None, limit: int = DEFAULT_LIMIT, offset: int = DEFAULT_OFFSET, order_by: str = "update_date", order_desc: bool = False, - filter_by: Optional[Literal["submission_date", "last_update_date"]] = None, - filter_start_date: Optional[str] = None, - filter_end_date: Optional[str] = None, - pep_type: Optional[Literal["pep", "pop"]] = None, - ) -> List[RegistryPath]: + filter_by: Literal["submission_date", "last_update_date"] | None = None, + filter_start_date: str | None = None, + filter_end_date: str | None = None, + pep_type: Literal["pep", "pop"] | None = None, + ) -> list[RegistryPath]: """ Retrieve a list of projects by providing a search string. This function serves as a lightweight version of the full 'get' function, diff --git a/pepdbagent/modules/namespace.py b/pepdbagent/modules/namespace.py index 44dbc441..8fd8c4a8 100644 --- a/pepdbagent/modules/namespace.py +++ b/pepdbagent/modules/namespace.py @@ -1,7 +1,6 @@ import logging from collections import Counter from datetime import datetime, timedelta -from typing import List, Tuple, Union from sqlalchemy import delete, distinct, func, or_, select from sqlalchemy.orm import Session @@ -42,7 +41,7 @@ def __init__(self, pep_db_engine: BaseEngine): def get( self, query: str = "", - admin: Union[List[str], str] = None, + admin: list[str] | str | None = None, limit: int = DEFAULT_LIMIT, offset: int = DEFAULT_OFFSET, ) -> NamespaceList: @@ -80,7 +79,7 @@ def _get_namespace( admin_nsp: tuple = None, limit: int = DEFAULT_LIMIT, offset: int = DEFAULT_OFFSET, - ) -> List[Namespace]: + ) -> list[Namespace]: """ Search for namespace by providing search string. @@ -153,7 +152,7 @@ def _count_namespace(self, search_str: str = None, admin_nsp: tuple = tuple()) - def _add_condition( statement: Select, search_str: str = None, - admin_list: Union[Tuple[str], List[str], str] = None, + admin_list: tuple[str, ...] | list[str] | str | None = None, ) -> Select: """ Add where clause to sqlalchemy statement (in namespace search) diff --git a/pepdbagent/modules/project.py b/pepdbagent/modules/project.py index 0f92344f..5b87948e 100644 --- a/pepdbagent/modules/project.py +++ b/pepdbagent/modules/project.py @@ -1,7 +1,5 @@ import datetime -import json import logging -from typing import Dict, List, NoReturn, Union import numpy as np import peprs @@ -86,7 +84,7 @@ def get( tag: str = DEFAULT_TAG, raw: bool = True, with_id: bool = False, - ) -> Union[peprs.Project, dict, None]: + ) -> peprs.Project | dict | None: """ Retrieve project from database by specifying namespace, name and tag @@ -151,7 +149,7 @@ def get( except NoResultFound: raise ProjectNotFoundError - def _get_samples(self, session: Session, prj_id: int, with_id: bool) -> List[Dict]: + def _get_samples(self, session: Session, prj_id: int, with_id: bool) -> list[dict]: """ Get samples from the project. This method is used to retrieve samples from the project, with open session object. @@ -168,7 +166,7 @@ def _get_samples(self, session: Session, prj_id: int, with_id: bool) -> List[Dic return ordered_samples_list @staticmethod - def _get_samples_dict(prj_id: int, session: Session, with_id: bool) -> Dict: + def _get_samples_dict(prj_id: int, session: Session, with_id: bool) -> dict: """ Get not ordered samples from the project. This method is used to retrieve samples from the project @@ -224,7 +222,7 @@ def get_by_rp( self, registry_path: str, raw: bool = False, - ) -> Union[peprs.Project, dict, None]: + ) -> peprs.Project | dict | None: """ Retrieve project from database by specifying project registry_path @@ -296,7 +294,7 @@ def delete_by_rp( def create( self, - project: Union[peprs.Project, dict], + project: peprs.Project | dict, namespace: str, name: str = None, tag: str = DEFAULT_TAG, @@ -482,7 +480,7 @@ def create( def _overwrite( self, - project_dict: json, + project_dict: dict, namespace: str, proj_name: str, tag: str, @@ -565,7 +563,7 @@ def _overwrite( def update( self, - update_dict: Union[dict, UpdateItems], + update_dict: dict | UpdateItems, namespace: str, name: str, tag: str = DEFAULT_TAG, @@ -692,7 +690,7 @@ def update( raise ProjectNotFoundError("No items will be updated!") @staticmethod - def _convert_update_schema_id(session: Session, update_values: dict): + def _convert_update_schema_id(session: Session, update_values: dict) -> None: """ Convert schema path to schema_id in update_values and update it in update dict @@ -740,9 +738,9 @@ def _convert_update_schema_id(session: Session, update_values: dict): def _update_samples( self, project_id: int, - samples_list: List[Dict[str, str]], + samples_list: list[dict[str, str]], sample_name_key: str = "sample_name", - history_sa_model: Union[HistoryProjects, None] = None, + history_sa_model: HistoryProjects | None = None, ) -> None: """ Update samples in the project @@ -762,7 +760,7 @@ def _update_samples( old_samples_mapping: dict = {sample.guid: sample for sample in old_samples} # old_child_parent_id needed because of the parent_guid is sometimes set to none in sqlalchemy mapping :( bug - old_child_parent_id: Dict[str, str] = { + old_child_parent_id: dict[str, str] = { child: mapping.parent_guid for child, mapping in old_samples_mapping.items() } @@ -974,7 +972,7 @@ def exists( @staticmethod def _add_samples_to_project( - projects_sa: Projects, samples: List[dict], sample_table_index: str = "sample_name" + projects_sa: Projects, samples: list[dict], sample_table_index: str = "sample_name" ) -> None: """ Add samples to the project sa object. (With commit this samples will be added to the 'samples table') @@ -999,8 +997,8 @@ def _add_samples_to_project( @staticmethod def _add_subsamples_to_project( - projects_sa: Projects, subsamples: List[List[dict]] - ) -> NoReturn: + projects_sa: Projects, subsamples: list[list[dict]] + ) -> None: """ Add subsamples to the project sa object. (With commit this samples will be added to the 'subsamples table') @@ -1014,7 +1012,7 @@ def _add_subsamples_to_project( Subsamples(subsample=sub_item, subsample_number=i, row_number=row_number) ) - def get_project_id(self, namespace: str, name: str, tag: str) -> Union[int, None]: + def get_project_id(self, namespace: str, name: str, tag: str) -> int | None: """ Get Project id by providing namespace, name, and tag @@ -1098,7 +1096,7 @@ def fork( session.commit() - def get_config(self, namespace: str, name: str, tag: str) -> Union[dict, None]: + def get_config(self, namespace: str, name: str, tag: str) -> dict | None: """ Get project configuration by providing namespace, name, and tag @@ -1117,7 +1115,7 @@ def get_config(self, namespace: str, name: str, tag: str) -> Union[dict, None]: return result[0] return None - def get_subsamples(self, namespace: str, name: str, tag: str) -> Union[list, None]: + def get_subsamples(self, namespace: str, name: str, tag: str) -> list | None: """ Get project subsamples by providing namespace, name, and tag @@ -1203,7 +1201,7 @@ def get_history(self, namespace: str, name: str, tag: str) -> HistoryAnnotationM .order_by(HistoryProjects.update_time.desc()) ) results = session.scalars(statement) - return_results: List = [] + return_results: list = [] if results: for result in results: @@ -1229,7 +1227,7 @@ def get_project_from_history( history_id: int, raw: bool = True, with_id: bool = False, - ) -> Union[dict, peprs.Project]: + ) -> dict | peprs.Project: """ Get project sample history annotation by providing namespace, name, and tag @@ -1355,7 +1353,7 @@ def _apply_history_changes(sample_dict: dict, change: HistoryProjects) -> dict: return sample_dict def delete_history( - self, namespace: str, name: str, tag: str, history_id: Union[int, None] = None + self, namespace: str, name: str, tag: str, history_id: int | None = None ) -> None: """ Delete history from the project diff --git a/pepdbagent/modules/sample.py b/pepdbagent/modules/sample.py index 56c1339f..f7dc76d6 100644 --- a/pepdbagent/modules/sample.py +++ b/pepdbagent/modules/sample.py @@ -1,6 +1,5 @@ import datetime import logging -from typing import Union import peprs from sqlalchemy import and_, select @@ -36,7 +35,7 @@ def get( sample_name: str, tag: str = DEFAULT_TAG, raw: bool = True, - ) -> Union[peprs.Sample, dict, None]: + ) -> peprs.Sample | dict | None: """ Retrieve sample from the database using namespace, name, tag, and sample_name diff --git a/pepdbagent/modules/schema.py b/pepdbagent/modules/schema.py index 4362ed67..4c251e4f 100644 --- a/pepdbagent/modules/schema.py +++ b/pepdbagent/modules/schema.py @@ -1,5 +1,4 @@ import logging -from typing import Dict, List, Optional, Union from sqlalchemy import Select, and_, func, or_, select from sqlalchemy.orm import Session @@ -105,7 +104,7 @@ def create( maintainers: str = "", contributors: str = "", release_notes: str = "", - tags: Optional[Union[List[str], str, Dict[str, str], List[Dict[str, str]]]] = None, + tags: list[str] | str | dict[str, str] | list[dict[str, str]] | None = None, private: bool = False, # TODO: for simplicity was not implemented yet ) -> None: """ @@ -188,7 +187,7 @@ def add_version( release_notes: str = "", contributors: str = "", overwrite: bool = False, - tags: Optional[Union[List[str], str, Dict[str, str], List[Dict[str, str]]]] = None, + tags: list[str] | str | dict[str, str] | list[dict[str, str]] | None = None, ) -> None: tags = self._unify_tags(tags) @@ -259,7 +258,7 @@ def update_schema_version( namespace: str, name: str, version: str, - update_fields: Union[UpdateSchemaVersionFields, dict], + update_fields: UpdateSchemaVersionFields | dict, ) -> None: """ Update schema version in the database. @@ -306,7 +305,7 @@ def update_schema_record( self, namespace: str, name: str, - update_fields: Union[UpdateSchemaRecordFields, dict], + update_fields: UpdateSchemaRecordFields | dict, ) -> None: """ Update schema record in the database. @@ -787,7 +786,7 @@ def add_tag_to_schema( namespace: str, name: str, version: str, - tag: Optional[Union[List[str], str, Dict[str, str]]], + tag: list[str] | str | dict[str, str] | None, ) -> None: """ Add tag to the schema @@ -909,8 +908,8 @@ def _add_order_by_schemas_keyword( return statement.order_by(order_by_obj) def _unify_tags( - self, tags: Optional[Union[List[str], str, Dict[str, str], List[Dict[str, str]]]] - ) -> [Dict[str, str]]: + self, tags: list[str] | str | dict[str, str] | list[dict[str, str]] | None + ) -> dict[str, str]: """ Convert provided tags to one standard diff --git a/pepdbagent/modules/user.py b/pepdbagent/modules/user.py index 416df88c..636f61d4 100644 --- a/pepdbagent/modules/user.py +++ b/pepdbagent/modules/user.py @@ -1,5 +1,4 @@ import logging -from typing import Union from sqlalchemy import and_, delete, select from sqlalchemy.exc import IntegrityError @@ -46,7 +45,7 @@ def create_user(self, namespace: str) -> int: user_id = new_user_raw.id return user_id - def get_user_id(self, namespace: str) -> Union[int, None]: + def get_user_id(self, namespace: str) -> int | None: """ Get user id using username diff --git a/pepdbagent/modules/view.py b/pepdbagent/modules/view.py index 3c6a07da..5ea1ca94 100644 --- a/pepdbagent/modules/view.py +++ b/pepdbagent/modules/view.py @@ -1,7 +1,6 @@ # View of the PEP. In other words, it is a part of the PEP, or subset of the samples in the PEP. import logging -from typing import List, Union import peprs from sqlalchemy import and_, delete, select @@ -50,7 +49,7 @@ def get( tag: str = DEFAULT_TAG, view_name: str = None, raw: bool = True, - ) -> Union[peprs.Project, dict, None]: + ) -> peprs.Project | dict | None: """ Retrieve view of the project from the database. View is a subset of the samples in the project. e.g. bed-db project has all the samples in bedbase, @@ -129,7 +128,7 @@ def get_annotation( def create( self, view_name: str, - view_dict: Union[dict, CreateViewDictModel], + view_dict: dict | CreateViewDictModel, description: str = None, no_fail: bool = False, ) -> None: @@ -242,8 +241,8 @@ def add_sample( name: str, tag: str, view_name: str, - sample_name: Union[str, List[str]], - ): + sample_name: str | list[str], + ) -> None: """ Add sample to the view. @@ -347,8 +346,8 @@ def remove_sample( sa_session.commit() def get_snap_view( - self, namespace: str, name: str, tag: str, sample_name_list: List[str], raw: bool = False - ) -> Union[peprs.Project, dict]: + self, namespace: str, name: str, tag: str, sample_name_list: list[str], raw: bool = False + ) -> peprs.Project | dict: """ Get a snap view of the project. Snap view is a view of the project with only the samples in the list. This view won't be saved in the database. @@ -395,7 +394,7 @@ def get_snap_view( def get_views_annotation( self, namespace: str, name: str, tag: str = DEFAULT_TAG - ) -> Union[ProjectViews, None]: + ) -> ProjectViews | None: """ Get list of views of the project diff --git a/pepdbagent/pepdbagent.py b/pepdbagent/pepdbagent.py index 07d66afc..1eeeb926 100644 --- a/pepdbagent/pepdbagent.py +++ b/pepdbagent/pepdbagent.py @@ -12,15 +12,15 @@ class PEPDatabaseAgent(object): def __init__( self, - host="localhost", - port=5432, - database="pep-db", - user=None, - password=None, - drivername=POSTGRES_DIALECT, - dsn=None, - echo=False, - run_migrations=False, + host: str = "localhost", + port: int = 5432, + database: str = "pep-db", + user: str | None = None, + password: str | None = None, + drivername: str = POSTGRES_DIALECT, + dsn: str | None = None, + echo: bool = False, + run_migrations: bool = False, ): """ Initialize connection to the pep_db database. You can use The basic connection parameters @@ -98,5 +98,5 @@ def __exit__(self): self._sa_engine.__exit__() @property - def connection(self): + def connection(self) -> BaseEngine: return self._sa_engine diff --git a/pepdbagent/utils.py b/pepdbagent/utils.py index a15809ad..b19635e1 100644 --- a/pepdbagent/utils.py +++ b/pepdbagent/utils.py @@ -3,7 +3,6 @@ import uuid from collections.abc import Iterable from hashlib import md5 -from typing import List, Tuple, Union import ubiquerg from peprs.const import SAMPLE_RAW_DICT_KEY @@ -63,7 +62,7 @@ def create_digest(project_dict: dict) -> str: return sample_digest -def registry_path_converter(registry_path: str) -> Tuple[str, str, str]: +def registry_path_converter(registry_path: str) -> tuple[str, str, str]: """ Convert registry path to namespace, name, tag @@ -80,7 +79,7 @@ def registry_path_converter(registry_path: str) -> Tuple[str, str, str]: raise RegistryPathError(f"Error in: '{registry_path}'") -def schema_path_converter(schema_path: str) -> Tuple[str, str, str]: +def schema_path_converter(schema_path: str) -> tuple[str, str, str]: """ Convert schema path to namespace, name @@ -97,7 +96,7 @@ def schema_path_converter(schema_path: str) -> Tuple[str, str, str]: raise RegistryPathError(f"Error in: '{schema_path}'") -def tuple_converter(value: Union[tuple, list, str, None]) -> tuple: +def tuple_converter(value: tuple | list | str | None) -> tuple: """ Convert string list or tuple to tuple. # is used to create admin tuple. @@ -124,7 +123,7 @@ def convert_date_string_to_date(date_string: str) -> datetime.datetime: return datetime.datetime.strptime(date_string, "%Y/%m/%d") + datetime.timedelta(days=1) -def order_samples(results: dict) -> List[dict]: +def order_samples(results: dict) -> list[dict]: """ Order samples by their parent_guid From ae06d09e277b7c307fcea5093394dd13fa0d90d2 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Mon, 29 Jun 2026 18:30:15 -0400 Subject: [PATCH 06/13] Modernization step 6 --- pep_db/Dockerfile | 5 - .../44cb1e7a80de_initial_migration.py | 24 +- .../8a037f13b4e5_upgrading_schemas.py | 45 +- pepdbagent/db_utils.py | 147 ++-- pepdbagent/exceptions.py | 4 +- pepdbagent/models.py | 8 +- pepdbagent/modules/annotation.py | 317 ++++----- pepdbagent/modules/namespace.py | 170 ++--- pepdbagent/modules/project.py | 637 ++++++++++-------- pepdbagent/modules/sample.py | 119 ++-- pepdbagent/modules/schema.py | 466 +++++++------ pepdbagent/modules/user.py | 96 +-- pepdbagent/modules/view.py | 208 +++--- pepdbagent/pepdbagent.py | 25 +- pepdbagent/utils.py | 99 +-- tests/test_annotation.py | 10 +- tests/test_namespace.py | 31 +- tests/test_project.py | 53 +- tests/test_project_history.py | 8 +- tests/test_samples.py | 17 +- tests/test_schema.py | 42 +- tests/test_tar_meta.py | 1 - tests/test_updates.py | 69 +- tests/test_views.py | 66 +- tests/utils.py | 4 +- 25 files changed, 1577 insertions(+), 1094 deletions(-) delete mode 100644 pep_db/Dockerfile diff --git a/pep_db/Dockerfile b/pep_db/Dockerfile deleted file mode 100644 index 5d04731d..00000000 --- a/pep_db/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM postgres -ENV POSTGRES_USER postgres -ENV POSTGRES_PASSWORD docker -ENV POSTGRES_DB pep-db -#COPY pep_db.sql /docker-entrypoint-initdb.d/ \ No newline at end of file diff --git a/pepdbagent/alembic/versions/44cb1e7a80de_initial_migration.py b/pepdbagent/alembic/versions/44cb1e7a80de_initial_migration.py index 64e61acb..e8521a59 100644 --- a/pepdbagent/alembic/versions/44cb1e7a80de_initial_migration.py +++ b/pepdbagent/alembic/versions/44cb1e7a80de_initial_migration.py @@ -64,7 +64,9 @@ def upgrade() -> None: sa.UniqueConstraint("namespace", "name"), ) op.create_index(op.f("ix_schema_groups_id"), "schema_groups", ["id"], unique=False) - op.create_index(op.f("ix_schema_groups_name"), "schema_groups", ["name"], unique=False) + op.create_index( + op.f("ix_schema_groups_name"), "schema_groups", ["name"], unique=False + ) op.create_index( op.f("ix_schema_groups_namespace"), "schema_groups", ["namespace"], unique=False ) @@ -87,7 +89,9 @@ def upgrade() -> None: sa.PrimaryKeyConstraint("id"), sa.UniqueConstraint("namespace", "name"), ) - op.create_index(op.f("ix_schemas_description"), "schemas", ["description"], unique=False) + op.create_index( + op.f("ix_schemas_description"), "schemas", ["description"], unique=False + ) op.create_index(op.f("ix_schemas_id"), "schemas", ["id"], unique=False) op.create_index(op.f("ix_schemas_name"), "schemas", ["name"], unique=False) op.create_table( @@ -113,7 +117,9 @@ def upgrade() -> None: sa.Column("schema_id", sa.Integer(), nullable=True), sa.Column("pop", sa.Boolean(), nullable=True), sa.Column("forked_from_id", sa.Integer(), nullable=True), - sa.ForeignKeyConstraint(["forked_from_id"], ["projects.id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint( + ["forked_from_id"], ["projects.id"], ondelete="SET NULL" + ), sa.ForeignKeyConstraint(["namespace"], ["users.namespace"], ondelete="CASCADE"), sa.ForeignKeyConstraint(["schema_id"], ["schemas.id"], ondelete="SET NULL"), sa.PrimaryKeyConstraint("id"), @@ -225,7 +231,9 @@ def upgrade() -> None: sa.Enum("UPDATE", "INSERT", "DELETE", name="updatetypes"), nullable=False, ), - sa.ForeignKeyConstraint(["history_id"], ["project_history.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint( + ["history_id"], ["project_history.id"], ondelete="CASCADE" + ), sa.PrimaryKeyConstraint("id"), ) op.create_table( @@ -249,8 +257,12 @@ def downgrade() -> None: op.drop_table("stars") op.drop_table("samples") op.drop_table("project_history") - op.drop_index(op.f("ix_schema_group_relations_schema_id"), table_name="schema_group_relations") - op.drop_index(op.f("ix_schema_group_relations_group_id"), table_name="schema_group_relations") + op.drop_index( + op.f("ix_schema_group_relations_schema_id"), table_name="schema_group_relations" + ) + op.drop_index( + op.f("ix_schema_group_relations_group_id"), table_name="schema_group_relations" + ) op.drop_table("schema_group_relations") op.drop_table("projects") op.drop_index(op.f("ix_schemas_name"), table_name="schemas") diff --git a/pepdbagent/alembic/versions/8a037f13b4e5_upgrading_schemas.py b/pepdbagent/alembic/versions/8a037f13b4e5_upgrading_schemas.py index 752d86c1..ff6bf953 100644 --- a/pepdbagent/alembic/versions/8a037f13b4e5_upgrading_schemas.py +++ b/pepdbagent/alembic/versions/8a037f13b4e5_upgrading_schemas.py @@ -52,7 +52,9 @@ def upgrade() -> None: sa.Column("last_update_date", sa.TIMESTAMP(timezone=True), nullable=True), sa.Column("contributors", sa.String(), nullable=True), sa.Column("release_notes", sa.String(), nullable=True), - sa.ForeignKeyConstraint(["schema_id"], ["schema_records.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint( + ["schema_id"], ["schema_records.id"], ondelete="CASCADE" + ), sa.PrimaryKeyConstraint("id"), sa.UniqueConstraint("schema_id", "version"), ) @@ -62,7 +64,9 @@ def upgrade() -> None: sa.Column("tag_name", sa.String(), nullable=False), sa.Column("tag_value", sa.String(), nullable=True), sa.Column("schema_version_id", sa.Integer(), nullable=False), - sa.ForeignKeyConstraint(["schema_version_id"], ["schema_versions.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint( + ["schema_version_id"], ["schema_versions.id"], ondelete="CASCADE" + ), sa.PrimaryKeyConstraint("id"), ) op.drop_index("ix_schemas_description", table_name="schemas") @@ -76,8 +80,12 @@ def upgrade() -> None: op.drop_index("ix_schema_groups_namespace", table_name="schema_groups") # op.drop_table('schema_groups') op.execute("DROP TABLE schema_groups CASCADE;") - op.drop_index("ix_schema_group_relations_group_id", table_name="schema_group_relations") - op.drop_index("ix_schema_group_relations_schema_id", table_name="schema_group_relations") + op.drop_index( + "ix_schema_group_relations_group_id", table_name="schema_group_relations" + ) + op.drop_index( + "ix_schema_group_relations_schema_id", table_name="schema_group_relations" + ) op.drop_table("schema_group_relations") op.execute("UPDATE projects SET schema_id = NULL;") @@ -85,7 +93,9 @@ def upgrade() -> None: None, "projects", "schema_versions", ["schema_id"], ["id"], ondelete="SET NULL" ) op.drop_column("projects", "pep_schema") - op.add_column("users", sa.Column("number_of_schemas", sa.Integer(), nullable=True, default=0)) + op.add_column( + "users", sa.Column("number_of_schemas", sa.Integer(), nullable=True, default=0) + ) # ### end Alembic commands ### @@ -94,7 +104,8 @@ def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_column("users", "number_of_schemas") op.add_column( - "projects", sa.Column("pep_schema", sa.VARCHAR(), autoincrement=False, nullable=True) + "projects", + sa.Column("pep_schema", sa.VARCHAR(), autoincrement=False, nullable=True), ) op.drop_constraint(None, "projects", type_="foreignkey") op.create_foreign_key( @@ -121,7 +132,9 @@ def downgrade() -> None: name="schema_group_relations_schema_id_fkey", ondelete="CASCADE", ), - sa.PrimaryKeyConstraint("schema_id", "group_id", name="schema_group_relations_pkey"), + sa.PrimaryKeyConstraint( + "schema_id", "group_id", name="schema_group_relations_pkey" + ), ) op.create_index( "ix_schema_group_relations_schema_id", @@ -130,7 +143,10 @@ def downgrade() -> None: unique=False, ) op.create_index( - "ix_schema_group_relations_group_id", "schema_group_relations", ["group_id"], unique=False + "ix_schema_group_relations_group_id", + "schema_group_relations", + ["group_id"], + unique=False, ) op.create_table( "schema_groups", @@ -145,9 +161,13 @@ def downgrade() -> None: ondelete="CASCADE", ), sa.PrimaryKeyConstraint("id", name="schema_groups_pkey"), - sa.UniqueConstraint("namespace", "name", name="schema_groups_namespace_name_key"), + sa.UniqueConstraint( + "namespace", "name", name="schema_groups_namespace_name_key" + ), + ) + op.create_index( + "ix_schema_groups_namespace", "schema_groups", ["namespace"], unique=False ) - op.create_index("ix_schema_groups_namespace", "schema_groups", ["namespace"], unique=False) op.create_index("ix_schema_groups_name", "schema_groups", ["name"], unique=False) op.create_index("ix_schema_groups_id", "schema_groups", ["id"], unique=False) op.create_table( @@ -176,7 +196,10 @@ def downgrade() -> None: nullable=True, ), sa.ForeignKeyConstraint( - ["namespace"], ["users.namespace"], name="schemas_namespace_fkey", ondelete="CASCADE" + ["namespace"], + ["users.namespace"], + name="schemas_namespace_fkey", + ondelete="CASCADE", ), sa.PrimaryKeyConstraint("id", name="schemas_pkey"), sa.UniqueConstraint("namespace", "name", name="schemas_namespace_name_key"), diff --git a/pepdbagent/db_utils.py b/pepdbagent/db_utils.py index 07a55fe5..3a6cf5d8 100644 --- a/pepdbagent/db_utils.py +++ b/pepdbagent/db_utils.py @@ -50,9 +50,7 @@ class Base(DeclarativeBase): @event.listens_for(Base.metadata, "after_create") def receive_after_create(target, connection, tables, **kw) -> None: - """ - listen for the 'after_create' event - """ + """Listen for the 'after_create' event.""" if tables: _LOGGER.info("A table was created") else: @@ -75,7 +73,9 @@ class Projects(Base): __tablename__ = "projects" id: Mapped[int] = mapped_column(primary_key=True) - namespace: Mapped[str] = mapped_column(ForeignKey("users.namespace", ondelete="CASCADE")) + namespace: Mapped[str] = mapped_column( + ForeignKey("users.namespace", ondelete="CASCADE") + ) name: Mapped[str] = mapped_column() tag: Mapped[str] = mapped_column() digest: Mapped[str] = mapped_column(String(32)) @@ -92,7 +92,9 @@ class Projects(Base): schema_id: Mapped[int | None] = mapped_column( ForeignKey("schema_versions.id", ondelete="SET NULL"), nullable=True ) - schema_mapping: Mapped["SchemaVersions"] = relationship("SchemaVersions", lazy="joined") + schema_mapping: Mapped["SchemaVersions"] = relationship( + "SchemaVersions", lazy="joined" + ) pop: Mapped[bool | None] = mapped_column(default=False) samples_mapping: Mapped[list["Samples"]] = relationship( @@ -126,7 +128,9 @@ class Projects(Base): cascade="save-update, merge, refresh-expire", ) - namespace_mapping: Mapped["User"] = relationship("User", back_populates="projects_mapping") + namespace_mapping: Mapped["User"] = relationship( + "User", back_populates="projects_mapping" + ) history_mapping: Mapped[list["HistoryProjects"]] = relationship( back_populates="project_mapping", cascade="all, delete-orphan" @@ -149,7 +153,9 @@ class Samples(Base): sample_name: Mapped[str | None] = mapped_column() guid: Mapped[str | None] = mapped_column(nullable=False, unique=True) - submission_date: Mapped[datetime.datetime] = mapped_column(default=deliver_update_date) + submission_date: Mapped[datetime.datetime] = mapped_column( + default=deliver_update_date + ) last_update_date: Mapped[datetime.datetime | None] = mapped_column( default=deliver_update_date, onupdate=deliver_update_date, @@ -164,7 +170,9 @@ class Samples(Base): parent_mapping: Mapped["Samples"] = relationship( "Samples", remote_side=guid, back_populates="child_mapping" ) - child_mapping: Mapped["Samples"] = relationship("Samples", back_populates="parent_mapping") + child_mapping: Mapped["Samples"] = relationship( + "Samples", back_populates="parent_mapping" + ) views: Mapped[list["ViewSampleAssociation"] | None] = relationship( back_populates="sample", cascade="all, delete-orphan" @@ -183,7 +191,9 @@ class Subsamples(Base): subsample_number: Mapped[int] row_number: Mapped[int] project_id = mapped_column(ForeignKey("projects.id", ondelete="CASCADE")) - subsample_mapping: Mapped["Projects"] = relationship(back_populates="subsamples_mapping") + subsample_mapping: Mapped["Projects"] = relationship( + back_populates="subsamples_mapping" + ) class User(Base): @@ -218,8 +228,12 @@ class Stars(Base): __tablename__ = "stars" - user_id = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), primary_key=True) - project_id = mapped_column(ForeignKey("projects.id", ondelete="CASCADE"), primary_key=True) + user_id = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), primary_key=True + ) + project_id = mapped_column( + ForeignKey("projects.id", ondelete="CASCADE"), primary_key=True + ) user_mapping: Mapped[list["User"]] = relationship(back_populates="stars_mapping") project_mapping: Mapped["Projects"] = relationship(back_populates="stars_mapping") star_date: Mapped[datetime.datetime] = mapped_column( @@ -255,19 +269,26 @@ class ViewSampleAssociation(Base): __tablename__ = "views_samples" - sample_id = mapped_column(ForeignKey("samples.id", ondelete="CASCADE"), primary_key=True) - view_id = mapped_column(ForeignKey("views.id", ondelete="CASCADE"), primary_key=True) + sample_id = mapped_column( + ForeignKey("samples.id", ondelete="CASCADE"), primary_key=True + ) + view_id = mapped_column( + ForeignKey("views.id", ondelete="CASCADE"), primary_key=True + ) sample: Mapped["Samples"] = relationship(back_populates="views") view: Mapped["Views"] = relationship(back_populates="samples") class HistoryProjects(Base): - __tablename__ = "project_history" id: Mapped[int] = mapped_column(primary_key=True) - project_id: Mapped[int] = mapped_column(ForeignKey("projects.id", ondelete="CASCADE")) - user: Mapped[str] = mapped_column(ForeignKey("users.namespace", ondelete="SET NULL")) + project_id: Mapped[int] = mapped_column( + ForeignKey("projects.id", ondelete="CASCADE") + ) + user: Mapped[str] = mapped_column( + ForeignKey("users.namespace", ondelete="SET NULL") + ) update_time: Mapped[datetime.datetime] = mapped_column( TIMESTAMP(timezone=True), default=deliver_update_date ) @@ -292,11 +313,12 @@ class UpdateTypes(enum.Enum): class HistorySamples(Base): - __tablename__ = "sample_history" id: Mapped[int] = mapped_column(primary_key=True) - history_id: Mapped[int] = mapped_column(ForeignKey("project_history.id", ondelete="CASCADE")) + history_id: Mapped[int] = mapped_column( + ForeignKey("project_history.id", ondelete="CASCADE") + ) guid: Mapped[str] = mapped_column(nullable=False) parent_guid: Mapped[str | None] = mapped_column(nullable=True) sample_json: Mapped[dict] = mapped_column(JSON, server_default=FetchedValue()) @@ -311,7 +333,9 @@ class SchemaRecords(Base): __tablename__ = "schema_records" id: Mapped[int] = mapped_column(primary_key=True) - namespace: Mapped[str] = mapped_column(ForeignKey("users.namespace", ondelete="CASCADE")) + namespace: Mapped[str] = mapped_column( + ForeignKey("users.namespace", ondelete="CASCADE") + ) name: Mapped[str] = mapped_column(nullable=False) maintainers: Mapped[str] = mapped_column(nullable=True) lifecycle_stage: Mapped[str] = mapped_column(nullable=True) @@ -329,14 +353,18 @@ class SchemaRecords(Base): cascade="all, delete-orphan", order_by="SchemaVersions.version.desc()", ) - user_mapping: Mapped["User"] = relationship("User", back_populates="schemas_mapping") + user_mapping: Mapped["User"] = relationship( + "User", back_populates="schemas_mapping" + ) class SchemaVersions(Base): __tablename__ = "schema_versions" id: Mapped[int] = mapped_column(primary_key=True) - schema_id: Mapped[int] = mapped_column(ForeignKey("schema_records.id", ondelete="CASCADE")) + schema_id: Mapped[int] = mapped_column( + ForeignKey("schema_records.id", ondelete="CASCADE") + ) version: Mapped[str] = mapped_column(nullable=False) schema_value: Mapped[dict] = mapped_column(JSON, server_default=FetchedValue()) release_date: Mapped[datetime.datetime] = mapped_column(default=deliver_update_date) @@ -353,7 +381,10 @@ class SchemaVersions(Base): ) tags_mapping: Mapped[list["SchemaTags"]] = relationship( - "SchemaTags", back_populates="schema_mapping", lazy="joined", cascade="all, delete-orphan" + "SchemaTags", + back_populates="schema_mapping", + lazy="joined", + cascade="all, delete-orphan", ) @@ -373,13 +404,16 @@ class SchemaTags(Base): class TarNamespace(Base): - __tablename__ = "namespace_archives" id: Mapped[int] = mapped_column(primary_key=True) - namespace: Mapped[str] = mapped_column(ForeignKey("users.namespace", ondelete="CASCADE")) + namespace: Mapped[str] = mapped_column( + ForeignKey("users.namespace", ondelete="CASCADE") + ) file_path: Mapped[str] = mapped_column(nullable=False) - creation_date: Mapped[datetime.datetime] = mapped_column(default=deliver_update_date) + creation_date: Mapped[datetime.datetime] = mapped_column( + default=deliver_update_date + ) number_of_projects: Mapped[int] = mapped_column(default=0) file_size: Mapped[int] = mapped_column(nullable=False) @@ -414,20 +448,18 @@ def __init__( echo: bool = False, run_migrations: bool = False, ) -> None: - """ - Initialize connection to the pep_db database. You can use The basic connection parameters - or libpq connection string. - :param host: database server address e.g., localhost or an IP address. - :param port: the port number that defaults to 5432 if it is not provided. - :param database: the name of the database that you want to connect. - :param user: the username used to authenticate. - :param password: password used to authenticate. - :param drivername: driver used in - :param dsn: libpq connection string using the dsn parameter - (e.g. 'postgresql://user_name:password@host_name:port/db_name') - - :param echo: If True, the Engine will log all statements as well as a repr() of their parameter lists to the - :param run_migrations: If True, run database migrations + """Initialize connection to the pep_db database. + + Args: + host: Database server address. + port: Port number (default: 5432). + database: Database name. + user: Username for authentication. + password: Password for authentication. + drivername: Database driver. + dsn: libpq connection string, e.g., "postgresql://user:pass@host:port/db". + echo: Log all SQL statements if True. + run_migrations: Run database migrations if True. """ if not dsn: dsn = URL.create( @@ -439,7 +471,9 @@ def __init__( drivername=drivername, ) if run_migrations: - dsn_with_password = f"{drivername}://{user}:{password}@{host}:{port}/{database}" + dsn_with_password = ( + f"{drivername}://{user}:{password}@{host}:{port}/{database}" + ) self.run_db_migration(dsn_with_password) self._engine = create_engine(dsn, echo=echo) @@ -447,11 +481,10 @@ def __init__( self.check_db_connection() def create_schema(self, engine=None) -> None: - """ - Create sql schema in the database. + """Create SQL schema in the database. - :param engine: sqlalchemy engine [Default: None] - :return: None + Args: + engine: SQLAlchemy engine (default: uses internal engine). """ if not engine: engine = self._engine @@ -459,12 +492,13 @@ def create_schema(self, engine=None) -> None: return None def session_execute(self, statement: Select) -> Result: - """ - Execute statement using sqlalchemy statement + """Execute a SQLAlchemy statement. - :param statement: SQL query or a SQL expression that is constructed using - SQLAlchemy's SQL expression language - :return: query result represented with declarative base + Args: + statement: SQL query or expression to execute. + + Returns: + Query result. """ _LOGGER.debug(f"Executing statement: {statement}") with Session(self._engine) as session: @@ -474,9 +508,7 @@ def session_execute(self, statement: Select) -> Result: @property def session(self) -> Session: - """ - :return: started sqlalchemy session - """ + """Return a started SQLAlchemy session.""" return self._start_session() @property @@ -499,11 +531,10 @@ def check_db_connection(self) -> None: raise SchemaError() def delete_schema(self, engine=None) -> None: - """ - Delete sql schema in the database. + """Delete SQL schema from the database. - :param engine: sqlalchemy engine [Default: None] - :return: None + Args: + engine: SQLAlchemy engine (default: uses internal engine). """ if not engine: engine = self._engine @@ -511,9 +542,7 @@ def delete_schema(self, engine=None) -> None: return None def run_db_migration(self, database_url: str) -> None: - """ - Migrate the database to the required version. - """ + """Migrate the database to the latest version.""" script_directory = os.path.dirname(os.path.abspath(__file__)) script_location = os.path.join(script_directory, "alembic") diff --git a/pepdbagent/exceptions.py b/pepdbagent/exceptions.py index 5ef92f66..4f6e4ace 100644 --- a/pepdbagent/exceptions.py +++ b/pepdbagent/exceptions.py @@ -10,7 +10,9 @@ def __init__(self, msg: str) -> None: class SchemaError(PEPDatabaseAgentError): def __init__(self) -> None: - super().__init__("""PEP_db connection error! The schema of connected db is incorrect""") + super().__init__( + """PEP_db connection error! The schema of connected db is incorrect""" + ) class RegistryPathError(PEPDatabaseAgentError): diff --git a/pepdbagent/models.py b/pepdbagent/models.py index a78542c1..90a73782 100644 --- a/pepdbagent/models.py +++ b/pepdbagent/models.py @@ -44,7 +44,7 @@ class AnnotationModel(BaseModel): ) @field_validator("is_private") - def is_private_should_be_bool(cls, v): + def is_private_should_be_bool(cls, v) -> bool: if not isinstance(v, bool): return False else: @@ -136,19 +136,19 @@ class UpdateModel(BaseModel): pop: bool | None = False @field_validator("tag", "name") - def value_must_not_be_empty(cls, v): + def value_must_not_be_empty(cls, v) -> str | None: if "" == v: return None return v @field_validator("tag", "name") - def value_must_be_lowercase(cls, v): + def value_must_be_lowercase(cls, v) -> str | None: if v: return v.lower() return v @field_validator("tag", "name") - def value_should_not_contain_question(cls, v): + def value_should_not_contain_question(cls, v) -> str: if "?" in v: return ValueError("Question mark (?) is prohibited in name and tag.") return v diff --git a/pepdbagent/modules/annotation.py b/pepdbagent/modules/annotation.py index b53ead78..ee7c6f21 100644 --- a/pepdbagent/modules/annotation.py +++ b/pepdbagent/modules/annotation.py @@ -35,7 +35,8 @@ class PEPDatabaseAnnotation: def __init__(self, pep_db_engine: BaseEngine): """ - :param pep_db_engine: pepdbengine object with sa engine + Args: + pep_db_engine: PEPDatabaseAgent engine object. """ self._sa_engine = pep_db_engine.engine self._pep_db_engine = pep_db_engine @@ -56,34 +57,32 @@ def get( filter_end_date: str | None = None, pep_type: Literal["pep", "pop"] | None = None, ) -> AnnotationList: - """ - Get project annotations. - - There is 5 scenarios how to get project or projects annotations: - - provide name, namespace and tag. Return: project annotations of exact provided PK(namespace, name, tag) - - provide only namespace. Return: list of projects annotations in specified namespace - - Nothing is provided. Return: list of projects annotations in all database - - provide query. Return: list of projects annotations find in database that have query pattern. - - provide query and namespace. Return: list of projects annotations find in specific namespace - that have query pattern. - :param namespace: Namespace - :param name: Project name - :param tag: tag - :param query: query (search string): Pattern of name, tag or description - :param admin: admin name (namespace), or list of namespaces, where user is admin - :param limit: return limit - :param offset: return offset - :param order_by: sort the result-set by the information - Options: ["name", "update_date", "submission_date"] - [Default: update_date] - :param order_desc: Sort the records in descending order. [Default: False] - :param filter_by: data to use filter on. - Options: ["submission_date", "last_update_date"] - [Default: filter won't be used] - :param filter_start_date: Filter start date. Format: "YYYY/MM/DD" - :param filter_end_date: Filter end date. Format: "YYYY/MM/DD". if None: present date will be used - :param pep_type: Get pep with specified type. Options: ["pep", "pop"]. Default: None, get all peps - :return: pydantic model: AnnotationList + """Get project annotations. + + Five retrieval scenarios: + - namespace + name + tag: exact match. + - namespace only: all projects in that namespace. + - nothing: all projects in the database. + - query: full-text search across name, tag, description. + - query + namespace: full-text search within a namespace. + + Args: + namespace: Namespace to filter by. + name: Project name (use with namespace and tag for exact lookup). + tag: Project tag. + query: Search string matched against name, tag, and description. + admin: Namespace(s) where the caller has admin rights. + limit: Maximum number of results. + offset: Number of results to skip. + order_by: Sort field — "name", "update_date", or "submission_date". + order_desc: Sort in descending order if True. + filter_by: Date field to apply range filter on — "submission_date" or "last_update_date". + filter_start_date: Range start in YYYY/MM/DD format. + filter_end_date: Range end in YYYY/MM/DD format (default: today). + pep_type: Restrict to "pep" or "pop" (default: all). + + Returns: + AnnotationList with count, limit, offset, and results. """ if all([namespace, name, tag]): found_annotation = [ @@ -102,7 +101,9 @@ def get( ) if pep_type not in [None, "pep", "pop"]: - raise ValueError(f"pep_type should be one of ['pep', 'pop'], got {pep_type}") + raise ValueError( + f"pep_type should be one of ['pep', 'pop'], got {pep_type}" + ) return AnnotationList( limit=limit, @@ -138,15 +139,14 @@ def get_by_rp( registry_paths: list[str] | str, admin: list[str] | str | None = None, ) -> AnnotationList: - """ - Get project annotations by providing registry_path or list of registry paths. - :param registry_paths: registry path string or list of registry paths - :param admin: list of namespaces where user is admin - :return: pydantic model: AnnotationReturnModel( - limit: - offset: - count: - result: List [AnnotationModel]) + """Get project annotations by registry path or list of registry paths. + + Args: + registry_paths: Single registry path or list of paths ("namespace/name:tag"). + admin: Namespace(s) where the caller has admin rights. + + Returns: + AnnotationList with count, limit, offset, and results. """ if isinstance(registry_paths, list): anno_results = [] @@ -157,7 +157,9 @@ def get_by_rp( _LOGGER.error(str(err), registry_paths) continue try: - single_return = self._get_single_annotation(namespace, name, tag, admin) + single_return = self._get_single_annotation( + namespace, name, tag, admin + ) if single_return: anno_results.append(single_return) except ProjectNotFoundError: @@ -181,13 +183,19 @@ def _get_single_annotation( tag: str = DEFAULT_TAG, admin: list[str] | str | None = None, ) -> AnnotationModel | None: - """ - Retrieving project annotation dict by specifying project name - :param namespace: project registry_path - will return dict of project annotations - :param name: project name in database - :param tag: tag of the projects - :param admin: string or list of admins [e.g. "Khoroshevskyi", or ["doc_adin","Khoroshevskyi"]] - :return: pydantic Annotation Model of annotations of current project + """Retrieve annotation for a single project. + + Args: + namespace: Project namespace. + name: Project name. + tag: Project tag. + admin: Namespace(s) with admin/private access. + + Returns: + AnnotationModel for the project, or None if not found. + + Raises: + ProjectNotFoundError: If the project does not exist or is not accessible. """ _LOGGER.info(f"Getting annotation of the project: '{namespace}/{name}:{tag}'") admin_tuple = tuple_converter(admin) @@ -235,7 +243,9 @@ def _get_single_annotation( ) return annot else: - raise ProjectNotFoundError(f"Project '{namespace}/{name}:{tag}' was not found.") + raise ProjectNotFoundError( + f"Project '{namespace}/{name}:{tag}' was not found." + ) def _count_projects( self, @@ -248,21 +258,20 @@ def _count_projects( filter_end_date: str | None = None, pep_type: Literal["pep", "pop"] | None = None, ) -> int: - """ - Count projects. [This function is related to _find_projects] - - :param namespace: namespace where to search for a project - :param search_str: search string. will be searched in name, tag and description information - :param tag: tag of the projects (find projects with specific tag) - :param admin: string or list of admins [e.g. "Khoroshevskyi", or ["doc_adin","Khoroshevskyi"]] - :param filter_by: data to use filter on. - Options: ["submission_date", "last_update_date"] - [Default: filter won't be used] - :param filter_start_date: Filter start date. Format: "YYYY:MM:DD" - :param filter_end_date: Filter end date. Format: "YYYY:MM:DD". if None: present date will be used - :param pep_type: Get pep with specified type. Options: ["pep", "pop"]. Default: None, get all peps - - :return: number of found project in specified namespace + """Count projects matching the given filters. + + Args: + namespace: Namespace to restrict the search to. + search_str: String to search in name, tag, and description. + tag: Exact tag to match. + admin: Namespace(s) with admin/private access. + filter_by: Date field for range filter — "submission_date" or "last_update_date". + filter_start_date: Range start in YYYY/MM/DD format. + filter_end_date: Range end in YYYY/MM/DD format (default: today). + pep_type: Restrict to "pep" or "pop" (default: all). + + Returns: + Number of matching projects. """ if admin is None: admin = [] @@ -302,28 +311,28 @@ def _get_projects( filter_end_date: str | None = None, pep_type: Literal["pep", "pop"] | None = None, ) -> list[AnnotationModel]: + """Get projects matching the given filters. + + Args: + namespace: Namespace to restrict the search to. + tag: Exact tag to match. + search_str: String to search in name, tag, and description. + admin: Namespace(s) with admin/private access. + limit: Maximum number of results. + offset: Number of results to skip. + order_by: Sort field — "update_date", "name", "submission_date", or "stars". + order_desc: Sort in descending order if True. + filter_by: Date field for range filter — "submission_date" or "last_update_date". + filter_start_date: Range start in YYYY/MM/DD format. + filter_end_date: Range end in YYYY/MM/DD format (default: today). + pep_type: Restrict to "pep" or "pop" (default: all). + + Returns: + List of AnnotationModel objects. """ - Get projects by providing search string. - - :param namespace: namespace where to search for a project - :param tag: tag of the projects (find projects with specific tag) - :param search_str: search string that has to be found in the name or tag - :param admin: True, if user is admin of the namespace [Default: False] - :param limit: limit of return results - :param offset: number of results off set (that were already showed) - :param order_by: sort the result-set by the information - Options: ["update_date", "name", "submission_date", "stars"] - [Default: "update_date"] - :param order_desc: Sort the records in descending order. [Default: False] - :param filter_by: data to use filter on. - Options: ["submission_date", "last_update_date"] - [Default: filter won't be used] - :param filter_start_date: Filter start date. Format: "YYYY:MM:DD" - :param filter_end_date: Filter end date. Format: "YYYY:MM:DD". if None: present date will be used - :param pep_type: Get pep with specified type. Options: ["pep", "pop"]. Default: None, get all peps - :return: list of found projects with their annotations. - """ - _LOGGER.info(f"Running annotation search: (namespace: {namespace}, query: {search_str}.") + _LOGGER.info( + f"Running annotation search: (namespace: {namespace}, query: {search_str}." + ) if admin is None: admin = [] @@ -381,15 +390,15 @@ def _get_projects( def _add_order_by_keyword( statement: Select, by: str = "update_date", desc: bool = False ) -> Select: - """ - Add order by clause to sqlalchemy statement - - :param statement: sqlalchemy representation of a SELECT statement. - :param by: sort the result-set by the information - Options: ["name", "update_date", "submission_date", "stars"] - [Default: "update_date"] - :param desc: Sort the records in descending order. [Default: False] - :return: sqlalchemy representation of a SELECT statement with order by keyword + """Add an ORDER BY clause to a SELECT statement. + + Args: + statement: SQLAlchemy SELECT statement to augment. + by: Sort field — "name", "update_date", "submission_date", or "stars". + desc: Sort in descending order if True. + + Returns: + Statement with ORDER BY applied. """ if by == "update_date": order_by_obj = Projects.last_update_date @@ -421,15 +430,17 @@ def _add_condition( admin_list: str | list[str] | None = None, tag: str = None, ) -> Select: - """ - Add where clause to sqlalchemy statement (in project search) - - :param statement: sqlalchemy representation of a SELECT statement. - :param namespace: project namespace sql:(where namespace = "") - :param search_str: search string that has to be found in the name or tag - :param admin_list: list or string of admin rights to namespace - :param tag: tag of the projects (find projects with specific tag) - :return: sqlalchemy representation of a SELECT statement with where clause. + """Add a WHERE clause to a project search statement. + + Args: + statement: SQLAlchemy SELECT statement to augment. + namespace: Filter to this namespace. + search_str: String to search in name, tag, and description. + admin_list: Namespace(s) with admin/private access. + tag: Exact tag to match. + + Returns: + Statement with WHERE clause applied. """ admin_list = tuple_converter(admin_list) if search_str: @@ -459,15 +470,16 @@ def _add_date_filter_if_provided( filter_start_date: str | None, filter_end_date: str | None = None, ) -> Select: - """ - Add filter to where clause to sqlalchemy statement (in project search) - - :param statement: sqlalchemy representation of a SELECT statement with where clause - :param filter_by: data to use filter on. - Options: ["submission_date", "last_update_date"] - :param filter_start_date: Filter start date. Format: "YYYY:MM:DD" - :param filter_end_date: Filter end date. Format: "YYYY:MM:DD". if None: present date will be used - :return: sqlalchemy representation of a SELECT statement with where clause with added filter + """Add a date range filter to a SELECT statement. + + Args: + statement: SQLAlchemy SELECT statement to augment. + filter_by: Date field to filter on — "submission_date" or "last_update_date". + filter_start_date: Range start in YYYY/MM/DD format. + filter_end_date: Range end in YYYY/MM/DD format (default: today). + + Returns: + Statement with date filter applied. """ if filter_by and filter_start_date: start_date = convert_date_string_to_date(filter_start_date) @@ -488,7 +500,9 @@ def _add_date_filter_if_provided( return statement else: if filter_by: - _LOGGER.warning("filter_start_date was not provided, skipping filter...") + _LOGGER.warning( + "filter_start_date was not provided, skipping filter..." + ) return statement def get_project_number_in_namespace( @@ -496,17 +510,21 @@ def get_project_number_in_namespace( namespace: str, admin: str | list[str] | None = None, ) -> int: - """ - Get number of found projects by providing search string. + """Get the number of projects in a namespace. + + Args: + namespace: Namespace to count projects in. + admin: Namespace(s) with admin/private access. - :param namespace: namespace where to search for a project - :param admin: True, if user is admin of the namespace [Default: False] - :return Integer: number of projects in the namepsace + Returns: + Number of accessible projects in the namespace. """ if admin is None: admin = [] statement = ( - select(func.count()).select_from(Projects).where(Projects.namespace == namespace) + select(func.count()) + .select_from(Projects) + .where(Projects.namespace == namespace) ) statement = statement.where( or_(Projects.private.is_(False), Projects.namespace.in_(admin)) @@ -524,16 +542,14 @@ def get_by_rp_list( registry_paths: list[str], admin: str | list[str] | None = None, ) -> AnnotationList: - """ - Get project annotations by providing list of registry paths. - - :param registry_paths: registry path string or list of registry paths - :param admin: list of namespaces where user is admin - :return: pydantic model: AnnotationReturnModel( - limit: - offset: - count: - result: List [AnnotationModel]) + """Get project annotations for a list of registry paths in a single query. + + Args: + registry_paths: List of registry paths ("namespace/name:tag"). + admin: Namespace(s) where the caller has admin rights. + + Returns: + AnnotationList preserving the input order (missing projects returned as None). """ admin_tuple = tuple_converter(admin) @@ -624,30 +640,29 @@ def get_projects_list( filter_end_date: str | None = None, pep_type: Literal["pep", "pop"] | None = None, ) -> list[RegistryPath]: + """Retrieve a list of registry paths matching the given filters. + + Lightweight alternative to get() — returns only registry paths, no annotation data. + + Args: + namespace: Namespace to restrict the search to. + search_str: String to search in name, tag, and description. + admin: Namespace(s) with admin/private access. + limit: Maximum number of results. + offset: Number of results to skip. + order_by: Sort field — "name", "update_date", "submission_date", or "stars". + order_desc: Sort in descending order if True. + filter_by: Date field for range filter — "submission_date" or "last_update_date". + filter_start_date: Range start in YYYY/MM/DD format. + filter_end_date: Range end in YYYY/MM/DD format (default: today). + pep_type: Restrict to "pep" or "pop" (default: all). + + Returns: + List of RegistryPath objects. """ - Retrieve a list of projects by providing a search string. - This function serves as a lightweight version of the full 'get' function, - returning only a list of registry paths without annotations. - It is designed for use cases where a large list of projects is needed with minimal processing time. - - :param namespace: namespace where to search for a project - :param search_str: search string that has to be found in the name or tag - :param admin: True, if user is admin of the namespace [Default: False] - :param limit: limit of return results - :param offset: number of results off set (that were already showed) - :param order_by: sort the result-set by the information - Options: ["name", "update_date", "submission_date", "stars"] - [Default: "update_date"] - :param order_desc: Sort the records in descending order. [Default: False] - :param filter_by: data to use filter on. - Options: ["submission_date", "last_update_date"] - [Default: filter won't be used] - :param filter_start_date: Filter start date. Format: "YYYY:MM:DD" - :param filter_end_date: Filter end date. Format: "YYYY:MM:DD". if None: present date will be used - :param pep_type: Get pep with specified type. Options: ["pep", "pop"]. Default: None, get all peps - :return: list of found projects with their annotations. - """ - _LOGGER.info(f"Running project search: (namespace: {namespace}, query: {search_str}.") + _LOGGER.info( + f"Running project search: (namespace: {namespace}, query: {search_str}." + ) if admin is None: admin = [] diff --git a/pepdbagent/modules/namespace.py b/pepdbagent/modules/namespace.py index 8fd8c4a8..8e4a7506 100644 --- a/pepdbagent/modules/namespace.py +++ b/pepdbagent/modules/namespace.py @@ -33,7 +33,8 @@ class PEPDatabaseNamespace: def __init__(self, pep_db_engine: BaseEngine): """ - :param pep_db_engine: pepdbengine object with sa engine + Args: + pep_db_engine: PEPDatabaseAgent engine object. """ self._sa_engine = pep_db_engine.engine self._pep_db_engine = pep_db_engine @@ -45,21 +46,20 @@ def get( limit: int = DEFAULT_LIMIT, offset: int = DEFAULT_OFFSET, ) -> NamespaceList: + """Search available namespaces in the database. + + Args: + query: Search string. + admin: Namespaces where the user has admin rights. + limit: Maximum number of results to return. + offset: Number of results to skip. + + Returns: + NamespaceList with count, limit, offset, and results. """ - Search available namespaces in the database - :param query: search string - :param admin: list of namespaces where user is admin - :param offset: offset of the search - :param limit: limit of the search - :return: Search result: - { - total number of results - search limit - search offset - search results - } - """ - _LOGGER.info(f"Getting namespaces annotation with provided info: (query: {query})") + _LOGGER.info( + f"Getting namespaces annotation with provided info: (query: {query})" + ) admin_tuple = tuple_converter(admin) return NamespaceList( count=self._count_namespace(search_str=query, admin_nsp=admin_tuple), @@ -80,19 +80,16 @@ def _get_namespace( limit: int = DEFAULT_LIMIT, offset: int = DEFAULT_OFFSET, ) -> list[Namespace]: - """ - Search for namespace by providing search string. - - :param search_str: string of symbols, words, keywords to search in the - namespace name. - :param admin_nsp: tuple of namespaces where project can be retrieved if they are privet - :param limit: limit of return results - :param offset: number of results off set (that were already showed) - :return: list of dict with structure { - namespace, - number_of_projects, - number_of_samples, - } + """Search for namespaces matching a search string. + + Args: + search_str: Keywords to search in namespace names. + admin_nsp: Namespaces accessible even when private. + limit: Maximum number of results. + offset: Number of results to skip. + + Returns: + List of Namespace objects. """ statement = ( select( @@ -126,14 +123,17 @@ def _get_namespace( ) return results_list - def _count_namespace(self, search_str: str = None, admin_nsp: tuple = tuple()) -> int: - """ - Get number of found namespace. [This function is related to _get_namespaces] + def _count_namespace( + self, search_str: str = None, admin_nsp: tuple = tuple() + ) -> int: + """Count namespaces matching a search string. + + Args: + search_str: Keywords to search in namespace names. + admin_nsp: Namespaces accessible even when private. - :param search_str: string of symbols, words, keywords to search in the - namespace name. - :param admin_nsp: tuple of namespaces where project can be retrieved if they are privet - :return: number of found namespaces + Returns: + Number of matching namespaces. """ statement = select( func.count(distinct(Projects.namespace)).label("number_of_namespaces") @@ -154,13 +154,15 @@ def _add_condition( search_str: str = None, admin_list: tuple[str, ...] | list[str] | str | None = None, ) -> Select: - """ - Add where clause to sqlalchemy statement (in namespace search) + """Add a WHERE clause to a namespace search statement. + + Args: + statement: SQLAlchemy SELECT statement to augment. + search_str: String to search in namespace names. + admin_list: Namespaces with admin/private access. - :param statement: sqlalchemy representation of a SELECT statement. - :param search_str: search string that has to be found namespace - :param admin_list: list or string of admin rights to namespace - :return: sqlalchemy representation of a SELECT statement with where clause. + Returns: + Statement with WHERE clause applied. """ if search_str: sql_search_str = f"%{search_str}%" @@ -180,21 +182,17 @@ def info( page_size: int = DEFAULT_LIMIT_INFO, order_by: str = "number_of_projects", ) -> ListOfNamespaceInfo: - """ - Get list of top n namespaces in the database - ! Warning: this function counts number of all projects in namespaces. - ! it does not filter private projects (It was done for efficiency reasons) - - :param page: page number - :param page_size: number of namespaces to show - :param order_by: order by field. Options: number_of_projects, number_of_schemas [Default: number_of_projects] - - :return: number_of_namespaces: int - limit: int - results: { namespace: str - number_of_projects: int - number_of_schemas: int - } + """Get a paginated list of top namespaces. + + Warning: counts all projects including private ones (by design, for efficiency). + + Args: + page: Page number (zero-based). + page_size: Number of namespaces per page. + order_by: Sort field — "number_of_projects" or "number_of_schemas". + + Returns: + ListOfNamespaceInfo with pagination metadata and results. """ statement = select(User) @@ -205,8 +203,12 @@ def info( statement = statement.order_by(User.number_of_schemas.desc()) with Session(self._sa_engine) as session: - results = session.scalars(statement.limit(page_size).offset(page_size * page)) - total_number_of_namespaces = session.execute(select(func.count(User.id))).one()[0] + results = session.scalars( + statement.limit(page_size).offset(page_size * page) + ) + total_number_of_namespaces = session.execute( + select(func.count(User.id)) + ).one()[0] list_of_results = [] for result in results: @@ -228,11 +230,14 @@ def info( ) def stats(self, namespace: str = None, monthly: bool = False) -> NamespaceStats: - """ - Get statistics for project in the namespace or for all projects in the database. + """Get submission/update statistics for a namespace or the whole database. + + Args: + namespace: Namespace to filter by (default: all namespaces). + monthly: Return monthly stats for 3 years if True, daily stats for 3 months if False. - :param namespace: namespace name [Default: None (all projects)] - :param monthly: if True, get statistics for the last 3 years monthly, else for the last 3 months daily. + Returns: + NamespaceStats with projects_updated and projects_created histograms. """ if monthly: number_of_month = 12 * 3 @@ -247,15 +252,21 @@ def stats(self, namespace: str = None, monthly: bool = False) -> NamespaceStats: Projects.submission_date.between(three_month_ago, today_date) ) if namespace: - statement_last_update = statement_last_update.where(Projects.namespace == namespace) - statement_create_date = statement_create_date.where(Projects.namespace == namespace) + statement_last_update = statement_last_update.where( + Projects.namespace == namespace + ) + statement_create_date = statement_create_date.where( + Projects.namespace == namespace + ) with Session(self._sa_engine) as session: update_results = session.execute(statement_last_update).all() create_results = session.execute(statement_create_date).all() if not update_results: - raise NamespaceNotFoundError(f"Namespace {namespace} not found in the database") + raise NamespaceNotFoundError( + f"Namespace {namespace} not found in the database" + ) if monthly: year_month_str_submission = [ @@ -282,11 +293,10 @@ def stats(self, namespace: str = None, monthly: bool = False) -> NamespaceStats: ) def upload_tar_info(self, tar_info: TarNamespaceModel) -> None: - """ - Upload metadata of tar GEO files + """Upload metadata for a namespace tar archive. - tar_info: TarNamespaceModel - :return: None + Args: + tar_info: Tar archive metadata. """ with Session(self._sa_engine) as session: @@ -303,12 +313,13 @@ def upload_tar_info(self, tar_info: TarNamespaceModel) -> None: _LOGGER.info("Geo tar info was uploaded successfully!") def get_tar_info(self, namespace: str) -> TarNamespaceModelReturn: - """ - Get metadata of tar GEO files + """Get metadata for namespace tar archives. - :param namespace: namespace of the tar files + Args: + namespace: Namespace of the tar files. - :return: list with geo data + Returns: + TarNamespaceModelReturn with count and list of archive metadata. """ with Session(self._sa_engine) as session: @@ -334,19 +345,18 @@ def get_tar_info(self, namespace: str) -> TarNamespaceModelReturn: return TarNamespaceModelReturn(count=len(results), results=results) def delete_tar_info(self, namespace: str = None) -> None: - """ - Delete all metadata of tar GEO files + """Delete tar archive metadata for a namespace. - :param namespace: namespace of the tar files - - :return: None + Args: + namespace: Namespace to delete archives for (default: all namespaces). """ with Session(self._sa_engine) as session: - delete_statement = delete(TarNamespace) if namespace: - delete_statement = delete_statement.where(TarNamespace.namespace == namespace) + delete_statement = delete_statement.where( + TarNamespace.namespace == namespace + ) session.execute(delete_statement) session.commit() _LOGGER.info("Geo tar info was deleted successfully!") diff --git a/pepdbagent/modules/project.py b/pepdbagent/modules/project.py index 5b87948e..54ccaae0 100644 --- a/pepdbagent/modules/project.py +++ b/pepdbagent/modules/project.py @@ -72,7 +72,8 @@ class PEPDatabaseProject: def __init__(self, pep_db_engine: BaseEngine): """ - :param pep_db_engine: pepdbengine object with sa engine + Args: + pep_db_engine: PEPDatabaseAgent engine object. """ self._sa_engine = pep_db_engine.engine self._pep_db_engine = pep_db_engine @@ -85,22 +86,20 @@ def get( raw: bool = True, with_id: bool = False, ) -> peprs.Project | dict | None: - """ - Retrieve project from database by specifying namespace, name and tag - - :param namespace: namespace of the project - :param name: name of the project (Default: name is taken from the project object) - :param tag: tag (or version) of the project. - :param raw: retrieve unprocessed (raw) PEP dict. - :param with_id: retrieve project with id [default: False] - :return: peprs.Project object with found project or dict with unprocessed - PEP elements: { - name: str - description: str - _config: dict - _sample_dict: dict - _subsample_dict: dict - } + """Retrieve a project from the database. + + Args: + namespace: Namespace of the project. + name: Name of the project. + tag: Tag of the project. + raw: Return raw dict if True, peprs.Project object if False. + with_id: Include pephub_sample_id in each sample dict if True. + + Returns: + Raw dict or peprs.Project object. + + Raises: + ProjectNotFoundError: If the project does not exist. """ # name = name.lower() namespace = namespace.lower() @@ -119,7 +118,9 @@ def get( for subsample in found_prj.subsamples_mapping: if subsample.subsample_number not in subsample_dict.keys(): subsample_dict[subsample.subsample_number] = [] - subsample_dict[subsample.subsample_number].append(subsample.subsample) + subsample_dict[subsample.subsample_number].append( + subsample.subsample + ) subsample_list = list(subsample_dict.values()) else: subsample_list = [] @@ -150,13 +151,15 @@ def get( raise ProjectNotFoundError def _get_samples(self, session: Session, prj_id: int, with_id: bool) -> list[dict]: - """ - Get samples from the project. This method is used to retrieve samples from the project, - with open session object. + """Get ordered samples from a project using an open session. - :param session: open session object - :param prj_id: project id - :param with_id: retrieve sample with id + Args: + session: Open SQLAlchemy session. + prj_id: Project id. + with_id: Include pephub_sample_id in each sample dict if True. + + Returns: + Ordered list of sample dicts. """ result_dict = self._get_samples_dict(prj_id, session, with_id) @@ -167,23 +170,19 @@ def _get_samples(self, session: Session, prj_id: int, with_id: bool) -> list[dic @staticmethod def _get_samples_dict(prj_id: int, session: Session, with_id: bool) -> dict: + """Get unordered samples from a project keyed by guid. + + Args: + prj_id: Project id. + session: Open SQLAlchemy session. + with_id: Include pephub_sample_id in each sample dict if True. + + Returns: + Dict mapping guid → {sample, guid, parent_guid}. """ - Get not ordered samples from the project. This method is used to retrieve samples from the project - - :param prj_id: project id - :param session: open session object - :param with_id: retrieve sample with id - - :return: dictionary with samples: - {guid: - { - "sample": sample_dict, - "guid": guid, - "parent_guid": parent_guid - } - } - """ - samples_results = session.scalars(select(Samples).where(Samples.project_id == prj_id)) + samples_results = session.scalars( + select(Samples).where(Samples.project_id == prj_id) + ) result_dict = {} for sample in samples_results: sample_dict = sample.sample @@ -199,14 +198,18 @@ def _get_samples_dict(prj_id: int, session: Session, with_id: bool) -> dict: return result_dict @staticmethod - def _create_select_statement(name: str, namespace: str, tag: str = DEFAULT_TAG) -> Select: - """ - Create simple select statement for retrieving project from database - - :param name: name of the project - :param namespace: namespace of the project - :param tag: tag of the project - :return: select statement + def _create_select_statement( + name: str, namespace: str, tag: str = DEFAULT_TAG + ) -> Select: + """Create a SELECT statement for a single project. + + Args: + name: Project name. + namespace: Project namespace. + tag: Project tag. + + Returns: + SQLAlchemy SELECT statement for the Projects table. """ statement = select(Projects) statement = statement.where( @@ -223,19 +226,17 @@ def get_by_rp( registry_path: str, raw: bool = False, ) -> peprs.Project | dict | None: - """ - Retrieve project from database by specifying project registry_path - - :param registry_path: project registry_path [e.g. namespace/name:tag] - :param raw: retrieve unprocessed (raw) PEP dict. - :return: peprs.Project object with found project or dict with unprocessed - PEP elements: { - name: str - description: str - _config: dict - _sample_dict: dict - _subsample_dict: dict - } + """Retrieve a project from the database by registry path. + + Args: + registry_path: Registry path, e.g., "namespace/name:tag". + raw: Return raw dict if True, peprs.Project object if False. + + Returns: + Raw dict or peprs.Project object. + + Raises: + ProjectNotFoundError: If the project does not exist. """ namespace, name, tag = registry_path_converter(registry_path) return self.get(namespace=namespace, name=name, tag=tag, raw=raw) @@ -246,13 +247,15 @@ def delete( name: str = None, tag: str = None, ) -> None: - """ - Delete record from database + """Delete a project from the database. + + Args: + namespace: Project namespace. + name: Project name. + tag: Project tag. - :param namespace: Namespace - :param name: Name - :param tag: Tag - :return: None + Raises: + ProjectNotFoundError: If the project does not exist. """ # name = name.lower() namespace = namespace.lower() @@ -283,11 +286,13 @@ def delete_by_rp( self, registry_path: str, ) -> None: - """ - Delete record from database by using registry_path + """Delete a project from the database by registry path. + + Args: + registry_path: Registry path of the project, e.g., "namespace/name:tag". - :param registry_path: Registry path of the project ('namespace/name:tag') - :return: None + Raises: + ProjectNotFoundError: If the project does not exist. """ namespace, name, tag = registry_path_converter(registry_path) return self.delete(namespace=namespace, name=name, tag=tag) @@ -305,24 +310,23 @@ def create( overwrite: bool = False, update_only: bool = False, ) -> None: - """ - Upload project to the database. - Project with the key, that already exists won't be uploaded(but case, when argument - update is set True) - - :param project: peprs.Project object or dict with PEP elements - ({config: dict, samples: list, subsamples: list}) - :param namespace: namespace of the project (Default: 'other') - :param name: name of the project (Default: name is taken from the project object) - :param tag: tag (or version) of the project. - :param is_private: boolean value if the project should be visible just for user that creates it. - :param pep_schema: assign PEP to a specific schema. Example: 'namespace/name' [Default: None] - :param pop: if project is a pep of peps (POP) [Default: False] - :param overwrite: if project exists overwrite the project, otherwise upload it. - [Default: False - project won't be overwritten if it exists in db] - :param update_only: if project exists overwrite it, otherwise do nothing. [Default: False] - :param description: description of the project - :return: None + """Upload a project to the database. + + Args: + project: peprs.Project or dict with config, samples, and subsamples keys. + namespace: Namespace to upload to. + name: Project name (default: taken from project config). + tag: Project tag. + description: Project description. + is_private: Mark project as private if True. + pop: Mark as a PEP-of-PEPs (POP) if True. + pep_schema: Schema to associate, e.g., "namespace/name". + overwrite: Overwrite the project if it already exists. + update_only: Update the project if it exists; do nothing otherwise. + + Raises: + ProjectUniqueNameError: If the project already exists and overwrite is False. + SchemaDoesNotExistError: If pep_schema is provided but does not exist. """ if isinstance(project, peprs.Project): proj_dict = project.to_dict(raw=True, by_sample=True) @@ -351,7 +355,9 @@ def create( elif proj_dict[CONFIG_KEY][NAME_KEY]: proj_name = proj_dict[CONFIG_KEY][NAME_KEY].lower() else: - raise ValueError("Name of the project wasn't provided. Project will not be uploaded.") + raise ValueError( + "Name of the project wasn't provided. Project will not be uploaded." + ) proj_dict[CONFIG_KEY][NAME_KEY] = proj_name @@ -362,13 +368,16 @@ def create( number_of_samples = len(proj_dict[SAMPLE_RAW_DICT_KEY]) if pep_schema: - schema_namespace, schema_name, schema_version = schema_path_converter(pep_schema) + schema_namespace, schema_name, schema_version = schema_path_converter( + pep_schema + ) with Session(self._sa_engine) as session: - if schema_version == LATEST_SCHEMA_VERSION: schema_mapping = session.scalar( select(SchemaVersions) - .join(SchemaRecords, SchemaRecords.id == SchemaVersions.schema_id) + .join( + SchemaRecords, SchemaRecords.id == SchemaVersions.schema_id + ) .where( and_( SchemaRecords.namespace == schema_namespace, @@ -396,7 +405,9 @@ def create( pep_schema = schema_mapping.id if update_only: - _LOGGER.info(f"Update_only argument is set True. Updating project {proj_name} ...") + _LOGGER.info( + f"Update_only argument is set True. Updating project {proj_name} ..." + ) self._overwrite( project_dict=proj_dict, namespace=namespace, @@ -442,7 +453,9 @@ def create( self._add_subsamples_to_project(new_prj, subsamples) with Session(self._sa_engine) as session: - user = session.scalar(select(User).where(User.namespace == namespace)) + user = session.scalar( + select(User).where(User.namespace == namespace) + ) if not user: user = User(namespace=namespace) @@ -491,20 +504,22 @@ def _overwrite( description: str = "", pop: bool = False, ) -> None: - """ - Update existing project by providing all necessary information. - - :param project_dict: project dictionary in json format - :param namespace: project namespace - :param proj_name: project name - :param tag: project tag - :param project_digest: project digest - :param number_of_samples: number of samples in project - :param private: boolean value if the project should be visible just for user that creates it. - :param pep_schema: assign PEP to a specific schema. [DefaultL: None] - :param description: project description - :param pop: if project is a pep of peps, simply POP [Default: False] - :return: None + """Overwrite an existing project with new content. + + Args: + project_dict: Full project dict (config, samples, subsamples). + namespace: Project namespace. + proj_name: Project name. + tag: Project tag. + project_digest: Pre-computed digest. + number_of_samples: Sample count. + private: Mark as private if True. + pep_schema: Schema id to associate (integer FK). + description: Project description. + pop: Mark as POP if True. + + Raises: + ProjectNotFoundError: If the project does not exist. """ proj_name = proj_name.lower() namespace = namespace.lower() @@ -527,7 +542,9 @@ def _overwrite( found_prj.schema_id = pep_schema found_prj.config = project_dict[CONFIG_KEY] found_prj.description = description - found_prj.last_update_date = datetime.datetime.now(datetime.timezone.utc) + found_prj.last_update_date = datetime.datetime.now( + datetime.timezone.utc + ) found_prj.pop = pop # Deleting old samples and subsamples @@ -545,7 +562,9 @@ def _overwrite( self._add_samples_to_project( found_prj, project_dict[SAMPLE_RAW_DICT_KEY], - sample_table_index=project_dict[CONFIG_KEY].get(SAMPLE_TABLE_INDEX_KEY), + sample_table_index=project_dict[CONFIG_KEY].get( + SAMPLE_TABLE_INDEX_KEY + ), ) if project_dict.get(SUBSAMPLE_RAW_DICT_KEY): @@ -555,11 +574,15 @@ def _overwrite( session.commit() - _LOGGER.info(f"Project '{namespace}/{proj_name}:{tag}' has been successfully updated!") + _LOGGER.info( + f"Project '{namespace}/{proj_name}:{tag}' has been successfully updated!" + ) return None else: - raise ProjectNotFoundError("Project does not exist! No project will be updated!") + raise ProjectNotFoundError( + "Project does not exist! No project will be updated!" + ) def update( self, @@ -569,38 +592,32 @@ def update( tag: str = DEFAULT_TAG, user: str = None, ) -> None: - """ - Update partial parts of the record in db - - :param update_dict: dict with update key->values. Dict structure: - { - project: Optional[peprs.Project] - is_private: Optional[bool] - tag: Optional[str] - name: Optional[str] - description: Optional[str] - is_private: Optional[bool] - pep_schema: Optional[str] - config: Optional[dict] - samples: Optional[List[dict]] - subsamples: Optional[List[List[dict]]] - pop: Optional[bool] - } - :param namespace: project namespace - :param name: project name - :param tag: project tag - :param user: user that updates the project if user is not provided, user will be set as Namespace - :return: None + """Update specific fields of a project. + + Args: + update_dict: Fields to update — any combination of project, is_private, tag, name, + description, pep_schema, config, samples, subsamples, pop. + namespace: Project namespace. + name: Project name. + tag: Project tag. + user: User performing the update (default: namespace). + + Raises: + ProjectNotFoundError: If the project does not exist. """ if self.exists(namespace=namespace, name=name, tag=tag): if isinstance(update_dict, UpdateItems): update_values = update_dict else: if "project" in update_dict: - project_dict = update_dict.pop("project").to_dict(raw=True, by_sample=True) + project_dict = update_dict.pop("project").to_dict( + raw=True, by_sample=True + ) update_dict["config"] = project_dict[CONFIG_KEY] update_dict["samples"] = project_dict[SAMPLE_RAW_DICT_KEY] - update_dict["subsamples"] = project_dict.get(SUBSAMPLE_RAW_DICT_KEY, []) + update_dict["subsamples"] = project_dict.get( + SUBSAMPLE_RAW_DICT_KEY, [] + ) update_values = UpdateItems(**update_dict) @@ -640,7 +657,6 @@ def update( flag_modified(found_prj, "config") if "samples" in update_dict: - if PEPHUB_SAMPLE_ID_KEY not in update_dict["samples"][0]: raise SampleTableUpdateError( f"pephub_sample_id '{PEPHUB_SAMPLE_ID_KEY}' is missing in samples." @@ -678,9 +694,13 @@ def update( # Adding new subsamples if update_dict["subsamples"]: - self._add_subsamples_to_project(found_prj, update_dict["subsamples"]) + self._add_subsamples_to_project( + found_prj, update_dict["subsamples"] + ) - found_prj.last_update_date = datetime.datetime.now(datetime.timezone.utc) + found_prj.last_update_date = datetime.datetime.now( + datetime.timezone.utc + ) session.commit() @@ -691,14 +711,14 @@ def update( @staticmethod def _convert_update_schema_id(session: Session, update_values: dict) -> None: - """ - Convert schema path to schema_id in update_values and update it in update dict - + """Resolve pep_schema path to a schema_id in the update dict. - :param session: open session object - :param update_values: dict with update key->values + Args: + session: Open SQLAlchemy session. + update_values: Update dict that may contain a "pep_schema" key. - return None + Raises: + SchemaDoesNotExistError: If the schema path does not resolve to an existing schema. """ if "pep_schema" in update_values: schema_namespace, schema_name, schema_version = schema_path_converter( @@ -742,26 +762,32 @@ def _update_samples( sample_name_key: str = "sample_name", history_sa_model: HistoryProjects | None = None, ) -> None: - """ - Update samples in the project - This is linked list method, that first finds differences in old and new samples list - and then updates, adds, inserts, deletes, or changes the order. - - :param project_id: project id in PEPhub database - :param samples_list: list of samples to be updated - :param sample_name_key: key of the sample name - :param history_sa_model: HistoryProjects object, to write to the history table - :return: None + """Update samples in a project using a linked-list diff approach. + + Finds differences between old and new sample lists, then applies inserts, updates, + deletes, and order changes. + + Args: + project_id: Project id in the database. + samples_list: New list of sample dicts (must include pephub_sample_id). + sample_name_key: Sample table index key. + history_sa_model: HistoryProjects row to append change records to. + + Raises: + ProjectDuplicatedSampleGUIDsError: If any pephub_sample_id is duplicated. """ with Session(self._sa_engine) as session: - old_samples = session.scalars(select(Samples).where(Samples.project_id == project_id)) + old_samples = session.scalars( + select(Samples).where(Samples.project_id == project_id) + ) old_samples_mapping: dict = {sample.guid: sample for sample in old_samples} # old_child_parent_id needed because of the parent_guid is sometimes set to none in sqlalchemy mapping :( bug old_child_parent_id: dict[str, str] = { - child: mapping.parent_guid for child, mapping in old_samples_mapping.items() + child: mapping.parent_guid + for child, mapping in old_samples_mapping.items() } old_samples_ids_set: set = set(old_samples_mapping.keys()) @@ -789,7 +815,6 @@ def _update_samples( del new_samples_ids_list, new_samples_ids_set for remove_id in deleted_ids: - if history_sa_model: history_sa_model.sample_changes_mapping.append( HistorySamples( @@ -832,7 +857,6 @@ def _update_samples( else: current_history = None if old_samples_mapping[current_id].sample != sample_value: - if history_sa_model: current_history = HistorySamples( guid=old_samples_mapping[current_id].guid, @@ -842,7 +866,9 @@ def _update_samples( ) old_samples_mapping[current_id].sample = sample_value - old_samples_mapping[current_id].sample_name = sample_value[sample_name_key] + old_samples_mapping[current_id].sample_name = sample_value[ + sample_name_key + ] # !bug workaround: if project was deleted and sometimes old_samples_mapping[current_id].parent_guid # and it can cause an error in history. For this we have `old_child_parent_id` dict @@ -869,13 +895,13 @@ def _update_samples( @staticmethod def __create_update_dict(update_values: UpdateItems) -> dict: - """ - Modify keys and values that set for update and create unified - dictionary of the values that have to be updated + """Build a normalised update dict from an UpdateItems model. - :param update_values: UpdateItems (pydantic class) with - updating values - :return: unified update dict + Args: + update_values: UpdateItems with fields to apply. + + Returns: + Dict of field→value pairs ready for setattr on the Projects row. """ update_final = UpdateModel.model_construct() @@ -896,7 +922,8 @@ def __create_update_dict(update_values: UpdateItems) -> dict: ) if update_values.config is not None: update_final = UpdateModel( - config=update_values.config, **update_final.model_dump(exclude_unset=True) + config=update_values.config, + **update_final.model_dump(exclude_unset=True), ) name = update_values.config.get(NAME_KEY) description = update_values.config.get(DESCRIPTION_KEY) @@ -908,7 +935,9 @@ def __create_update_dict(update_values: UpdateItems) -> dict: if description: update_final = UpdateModel( description=description, - **update_final.model_dump(exclude_unset=True, exclude={DESCRIPTION_KEY}), + **update_final.model_dump( + exclude_unset=True, exclude={DESCRIPTION_KEY} + ), ) if update_values.tag is not None: @@ -947,12 +976,15 @@ def exists( name: str, tag: str = DEFAULT_TAG, ) -> bool: - """ - Check if project exists in the database. - :param namespace: project namespace - :param name: project name - :param tag: project tag - :return: Returning True if project exist + """Check if a project exists in the database. + + Args: + namespace: Project namespace. + name: Project name. + tag: Project tag. + + Returns: + True if the project exists. """ statement = select(Projects.id) @@ -972,18 +1004,19 @@ def exists( @staticmethod def _add_samples_to_project( - projects_sa: Projects, samples: list[dict], sample_table_index: str = "sample_name" + projects_sa: Projects, + samples: list[dict], + sample_table_index: str = "sample_name", ) -> None: - """ - Add samples to the project sa object. (With commit this samples will be added to the 'samples table') - :param projects_sa: Projects sa object, in open session - :param samples: list of samles to be added to the database - :param sample_table_index: index of the sample table - :return: NoReturn + """Append sample rows to a Projects ORM object. + + Args: + projects_sa: Projects ORM object in an open session. + samples: List of sample dicts to add. + sample_table_index: Column name to use as the sample identifier. """ previous_sample_guid = None for sample in samples: - sample = Samples( sample=sample, sample_name=sample.get(sample_table_index), @@ -999,30 +1032,37 @@ def _add_samples_to_project( def _add_subsamples_to_project( projects_sa: Projects, subsamples: list[list[dict]] ) -> None: - """ - Add subsamples to the project sa object. (With commit this samples will be added to the 'subsamples table') + """Append subsample rows to a Projects ORM object. - :param projects_sa: Projects sa object, in open session - :param subsamples: list of subsamles to be added to the database - :return: NoReturn + Args: + projects_sa: Projects ORM object in an open session. + subsamples: List of subsample groups (list of list of dicts). """ for i, subs in enumerate(subsamples): for row_number, sub_item in enumerate(subs): projects_sa.subsamples_mapping.append( - Subsamples(subsample=sub_item, subsample_number=i, row_number=row_number) + Subsamples( + subsample=sub_item, subsample_number=i, row_number=row_number + ) ) def get_project_id(self, namespace: str, name: str, tag: str) -> int | None: - """ - Get Project id by providing namespace, name, and tag + """Get the database id of a project. - :param namespace: project namespace - :param name: project name - :param tag: project tag - :return: projects id + Args: + namespace: Project namespace. + name: Project name. + tag: Project tag. + + Returns: + Project id, or None if the project does not exist. """ statement = select(Projects.id).where( - and_(Projects.namespace == namespace, Projects.name == name, Projects.tag == tag) + and_( + Projects.namespace == namespace, + Projects.name == name, + Projects.tag == tag, + ) ) with Session(self._sa_engine) as session: result = session.execute(statement).one_or_none() @@ -1042,18 +1082,20 @@ def fork( description: str = None, private: bool = False, ) -> None: - """ - Fork project from one namespace to another - - :param original_namespace: namespace of the project to be forked - :param original_name: name of the project to be forked - :param original_tag: tag of the project to be forked - :param fork_namespace: namespace of the forked project - :param fork_name: name of the forked project - :param fork_tag: tag of the forked project - :param description: description of the forked project - :param private: boolean value if the project should be visible just for user that creates it. - :return: None + """Fork a project into a new namespace. + + Args: + original_namespace: Namespace of the source project. + original_name: Name of the source project. + original_tag: Tag of the source project. + fork_namespace: Target namespace. + fork_name: Name for the fork (default: same as original). + fork_tag: Tag for the fork (default: same as original). + description: Description for the fork. + private: Mark fork as private if True. + + Raises: + ProjectNotFoundError: If the source project does not exist. """ self.create( @@ -1097,16 +1139,22 @@ def fork( session.commit() def get_config(self, namespace: str, name: str, tag: str) -> dict | None: - """ - Get project configuration by providing namespace, name, and tag + """Get the config dict for a project. + + Args: + namespace: Project namespace. + name: Project name. + tag: Project tag. - :param namespace: project namespace - :param name: project name - :param tag: project tag - :return: project configuration + Returns: + Config dict, or None if the project does not exist. """ statement = select(Projects.config).where( - and_(Projects.namespace == namespace, Projects.name == name, Projects.tag == tag) + and_( + Projects.namespace == namespace, + Projects.name == name, + Projects.tag == tag, + ) ) with Session(self._sa_engine) as session: result = session.execute(statement).one_or_none() @@ -1116,28 +1164,36 @@ def get_config(self, namespace: str, name: str, tag: str) -> dict | None: return None def get_subsamples(self, namespace: str, name: str, tag: str) -> list | None: - """ - Get project subsamples by providing namespace, name, and tag + """Get the subsample list for a project. + + Args: + namespace: Project namespace. + name: Project name. + tag: Project tag. + + Returns: + List of subsample groups, or an empty list if there are no subsamples. - :param namespace: project namespace - :param name: project name - :param tag: project tag - :return: list with project subsamples + Raises: + ProjectNotFoundError: If the project does not exist. """ statement = self._create_select_statement(name, namespace, tag) with Session(self._sa_engine) as session: - found_prj = session.scalar(statement) if found_prj: - _LOGGER.info(f"Project has been found: {found_prj.namespace}, {found_prj.name}") + _LOGGER.info( + f"Project has been found: {found_prj.namespace}, {found_prj.name}" + ) subsample_dict = {} if found_prj.subsamples_mapping: for subsample in found_prj.subsamples_mapping: if subsample.subsample_number not in subsample_dict.keys(): subsample_dict[subsample.subsample_number] = [] - subsample_dict[subsample.subsample_number].append(subsample.subsample) + subsample_dict[subsample.subsample_number].append( + subsample.subsample + ) return list(subsample_dict.values()) else: return [] @@ -1148,39 +1204,50 @@ def get_subsamples(self, namespace: str, name: str, tag: str) -> list | None: ) def get_samples( - self, namespace: str, name: str, tag: str, raw: bool = True, with_ids: bool = False + self, + namespace: str, + name: str, + tag: str, + raw: bool = True, + with_ids: bool = False, ) -> list: - """ - Get project samples by providing namespace, name, and tag + """Get samples for a project. - :param namespace: project namespace - :param name: project name - :param tag: project tag - :param raw: if True, retrieve unprocessed (raw) PEP dict. [Default: True] - :param with_ids: if True, retrieve samples with ids. [Default: False] + Args: + namespace: Project namespace. + name: Project name. + tag: Project tag. + raw: Return raw dicts if True, records-orient pandas dicts if False. + with_ids: Include pephub_sample_id in each sample if True. - :return: list with project samples + Returns: + List of sample dicts. """ if raw: return self.get( namespace=namespace, name=name, tag=tag, raw=True, with_id=with_ids ).get(SAMPLE_RAW_DICT_KEY) return ( - self.get(namespace=namespace, name=name, tag=tag, raw=False, with_id=with_ids) + self.get( + namespace=namespace, name=name, tag=tag, raw=False, with_id=with_ids + ) .to_pandas() .replace({np.nan: None}) .to_dict(orient="records") ) - def get_history(self, namespace: str, name: str, tag: str) -> HistoryAnnotationModel: - """ - Get project history annotation by providing namespace, name, and tag + def get_history( + self, namespace: str, name: str, tag: str + ) -> HistoryAnnotationModel: + """Get the list of history entries for a project. - :param namespace: project namespace - :param name: project name - :param tag: project tag + Args: + namespace: Project namespace. + name: Project name. + tag: Project tag. - :return: project history annotation + Returns: + HistoryAnnotationModel with a list of HistoryChangeModel entries. """ with Session(self._sa_engine) as session: @@ -1228,17 +1295,22 @@ def get_project_from_history( raw: bool = True, with_id: bool = False, ) -> dict | peprs.Project: - """ - Get project sample history annotation by providing namespace, name, and tag - - :param namespace: project namespace - :param name: project name - :param tag: project tag - :param history_id: history id - :param raw: if True, retrieve unprocessed (raw) PEP dict. [Default: True] - :param with_id: if True, retrieve samples with ids. [Default: False] - - :return: project sample history annotation + """Reconstruct a project at a past history checkpoint. + + Args: + namespace: Project namespace. + name: Project name. + tag: Project tag. + history_id: Id of the history entry to restore to. + raw: Return raw dict if True, peprs.Project object if False. + with_id: Include pephub_sample_id in each sample if True. + + Returns: + Raw dict or peprs.Project at the requested history state. + + Raises: + ProjectNotFoundError: If the project does not exist. + HistoryNotFoundError: If the history entry does not exist. """ with Session(self._sa_engine) as session: @@ -1325,19 +1397,23 @@ def get_project_from_history( @staticmethod def _apply_history_changes(sample_dict: dict, change: HistoryProjects) -> dict: - """ - Apply changes from the history to the sample list + """Apply a history change record to a sample dict. + + Args: + sample_dict: Current sample dict keyed by guid. + change: HistoryProjects entry whose sample_changes_mapping to apply. - :param sample_dict: dictionary with samples - :param change: history change - :return: updated sample list + Returns: + Updated sample dict. """ for sample_change in change.sample_changes_mapping: sample_id = sample_change.guid if sample_change.change_type == UpdateTypes.UPDATE: sample_dict[sample_id]["sample"] = sample_change.sample_json - sample_dict[sample_id]["sample"][PEPHUB_SAMPLE_ID_KEY] = sample_change.guid + sample_dict[sample_id]["sample"][PEPHUB_SAMPLE_ID_KEY] = ( + sample_change.guid + ) sample_dict[sample_id]["parent_guid"] = sample_change.parent_guid elif sample_change.change_type == UpdateTypes.DELETE: @@ -1355,15 +1431,17 @@ def _apply_history_changes(sample_dict: dict, change: HistoryProjects) -> dict: def delete_history( self, namespace: str, name: str, tag: str, history_id: int | None = None ) -> None: - """ - Delete history from the project + """Delete one or all history entries for a project. - :param namespace: project namespace - :param name: project name - :param tag: project tag - :param history_id: history id. If none is provided, all history will be deleted + Args: + namespace: Project namespace. + name: Project name. + tag: Project tag. + history_id: Id of the entry to delete; deletes all history if None. - :return: None + Raises: + ProjectNotFoundError: If the project does not exist. + HistoryNotFoundError: If the specified history entry does not exist. """ with Session(self._sa_engine) as session: project_mapping = session.scalar( @@ -1383,7 +1461,9 @@ def delete_history( if history_id is None: session.execute( - delete(HistoryProjects).where(HistoryProjects.project_id == project_mapping.id) + delete(HistoryProjects).where( + HistoryProjects.project_id == project_mapping.id + ) ) session.commit() return None @@ -1413,16 +1493,14 @@ def restore( history_id: int, user: str = None, ) -> None: - """ - Restore project to the specific history state - - :param namespace: project namespace - :param name: project name - :param tag: project tag - :param history_id: history id - :param user: user that restores the project if user is not provided, user will be set as Namespace - - :return: None + """Restore a project to a past history state. + + Args: + namespace: Project namespace. + name: Project name. + tag: Project tag. + history_id: Id of the history entry to restore to. + user: User performing the restore (default: namespace). """ restore_project = self.get_project_from_history( @@ -1442,11 +1520,10 @@ def restore( ) def clean_history(self, days: int = 90) -> None: - """ - Delete all history data that is older than 3 month, or specific number of days + """Delete history entries older than a given number of days. - :param days: number of days to keep history data - :return: None + Args: + days: Retain history newer than this many days (default: 90). """ with Session(self._sa_engine) as session: diff --git a/pepdbagent/modules/sample.py b/pepdbagent/modules/sample.py index f7dc76d6..cc5afe43 100644 --- a/pepdbagent/modules/sample.py +++ b/pepdbagent/modules/sample.py @@ -23,7 +23,8 @@ class PEPDatabaseSample: def __init__(self, pep_db_engine: BaseEngine): """ - :param pep_db_engine: pepdbengine object with sa engine + Args: + pep_db_engine: PEPDatabaseAgent engine object. """ self._sa_engine = pep_db_engine.engine self._pep_db_engine = pep_db_engine @@ -36,15 +37,20 @@ def get( tag: str = DEFAULT_TAG, raw: bool = True, ) -> peprs.Sample | dict | None: - """ - Retrieve sample from the database using namespace, name, tag, and sample_name - - :param namespace: namespace of the project - :param name: name of the project (Default: name is taken from the project object) - :param tag: tag (or version) of the project. - :param sample_name: sample_name of the sample - :param raw: return raw dict or peprs.Sample object [Default: True] - :return: peprs.Sample object or raw dict + """Retrieve a sample from the database. + + Args: + namespace: Namespace of the project. + name: Name of the project. + sample_name: Name of the sample. + tag: Tag of the project. + raw: Return raw dict if True, peprs.Sample object if False. + + Returns: + Raw sample dict or peprs.Sample object. + + Raises: + SampleNotFoundError: If the sample does not exist. """ statement_sample = select(Samples).where( and_( @@ -97,18 +103,18 @@ def update( update_dict: dict, full_update: bool = False, ) -> None: - """ - Update one sample in the database - - :param namespace: namespace of the project - :param name: name of the project (Default: name is taken from the project object) - :param tag: tag (or version) of the project. - :param sample_name: sample_name of the sample - :param update_dict: dictionary with sample data (key: value pairs). e.g. - {"sample_name": "sample1", - "sample_protocol": "sample1 protocol"} - :param full_update: if True, update all sample fields, if False, update only fields from update_dict - :return: None + """Update a sample in the database. + + Args: + namespace: Namespace of the project. + name: Name of the project. + tag: Tag of the project. + sample_name: Name of the sample. + update_dict: Dict of fields to update, e.g., {"sample_name": "s1", "protocol": "rna"}. + full_update: Replace all sample fields if True, merge if False. + + Raises: + SampleNotFoundError: If the sample does not exist. """ statement = select(Samples).where( and_( @@ -143,7 +149,9 @@ def update( sample_mapping.sample.update(update_dict) try: sample_mapping.sample_name = sample_mapping.sample[ - project_mapping.config.get(SAMPLE_TABLE_INDEX_KEY, "sample_name") + project_mapping.config.get( + SAMPLE_TABLE_INDEX_KEY, "sample_name" + ) ] except KeyError: raise KeyError( @@ -153,7 +161,9 @@ def update( # This line needed due to: https://github.com/sqlalchemy/sqlalchemy/issues/5218 flag_modified(sample_mapping, "sample") - project_mapping.last_update_date = datetime.datetime.now(datetime.timezone.utc) + project_mapping.last_update_date = datetime.datetime.now( + datetime.timezone.utc + ) session.commit() else: @@ -169,17 +179,17 @@ def add( sample_dict: dict, overwrite: bool = False, ) -> None: - """ - Add one sample to the project in the database - - :param namespace: namespace of the project - :param name: name of the project - :param tag: tag (or version) of the project. - :param overwrite: overwrite sample if it already exists - :param sample_dict: dictionary with sample data (key: value pairs). e.g. - {"sample_name": "sample1", - "sample_protocol": "sample1 protocol"} - :return: None + """Add a sample to a project in the database. + + Args: + namespace: Namespace of the project. + name: Name of the project. + tag: Tag of the project. + sample_dict: Sample data, e.g., {"sample_name": "s1", "protocol": "rna"}. + overwrite: Overwrite the sample if it already exists. + + Raises: + SampleAlreadyExistsError: If the sample exists and overwrite is False. """ with Session(self._sa_engine) as session: @@ -201,7 +211,10 @@ def add( f"Sample index key {project_mapping.config.get('sample_table_index', 'sample_name')} not found in sample dict" ) statement = select(Samples).where( - and_(Samples.project_id == project_mapping.id, Samples.sample_name == sample_name) + and_( + Samples.project_id == project_mapping.id, + Samples.sample_name == sample_name, + ) ) sample_mapping = session.scalar(statement) @@ -228,17 +241,21 @@ def add( parent_guid=self._get_last_sample_guid(project_mapping.id), ) project_mapping.number_of_samples += 1 - project_mapping.last_update_date = datetime.datetime.now(datetime.timezone.utc) + project_mapping.last_update_date = datetime.datetime.now( + datetime.timezone.utc + ) session.add(sample_mapping) session.commit() def _get_last_sample_guid(self, project_id: int) -> str: - """ - Get last sample guid from the project + """Get the guid of the last sample in the project chain. - :param project_id: project_id of the project - :return: guid of the last sample + Args: + project_id: Database id of the project. + + Returns: + GUID of the last sample. """ statement = select(Samples).where(Samples.project_id == project_id) with Session(self._sa_engine) as session: @@ -262,14 +279,16 @@ def delete( tag: str, sample_name: str, ) -> None: - """ - Delete one sample from the database + """Delete a sample from the database. - :param namespace: namespace of the project - :param name: name of the project - :param tag: tag (or version) of the project. - :param sample_name: sample_name of the sample - :return: None + Args: + namespace: Namespace of the project. + name: Name of the project. + tag: Tag of the project. + sample_name: Name of the sample. + + Raises: + SampleNotFoundError: If the sample does not exist. """ statement = select(Samples).where( and_( @@ -304,7 +323,9 @@ def delete( if child_mapping: child_mapping.parent_mapping = parent_mapping project_mapping.number_of_samples -= 1 - project_mapping.last_update_date = datetime.datetime.now(datetime.timezone.utc) + project_mapping.last_update_date = datetime.datetime.now( + datetime.timezone.utc + ) session.commit() else: raise SampleNotFoundError( diff --git a/pepdbagent/modules/schema.py b/pepdbagent/modules/schema.py index 4c251e4f..51421ee0 100644 --- a/pepdbagent/modules/schema.py +++ b/pepdbagent/modules/schema.py @@ -42,20 +42,25 @@ class PEPDatabaseSchema: def __init__(self, pep_db_engine: BaseEngine): """ - :param pep_db_engine: pepdbengine object with sa engine + Args: + pep_db_engine: PEPDatabaseAgent engine object. """ self._sa_engine = pep_db_engine.engine self._pep_db_engine = pep_db_engine def get(self, namespace: str, name: str, version: str) -> dict: - """ - Get schema from the database. + """Get a schema value from the database. + + Args: + namespace: User namespace. + name: Schema name. + version: Schema version (use "latest" for the most recent). - :param namespace: user namespace - :param name: schema name - :param version: schema version + Returns: + Schema value dict. - :return: schema dict + Raises: + SchemaVersionDoesNotExistError: If the schema or version does not exist. """ with Session(self._sa_engine) as session: @@ -73,7 +78,6 @@ def get(self, namespace: str, name: str, version: str) -> dict: ) else: - schema_obj = session.scalar( select(SchemaVersions) .join(SchemaRecords, SchemaRecords.id == SchemaVersions.schema_id) @@ -107,22 +111,23 @@ def create( tags: list[str] | str | dict[str, str] | list[dict[str, str]] | None = None, private: bool = False, # TODO: for simplicity was not implemented yet ) -> None: - """ - Create or update schema in the database. - - :param namespace: user namespace - :param name: schema name - :param schema_value: schema dict - :param version: schema version [Default: "1.0.0"] - :param description: schema description [Default: ""] - :param lifecycle_stage: schema lifecycle stage [Default: ""] - :param maintainers: schema maintainers [Default: ""] - :param contributors: schema contributors [Default: ""] - :param release_notes: schema release notes [Default: ""] - :param tags: schema tags [Default: None] - :param private: schema privacy [Default: False] - - :return: None + """Create a new schema record in the database. + + Args: + namespace: User namespace. + name: Schema name. + schema_value: Schema content as a dict. + version: Initial version string. + description: Schema description. + lifecycle_stage: Lifecycle stage, e.g., "stable" or "deprecated". + maintainers: Comma-separated maintainer names. + contributors: Comma-separated contributor names. + release_notes: Release notes for this version. + tags: Tags to associate with the schema version. + private: Mark schema as private if True. + + Raises: + SchemaAlreadyExistsError: If a schema with the same name exists in the namespace. """ tags = self._unify_tags(tags) @@ -130,12 +135,16 @@ def create( with Session(self._sa_engine) as session: schema_obj = session.scalar( select(SchemaRecords).where( - and_(SchemaRecords.namespace == namespace, SchemaRecords.name == name) + and_( + SchemaRecords.namespace == namespace, SchemaRecords.name == name + ) ) ) if schema_obj: - raise SchemaAlreadyExistsError(f"Schema '{name}' already exists in the database") + raise SchemaAlreadyExistsError( + f"Schema '{name}' already exists in the database" + ) user = session.scalar(select(User).where(User.namespace == namespace)) @@ -166,10 +175,14 @@ def create( ) for tag_name, tag_value in tags.items(): - tag_obj = session.scalar(select(SchemaTags).where(SchemaTags.tag_name == tag_name)) + tag_obj = session.scalar( + select(SchemaTags).where(SchemaTags.tag_name == tag_name) + ) if not tag_obj: tag_obj = SchemaTags( - tag_name=tag_name, tag_value=tag_value, schema_mapping=schema_version_obj + tag_name=tag_name, + tag_value=tag_value, + schema_mapping=schema_version_obj, ) session.add(tag_obj) @@ -195,7 +208,9 @@ def add_version( with Session(self._sa_engine) as session: schema_obj = session.scalar( select(SchemaRecords).where( - and_(SchemaRecords.namespace == namespace, SchemaRecords.name == name) + and_( + SchemaRecords.namespace == namespace, SchemaRecords.name == name + ) ) ) if not schema_obj: @@ -244,7 +259,9 @@ def add_version( for tag_name, tag_value in tags.items(): tag_obj = SchemaTags( - tag_name=tag_name, tag_value=tag_value, schema_mapping=schema_version_obj + tag_name=tag_name, + tag_value=tag_value, + schema_mapping=schema_version_obj, ) session.add(tag_obj) @@ -260,20 +277,22 @@ def update_schema_version( version: str, update_fields: UpdateSchemaVersionFields | dict, ) -> None: - """ - Update schema version in the database. - - :param namespace: user namespace - :param name: schema name - :param version: schema version - :param update_fields: fields to be updated. Fields are optional, and include: - - contributors: str - - schema_value: dict - - release_notes: str + """Update fields of an existing schema version. + + Args: + namespace: User namespace. + name: Schema name. + version: Schema version to update. + update_fields: Fields to update — contributors, schema_value, and/or release_notes. + + Raises: + SchemaVersionDoesNotExistError: If the version does not exist. """ if isinstance(update_fields, dict): update_fields = UpdateSchemaVersionFields(**update_fields) - update_fields = update_fields.model_dump(exclude_unset=True, exclude_defaults=True) + update_fields = update_fields.model_dump( + exclude_unset=True, exclude_defaults=True + ) with Session(self._sa_engine) as session: schema_obj = session.scalar( @@ -307,32 +326,37 @@ def update_schema_record( name: str, update_fields: UpdateSchemaRecordFields | dict, ) -> None: - """ - Update schema record in the database. - - :param namespace: user namespace - :param name: schema name - :param update_fields: fields to be updated. Fields are optional, and include: - - maintainers: str - - lifecycle_stage: str - - private: bool - - name: str + """Update metadata fields of a schema record. + + Args: + namespace: User namespace. + name: Schema name. + update_fields: Fields to update — maintainers, lifecycle_stage, private, and/or name. + + Raises: + SchemaDoesNotExistError: If the schema does not exist. """ if isinstance(update_fields, dict): update_fields = UpdateSchemaRecordFields(**update_fields) - update_fields = update_fields.model_dump(exclude_unset=True, exclude_defaults=True) + update_fields = update_fields.model_dump( + exclude_unset=True, exclude_defaults=True + ) with Session(self._sa_engine) as session: schema_obj = session.scalar( select(SchemaRecords).where( - and_(SchemaRecords.namespace == namespace, SchemaRecords.name == name) + and_( + SchemaRecords.namespace == namespace, SchemaRecords.name == name + ) ) ) if not schema_obj: - raise SchemaDoesNotExistError(f"Schema '{name}' does not exist in the database") + raise SchemaDoesNotExistError( + f"Schema '{name}' does not exist in the database" + ) for field, value in update_fields.items(): setattr(schema_obj, field, value) @@ -340,32 +364,36 @@ def update_schema_record( session.commit() def schema_exist(self, namespace: str, name: str) -> bool: - """ - Check if schema exists in the database. + """Check whether a schema exists in the database. - :param namespace: user namespace - :param name: schema name + Args: + namespace: User namespace. + name: Schema name. - :return: True if schema exists, False otherwise + Returns: + True if the schema exists. """ with Session(self._sa_engine) as session: schema_obj = session.scalar( select(SchemaRecords).where( - and_(SchemaRecords.namespace == namespace, SchemaRecords.name == name) + and_( + SchemaRecords.namespace == namespace, SchemaRecords.name == name + ) ) ) return True if schema_obj else False def version_exist(self, namespace: str, name: str, version: str) -> bool: - """ - Check if schema version exists in the database. + """Check whether a specific schema version exists. - :param namespace: user namespace - :param name: schema name - :param version: schema version + Args: + namespace: User namespace. + name: Schema name. + version: Schema version. - :return: True if schema version exists, False otherwise + Returns: + True if the version exists. """ with Session(self._sa_engine) as session: @@ -383,24 +411,32 @@ def version_exist(self, namespace: str, name: str, version: str) -> bool: return True if schema_obj else False def get_schema_info(self, namespace: str, name: str) -> SchemaRecordAnnotation: - """ - Get schema information from the database. + """Get metadata for a schema record. + + Args: + namespace: User namespace. + name: Schema name. - :param namespace: user namespace - :param name: schema name + Returns: + SchemaRecordAnnotation with schema metadata. - :return: SchemaRecordAnnotation + Raises: + SchemaDoesNotExistError: If the schema does not exist. """ with Session(self._sa_engine) as session: schema_obj = session.scalar( select(SchemaRecords).where( - and_(SchemaRecords.namespace == namespace, SchemaRecords.name == name) + and_( + SchemaRecords.namespace == namespace, SchemaRecords.name == name + ) ) ) if not schema_obj: - raise SchemaDoesNotExistError(f"Schema '{name}' does not exist in the database") + raise SchemaDoesNotExistError( + f"Schema '{name}' does not exist in the database" + ) return SchemaRecordAnnotation( namespace=schema_obj.namespace, @@ -413,25 +449,35 @@ def get_schema_info(self, namespace: str, name: str) -> SchemaRecordAnnotation: lifecycle_stage=schema_obj.lifecycle_stage, ) - def get_version_info(self, namespace: str, name: str, version: str) -> SchemaVersionAnnotation: - """ - Get schema version information from the database. + def get_version_info( + self, namespace: str, name: str, version: str + ) -> SchemaVersionAnnotation: + """Get metadata for a specific schema version. - :param namespace: user namespace - :param name: schema name - :param version: schema version + Args: + namespace: User namespace. + name: Schema name. + version: Schema version (use "latest" for the most recent). - :return: SchemaVersionAnnotation + Returns: + SchemaVersionAnnotation with version metadata and tags. + + Raises: + SchemaVersionDoesNotExistError: If the version does not exist. """ with Session(self._sa_engine) as session: - # if user provided "latest" version if version == LATEST_SCHEMA_VERSION: version_obj = session.scalar( select(SchemaVersions) .join(SchemaRecords, SchemaRecords.id == SchemaVersions.schema_id) - .where(and_(SchemaRecords.namespace == namespace, SchemaRecords.name == name)) + .where( + and_( + SchemaRecords.namespace == namespace, + SchemaRecords.name == name, + ) + ) .order_by(SchemaVersions.version.desc()) .limit(1) ) @@ -476,28 +522,23 @@ def fetch_schemas( order_by: str = "update_date", order_desc: bool = False, ) -> SchemaSearchResult: - """ - Get schemas with providing filters. - If not filters provided, return all schemas. - - :param namespace: user namespace [Default: None]. If None, search in all namespaces - :param name: schema name [Default: None] - :param maintainer: schema maintainer [Default: None] - :param lifecycle_stage: schema lifecycle stage [Default: None] - :param latest_version: schema latest version [Default: None] - - :param page: page number [Default: 0] - :param page_size: number of schemas per page [Default: 0] - :param order_by: sort the result-set by the information - Options: ["name", "update_date"] - [Default: update_date] - :param order_desc: Sort the records in descending order. [Default: False] - - :return: { - pagination: {page: int, - page_size: int, - total: int}, - results: [SchemaRecordAnnotation] + """Get schemas matching the given filters. + + Returns all schemas if no filters are provided. + + Args: + namespace: Restrict to this namespace (default: all namespaces). + name: Partial name match. + maintainer: Partial maintainer match. + lifecycle_stage: Partial lifecycle stage match. + latest_version: Partial latest version match. + page: Page number (zero-based). + page_size: Number of results per page. + order_by: Sort field — "name" or "update_date". + order_desc: Sort in descending order if True. + + Returns: + SchemaSearchResult with pagination and list of SchemaRecordAnnotation objects. """ # filters = [ @@ -521,7 +562,9 @@ def fetch_schemas( conditions = [f for f in filters if f is not None] statement = ( - select(SchemaRecords).where(and_(*conditions)) if conditions else select(SchemaRecords) + select(SchemaRecords).where(and_(*conditions)) + if conditions + else select(SchemaRecords) ) statement_count = ( select(func.count(SchemaRecords.id)).where(and_(*conditions)) @@ -532,9 +575,13 @@ def fetch_schemas( with Session(self._sa_engine) as session: total = session.scalar(statement_count) - statement = self._add_order_by_schemas_keyword(statement, by=order_by, desc=order_desc) + statement = self._add_order_by_schemas_keyword( + statement, by=order_by, desc=order_desc + ) - results_objects = session.scalars(statement.limit(page_size).offset(page * page_size)) + results_objects = session.scalars( + statement.limit(page_size).offset(page * page_size) + ) return SchemaSearchResult( pagination=PaginationResult( page=page, @@ -564,23 +611,18 @@ def query_schemas( order_by: str = "update_date", order_desc: bool = False, ) -> SchemaSearchResult: - """ - Search schemas in the database with pagination. - - :param namespace: user namespace [Default: None]. If None, search in all namespaces - :param search_str: query string. [Default: ""]. If empty, return all schemas - :param page: page number [Default: 0] - :param page_size: number of schemas per page [Default: 0] - :param order_by: sort the result-set by the information - Options: ["name", "update_date"] - [Default: update_date] - :param order_desc: Sort the records in descending order. [Default: False] - - :return: { - pagination: {page: int, - page_size: int, - total: int}, - results: [SchemaRecordAnnotation] + """Search schemas by name and description with pagination. + + Args: + namespace: Restrict to this namespace (default: all namespaces). + search_str: Text to search in name and description (default: all schemas). + page: Page number (zero-based). + page_size: Number of results per page. + order_by: Sort field — "name" or "update_date". + order_desc: Sort in descending order if True. + + Returns: + SchemaSearchResult with pagination and list of SchemaRecordAnnotation objects. """ search_str = search_str.lower() if search_str else "" @@ -590,17 +632,23 @@ def query_schemas( SchemaRecords.description.ilike(f"%{search_str}%"), ) if namespace: - where_statement = and_(where_statement, SchemaRecords.namespace == namespace) + where_statement = and_( + where_statement, SchemaRecords.namespace == namespace + ) with Session(self._sa_engine) as session: - total = session.scalar(select(func.count(SchemaRecords.id)).where(where_statement)) + total = session.scalar( + select(func.count(SchemaRecords.id)).where(where_statement) + ) statement = ( select(SchemaRecords) .where(where_statement) .limit(page_size) .offset(page * page_size) ) - statement = self._add_order_by_schemas_keyword(statement, by=order_by, desc=order_desc) + statement = self._add_order_by_schemas_keyword( + statement, by=order_by, desc=order_desc + ) results_objects = session.scalars(statement) return SchemaSearchResult( @@ -632,21 +680,21 @@ def query_schema_version( page: int = 0, page_size: int = 10, ) -> SchemaVersionSearchResult: - """ - Search schema versions in the database with pagination. - - :param namespace: user namespace - :param name: schema name - :param tag: tag name. [Default: None]. If None, return versions with all tags - :param search_str: query string. [Default: ""]. If empty, return all schemas - :param page: result page number [Default: 10] - :param page_size: number of schemas per page [Default: 10] - - :return: { - pagination: {page: int, - page_size: int, - total: int}, - results: [SchemaVersionAnnotation] + """Search versions of a schema with pagination. + + Args: + namespace: User namespace. + name: Schema name. + tag: Filter by tag name (default: all tags). + search_str: Text to search in version and release_notes. + page: Page number (zero-based). + page_size: Number of results per page. + + Returns: + SchemaVersionSearchResult with pagination and list of SchemaVersionAnnotation objects. + + Raises: + SchemaDoesNotExistError: If the schema does not exist. """ search_str = search_str.lower() if search_str else "" @@ -654,12 +702,16 @@ def query_schema_version( with Session(self._sa_engine) as session: schema_obj = session.scalar( select(SchemaRecords).where( - and_(SchemaRecords.namespace == namespace, SchemaRecords.name == name) + and_( + SchemaRecords.namespace == namespace, SchemaRecords.name == name + ) ) ) if not schema_obj: - raise SchemaDoesNotExistError(f"Schema '{name}' does not exist in the database") + raise SchemaDoesNotExistError( + f"Schema '{name}' does not exist in the database" + ) where_statement = and_( SchemaRecords.namespace == namespace, @@ -691,7 +743,9 @@ def query_schema_version( .join(SchemaRecords) .where(where_statement) ) - find_statement = select(SchemaVersions).join(SchemaRecords).where(where_statement) + find_statement = ( + select(SchemaVersions).join(SchemaRecords).where(where_statement) + ) total = session.scalar(total_statement) @@ -714,7 +768,9 @@ def query_schema_version( version=result.version, contributors=result.contributors, release_notes=result.release_notes, - tags={tag.tag_name: tag.tag_value for tag in result.tags_mapping}, + tags={ + tag.tag_name: tag.tag_value for tag in result.tags_mapping + }, release_date=result.release_date, last_update_date=result.last_update_date, ) @@ -723,23 +779,29 @@ def query_schema_version( ) def delete_schema(self, namespace: str, name: str) -> None: - """ - Delete schema from the database. + """Delete a schema record and all its versions from the database. + + Args: + namespace: User namespace. + name: Schema name. - :param namespace: user namespace - :param name: schema name - :return: None + Raises: + SchemaDoesNotExistError: If the schema does not exist. """ with Session(self._sa_engine) as session: schema_obj = session.scalar( select(SchemaRecords).where( - and_(SchemaRecords.namespace == namespace, SchemaRecords.name == name) + and_( + SchemaRecords.namespace == namespace, SchemaRecords.name == name + ) ) ) if not schema_obj: - raise SchemaDoesNotExistError(f"Schema '{name}' does not exist in the database") + raise SchemaDoesNotExistError( + f"Schema '{name}' does not exist in the database" + ) statement = select(User).where(User.namespace == namespace) user = session.scalar(statement) @@ -751,15 +813,15 @@ def delete_schema(self, namespace: str, name: str) -> None: session.commit() def delete_version(self, namespace: str, name: str, version: str) -> None: - """ - Delete version of the schema + """Delete a specific schema version. - :param namespace: Namespace of the schema - :param name: Name of the schema - :param version: Version of the Schema + Args: + namespace: Namespace of the schema. + name: Name of the schema. + version: Version to delete. - :raise: SchemaVersionDoesNotExistError if version doesn't exist - :return: None + Raises: + SchemaVersionDoesNotExistError: If the version does not exist. """ with Session(self._sa_engine) as session: schema_obj = session.scalar( @@ -788,16 +850,17 @@ def add_tag_to_schema( version: str, tag: list[str] | str | dict[str, str] | None, ) -> None: - """ - Add tag to the schema + """Add a tag to a schema version. - :param namespace: Namespace of the schema - :param name: Name of the schema - :param version: Version of the Schema - :param tag: Tag to be added. Can be a string, list of strings or dictionaries + Args: + namespace: Namespace of the schema. + name: Name of the schema. + version: Schema version. + tag: Tag to add — string, list of strings, or dict of name→value pairs. - :raise: SchemaVersionDoesNotExistError if version doesn't exist - :return: None + Raises: + SchemaVersionDoesNotExistError: If the version does not exist. + SchemaTagAlreadyExistsError: If the tag already exists on the version. """ tag = self._unify_tags(tag) @@ -822,10 +885,14 @@ def add_tag_to_schema( tag = [tag] for tag_name, tag_value in tag.items(): - tag_obj = session.scalar(select(SchemaTags).where(SchemaTags.tag_name == tag_name)) + tag_obj = session.scalar( + select(SchemaTags).where(SchemaTags.tag_name == tag_name) + ) if not tag_obj: tag_obj = SchemaTags( - tag_name=tag_name, tag_value=tag_value, schema_mapping=schema_obj + tag_name=tag_name, + tag_value=tag_value, + schema_mapping=schema_obj, ) session.add(tag_obj) else: @@ -835,17 +902,20 @@ def add_tag_to_schema( session.commit() - def remove_tag_from_schema(self, namespace: str, name: str, version: str, tag: str) -> None: - """ - Remove tag from the schema + def remove_tag_from_schema( + self, namespace: str, name: str, version: str, tag: str + ) -> None: + """Remove a tag from a schema version. - :param namespace: Namespace of the schema - :param name: Name of the schema - :param version: Version of the Schema - :param tag: Tag to be removed + Args: + namespace: Namespace of the schema. + name: Name of the schema. + version: Schema version. + tag: Name of the tag to remove. - :raise: SchemaVersionDoesNotExistError if version doesn't exist - :return: None + Raises: + SchemaVersionDoesNotExistError: If the version does not exist. + SchemaTagDoesNotExistError: If the tag does not exist on the version. """ with Session(self._sa_engine) as session: schema_obj = session.scalar( @@ -866,11 +936,14 @@ def remove_tag_from_schema(self, namespace: str, name: str, version: str, tag: s tag_obj = session.scalar( select(SchemaTags).where( - SchemaTags.tag_name == tag, SchemaTags.schema_version_id == schema_obj.id + SchemaTags.tag_name == tag, + SchemaTags.schema_version_id == schema_obj.id, ) ) if not tag_obj: - raise SchemaTagDoesNotExistError(f"Tag '{tag}' does not exist in the schema") + raise SchemaTagDoesNotExistError( + f"Tag '{tag}' does not exist in the schema" + ) session.delete(tag_obj) session.commit() @@ -879,15 +952,15 @@ def remove_tag_from_schema(self, namespace: str, name: str, version: str, tag: s def _add_order_by_schemas_keyword( statement: Select, by: str = "update_date", desc: bool = False ) -> Select: - """ - Add order by clause to sqlalchemy statement - - :param statement: sqlalchemy representation of a SELECT statement. - :param by: sort the result-set by the information - Options: ["name", "update_date"] - [Default: "update_date"] - :param desc: Sort the records in descending order. [Default: False] - :return: sqlalchemy representation of a SELECT statement with order by keyword + """Add an ORDER BY clause for schema queries. + + Args: + statement: SQLAlchemy SELECT statement to augment. + by: Sort field — "name" or "update_date". + desc: Sort in descending order if True. + + Returns: + Statement with ORDER BY applied. """ if by == "update_date": order_by_obj = SchemaRecords.last_update_date @@ -910,13 +983,16 @@ def _add_order_by_schemas_keyword( def _unify_tags( self, tags: list[str] | str | dict[str, str] | list[dict[str, str]] | None ) -> dict[str, str]: - """ - Convert provided tags to one standard + """Normalise tags to a dict[str, str] representation. + + Args: + tags: Tags as a string, list of strings, dict, or list of dicts. - :param tags: tags to be converted from types: str, dict, list of str, list of dict + Returns: + Dict mapping tag names to tag values. - :raise: ValueError if tags are not in the correct format - :return: dictionary of tags + Raises: + ValueError: If tags are in an unsupported format. """ if not tags: tags = {} diff --git a/pepdbagent/modules/user.py b/pepdbagent/modules/user.py index 636f61d4..be334fdb 100644 --- a/pepdbagent/modules/user.py +++ b/pepdbagent/modules/user.py @@ -25,17 +25,20 @@ class PEPDatabaseUser: def __init__(self, pep_db_engine: BaseEngine): """ - :param pep_db_engine: pepdbengine object with sa engine + Args: + pep_db_engine: PEPDatabaseAgent engine object. """ self._sa_engine = pep_db_engine.engine self._pep_db_engine = pep_db_engine def create_user(self, namespace: str) -> int: - """ - Create new user + """Create a new user. + + Args: + namespace: User namespace. - :param namespace: user namespace - :return: user id + Returns: + New user id. """ new_user_raw = User(namespace=namespace) @@ -46,11 +49,13 @@ def create_user(self, namespace: str) -> int: return user_id def get_user_id(self, namespace: str) -> int | None: - """ - Get user id using username + """Get user id by namespace. - :param namespace: user namespace - :return: user id + Args: + namespace: User namespace. + + Returns: + User id, or None if the user does not exist. """ statement = select(User.id).where(User.namespace == namespace) with Session(self._sa_engine) as session: @@ -61,16 +66,19 @@ def get_user_id(self, namespace: str) -> int | None: return None def add_project_to_favorites( - self, namespace: str, project_namespace: str, project_name: str, project_tag: str + self, + namespace: str, + project_namespace: str, + project_name: str, + project_tag: str, ) -> None: - """ - Add project to favorites + """Add a project to a user's favorites. - :param namespace: namespace of the user - :param project_namespace: namespace of the project - :param project_name: name of the project - :param project_tag: tag of the project - :return: None + Args: + namespace: Namespace of the user. + project_namespace: Namespace of the project. + project_name: Name of the project. + project_tag: Tag of the project. """ user_id = self.get_user_id(namespace) @@ -90,7 +98,9 @@ def add_project_to_favorites( ) ) - new_favorites_raw = Stars(user_id=user_id, project_id=project_mapping.id) + new_favorites_raw = Stars( + user_id=user_id, project_id=project_mapping.id + ) session.add(new_favorites_raw) project_mapping.number_of_stars += 1 @@ -100,16 +110,19 @@ def add_project_to_favorites( return None def remove_project_from_favorites( - self, namespace: str, project_namespace: str, project_name: str, project_tag: str + self, + namespace: str, + project_namespace: str, + project_name: str, + project_tag: str, ) -> None: - """ - Remove project from favorites + """Remove a project from a user's favorites. - :param namespace: namespace of the user - :param project_namespace: namespace of the project - :param project_name: name of the project - :param project_tag: tag of the project - :return: None + Args: + namespace: Namespace of the user. + project_namespace: Namespace of the project. + project_name: Name of the project. + project_tag: Tag of the project. """ _LOGGER.debug( f"Removing project {project_namespace}/{project_name}:{project_tag} from favorites in {namespace}" @@ -144,11 +157,13 @@ def remove_project_from_favorites( return None def get_favorites(self, namespace: str) -> AnnotationList: - """ - Get list of favorites for user + """Get list of favorite projects for a user. + + Args: + namespace: Namespace of the user. - :param namespace: namespace of the user - :return: list of favorite projects with annotations + Returns: + AnnotationList of favorite projects. """ _LOGGER.debug(f"Getting favorites for user {namespace}") if not self.exists(namespace): @@ -161,7 +176,9 @@ def get_favorites(self, namespace: str) -> AnnotationList: statement = select(User).where(User.namespace == namespace) with Session(self._sa_engine) as session: query_result = session.scalar(statement) - number_of_projects = len([kk.project_mapping for kk in query_result.stars_mapping]) + number_of_projects = len( + [kk.project_mapping for kk in query_result.stars_mapping] + ) project_list = [] for prj_list in query_result.stars_mapping: prj = prj_list.project_mapping @@ -202,11 +219,13 @@ def exists( self, namespace: str, ) -> bool: - """ - Check if user exists in the database. + """Check if a user exists in the database. - :param namespace: project namespace - :return: Returning True if project exist + Args: + namespace: User namespace. + + Returns: + True if the user exists. """ statement = select(User.id) @@ -223,11 +242,10 @@ def exists( return False def delete(self, namespace: str) -> None: - """ - Delete user from the database with all related data + """Delete a user and all related data from the database. - :param namespace: user namespace - :return: None + Args: + namespace: User namespace. """ if not self.exists(namespace): raise UserNotFoundError diff --git a/pepdbagent/modules/view.py b/pepdbagent/modules/view.py index 5ea1ca94..65301161 100644 --- a/pepdbagent/modules/view.py +++ b/pepdbagent/modules/view.py @@ -37,7 +37,8 @@ class PEPDatabaseView: def __init__(self, pep_db_engine: BaseEngine): """ - :param pep_db_engine: pepdbengine object with sa engine + Args: + pep_db_engine: PEPDatabaseAgent engine object. """ self._sa_engine = pep_db_engine.engine self._pep_db_engine = pep_db_engine @@ -50,17 +51,22 @@ def get( view_name: str = None, raw: bool = True, ) -> peprs.Project | dict | None: - """ - Retrieve view of the project from the database. - View is a subset of the samples in the project. e.g. bed-db project has all the samples in bedbase, - bedset is a view of the bedbase project with only the samples in the bedset. - - :param namespace: namespace of the project - :param name: name of the project (Default: name is taken from the project object) - :param tag: tag of the project (Default: tag is taken from the project object) - :param view_name: name of the view - :param raw: retrieve unprocessed (raw) PEP dict. [Default: True] - :return: peprs.Project object or raw dict + """Retrieve a view of the project from the database. + + A view is a subset of samples in the project. + + Args: + namespace: Namespace of the project. + name: Name of the project. + tag: Tag of the project. + view_name: Name of the view. + raw: Return raw dict if True, peprs.Project object if False. + + Returns: + Raw dict or peprs.Project with only the view's samples. + + Raises: + ViewNotFoundError: If the view does not exist. """ _LOGGER.debug(f"Get view {view_name} from {namespace}/{name}:{tag}") view_statement = select(Views).where( @@ -87,22 +93,23 @@ def get( def get_annotation( self, namespace: str, name: str, tag: str = DEFAULT_TAG, view_name: str = None ) -> ViewAnnotation: + """Get annotation for a view. + + Args: + namespace: Namespace of the project. + name: Name of the project. + tag: Tag of the project. + view_name: Name of the view. + + Returns: + ViewAnnotation with project coordinates, name, description, and sample count. + + Raises: + ViewNotFoundError: If the view does not exist. """ - Get annotation of the view. - - :param namespace: namespace of the project - :param name: name of the project - :param tag: tag of the project - :param view_name: name of the sample - :return: ViewAnnotation object: - {project_namespace: str, - project_name: str, - project_tag: str, - name: str, - description: str, - number_of_samples: int} - """ - _LOGGER.debug(f"Get annotation for view {view_name} in {namespace}/{name}:{tag}") + _LOGGER.debug( + f"Get annotation for view {view_name} in {namespace}/{name}:{tag}" + ) view_statement = select(Views).where( and_( Views.project_mapping.has(namespace=namespace, name=name, tag=tag), @@ -132,23 +139,23 @@ def create( description: str = None, no_fail: bool = False, ) -> None: + """Create a view of a project in the database. + + Args: + view_name: Name for the new view. + view_dict: View definition with project_namespace, project_name, project_tag, + and sample_list keys (or a CreateViewDictModel). + description: Optional description of the view. + no_fail: Skip samples that do not exist instead of raising an error. + + Raises: + ProjectNotFoundError: If the project does not exist. + SampleNotFoundError: If a sample does not exist and no_fail is False. + ViewAlreadyExistsError: If a view with the same name already exists. """ - Create a view of the project in the database. - - :param view_name: namespace of the project - :param view_dict: dict or CreateViewDictModel object with view samples. - Dict should have the following structure: - { - project_namespace: str - project_name: str - project_tag: str - sample_list: List[str] # list of sample names - } - :param description: description of the view - :param no_fail: if True, skip samples that doesn't exist in the project - retrun: None - """ - _LOGGER.debug(f"Creating view {view_name} with provided info: (view_dict: {view_dict})") + _LOGGER.debug( + f"Creating view {view_name} with provided info: (view_dict: {view_dict})" + ) if isinstance(view_dict, dict): view_dict = CreateViewDictModel(**view_dict) @@ -190,7 +197,9 @@ def create( else: continue - sa_session.add(ViewSampleAssociation(sample_id=sample_id, view=view)) + sa_session.add( + ViewSampleAssociation(sample_id=sample_id, view=view) + ) sa_session.commit() except IntegrityError: @@ -205,14 +214,16 @@ def delete( project_tag: str = DEFAULT_TAG, view_name: str = None, ) -> None: - """ - Delete a view of the project in the database. + """Delete a view from the database. - :param project_namespace: namespace of the project - :param project_name: name of the project - :param project_tag: tag of the project - :param view_name: name of the view - :return: None + Args: + project_namespace: Namespace of the project. + project_name: Name of the project. + project_tag: Tag of the project. + view_name: Name of the view to delete. + + Raises: + ViewNotFoundError: If the view does not exist. """ _LOGGER.debug( f"Deleting view {view_name} from {project_namespace}/{project_name}:{project_tag}" @@ -243,15 +254,19 @@ def add_sample( view_name: str, sample_name: str | list[str], ) -> None: - """ - Add sample to the view. - - :param namespace: namespace of the project - :param name: name of the project - :param tag: tag of the project - :param view_name: name of the view - :param sample_name: sample name - :return: None + """Add one or more samples to a view. + + Args: + namespace: Namespace of the project. + name: Name of the project. + tag: Tag of the project. + view_name: Name of the view. + sample_name: Sample name or list of sample names. + + Raises: + ViewNotFoundError: If the view does not exist. + SampleNotFoundError: If a sample does not exist. + SampleAlreadyInView: If a sample is already in the view. """ _LOGGER.debug( f"Adding sample {sample_name} to view {view_name} in {namespace}/{name}:{tag}" @@ -299,15 +314,18 @@ def remove_sample( view_name: str, sample_name: str, ) -> None: - """ - Remove sample from the view. - - :param namespace: namespace of the project - :param name: name of the project - :param tag: tag of the project - :param view_name: name of the view - :param sample_name: sample name - :return: None + """Remove a sample from a view. + + Args: + namespace: Namespace of the project. + name: Name of the project. + tag: Tag of the project. + view_name: Name of the view. + sample_name: Name of the sample to remove. + + Raises: + ViewNotFoundError: If the view does not exist. + SampleNotInViewError: If the sample is not in the view. """ _LOGGER.debug( f"Removing sample {sample_name} from view {view_name} in {namespace}/{name}:{tag}" @@ -346,18 +364,30 @@ def remove_sample( sa_session.commit() def get_snap_view( - self, namespace: str, name: str, tag: str, sample_name_list: list[str], raw: bool = False + self, + namespace: str, + name: str, + tag: str, + sample_name_list: list[str], + raw: bool = False, ) -> peprs.Project | dict: - """ - Get a snap view of the project. Snap view is a view of the project - with only the samples in the list. This view won't be saved in the database. - - :param namespace: project namespace - :param name: name of the project - :param tag: tag of the project - :param sample_name_list: list of sample names e.g. ["sample1", "sample2"] - :param raw: retrieve unprocessed (raw) PEP dict. - :return: peprs.Project object + """Get an ephemeral view of a project limited to a set of samples. + + The snap view is not persisted in the database. + + Args: + namespace: Namespace of the project. + name: Name of the project. + tag: Tag of the project. + sample_name_list: Sample names to include, e.g., ["sample1", "sample2"]. + raw: Return raw dict if True, peprs.Project object if False. + + Returns: + Raw dict or peprs.Project containing only the requested samples. + + Raises: + ProjectNotFoundError: If the project does not exist. + SampleNotFoundError: If any sample does not exist. """ _LOGGER.debug(f"Creating snap view for {namespace}/{name}:{tag}") project_statement = select(Projects).where( @@ -370,7 +400,9 @@ def get_snap_view( with Session(self._sa_engine) as sa_session: project = sa_session.scalar(project_statement) if not project: - raise ProjectNotFoundError(f"Project {namespace}/{name}:{tag} does not exist") + raise ProjectNotFoundError( + f"Project {namespace}/{name}:{tag} does not exist" + ) samples = [] for sample_name in sample_name_list: sample_statement = select(Samples).where( @@ -395,13 +427,15 @@ def get_snap_view( def get_views_annotation( self, namespace: str, name: str, tag: str = DEFAULT_TAG ) -> ProjectViews | None: - """ - Get list of views of the project + """Get annotation for all views of a project. + + Args: + namespace: Namespace of the project. + name: Name of the project. + tag: Tag of the project. - :param namespace: namespace of the project - :param name: name of the project - :param tag: tag of the project - :return: list of views of the project + Returns: + ProjectViews with a list of ViewAnnotation objects. """ _LOGGER.debug(f"Get views annotation for {namespace}/{name}:{tag}") statement = select(Views).where( diff --git a/pepdbagent/pepdbagent.py b/pepdbagent/pepdbagent.py index 1eeeb926..6fc82add 100644 --- a/pepdbagent/pepdbagent.py +++ b/pepdbagent/pepdbagent.py @@ -22,19 +22,18 @@ def __init__( echo: bool = False, run_migrations: bool = False, ): - """ - Initialize connection to the pep_db database. You can use The basic connection parameters - or libpq connection string. - :param host: database server address e.g., localhost or an IP address. - :param port: the port number that defaults to 5432 if it is not provided. - :param database: the name of the database that you want to connect. - :param user: the username used to authenticate. - :param password: password used to authenticate. - :param drivername: driver of the database [Default: postgresql] - :param dsn: libpq connection string using the dsn parameter - (e.g. "localhost://username:password@pdp_db:5432") - - :param run_migrations: run migrations on the database + """Initialize connection to the pep_db database. + + Args: + host: Database server address, e.g., localhost or an IP address. + port: Port number (default: 5432). + database: Name of the database to connect to. + user: Username for authentication. + password: Password for authentication. + drivername: Database driver (default: postgresql). + dsn: libpq connection string, e.g., "localhost://username:password@pdp_db:5432". + echo: Log all SQL statements if True. + run_migrations: Run migrations on the database if True. """ pep_db_engine = BaseEngine( diff --git a/pepdbagent/utils.py b/pepdbagent/utils.py index b19635e1..87b5e7a0 100644 --- a/pepdbagent/utils.py +++ b/pepdbagent/utils.py @@ -11,13 +11,15 @@ def is_valid_registry_path(rpath: str) -> bool: - """ - Verify that a registry path is valid. Checks for two things: - 1. Contains forward slash ("/"), and - 2. Forward slash divides two strings + """Verify that a registry path is valid. + + Checks that the path contains a forward slash dividing two non-empty strings. + + Args: + rpath: Registry path to test. - :param str rpath: registry path to test - :return bool: Is it a valid registry or not. + Returns: + True if the path is a valid registry path. """ # check for string if not isinstance(rpath, str): @@ -32,11 +34,13 @@ def is_valid_registry_path(rpath: str) -> bool: def all_elements_are_strings(iterable: Iterable) -> bool: - """ - Helper method to determine if an iterable only contains `str` objects. + """Check if every element of an iterable is a str. + + Args: + iterable: An iterable item. - :param Iterable iterable: An iterable item - :returns bool: Boolean value indicating if the iterable only contains strings. + Returns: + True if every element is a string. """ if not isinstance(iterable, Iterable): return False @@ -44,11 +48,13 @@ def all_elements_are_strings(iterable: Iterable) -> bool: def create_digest(project_dict: dict) -> str: - """ - Create digest for PEP project + """Create an MD5 digest for a PEP project. + + Args: + project_dict: Project dictionary. - :param project_dict: project dict - :return: digest string + Returns: + MD5 hex digest of the sample table. """ sample_digest = md5( json.dumps( @@ -63,11 +69,16 @@ def create_digest(project_dict: dict) -> str: def registry_path_converter(registry_path: str) -> tuple[str, str, str]: - """ - Convert registry path to namespace, name, tag + """Convert a registry path to (namespace, name, tag). + + Args: + registry_path: Registry path with structure "namespace/name:tag". + + Returns: + Tuple of (namespace, name, tag). - :param registry_path: registry path that has structure: "namespace/name:tag" - :return: tuple(namespace, name, tag) + Raises: + RegistryPathError: If the path is not a valid registry path. """ if is_valid_registry_path(registry_path): reg = ubiquerg.parse_registry_path(registry_path) @@ -80,11 +91,16 @@ def registry_path_converter(registry_path: str) -> tuple[str, str, str]: def schema_path_converter(schema_path: str) -> tuple[str, str, str]: - """ - Convert schema path to namespace, name + """Convert a schema path to (namespace, name, version). + + Args: + schema_path: Schema path with structure "namespace/name:version". + + Returns: + Tuple of (namespace, name, version). - :param schema_path: schema path that has structure: "namespace/name.yaml" - :return: tuple(namespace, name, version) + Raises: + RegistryPathError: If the path is invalid. """ if "/" in schema_path: namespace, name_tag = schema_path.split("/") @@ -97,12 +113,13 @@ def schema_path_converter(schema_path: str) -> tuple[str, str, str]: def tuple_converter(value: tuple | list | str | None) -> tuple: - """ - Convert string list or tuple to tuple. - # is used to create admin tuple. + """Convert a string, list, or tuple to a tuple. + + Args: + value: Value to convert. - :param value: Any value that has to be converted to tuple - :return: tuple of strings + Returns: + Tuple of strings. """ if isinstance(value, str): value = [value] @@ -114,28 +131,30 @@ def tuple_converter(value: tuple | list | str | None) -> tuple: def convert_date_string_to_date(date_string: str) -> datetime.datetime: - """ - Convert string into datetime format + """Convert a date string to a datetime. + + Args: + date_string: Date string in format YYYY/MM/DD, e.g., "2022/02/22". - :param date_string: date string in format [YYYY/MM/DD]. e.g. 2022/02/22 - :return: datetime format + Returns: + Parsed datetime offset by one day. """ - return datetime.datetime.strptime(date_string, "%Y/%m/%d") + datetime.timedelta(days=1) + return datetime.datetime.strptime(date_string, "%Y/%m/%d") + datetime.timedelta( + days=1 + ) def order_samples(results: dict) -> list[dict]: - """ - Order samples by their parent_guid + """Order samples by their parent_guid chain. # TODO: To make this function more efficient, we should write it in Rust! - :param results: dictionary of samples. Structure: { - "sample": sample_dict, - "guid": sample.guid, - "parent_guid": sample.parent_guid, - } + Args: + results: Dict of samples keyed by guid, each value a dict with + "sample", "guid", and "parent_guid" keys. - :return: ordered list of samples + Returns: + Ordered list of samples from root to leaf. """ # Find the Root Node # Create a lookup dictionary for nodes by their GUIDs diff --git a/tests/test_annotation.py b/tests/test_annotation.py index 9e5603e7..07d14668 100644 --- a/tests/test_annotation.py +++ b/tests/test_annotation.py @@ -114,7 +114,9 @@ def test_annotation_limit(self, namespace, limit, admin, n_projects): @pytest.mark.parametrize("admin", ["private_test"]) def test_order_by(self, namespace, admin, order_by, first_name): with PEPDBAgentContextManager(add_data=True) as agent: - result = agent.annotation.get(namespace=namespace, admin=admin, order_by=order_by) + result = agent.annotation.get( + namespace=namespace, admin=admin, order_by=order_by + ) assert result.results[0].name == first_name @pytest.mark.parametrize( @@ -164,7 +166,9 @@ def test_name_search(self, namespace, query, found_number): ) def test_name_search_private(self, namespace, query, found_number): with PEPDBAgentContextManager(add_data=True) as agent: - result = agent.annotation.get(namespace=namespace, query=query, admin="private_test") + result = agent.annotation.get( + namespace=namespace, query=query, admin="private_test" + ) assert len(result.results) == found_number @pytest.mark.parametrize( @@ -289,7 +293,6 @@ def test_search_incorrect_filter_by_string(self, namespace, query, found_number) ) def test_get_annotation_by_rp_list(self, rp_list, admin, found_number): with PEPDBAgentContextManager(add_data=True) as agent: - result = agent.annotation.get_by_rp_list(rp_list) assert len(result.results) == found_number @@ -306,7 +309,6 @@ def test_get_annotation_by_rp_enpty_list(self): ) def test_search_incorrect_incorrect_pep_type(self, namespace, query, found_number): with PEPDBAgentContextManager(add_data=True) as agent: - with pytest.raises(ValueError): agent.annotation.get(namespace=namespace, pep_type="incorrect") diff --git a/tests/test_namespace.py b/tests/test_namespace.py index d4fd309c..de36e2d4 100644 --- a/tests/test_namespace.py +++ b/tests/test_namespace.py @@ -109,11 +109,17 @@ def test_count_project_one(self, namespace, name): ) def test_remove_from_favorite(self, namespace, name): with PEPDBAgentContextManager(add_data=True) as agent: - agent.user.add_project_to_favorites("namespace1", namespace, name, "default") - agent.user.add_project_to_favorites("namespace1", namespace, "amendments2", "default") + agent.user.add_project_to_favorites( + "namespace1", namespace, name, "default" + ) + agent.user.add_project_to_favorites( + "namespace1", namespace, "amendments2", "default" + ) result = agent.user.get_favorites("namespace1") assert result.count == len(result.results) == 2 - agent.user.remove_project_from_favorites("namespace1", namespace, name, "default") + agent.user.remove_project_from_favorites( + "namespace1", namespace, name, "default" + ) result = agent.user.get_favorites("namespace1") assert result.count == len(result.results) == 1 @@ -126,7 +132,9 @@ def test_remove_from_favorite(self, namespace, name): def test_remove_from_favorite_error(self, namespace, name): with PEPDBAgentContextManager(add_data=True) as agent: with pytest.raises(ProjectNotInFavorites): - agent.user.remove_project_from_favorites("namespace1", namespace, name, "default") + agent.user.remove_project_from_favorites( + "namespace1", namespace, name, "default" + ) @pytest.mark.parametrize( "namespace, name", @@ -136,9 +144,13 @@ def test_remove_from_favorite_error(self, namespace, name): ) def test_favorites_duplication_error(self, namespace, name): with PEPDBAgentContextManager(add_data=True) as agent: - agent.user.add_project_to_favorites("namespace1", namespace, name, "default") + agent.user.add_project_to_favorites( + "namespace1", namespace, name, "default" + ) with pytest.raises(ProjectAlreadyInFavorites): - agent.user.add_project_to_favorites("namespace1", namespace, name, "default") + agent.user.add_project_to_favorites( + "namespace1", namespace, name, "default" + ) @pytest.mark.parametrize( "namespace, name", @@ -148,7 +160,9 @@ def test_favorites_duplication_error(self, namespace, name): ) def test_annotation_favorite_number(self, namespace, name): with PEPDBAgentContextManager(add_data=True) as agent: - agent.user.add_project_to_favorites("namespace1", namespace, name, "default") + agent.user.add_project_to_favorites( + "namespace1", namespace, name, "default" + ) annotations_in_namespace = agent.annotation.get("namespace1") for prj_annot in annotations_in_namespace.results: @@ -169,14 +183,12 @@ class TestUser: def test_create_user(self): with PEPDBAgentContextManager(add_data=True) as agent: - agent.user.create_user("test_user") assert agent.user.exists("test_user") def test_delete_user(self): with PEPDBAgentContextManager(add_data=True) as agent: - test_user = "test_user" agent.user.create_user(test_user) assert agent.user.exists(test_user) @@ -185,7 +197,6 @@ def test_delete_user(self): def test_delete_user_deletes_projects(self): with PEPDBAgentContextManager(add_data=True) as agent: - test_user = "namespace1" assert agent.user.exists(test_user) diff --git a/tests/test_project.py b/tests/test_project.py index 8df174fd..ea3df5e9 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -51,7 +51,9 @@ def test_create_project_from_dict(self): ) def test_get_project(self, namespace, name): with PEPDBAgentContextManager(add_data=True) as agent: - kk = agent.project.get(namespace=namespace, name=name, tag="default", raw=True) + kk = agent.project.get( + namespace=namespace, name=name, tag="default", raw=True + ) ff = peprs.Project(get_path_to_example_file(namespace, name)).to_dict( raw=True, by_sample=True ) @@ -97,7 +99,10 @@ def test_get_subsamples(self, namespace, name): ) orgiginal_prj = peprs.Project(get_path_to_example_file(namespace, name)) - assert prj_subtables == orgiginal_prj.to_dict(raw=True, by_sample=True)["subsamples"] + assert ( + prj_subtables + == orgiginal_prj.to_dict(raw=True, by_sample=True)["subsamples"] + ) @pytest.mark.parametrize( "namespace, name", @@ -112,7 +117,10 @@ def test_get_samples_raw(self, namespace, name): ) orgiginal_prj = peprs.Project(get_path_to_example_file(namespace, name)) - assert prj_samples == orgiginal_prj.to_dict(raw=True, by_sample=True)["samples"] + assert ( + prj_samples + == orgiginal_prj.to_dict(raw=True, by_sample=True)["samples"] + ) @pytest.mark.parametrize( "namespace, name", @@ -129,13 +137,20 @@ def test_get_samples_processed(self, namespace, name): raw=False, ) orgiginal_prj = peprs.Project(get_path_to_example_file(namespace, name)) - expected = orgiginal_prj.to_pandas().replace({np.nan: None}).to_dict(orient="records") + expected = ( + orgiginal_prj.to_pandas() + .replace({np.nan: None}) + .to_dict(orient="records") + ) # Normalize numpy arrays (used for subsample list columns) to plain lists # so dict equality works without raising "truth value is ambiguous". def _normalize(samples): return [ - {k: (v.tolist() if isinstance(v, np.ndarray) else v) for k, v in s.items()} + { + k: (v.tolist() if isinstance(v, np.ndarray) else v) + for k, v in s.items() + } for s in samples ] @@ -177,7 +192,9 @@ def test_overwrite_project(self, namespace, name): overwrite=True, ) - assert agent.project.get(namespace=namespace, name=name, raw=False) == new_prj + assert ( + agent.project.get(namespace=namespace, name=name, raw=False) == new_prj + ) @pytest.mark.parametrize( "namespace, name", @@ -196,7 +213,9 @@ def test_delete_project(self, namespace, name): def test_delete_not_existing_project(self): with PEPDBAgentContextManager(add_data=True) as agent: with pytest.raises(ProjectNotFoundError, match="Project does not exist."): - agent.project.delete(namespace="namespace1", name="nothing", tag="default") + agent.project.delete( + namespace="namespace1", name="nothing", tag="default" + ) @pytest.mark.parametrize( "namespace, name", @@ -217,7 +236,9 @@ def test_fork_projects(self, namespace, name): fork_tag="new_tag", ) - assert agent.project.exists(namespace="new_namespace", name="new_name", tag="new_tag") + assert agent.project.exists( + namespace="new_namespace", name="new_name", tag="new_tag" + ) result = agent.annotation.get( namespace="new_namespace", name="new_name", tag="new_tag" ) @@ -244,9 +265,13 @@ def test_parent_project_delete(self, namespace, name): fork_tag="new_tag", ) - assert agent.project.exists(namespace="new_namespace", name="new_name", tag="new_tag") + assert agent.project.exists( + namespace="new_namespace", name="new_name", tag="new_tag" + ) agent.project.delete(namespace=namespace, name=name, tag="default") - assert agent.project.exists(namespace="new_namespace", name="new_name", tag="new_tag") + assert agent.project.exists( + namespace="new_namespace", name="new_name", tag="new_tag" + ) @pytest.mark.parametrize( "namespace, name", @@ -269,9 +294,13 @@ def test_child_project_delete(self, namespace, name): fork_tag="new_tag", ) - assert agent.project.exists(namespace="new_namespace", name="new_name", tag="new_tag") + assert agent.project.exists( + namespace="new_namespace", name="new_name", tag="new_tag" + ) assert agent.project.exists(namespace=namespace, name=name, tag="default") - agent.project.delete(namespace="new_namespace", name="new_name", tag="new_tag") + agent.project.delete( + namespace="new_namespace", name="new_name", tag="new_tag" + ) assert agent.project.exists(namespace=namespace, name=name, tag="default") @pytest.mark.parametrize( diff --git a/tests/test_project_history.py b/tests/test_project_history.py index 20d00aeb..8bd38aff 100644 --- a/tests/test_project_history.py +++ b/tests/test_project_history.py @@ -208,7 +208,9 @@ def test_delete_all_history(self, namespace, name, sample_name): assert len(history.history) == 2 - agent.project.delete_history(namespace, name, tag="default", history_id=None) + agent.project.delete_history( + namespace, name, tag="default", history_id=None + ) history = agent.project.get_history(namespace, name, tag="default") assert len(history.history) == 0 @@ -300,6 +302,8 @@ def test_restore_project(self, namespace, name, sample_name): agent.project.restore(namespace, name, tag="default", history_id=1) - restored_project = agent.project.get(namespace, name, tag="default", with_id=False) + restored_project = agent.project.get( + namespace, name, tag="default", with_id=False + ) assert prj_org == restored_project diff --git a/tests/test_samples.py b/tests/test_samples.py index 15acc73d..d0b0d6e2 100644 --- a/tests/test_samples.py +++ b/tests/test_samples.py @@ -41,7 +41,9 @@ def test_retrieve_raw_sample(self, namespace, name, sample_name): ["namespace2", "custom_index", "frog_1"], ], ) - def test_retrieve_sample_with_modified_sample_id(self, namespace, name, sample_name): + def test_retrieve_sample_with_modified_sample_id( + self, namespace, name, sample_name + ): with PEPDBAgentContextManager(add_data=True) as agent: one_sample = agent.sample.get(namespace, name, sample_name, raw=False) assert isinstance(one_sample, peprs.Sample) @@ -142,7 +144,8 @@ def test_project_timestamp_was_changed(self, namespace, name, sample_name): annotation2 = agent.annotation.get(namespace, name, "default") assert ( - annotation1.results[0].last_update_date != annotation2.results[0].last_update_date + annotation1.results[0].last_update_date + != annotation2.results[0].last_update_date ) @pytest.mark.parametrize( @@ -205,7 +208,10 @@ def test_overwrite_sample(self, namespace, name, tag, sample_dict): with PEPDBAgentContextManager(add_data=True) as agent: # peprs/polars infers numeric values from the sample table, so the # original time is loaded as int (not string). - assert agent.project.get(namespace, name, raw=False).get_sample("pig_0h").time == 0 + assert ( + agent.project.get(namespace, name, raw=False).get_sample("pig_0h").time + == 0 + ) agent.sample.add(namespace, name, tag, sample_dict, overwrite=True) assert ( @@ -234,4 +240,7 @@ def test_delete_and_add(self, namespace, name, tag, sample_dict): agent.sample.delete(namespace, name, tag, "pig_0h") agent.sample.add(namespace, name, tag, sample_dict) prj2 = agent.project.get(namespace, name, raw=False) - assert prj.get_sample("pig_0h").to_dict() == prj2.get_sample("pig_0h").to_dict() + assert ( + prj.get_sample("pig_0h").to_dict() + == prj2.get_sample("pig_0h").to_dict() + ) diff --git a/tests/test_schema.py b/tests/test_schema.py index 0202fc9e..e8e11711 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -12,7 +12,6 @@ reason="DB is not setup", ) class TestSchemas: - @pytest.mark.parametrize( "namespace, name", [ @@ -64,7 +63,9 @@ def test_update_schema_update_date(self): }, } - first_time = agent.schema.get_schema_info("namespace1", "2.0.0").last_update_date + first_time = agent.schema.get_schema_info( + "namespace1", "2.0.0" + ).last_update_date agent.schema.add_version( "namespace1", @@ -98,10 +99,15 @@ def test_add_schema_version(self): release_notes=new_release_notes, ), ) - result = agent.schema.get_version_info("namespace1", "2.0.0", DEFAULT_SCHEMA_VERSION) + result = agent.schema.get_version_info( + "namespace1", "2.0.0", DEFAULT_SCHEMA_VERSION + ) assert result.contributors == new_contributors assert result.release_notes == new_release_notes - assert agent.schema.get("namespace1", "2.0.0", DEFAULT_SCHEMA_VERSION) == new_schema + assert ( + agent.schema.get("namespace1", "2.0.0", DEFAULT_SCHEMA_VERSION) + == new_schema + ) def test_search_schema_version(self): with PEPDBAgentContextManager(add_schemas=True) as agent: @@ -143,12 +149,16 @@ def test_search_schema_version_with_tags(self): release_notes="language", ) - result = agent.schema.query_schema_version("namespace1", "2.0.0", tag="tag1") + result = agent.schema.query_schema_version( + "namespace1", "2.0.0", tag="tag1" + ) assert result.pagination.total == 1 assert len(result.results) == 1 - result = agent.schema.query_schema_version("namespace1", "2.0.0", tag="bioinfo") + result = agent.schema.query_schema_version( + "namespace1", "2.0.0", tag="bioinfo" + ) assert result.pagination.total == 2 assert len(result.results) == 2 @@ -226,7 +236,9 @@ def test_insert_tags(self): "namespace1", "2.0.0", DEFAULT_SCHEMA_VERSION, tag=[new_tag1, new_tag2] ) - result = agent.schema.get_version_info("namespace1", "2.0.0", DEFAULT_SCHEMA_VERSION) + result = agent.schema.get_version_info( + "namespace1", "2.0.0", DEFAULT_SCHEMA_VERSION + ) assert new_tag1 in result.tags assert new_tag2 in result.tags @@ -237,7 +249,9 @@ def test_insert_one_tag(self): agent.schema.add_tag_to_schema( "namespace1", "2.0.0", DEFAULT_SCHEMA_VERSION, tag=new_tag1 ) - result = agent.schema.get_version_info("namespace1", "2.0.0", DEFAULT_SCHEMA_VERSION) + result = agent.schema.get_version_info( + "namespace1", "2.0.0", DEFAULT_SCHEMA_VERSION + ) assert new_tag1 in result.tags @pytest.mark.parametrize( @@ -249,11 +263,17 @@ def test_insert_one_tag(self): def test_delete_tag(self, namespace, name): with PEPDBAgentContextManager(add_schemas=True) as agent: new_tag1 = "new_tag" - agent.schema.add_tag_to_schema(namespace, name, DEFAULT_SCHEMA_VERSION, tag=new_tag1) - result = agent.schema.get_version_info(namespace, name, DEFAULT_SCHEMA_VERSION) + agent.schema.add_tag_to_schema( + namespace, name, DEFAULT_SCHEMA_VERSION, tag=new_tag1 + ) + result = agent.schema.get_version_info( + namespace, name, DEFAULT_SCHEMA_VERSION + ) assert new_tag1 in result.tags agent.schema.remove_tag_from_schema( namespace, name, DEFAULT_SCHEMA_VERSION, tag=new_tag1 ) - result = agent.schema.get_version_info(namespace, name, DEFAULT_SCHEMA_VERSION) + result = agent.schema.get_version_info( + namespace, name, DEFAULT_SCHEMA_VERSION + ) assert new_tag1 not in result.tags diff --git a/tests/test_tar_meta.py b/tests/test_tar_meta.py index 2488a4b1..99d2caa4 100644 --- a/tests/test_tar_meta.py +++ b/tests/test_tar_meta.py @@ -29,7 +29,6 @@ class TestGeoTar: def test_create_meta_tar(self): with PEPDBAgentContextManager(add_data=True) as agent: - agent.namespace.upload_tar_info(tar_info=self.tar_info) result = agent.namespace.get_tar_info(namespace=self.test_namespace) diff --git a/tests/test_updates.py b/tests/test_updates.py index a3fe0dd4..d762c12c 100644 --- a/tests/test_updates.py +++ b/tests/test_updates.py @@ -30,7 +30,9 @@ def test_update_project_name(self, namespace, name, new_name): tag="default", update_dict={"name": new_name}, ) - assert agent.project.exists(namespace=namespace, name=new_name, tag="default") + assert agent.project.exists( + namespace=namespace, name=new_name, tag="default" + ) @pytest.mark.parametrize( "namespace, name,new_name", @@ -41,7 +43,9 @@ def test_update_project_name(self, namespace, name, new_name): ) def test_update_project_name_in_config(self, namespace, name, new_name): with PEPDBAgentContextManager(add_data=True) as agent: - prj = agent.project.get(namespace=namespace, name=name, raw=False, with_id=True) + prj = agent.project.get( + namespace=namespace, name=name, raw=False, with_id=True + ) prj.name = new_name agent.project.update( namespace=namespace, @@ -49,7 +53,9 @@ def test_update_project_name_in_config(self, namespace, name, new_name): tag="default", update_dict={"project": prj}, ) - assert agent.project.exists(namespace=namespace, name=new_name, tag="default") + assert agent.project.exists( + namespace=namespace, name=new_name, tag="default" + ) @pytest.mark.parametrize( "namespace, name, new_tag", @@ -118,9 +124,13 @@ def test_update_project_schema(self, namespace, name, new_schema): ["namespace1", "amendments1", "desc1 f"], ], ) - def test_update_project_description_in_config(self, namespace, name, new_description): + def test_update_project_description_in_config( + self, namespace, name, new_description + ): with PEPDBAgentContextManager(add_data=True) as agent: - prj = agent.project.get(namespace=namespace, name=name, raw=False, with_id=True) + prj = agent.project.get( + namespace=namespace, name=name, raw=False, with_id=True + ) prj.description = new_description agent.project.update( namespace=namespace, @@ -249,7 +259,9 @@ def test_project_can_have_2_sample_names(self, namespace, name): ensure that update works correctly """ with PEPDBAgentContextManager(add_data=True) as agent: - new_prj = agent.project.get(namespace=namespace, name=name, raw=False, with_id=True) + new_prj = agent.project.get( + namespace=namespace, name=name, raw=False, with_id=True + ) prj_dict = new_prj.to_dict(raw=True, by_sample=True) prj_dict["samples"].append( @@ -298,7 +310,9 @@ def test_update_whole_project_with_id(self, namespace, name): """ with PEPDBAgentContextManager(add_data=True) as agent: peppy_prj = agent.project.get(namespace=namespace, name=name, raw=True) - prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) + prj = agent.project.get( + namespace=namespace, name=name, raw=True, with_id=True + ) new_sample = { "sample_name": "new_sample", @@ -336,7 +350,9 @@ def test_update_whole_project_with_id(self, namespace, name): def test_insert_new_row(self, namespace, name): with PEPDBAgentContextManager(add_data=True) as agent: peppy_prj = agent.project.get(namespace=namespace, name=name, raw=True) - prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) + prj = agent.project.get( + namespace=namespace, name=name, raw=True, with_id=True + ) new_sample = { "sample_name": "new_sample", @@ -370,7 +386,9 @@ def test_insert_new_row(self, namespace, name): def test_insert_new_multiple_rows(self, namespace, name): with PEPDBAgentContextManager(add_data=True) as agent: peppy_prj = agent.project.get(namespace=namespace, name=name, raw=True) - prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) + prj = agent.project.get( + namespace=namespace, name=name, raw=True, with_id=True + ) new_sample1 = { "sample_name": "new_sample", @@ -414,7 +432,9 @@ def test_insert_new_multiple_rows_duplicated_samples(self, namespace, name): original_count = len( agent.project.get(namespace=namespace, name=name, raw=True)["samples"] ) - prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) + prj = agent.project.get( + namespace=namespace, name=name, raw=True, with_id=True + ) new_sample1 = { "sample_name": "new_sample", @@ -449,7 +469,9 @@ def test_insert_new_multiple_rows_duplicated_samples(self, namespace, name): def test_delete_multiple_rows(self, namespace, name): with PEPDBAgentContextManager(add_data=True) as agent: peppy_prj = agent.project.get(namespace=namespace, name=name, raw=True) - prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) + prj = agent.project.get( + namespace=namespace, name=name, raw=True, with_id=True + ) del prj["samples"][1] del prj["samples"][2] @@ -478,7 +500,9 @@ def test_delete_multiple_rows(self, namespace, name): def test_modify_one_row(self, namespace, name): with PEPDBAgentContextManager(add_data=True) as agent: peppy_prj = agent.project.get(namespace=namespace, name=name, raw=True) - prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) + prj = agent.project.get( + namespace=namespace, name=name, raw=True, with_id=True + ) prj["samples"][0]["sample_name"] = "new_sample_name2" @@ -505,7 +529,9 @@ def test_modify_one_row(self, namespace, name): def test_modify_multiple_rows(self, namespace, name): with PEPDBAgentContextManager(add_data=True) as agent: peppy_prj = agent.project.get(namespace=namespace, name=name, raw=True) - prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) + prj = agent.project.get( + namespace=namespace, name=name, raw=True, with_id=True + ) prj["samples"][0]["sample_name"] = "new_sample_name2" prj["samples"][1]["sample_name"] = "new_sample_name3" @@ -534,7 +560,9 @@ def test_modify_multiple_rows(self, namespace, name): def test_add_new_first_sample(self, namespace, name): with PEPDBAgentContextManager(add_data=True) as agent: peppy_prj = agent.project.get(namespace=namespace, name=name, raw=True) - prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) + prj = agent.project.get( + namespace=namespace, name=name, raw=True, with_id=True + ) new_sample = { "sample_name": "new_sample", @@ -568,7 +596,9 @@ def test_add_new_first_sample(self, namespace, name): def test_change_sample_order(self, namespace, name): with PEPDBAgentContextManager(add_data=True) as agent: peppy_prj = agent.project.get(namespace=namespace, name=name, raw=True) - prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) + prj = agent.project.get( + namespace=namespace, name=name, raw=True, with_id=True + ) sample1 = prj["samples"][0].copy() sample2 = prj["samples"][1].copy() @@ -602,12 +632,13 @@ def test_change_sample_order(self, namespace, name): ) def test_update_porject_without_ids(self, namespace, name): with PEPDBAgentContextManager(add_data=True) as agent: - prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=False) + prj = agent.project.get( + namespace=namespace, name=name, raw=True, with_id=False + ) prj["samples"][0]["sample_name"] = "new_sample_name2" with pytest.raises(SampleTableUpdateError): - agent.project.update( namespace=namespace, name=name, @@ -623,7 +654,9 @@ def test_update_porject_without_ids(self, namespace, name): ) def test_update_project_with_duplicated_sample_guids(self, namespace, name): with PEPDBAgentContextManager(add_data=True) as agent: - new_prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) + new_prj = agent.project.get( + namespace=namespace, name=name, raw=True, with_id=True + ) new_prj["samples"].append(new_prj["samples"][0]) with pytest.raises(ProjectDuplicatedSampleGUIDsError): diff --git a/tests/test_views.py b/tests/test_views.py index 475936b5..85ee0de0 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -38,7 +38,9 @@ def test_create_view(self, namespace, name, sample_name, view_name): ) project = agent.project.get(namespace, name, raw=False) - view_project = agent.view.get(namespace, name, "default", view_name, raw=False) + view_project = agent.view.get( + namespace, name, "default", view_name, raw=False + ) assert len(view_project.samples) == 2 assert view_project != project @@ -48,7 +50,9 @@ def test_create_view(self, namespace, name, sample_name, view_name): ["namespace1", "amendments1", "pig_0h", "view1"], ], ) - def test_create_view_with_incorrect_sample(self, namespace, name, sample_name, view_name): + def test_create_view_with_incorrect_sample( + self, namespace, name, sample_name, view_name + ): with PEPDBAgentContextManager(add_data=True) as agent: with pytest.raises(SampleNotFoundError): agent.view.create( @@ -82,7 +86,9 @@ def test_create_view_with_incorrect_sample_no_fail( no_fail=True, ) project = agent.project.get(namespace, name, raw=False) - view_project = agent.view.get(namespace, name, "default", view_name, raw=False) + view_project = agent.view.get( + namespace, name, "default", view_name, raw=False + ) assert len(view_project.samples) == 2 assert view_project != project @@ -103,7 +109,14 @@ def test_delete_view(self, namespace, name, sample_name): "sample_list": [sample_name, "pig_1h"], }, ) - assert len(agent.view.get(namespace, name, "default", "view1", raw=False).samples) == 2 + assert ( + len( + agent.view.get( + namespace, name, "default", "view1", raw=False + ).samples + ) + == 2 + ) agent.view.delete(namespace, name, "default", "view1") with pytest.raises(ViewNotFoundError): agent.view.get(namespace, name, "default", "view1", raw=False) @@ -127,7 +140,14 @@ def test_add_sample_to_view(self, namespace, name, sample_name): }, ) agent.view.add_sample(namespace, name, "default", "view1", "pig_1h") - assert len(agent.view.get(namespace, name, "default", "view1", raw=False).samples) == 2 + assert ( + len( + agent.view.get( + namespace, name, "default", "view1", raw=False + ).samples + ) + == 2 + ) @pytest.mark.parametrize( "namespace, name, sample_name", @@ -146,8 +166,17 @@ def test_add_multiple_samples_to_view(self, namespace, name, sample_name): "sample_list": [sample_name], }, ) - agent.view.add_sample(namespace, name, "default", "view1", ["pig_1h", "frog_0h"]) - assert len(agent.view.get(namespace, name, "default", "view1", raw=False).samples) == 3 + agent.view.add_sample( + namespace, name, "default", "view1", ["pig_1h", "frog_0h"] + ) + assert ( + len( + agent.view.get( + namespace, name, "default", "view1", raw=False + ).samples + ) + == 3 + ) @pytest.mark.parametrize( "namespace, name, sample_name", @@ -167,11 +196,20 @@ def test_remove_sample_from_view(self, namespace, name, sample_name): }, ) agent.view.remove_sample(namespace, name, "default", "view1", sample_name) - assert len(agent.view.get(namespace, name, "default", "view1", raw=False).samples) == 1 + assert ( + len( + agent.view.get( + namespace, name, "default", "view1", raw=False + ).samples + ) + == 1 + ) assert len(agent.project.get(namespace, name, raw=False).samples) == 4 with pytest.raises(SampleNotInViewError): - agent.view.remove_sample(namespace, name, "default", "view1", sample_name) + agent.view.remove_sample( + namespace, name, "default", "view1", sample_name + ) @pytest.mark.parametrize( "namespace, name, sample_name", @@ -218,7 +256,10 @@ def test_get_snap_view(self, namespace, name, sample_name, view_name): ) def test_get_view_list_from_project(self, namespace, name, sample_name, view_name): with PEPDBAgentContextManager(add_data=True) as agent: - assert len(agent.view.get_views_annotation(namespace, name, "default").views) == 0 + assert ( + len(agent.view.get_views_annotation(namespace, name, "default").views) + == 0 + ) agent.view.create( "view1", { @@ -228,4 +269,7 @@ def test_get_view_list_from_project(self, namespace, name, sample_name, view_nam "sample_list": [sample_name, "pig_1h"], }, ) - assert len(agent.view.get_views_annotation(namespace, name, "default").views) == 1 + assert ( + len(agent.view.get_views_annotation(namespace, name, "default").views) + == 1 + ) diff --git a/tests/utils.py b/tests/utils.py index f440124f..26c321eb 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -70,7 +70,9 @@ class PEPDBAgentContextManager: Class with context manager to connect to database. Adds data and drops everything from the database upon exit to ensure. """ - def __init__(self, url: str = DSN, add_data: bool = False, add_schemas=True, echo=False): + def __init__( + self, url: str = DSN, add_data: bool = False, add_schemas=True, echo=False + ): """ :param url: database url e.g. "postgresql+psycopg://postgres:docker@localhost:5432/pep-db" :param add_data: add data to the database From 7cb401f5efee1034280ffd33992efdf8dac7835a Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Tue, 30 Jun 2026 13:22:04 -0400 Subject: [PATCH 07/13] Updated requirements --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 27ac75cf..d6a5a0fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ test = [ "python-dotenv", "coverage", "smokeshow", + "pyyaml", ] [build-system] From 4843f5986b27929c39397984d3e9e1a1379cff35 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Tue, 30 Jun 2026 13:49:48 -0400 Subject: [PATCH 08/13] Added polars as deps --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index d6a5a0fd..47159e8b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ dependencies = [ "psycopg>=3.1.15", "numpy>=1.24.4", "alembic>=1.15.1", + "polars>=1.0.0", ] [project.urls] From 7bc2d720e1efdaf30693ca9c85637d4b44f2a8cb Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Tue, 30 Jun 2026 15:49:14 -0400 Subject: [PATCH 09/13] Added pyarrow deps --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 47159e8b..4204271d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,9 @@ dependencies = [ "psycopg>=3.1.15", "numpy>=1.24.4", "alembic>=1.15.1", + # TODDO: delete this 2 deps after peprs is released "polars>=1.0.0", + "pyarrow" ] [project.urls] From 16ec1e7fa3506f3d922f7fd54e5e70788b4a6449 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Tue, 30 Jun 2026 15:54:45 -0400 Subject: [PATCH 10/13] updated test deps --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 4204271d..2aee2630 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ test = [ "coverage", "smokeshow", "pyyaml", + "pandas", ] [build-system] From 78f360e31eb011d8464f8e2205ed49d7d8ad0664 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Wed, 1 Jul 2026 14:59:46 -0400 Subject: [PATCH 11/13] Removed redundant dependencies --- pyproject.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2aee2630..95a4a603 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,9 +27,6 @@ dependencies = [ "psycopg>=3.1.15", "numpy>=1.24.4", "alembic>=1.15.1", - # TODDO: delete this 2 deps after peprs is released - "polars>=1.0.0", - "pyarrow" ] [project.urls] From 6a0d35ce4f375110830be50b0ad3b9a093043084 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Wed, 1 Jul 2026 15:34:52 -0400 Subject: [PATCH 12/13] updated peprs deps --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 95a4a603..f13d871d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ classifiers = [ dependencies = [ "sqlalchemy>=2.0.0", "logmuse>=0.2.7", - "peprs>=0.2.0", + "peprs>=0.2.1", "ubiquerg>=0.6.2", "coloredlogs>=15.0.1", "pydantic>=2.0", From ea8a303757d14180cfedaae9f2fcaef0faf2665b Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Wed, 1 Jul 2026 15:58:43 -0400 Subject: [PATCH 13/13] updated changelog --- docs/changelog.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index f97b6952..716a689e 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -4,8 +4,13 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## [0.13.0] -- 2026-07-01 -- Removed peppy from dependencies, using peprs instead -- Bug fixes and speed improvements in project update, validate and upload methods +- Migrated from `peppy` to `peprs` across all modules (project, sample, view, models, utils) +- Updated subsample key constant: `SUBSAMPLE_RAW_LIST_KEY` → `SUBSAMPLE_RAW_DICT_KEY` +- Added alembic database migration support; `run_migrations` parameter added to `PEPDatabaseAgent` and `BaseEngine` +- Fixed nullable fields in schema models (`SchemaVersionAnnotation`, `SchemaRecordAnnotation`, `UpdateSchemaRecordFields`, `UpdateSchemaVersionFields`) +- Added support for `"latest"` schema version resolution when uploading/updating projects +- Moved `SAMPLE_NAME_ATTR` and `SAMPLE_TABLE_INDEX_KEY` constants into `pepdbagent/const.py` +- Performed code cleanup and refactoring across modules, including modernization of the codestyle, installation and dependencies. ## [0.12.4] -- 2026-01-26 - Added project search by tag in annotation module