diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index 79ec0bb..093667e 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -1,6 +1,6 @@ name: Lint -on: [pull_request] +on: [push, pull_request] jobs: lint: @@ -8,4 +8,8 @@ jobs: steps: - uses: actions/checkout@v6 - uses: actions/setup-python@v6 - - uses: psf/black@stable + with: + python-version: "3.12" + - run: pip install ruff + - run: ruff check . + - run: ruff format --check . diff --git a/.github/workflows/cli-coverage.yml b/.github/workflows/cli-coverage.yml index cbd7cfe..dd6b92c 100644 --- a/.github/workflows/cli-coverage.yml +++ b/.github/workflows/cli-coverage.yml @@ -8,7 +8,7 @@ jobs: cli-coverage-report: strategy: matrix: - python-version: [ "3.11" ] + python-version: [ "3.10" ] os: [ ubuntu-latest ] # can't use macOS when using service containers or container jobs r: [ release ] runs-on: ${{ matrix.os }} @@ -31,10 +31,8 @@ jobs: with: python-version: '3.10' - - name: Install dev dependancies - run: if [ -f requirements/requirements-dev.txt ]; then pip install -r requirements/requirements-dev.txt; fi - - - run: pip install . + - name: Install package with test dependencies + run: pip install ".[test]" - name: Run tests run: coverage run -m pytest @@ -49,4 +47,4 @@ jobs: SMOKESHOW_GITHUB_CONTEXT: coverage SMOKESHOW_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SMOKESHOW_GITHUB_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} - SMOKESHOW_AUTH_KEY: ${{ secrets.SMOKESHOW_AUTH_KEY }} \ No newline at end of file + SMOKESHOW_AUTH_KEY: ${{ secrets.SMOKESHOW_AUTH_KEY }} diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 7771b34..cfd1c91 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -10,7 +10,7 @@ jobs: pytest: strategy: matrix: - python-version: ["3.9", "3.13"] + python-version: ["3.10", "3.13"] os: [ubuntu-latest] # can't use macOS when using service containers or container jobs r: [release] runs-on: ${{ matrix.os }} @@ -33,11 +33,8 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install dev dependencies - run: if [ -f requirements/requirements-dev.txt ]; then pip install -r requirements/requirements-dev.txt; fi - - - name: Install package - run: python -m pip install . + - name: Install package with test dependencies + run: python -m pip install ".[test]" - name: Run pytest tests run: pytest tests -x -vv diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 1aa6b6a..6368928 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -9,19 +9,18 @@ jobs: name: upload release to PyPI runs-on: ubuntu-latest permissions: + contents: read id-token: write + steps: - uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v6 with: python-version: '3.x' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - - name: Build and publish - run: | - python setup.py sdist bdist_wheel + - name: Install build dependencies + run: python -m pip install --upgrade pip build + - name: Build package + run: python -m build - name: Publish package distributions to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 \ No newline at end of file + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 814aee1..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,5 +0,0 @@ -include LICENSE.txt -include requirements/* -include README.md -include pepdbagent/alembic/* -include pepdbagent/alembic/versions/* \ No newline at end of file diff --git a/Makefile b/Makefile deleted file mode 100644 index 067ac8a..0000000 --- a/Makefile +++ /dev/null @@ -1,14 +0,0 @@ -lint: - # black should be last in the list, as it lint the code. Tests can fail if order will be different - flake8 && isort . && black . - -run-coverage: - coverage run -m pytest - -html-report: - coverage html - -open-coverage: - cd htmlcov && google-chrome index.html - -coverage: run-coverage html-report open-coverage \ No newline at end of file diff --git a/docs/changelog.md b/docs/changelog.md index 96b6a03..716a689 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,16 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format. + +## [0.13.0] -- 2026-07-01 +- Migrated from `peppy` to `peprs` across all modules (project, sample, view, models, utils) +- Updated subsample key constant: `SUBSAMPLE_RAW_LIST_KEY` → `SUBSAMPLE_RAW_DICT_KEY` +- Added alembic database migration support; `run_migrations` parameter added to `PEPDatabaseAgent` and `BaseEngine` +- Fixed nullable fields in schema models (`SchemaVersionAnnotation`, `SchemaRecordAnnotation`, `UpdateSchemaRecordFields`, `UpdateSchemaVersionFields`) +- Added support for `"latest"` schema version resolution when uploading/updating projects +- Moved `SAMPLE_NAME_ATTR` and `SAMPLE_TABLE_INDEX_KEY` constants into `pepdbagent/const.py` +- Performed code cleanup and refactoring across modules, including modernization of the codestyle, installation and dependencies. + ## [0.12.4] -- 2026-01-26 - Added project search by tag in annotation module - Updated github actions workflows diff --git a/pep_db/Dockerfile b/pep_db/Dockerfile deleted file mode 100644 index 5d04731..0000000 --- a/pep_db/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM postgres -ENV POSTGRES_USER postgres -ENV POSTGRES_PASSWORD docker -ENV POSTGRES_DB pep-db -#COPY pep_db.sql /docker-entrypoint-initdb.d/ \ No newline at end of file diff --git a/pepdbagent/__init__.py b/pepdbagent/__init__.py index 1d9283d..e332236 100644 --- a/pepdbagent/__init__.py +++ b/pepdbagent/__init__.py @@ -1,6 +1,19 @@ -"""Package-level data""" +"""Package-level data.""" + +from importlib.metadata import version + +import coloredlogs +import logmuse -from pepdbagent._version import __version__ from pepdbagent.pepdbagent import PEPDatabaseAgent +__version__ = version("pepdbagent") + __all__ = ["__version__", "PEPDatabaseAgent"] + +_LOGGER = logmuse.init_logger("pepdbagent") +coloredlogs.install( + logger=_LOGGER, + datefmt="%H:%M:%S", + fmt="[%(levelname)s] [%(asctime)s] %(message)s", +) diff --git a/pepdbagent/_version.py b/pepdbagent/_version.py deleted file mode 100644 index 6dd4954..0000000 --- a/pepdbagent/_version.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "0.12.4" diff --git a/pepdbagent/alembic/env.py b/pepdbagent/alembic/env.py index 2610126..1e979b0 100644 --- a/pepdbagent/alembic/env.py +++ b/pepdbagent/alembic/env.py @@ -1,9 +1,7 @@ from logging.config import fileConfig -from sqlalchemy import engine_from_config -from sqlalchemy import pool - from alembic import context +from sqlalchemy import engine_from_config, pool # this is the Alembic Config object, which provides # access to the values within the .ini file in use. diff --git a/pepdbagent/alembic/versions/44cb1e7a80de_initial_migration.py b/pepdbagent/alembic/versions/44cb1e7a80de_initial_migration.py index 2ec118c..e8521a5 100644 --- a/pepdbagent/alembic/versions/44cb1e7a80de_initial_migration.py +++ b/pepdbagent/alembic/versions/44cb1e7a80de_initial_migration.py @@ -8,8 +8,8 @@ from typing import Sequence, Union -from alembic import op import sqlalchemy as sa +from alembic import op from sqlalchemy.dialects import postgresql from sqlalchemy.schema import FetchedValue @@ -64,7 +64,9 @@ def upgrade() -> None: sa.UniqueConstraint("namespace", "name"), ) op.create_index(op.f("ix_schema_groups_id"), "schema_groups", ["id"], unique=False) - op.create_index(op.f("ix_schema_groups_name"), "schema_groups", ["name"], unique=False) + op.create_index( + op.f("ix_schema_groups_name"), "schema_groups", ["name"], unique=False + ) op.create_index( op.f("ix_schema_groups_namespace"), "schema_groups", ["namespace"], unique=False ) @@ -87,7 +89,9 @@ def upgrade() -> None: sa.PrimaryKeyConstraint("id"), sa.UniqueConstraint("namespace", "name"), ) - op.create_index(op.f("ix_schemas_description"), "schemas", ["description"], unique=False) + op.create_index( + op.f("ix_schemas_description"), "schemas", ["description"], unique=False + ) op.create_index(op.f("ix_schemas_id"), "schemas", ["id"], unique=False) op.create_index(op.f("ix_schemas_name"), "schemas", ["name"], unique=False) op.create_table( @@ -113,7 +117,9 @@ def upgrade() -> None: sa.Column("schema_id", sa.Integer(), nullable=True), sa.Column("pop", sa.Boolean(), nullable=True), sa.Column("forked_from_id", sa.Integer(), nullable=True), - sa.ForeignKeyConstraint(["forked_from_id"], ["projects.id"], ondelete="SET NULL"), + sa.ForeignKeyConstraint( + ["forked_from_id"], ["projects.id"], ondelete="SET NULL" + ), sa.ForeignKeyConstraint(["namespace"], ["users.namespace"], ondelete="CASCADE"), sa.ForeignKeyConstraint(["schema_id"], ["schemas.id"], ondelete="SET NULL"), sa.PrimaryKeyConstraint("id"), @@ -225,7 +231,9 @@ def upgrade() -> None: sa.Enum("UPDATE", "INSERT", "DELETE", name="updatetypes"), nullable=False, ), - sa.ForeignKeyConstraint(["history_id"], ["project_history.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint( + ["history_id"], ["project_history.id"], ondelete="CASCADE" + ), sa.PrimaryKeyConstraint("id"), ) op.create_table( @@ -249,8 +257,12 @@ def downgrade() -> None: op.drop_table("stars") op.drop_table("samples") op.drop_table("project_history") - op.drop_index(op.f("ix_schema_group_relations_schema_id"), table_name="schema_group_relations") - op.drop_index(op.f("ix_schema_group_relations_group_id"), table_name="schema_group_relations") + op.drop_index( + op.f("ix_schema_group_relations_schema_id"), table_name="schema_group_relations" + ) + op.drop_index( + op.f("ix_schema_group_relations_group_id"), table_name="schema_group_relations" + ) op.drop_table("schema_group_relations") op.drop_table("projects") op.drop_index(op.f("ix_schemas_name"), table_name="schemas") diff --git a/pepdbagent/alembic/versions/8a037f13b4e5_upgrading_schemas.py b/pepdbagent/alembic/versions/8a037f13b4e5_upgrading_schemas.py index 5e3e19c..ff6bf95 100644 --- a/pepdbagent/alembic/versions/8a037f13b4e5_upgrading_schemas.py +++ b/pepdbagent/alembic/versions/8a037f13b4e5_upgrading_schemas.py @@ -8,8 +8,8 @@ from typing import Sequence, Union -from alembic import op import sqlalchemy as sa +from alembic import op from sqlalchemy.dialects import postgresql from sqlalchemy.schema import FetchedValue @@ -52,7 +52,9 @@ def upgrade() -> None: sa.Column("last_update_date", sa.TIMESTAMP(timezone=True), nullable=True), sa.Column("contributors", sa.String(), nullable=True), sa.Column("release_notes", sa.String(), nullable=True), - sa.ForeignKeyConstraint(["schema_id"], ["schema_records.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint( + ["schema_id"], ["schema_records.id"], ondelete="CASCADE" + ), sa.PrimaryKeyConstraint("id"), sa.UniqueConstraint("schema_id", "version"), ) @@ -62,7 +64,9 @@ def upgrade() -> None: sa.Column("tag_name", sa.String(), nullable=False), sa.Column("tag_value", sa.String(), nullable=True), sa.Column("schema_version_id", sa.Integer(), nullable=False), - sa.ForeignKeyConstraint(["schema_version_id"], ["schema_versions.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint( + ["schema_version_id"], ["schema_versions.id"], ondelete="CASCADE" + ), sa.PrimaryKeyConstraint("id"), ) op.drop_index("ix_schemas_description", table_name="schemas") @@ -76,8 +80,12 @@ def upgrade() -> None: op.drop_index("ix_schema_groups_namespace", table_name="schema_groups") # op.drop_table('schema_groups') op.execute("DROP TABLE schema_groups CASCADE;") - op.drop_index("ix_schema_group_relations_group_id", table_name="schema_group_relations") - op.drop_index("ix_schema_group_relations_schema_id", table_name="schema_group_relations") + op.drop_index( + "ix_schema_group_relations_group_id", table_name="schema_group_relations" + ) + op.drop_index( + "ix_schema_group_relations_schema_id", table_name="schema_group_relations" + ) op.drop_table("schema_group_relations") op.execute("UPDATE projects SET schema_id = NULL;") @@ -85,7 +93,9 @@ def upgrade() -> None: None, "projects", "schema_versions", ["schema_id"], ["id"], ondelete="SET NULL" ) op.drop_column("projects", "pep_schema") - op.add_column("users", sa.Column("number_of_schemas", sa.Integer(), nullable=True, default=0)) + op.add_column( + "users", sa.Column("number_of_schemas", sa.Integer(), nullable=True, default=0) + ) # ### end Alembic commands ### @@ -94,7 +104,8 @@ def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_column("users", "number_of_schemas") op.add_column( - "projects", sa.Column("pep_schema", sa.VARCHAR(), autoincrement=False, nullable=True) + "projects", + sa.Column("pep_schema", sa.VARCHAR(), autoincrement=False, nullable=True), ) op.drop_constraint(None, "projects", type_="foreignkey") op.create_foreign_key( @@ -121,7 +132,9 @@ def downgrade() -> None: name="schema_group_relations_schema_id_fkey", ondelete="CASCADE", ), - sa.PrimaryKeyConstraint("schema_id", "group_id", name="schema_group_relations_pkey"), + sa.PrimaryKeyConstraint( + "schema_id", "group_id", name="schema_group_relations_pkey" + ), ) op.create_index( "ix_schema_group_relations_schema_id", @@ -130,7 +143,10 @@ def downgrade() -> None: unique=False, ) op.create_index( - "ix_schema_group_relations_group_id", "schema_group_relations", ["group_id"], unique=False + "ix_schema_group_relations_group_id", + "schema_group_relations", + ["group_id"], + unique=False, ) op.create_table( "schema_groups", @@ -145,9 +161,13 @@ def downgrade() -> None: ondelete="CASCADE", ), sa.PrimaryKeyConstraint("id", name="schema_groups_pkey"), - sa.UniqueConstraint("namespace", "name", name="schema_groups_namespace_name_key"), + sa.UniqueConstraint( + "namespace", "name", name="schema_groups_namespace_name_key" + ), + ) + op.create_index( + "ix_schema_groups_namespace", "schema_groups", ["namespace"], unique=False ) - op.create_index("ix_schema_groups_namespace", "schema_groups", ["namespace"], unique=False) op.create_index("ix_schema_groups_name", "schema_groups", ["name"], unique=False) op.create_index("ix_schema_groups_id", "schema_groups", ["id"], unique=False) op.create_table( @@ -176,7 +196,10 @@ def downgrade() -> None: nullable=True, ), sa.ForeignKeyConstraint( - ["namespace"], ["users.namespace"], name="schemas_namespace_fkey", ondelete="CASCADE" + ["namespace"], + ["users.namespace"], + name="schemas_namespace_fkey", + ondelete="CASCADE", ), sa.PrimaryKeyConstraint("id", name="schemas_pkey"), sa.UniqueConstraint("namespace", "name", name="schemas_namespace_name_key"), diff --git a/pepdbagent/const.py b/pepdbagent/const.py index cf0577b..52ef7f4 100644 --- a/pepdbagent/const.py +++ b/pepdbagent/const.py @@ -1,27 +1,29 @@ -PKG_NAME = "pepdbagent" +PKG_NAME: str = "pepdbagent" -DEFAULT_NAMESPACE = "_" -DEFAULT_TAG = "default" +DEFAULT_NAMESPACE: str = "_" +DEFAULT_TAG: str = "default" -DESCRIPTION_KEY = "description" -NAME_KEY = "name" +DESCRIPTION_KEY: str = "description" +NAME_KEY: str = "name" -# from peppy.const import SAMPLE_RAW_DICT_KEY, SUBSAMPLE_RAW_LIST_KEY -DEFAULT_OFFSET = 0 -DEFAULT_LIMIT = 100 +DEFAULT_OFFSET: int = 0 +DEFAULT_LIMIT: int = 100 # db_dialects -POSTGRES_DIALECT = "postgresql+psycopg" +POSTGRES_DIALECT: str = "postgresql+psycopg" -DEFAULT_LIMIT_INFO = 5 +DEFAULT_LIMIT_INFO: int = 5 -SUBMISSION_DATE_KEY = "submission_date" -LAST_UPDATE_DATE_KEY = "last_update_date" +SUBMISSION_DATE_KEY: str = "submission_date" +LAST_UPDATE_DATE_KEY: str = "last_update_date" -PEPHUB_SAMPLE_ID_KEY = "ph_id" +PEPHUB_SAMPLE_ID_KEY: str = "ph_id" -MAX_HISTORY_SAMPLES_NUMBER = 2000 +MAX_HISTORY_SAMPLES_NUMBER: int = 2000 -DEFAULT_TAG_VERSION = "1.0.0" -LATEST_SCHEMA_VERSION = "latest" +DEFAULT_TAG_VERSION: str = "1.0.0" +LATEST_SCHEMA_VERSION: str = "latest" + +SAMPLE_NAME_ATTR: str = "sample_name" +SAMPLE_TABLE_INDEX_KEY: str = "sample_table_index" diff --git a/pepdbagent/db_utils.py b/pepdbagent/db_utils.py index 610c574..3a6cf5d 100644 --- a/pepdbagent/db_utils.py +++ b/pepdbagent/db_utils.py @@ -1,12 +1,10 @@ import datetime import enum import logging -from typing import List, Optional import os from alembic import command from alembic.config import Config - from sqlalchemy import ( TIMESTAMP, BigInteger, @@ -21,7 +19,7 @@ select, ) from sqlalchemy.dialects.postgresql import JSON -from sqlalchemy.engine import URL, create_engine +from sqlalchemy.engine import URL, Engine, create_engine from sqlalchemy.exc import ProgrammingError from sqlalchemy.ext.compiler import compiles from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column, relationship @@ -37,12 +35,12 @@ class BIGSERIAL(BigInteger): @compiles(BIGSERIAL, POSTGRES_DIALECT) -def compile_bigserial_pg(type_, compiler, **kw): +def compile_bigserial_pg(type_, compiler, **kw) -> str: return "BIGSERIAL" @compiles(JSON, POSTGRES_DIALECT) -def compile_jsonb_pg(type_, compiler, **kw): +def compile_jsonb_pg(type_, compiler, **kw) -> str: return "JSON" @@ -51,10 +49,8 @@ class Base(DeclarativeBase): @event.listens_for(Base.metadata, "after_create") -def receive_after_create(target, connection, tables, **kw): - """ - listen for the 'after_create' event - """ +def receive_after_create(target, connection, tables, **kw) -> None: + """Listen for the 'after_create' event.""" if tables: _LOGGER.info("A table was created") else: @@ -65,7 +61,7 @@ def receive_after_create(target, connection, tables, **kw): # return context.get_current_parameters()["config"]["description"] -def deliver_update_date(context): +def deliver_update_date(context) -> datetime.datetime: return datetime.datetime.now(datetime.timezone.utc) @@ -77,41 +73,45 @@ class Projects(Base): __tablename__ = "projects" id: Mapped[int] = mapped_column(primary_key=True) - namespace: Mapped[str] = mapped_column(ForeignKey("users.namespace", ondelete="CASCADE")) + namespace: Mapped[str] = mapped_column( + ForeignKey("users.namespace", ondelete="CASCADE") + ) name: Mapped[str] = mapped_column() tag: Mapped[str] = mapped_column() digest: Mapped[str] = mapped_column(String(32)) - description: Mapped[Optional[str]] + description: Mapped[str | None] config: Mapped[dict] = mapped_column(JSON, server_default=FetchedValue()) private: Mapped[bool] number_of_samples: Mapped[int] number_of_stars: Mapped[int] = mapped_column(default=0) submission_date: Mapped[datetime.datetime] - last_update_date: Mapped[Optional[datetime.datetime]] = mapped_column( + last_update_date: Mapped[datetime.datetime | None] = mapped_column( default=deliver_update_date, # onupdate=deliver_update_date, # This field should not be updated, while we are adding project to favorites ) - schema_id: Mapped[Optional[int]] = mapped_column( + schema_id: Mapped[int | None] = mapped_column( ForeignKey("schema_versions.id", ondelete="SET NULL"), nullable=True ) - schema_mapping: Mapped["SchemaVersions"] = relationship("SchemaVersions", lazy="joined") + schema_mapping: Mapped["SchemaVersions"] = relationship( + "SchemaVersions", lazy="joined" + ) - pop: Mapped[Optional[bool]] = mapped_column(default=False) - samples_mapping: Mapped[List["Samples"]] = relationship( + pop: Mapped[bool | None] = mapped_column(default=False) + samples_mapping: Mapped[list["Samples"]] = relationship( back_populates="project_mapping", cascade="all, delete-orphan" ) - subsamples_mapping: Mapped[List["Subsamples"]] = relationship( + subsamples_mapping: Mapped[list["Subsamples"]] = relationship( back_populates="subsample_mapping", cascade="all, delete-orphan" ) - stars_mapping: Mapped[List["Stars"]] = relationship( + stars_mapping: Mapped[list["Stars"]] = relationship( back_populates="project_mapping", cascade="all, delete-orphan" ) - views_mapping: Mapped[List["Views"]] = relationship( + views_mapping: Mapped[list["Views"]] = relationship( back_populates="project_mapping", cascade="all, delete-orphan" ) # Self-referential relationship. The parent project is the one that was forked to create this one. - forked_from_id: Mapped[Optional[int]] = mapped_column( + forked_from_id: Mapped[int | None] = mapped_column( ForeignKey("projects.id", ondelete="SET NULL"), nullable=True ) forked_from_mapping = relationship( @@ -128,9 +128,11 @@ class Projects(Base): cascade="save-update, merge, refresh-expire", ) - namespace_mapping: Mapped["User"] = relationship("User", back_populates="projects_mapping") + namespace_mapping: Mapped["User"] = relationship( + "User", back_populates="projects_mapping" + ) - history_mapping: Mapped[List["HistoryProjects"]] = relationship( + history_mapping: Mapped[list["HistoryProjects"]] = relationship( back_populates="project_mapping", cascade="all, delete-orphan" ) @@ -148,16 +150,18 @@ class Samples(Base): sample: Mapped[dict] = mapped_column(JSON, server_default=FetchedValue()) project_id = mapped_column(ForeignKey("projects.id", ondelete="CASCADE")) project_mapping: Mapped["Projects"] = relationship(back_populates="samples_mapping") - sample_name: Mapped[Optional[str]] = mapped_column() - guid: Mapped[Optional[str]] = mapped_column(nullable=False, unique=True) + sample_name: Mapped[str | None] = mapped_column() + guid: Mapped[str | None] = mapped_column(nullable=False, unique=True) - submission_date: Mapped[datetime.datetime] = mapped_column(default=deliver_update_date) - last_update_date: Mapped[Optional[datetime.datetime]] = mapped_column( + submission_date: Mapped[datetime.datetime] = mapped_column( + default=deliver_update_date + ) + last_update_date: Mapped[datetime.datetime | None] = mapped_column( default=deliver_update_date, onupdate=deliver_update_date, ) - parent_guid: Mapped[Optional[str]] = mapped_column( + parent_guid: Mapped[str | None] = mapped_column( ForeignKey("samples.guid", ondelete="CASCADE"), nullable=True, doc="Parent sample id. Used to create a hierarchy of samples.", @@ -166,9 +170,11 @@ class Samples(Base): parent_mapping: Mapped["Samples"] = relationship( "Samples", remote_side=guid, back_populates="child_mapping" ) - child_mapping: Mapped["Samples"] = relationship("Samples", back_populates="parent_mapping") + child_mapping: Mapped["Samples"] = relationship( + "Samples", back_populates="parent_mapping" + ) - views: Mapped[Optional[List["ViewSampleAssociation"]]] = relationship( + views: Mapped[list["ViewSampleAssociation"] | None] = relationship( back_populates="sample", cascade="all, delete-orphan" ) @@ -185,7 +191,9 @@ class Subsamples(Base): subsample_number: Mapped[int] row_number: Mapped[int] project_id = mapped_column(ForeignKey("projects.id", ondelete="CASCADE")) - subsample_mapping: Mapped["Projects"] = relationship(back_populates="subsamples_mapping") + subsample_mapping: Mapped["Projects"] = relationship( + back_populates="subsamples_mapping" + ) class User(Base): @@ -197,7 +205,7 @@ class User(Base): id: Mapped[int] = mapped_column(primary_key=True) namespace: Mapped[str] = mapped_column(nullable=False, unique=True) - stars_mapping: Mapped[List["Stars"]] = relationship( + stars_mapping: Mapped[list["Stars"]] = relationship( back_populates="user_mapping", cascade="all, delete-orphan", order_by="Stars.star_date.desc()", @@ -205,10 +213,10 @@ class User(Base): number_of_projects: Mapped[int] = mapped_column(default=0) number_of_schemas: Mapped[int] = mapped_column(default=0) - projects_mapping: Mapped[List["Projects"]] = relationship( + projects_mapping: Mapped[list["Projects"]] = relationship( "Projects", back_populates="namespace_mapping" ) - schemas_mapping: Mapped[List["SchemaRecords"]] = relationship( + schemas_mapping: Mapped[list["SchemaRecords"]] = relationship( "SchemaRecords", back_populates="user_mapping" ) @@ -220,9 +228,13 @@ class Stars(Base): __tablename__ = "stars" - user_id = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), primary_key=True) - project_id = mapped_column(ForeignKey("projects.id", ondelete="CASCADE"), primary_key=True) - user_mapping: Mapped[List["User"]] = relationship(back_populates="stars_mapping") + user_id = mapped_column( + ForeignKey("users.id", ondelete="CASCADE"), primary_key=True + ) + project_id = mapped_column( + ForeignKey("projects.id", ondelete="CASCADE"), primary_key=True + ) + user_mapping: Mapped[list["User"]] = relationship(back_populates="stars_mapping") project_mapping: Mapped["Projects"] = relationship(back_populates="stars_mapping") star_date: Mapped[datetime.datetime] = mapped_column( onupdate=deliver_update_date, default=deliver_update_date @@ -238,12 +250,12 @@ class Views(Base): id: Mapped[int] = mapped_column(primary_key=True) name: Mapped[str] = mapped_column() - description: Mapped[Optional[str]] + description: Mapped[str | None] project_id = mapped_column(ForeignKey("projects.id", ondelete="CASCADE")) project_mapping = relationship("Projects", back_populates="views_mapping") - samples: Mapped[List["ViewSampleAssociation"]] = relationship( + samples: Mapped[list["ViewSampleAssociation"]] = relationship( back_populates="view", cascade="all, delete-orphan" ) @@ -257,19 +269,26 @@ class ViewSampleAssociation(Base): __tablename__ = "views_samples" - sample_id = mapped_column(ForeignKey("samples.id", ondelete="CASCADE"), primary_key=True) - view_id = mapped_column(ForeignKey("views.id", ondelete="CASCADE"), primary_key=True) + sample_id = mapped_column( + ForeignKey("samples.id", ondelete="CASCADE"), primary_key=True + ) + view_id = mapped_column( + ForeignKey("views.id", ondelete="CASCADE"), primary_key=True + ) sample: Mapped["Samples"] = relationship(back_populates="views") view: Mapped["Views"] = relationship(back_populates="samples") class HistoryProjects(Base): - __tablename__ = "project_history" id: Mapped[int] = mapped_column(primary_key=True) - project_id: Mapped[int] = mapped_column(ForeignKey("projects.id", ondelete="CASCADE")) - user: Mapped[str] = mapped_column(ForeignKey("users.namespace", ondelete="SET NULL")) + project_id: Mapped[int] = mapped_column( + ForeignKey("projects.id", ondelete="CASCADE") + ) + user: Mapped[str] = mapped_column( + ForeignKey("users.namespace", ondelete="SET NULL") + ) update_time: Mapped[datetime.datetime] = mapped_column( TIMESTAMP(timezone=True), default=deliver_update_date ) @@ -278,7 +297,7 @@ class HistoryProjects(Base): project_mapping: Mapped["Projects"] = relationship( "Projects", back_populates="history_mapping" ) - sample_changes_mapping: Mapped[List["HistorySamples"]] = relationship( + sample_changes_mapping: Mapped[list["HistorySamples"]] = relationship( back_populates="history_project_mapping", cascade="all, delete-orphan" ) @@ -294,13 +313,14 @@ class UpdateTypes(enum.Enum): class HistorySamples(Base): - __tablename__ = "sample_history" id: Mapped[int] = mapped_column(primary_key=True) - history_id: Mapped[int] = mapped_column(ForeignKey("project_history.id", ondelete="CASCADE")) + history_id: Mapped[int] = mapped_column( + ForeignKey("project_history.id", ondelete="CASCADE") + ) guid: Mapped[str] = mapped_column(nullable=False) - parent_guid: Mapped[Optional[str]] = mapped_column(nullable=True) + parent_guid: Mapped[str | None] = mapped_column(nullable=True) sample_json: Mapped[dict] = mapped_column(JSON, server_default=FetchedValue()) change_type: Mapped[UpdateTypes] = mapped_column(Enum(UpdateTypes), nullable=False) @@ -313,40 +333,46 @@ class SchemaRecords(Base): __tablename__ = "schema_records" id: Mapped[int] = mapped_column(primary_key=True) - namespace: Mapped[str] = mapped_column(ForeignKey("users.namespace", ondelete="CASCADE")) + namespace: Mapped[str] = mapped_column( + ForeignKey("users.namespace", ondelete="CASCADE") + ) name: Mapped[str] = mapped_column(nullable=False) maintainers: Mapped[str] = mapped_column(nullable=True) lifecycle_stage: Mapped[str] = mapped_column(nullable=True) - description: Mapped[Optional[str]] = mapped_column(nullable=True) + description: Mapped[str | None] = mapped_column(nullable=True) private: Mapped[bool] = mapped_column(default=False) - last_update_date: Mapped[Optional[datetime.datetime]] = mapped_column( + last_update_date: Mapped[datetime.datetime | None] = mapped_column( default=deliver_update_date, onupdate=deliver_update_date ) __table_args__ = (UniqueConstraint("namespace", "name"),) - versions_mapping: Mapped[List["SchemaVersions"]] = relationship( + versions_mapping: Mapped[list["SchemaVersions"]] = relationship( "SchemaVersions", back_populates="schema_mapping", cascade="all, delete-orphan", order_by="SchemaVersions.version.desc()", ) - user_mapping: Mapped["User"] = relationship("User", back_populates="schemas_mapping") + user_mapping: Mapped["User"] = relationship( + "User", back_populates="schemas_mapping" + ) class SchemaVersions(Base): __tablename__ = "schema_versions" id: Mapped[int] = mapped_column(primary_key=True) - schema_id: Mapped[int] = mapped_column(ForeignKey("schema_records.id", ondelete="CASCADE")) + schema_id: Mapped[int] = mapped_column( + ForeignKey("schema_records.id", ondelete="CASCADE") + ) version: Mapped[str] = mapped_column(nullable=False) schema_value: Mapped[dict] = mapped_column(JSON, server_default=FetchedValue()) release_date: Mapped[datetime.datetime] = mapped_column(default=deliver_update_date) - last_update_date: Mapped[Optional[datetime.datetime]] = mapped_column( + last_update_date: Mapped[datetime.datetime | None] = mapped_column( default=deliver_update_date, onupdate=deliver_update_date ) - contributors: Mapped[Optional[str]] = mapped_column(nullable=True) - release_notes: Mapped[Optional[str]] = mapped_column(nullable=True) + contributors: Mapped[str | None] = mapped_column(nullable=True) + release_notes: Mapped[str | None] = mapped_column(nullable=True) __table_args__ = (UniqueConstraint("schema_id", "version"),) @@ -354,8 +380,11 @@ class SchemaVersions(Base): "SchemaRecords", back_populates="versions_mapping" ) - tags_mapping: Mapped[List["SchemaTags"]] = relationship( - "SchemaTags", back_populates="schema_mapping", lazy="joined", cascade="all, delete-orphan" + tags_mapping: Mapped[list["SchemaTags"]] = relationship( + "SchemaTags", + back_populates="schema_mapping", + lazy="joined", + cascade="all, delete-orphan", ) @@ -375,13 +404,16 @@ class SchemaTags(Base): class TarNamespace(Base): - __tablename__ = "namespace_archives" id: Mapped[int] = mapped_column(primary_key=True) - namespace: Mapped[str] = mapped_column(ForeignKey("users.namespace", ondelete="CASCADE")) + namespace: Mapped[str] = mapped_column( + ForeignKey("users.namespace", ondelete="CASCADE") + ) file_path: Mapped[str] = mapped_column(nullable=False) - creation_date: Mapped[datetime.datetime] = mapped_column(default=deliver_update_date) + creation_date: Mapped[datetime.datetime] = mapped_column( + default=deliver_update_date + ) number_of_projects: Mapped[int] = mapped_column(default=0) file_size: Mapped[int] = mapped_column(nullable=False) @@ -393,9 +425,9 @@ class BedBaseStats(Base): gse: Mapped[str] = mapped_column() gsm: Mapped[str] = mapped_column() sample_name: Mapped[str] = mapped_column(nullable=True) - genome: Mapped[Optional[str]] = mapped_column(nullable=True, default="") - last_update_date: Mapped[Optional[str]] = mapped_column() - submission_date: Mapped[Optional[str]] = mapped_column() + genome: Mapped[str | None] = mapped_column(nullable=True, default="") + last_update_date: Mapped[str | None] = mapped_column() + submission_date: Mapped[str | None] = mapped_column() class BaseEngine: @@ -409,27 +441,25 @@ def __init__( host: str = "localhost", port: int = 5432, database: str = "pep-db", - user: str = None, - password: str = None, + user: str | None = None, + password: str | None = None, drivername: str = POSTGRES_DIALECT, - dsn: str = None, + dsn: str | None = None, echo: bool = False, run_migrations: bool = False, - ): - """ - Initialize connection to the pep_db database. You can use The basic connection parameters - or libpq connection string. - :param host: database server address e.g., localhost or an IP address. - :param port: the port number that defaults to 5432 if it is not provided. - :param database: the name of the database that you want to connect. - :param user: the username used to authenticate. - :param password: password used to authenticate. - :param drivername: driver used in - :param dsn: libpq connection string using the dsn parameter - (e.g. 'postgresql://user_name:password@host_name:port/db_name') - - :param echo: If True, the Engine will log all statements as well as a repr() of their parameter lists to the - :param run_migrations: If True, run database migrations + ) -> None: + """Initialize connection to the pep_db database. + + Args: + host: Database server address. + port: Port number (default: 5432). + database: Database name. + user: Username for authentication. + password: Password for authentication. + drivername: Database driver. + dsn: libpq connection string, e.g., "postgresql://user:pass@host:port/db". + echo: Log all SQL statements if True. + run_migrations: Run database migrations if True. """ if not dsn: dsn = URL.create( @@ -441,19 +471,20 @@ def __init__( drivername=drivername, ) if run_migrations: - dsn_with_password = f"{drivername}://{user}:{password}@{host}:{port}/{database}" + dsn_with_password = ( + f"{drivername}://{user}:{password}@{host}:{port}/{database}" + ) self.run_db_migration(dsn_with_password) self._engine = create_engine(dsn, echo=echo) self.create_schema(self._engine) self.check_db_connection() - def create_schema(self, engine=None): - """ - Create sql schema in the database. + def create_schema(self, engine=None) -> None: + """Create SQL schema in the database. - :param engine: sqlalchemy engine [Default: None] - :return: None + Args: + engine: SQLAlchemy engine (default: uses internal engine). """ if not engine: engine = self._engine @@ -461,12 +492,13 @@ def create_schema(self, engine=None): return None def session_execute(self, statement: Select) -> Result: - """ - Execute statement using sqlalchemy statement + """Execute a SQLAlchemy statement. - :param statement: SQL query or a SQL expression that is constructed using - SQLAlchemy's SQL expression language - :return: query result represented with declarative base + Args: + statement: SQL query or expression to execute. + + Returns: + Query result. """ _LOGGER.debug(f"Executing statement: {statement}") with Session(self._engine) as session: @@ -475,17 +507,15 @@ def session_execute(self, statement: Select) -> Result: return query_result @property - def session(self): - """ - :return: started sqlalchemy session - """ + def session(self) -> Session: + """Return a started SQLAlchemy session.""" return self._start_session() @property - def engine(self): + def engine(self) -> Engine: return self._engine - def _start_session(self): + def _start_session(self) -> Session: session = Session(self.engine) try: session.execute(select(Projects).limit(1)) @@ -494,28 +524,25 @@ def _start_session(self): return session - def check_db_connection(self): + def check_db_connection(self) -> None: try: self.session_execute(select(Projects).limit(1)) except ProgrammingError: raise SchemaError() def delete_schema(self, engine=None) -> None: - """ - Delete sql schema in the database. + """Delete SQL schema from the database. - :param engine: sqlalchemy engine [Default: None] - :return: None + Args: + engine: SQLAlchemy engine (default: uses internal engine). """ if not engine: engine = self._engine Base.metadata.drop_all(engine) return None - def run_db_migration(self, database_url: str): - """ - Migrate the database to the required version. - """ + def run_db_migration(self, database_url: str) -> None: + """Migrate the database to the latest version.""" script_directory = os.path.dirname(os.path.abspath(__file__)) script_location = os.path.join(script_directory, "alembic") diff --git a/pepdbagent/exceptions.py b/pepdbagent/exceptions.py index 17be697..4f6e4ac 100644 --- a/pepdbagent/exceptions.py +++ b/pepdbagent/exceptions.py @@ -4,37 +4,39 @@ class PEPDatabaseAgentError(Exception): """Base error type for pepdbagent custom errors.""" - def __init__(self, msg): + def __init__(self, msg: str) -> None: super(PEPDatabaseAgentError, self).__init__(msg) class SchemaError(PEPDatabaseAgentError): - def __init__(self): - super().__init__("""PEP_db connection error! The schema of connected db is incorrect""") + def __init__(self) -> None: + super().__init__( + """PEP_db connection error! The schema of connected db is incorrect""" + ) class RegistryPathError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Provided registry path is incorrect. {msg}""") class ProjectNotFoundError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Project does not exist. {msg}""") class ProjectUniqueNameError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""{msg}""") class IncorrectDateFormat(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Incorrect date format was provided. {msg}""") class FilterError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""pepdbagent filter error. {msg}""") @@ -43,7 +45,7 @@ class ProjectNotInFavorites(PEPDatabaseAgentError): Project doesn't exist in favorites """ - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Project is not in favorites list. {msg}""") @@ -52,37 +54,37 @@ class ProjectAlreadyInFavorites(PEPDatabaseAgentError): Project doesn't exist in favorites """ - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Project is already in favorites list. {msg}""") class SampleNotFoundError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Sample does not exist. {msg}""") class SampleTableUpdateError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Sample table update error. {msg}""") class ProjectDuplicatedSampleGUIDsError(SampleTableUpdateError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Project has duplicated sample GUIDs. {msg}""") class SampleAlreadyExistsError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Sample already exists. {msg}""") class ViewNotFoundError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""View does not exist. {msg}""") class SampleNotInViewError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Sample is not in the view. {msg}""") @@ -91,7 +93,7 @@ class SampleAlreadyInView(PEPDatabaseAgentError): Sample is already in the view exception """ - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Sample is already in the view. {msg}""") @@ -100,50 +102,50 @@ class ViewAlreadyExistsError(PEPDatabaseAgentError): View is already in the project exception """ - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""View already in the project. {msg}""") class NamespaceNotFoundError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Project does not exist. {msg}""") class HistoryNotFoundError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""History does not exist. {msg}""") class UserNotFoundError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""User does not exist. {msg}""") class SchemaDoesNotExistError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Schema does not exist. {msg}""") class SchemaAlreadyExistsError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Schema already exists. {msg}""") class SchemaVersionDoesNotExistError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Schema version does not exist. {msg}""") class SchemaVersionAlreadyExistsError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Schema version already exists. {msg}""") class SchemaTagDoesNotExistError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Schema tag does not exist. {msg}""") class SchemaTagAlreadyExistsError(PEPDatabaseAgentError): - def __init__(self, msg=""): + def __init__(self, msg: str = "") -> None: super().__init__(f"""Schema tag already exists. {msg}""") diff --git a/pepdbagent/models.py b/pepdbagent/models.py index 95aa783..90a7378 100644 --- a/pepdbagent/models.py +++ b/pepdbagent/models.py @@ -1,8 +1,7 @@ # file with pydantic models import datetime -from typing import Dict, List, Optional, Union -from peppy.const import CONFIG_KEY, SAMPLE_RAW_DICT_KEY, SUBSAMPLE_RAW_LIST_KEY +from peprs.const import CONFIG_KEY, SAMPLE_RAW_DICT_KEY, SUBSAMPLE_RAW_DICT_KEY from pydantic import BaseModel, ConfigDict, Field, field_validator from pepdbagent.const import DEFAULT_TAG @@ -14,7 +13,7 @@ class ProjectDict(BaseModel): """ config: dict = Field(alias=CONFIG_KEY) - subsample_list: Optional[Union[list, None]] = Field(alias=SUBSAMPLE_RAW_LIST_KEY) + subsample_list: list | None = Field(alias=SUBSAMPLE_RAW_DICT_KEY) sample_dict: list = Field(alias=SAMPLE_RAW_DICT_KEY) model_config = ConfigDict(populate_by_name=True, extra="forbid") @@ -25,19 +24,19 @@ class AnnotationModel(BaseModel): Project Annotation model. All meta metadata """ - namespace: Optional[str] - name: Optional[str] - tag: Optional[str] - is_private: Optional[bool] - number_of_samples: Optional[int] - description: Optional[str] - last_update_date: Optional[str] - submission_date: Optional[str] - digest: Optional[str] - pep_schema: Optional[str] - pop: Optional[bool] = False - stars_number: Optional[int] = 0 - forked_from: Optional[Union[str, None]] = None + namespace: str | None = None + name: str | None = None + tag: str | None = None + is_private: bool | None = None + number_of_samples: int | None = None + description: str | None = None + last_update_date: str | None = None + submission_date: str | None = None + digest: str | None = None + pep_schema: str | None = None + pop: bool | None = False + stars_number: int | None = 0 + forked_from: str | None = None model_config = ConfigDict( validate_assignment=True, @@ -45,7 +44,7 @@ class AnnotationModel(BaseModel): ) @field_validator("is_private") - def is_private_should_be_bool(cls, v): + def is_private_should_be_bool(cls, v) -> bool: if not isinstance(v, bool): return False else: @@ -66,7 +65,7 @@ class AnnotationList(BaseModel): count: int limit: int offset: int - results: List[Union[AnnotationModel, None]] + results: list[AnnotationModel | None] class Namespace(BaseModel): @@ -87,7 +86,7 @@ class NamespaceList(BaseModel): count: int limit: int offset: int - results: List[Namespace] + results: list[Namespace] class UpdateItems(BaseModel): @@ -95,17 +94,17 @@ class UpdateItems(BaseModel): Model used for updating individual items in db """ - name: Optional[str] = None - description: Optional[str] = None - tag: Optional[str] = None - is_private: Optional[bool] = None - pep_schema: Optional[str] = None - digest: Optional[str] = None - config: Optional[dict] = None - samples: Optional[List[dict]] = None - subsamples: Optional[List[List[dict]]] = None - pop: Optional[bool] = None - schema_id: Optional[int] = None + name: str | None = None + description: str | None = None + tag: str | None = None + is_private: bool | None = None + pep_schema: str | None = None + digest: str | None = None + config: dict | None = None + samples: list[dict] | None = None + subsamples: list[list[dict]] | None = None + pop: bool | None = None + schema_id: int | None = None model_config = ConfigDict( arbitrary_types_allowed=True, @@ -113,7 +112,7 @@ class UpdateItems(BaseModel): ) @property - def number_of_samples(self) -> Union[int, None]: + def number_of_samples(self) -> int | None: if self.samples: return len(self.samples) return None @@ -125,31 +124,31 @@ class UpdateModel(BaseModel): Model used for updating individual items and creating sql string in the code """ - config: Optional[dict] = None - name: Optional[str] = None - tag: Optional[str] = None - private: Optional[bool] = Field(alias="is_private", default=None) - digest: Optional[str] = None - number_of_samples: Optional[int] = None - pep_schema: Optional[str] = None - description: Optional[str] = "" - # last_update_date: Optional[datetime.datetime] = datetime.datetime.now(datetime.timezone.utc) - pop: Optional[bool] = False + config: dict | None = None + name: str | None = None + tag: str | None = None + private: bool | None = Field(alias="is_private", default=None) + digest: str | None = None + number_of_samples: int | None = None + pep_schema: str | None = None + description: str | None = "" + # last_update_date: datetime.datetime | None = datetime.datetime.now(datetime.timezone.utc) + pop: bool | None = False @field_validator("tag", "name") - def value_must_not_be_empty(cls, v): + def value_must_not_be_empty(cls, v) -> str | None: if "" == v: return None return v @field_validator("tag", "name") - def value_must_be_lowercase(cls, v): + def value_must_be_lowercase(cls, v) -> str | None: if v: return v.lower() return v @field_validator("tag", "name") - def value_should_not_contain_question(cls, v): + def value_should_not_contain_question(cls, v) -> str: if "?" in v: return ValueError("Question mark (?) is prohibited in name and tag.") return v @@ -163,7 +162,7 @@ class NamespaceInfo(BaseModel): """ namespace_name: str - contact_url: Optional[str] = None + contact_url: str | None = None number_of_projects: int number_of_schemas: int @@ -174,7 +173,7 @@ class ListOfNamespaceInfo(BaseModel): """ pagination: PaginationResult - results: List[NamespaceInfo] + results: list[NamespaceInfo] class ProjectRegistryPath(BaseModel): @@ -193,7 +192,7 @@ class ViewAnnotation(BaseModel): """ name: str - description: Optional[str] = None + description: str | None = None number_of_samples: int = 0 @@ -205,7 +204,7 @@ class ProjectViews(BaseModel): namespace: str name: str tag: str = DEFAULT_TAG - views: List[ViewAnnotation] = [] + views: list[ViewAnnotation] = [] class CreateViewDictModel(BaseModel): @@ -216,13 +215,13 @@ class CreateViewDictModel(BaseModel): project_namespace: str project_name: str project_tag: str - sample_list: List[str] + sample_list: list[str] class RegistryPath(BaseModel): namespace: str name: str - tag: Optional[str] = "default" + tag: str | None = "default" class NamespaceStats(BaseModel): @@ -230,9 +229,9 @@ class NamespaceStats(BaseModel): Namespace stats model """ - namespace: Union[str, None] = None - projects_updated: Dict[str, int] = None - projects_created: Dict[str, int] = None + namespace: str | None = None + projects_updated: dict[str, int] | None = None + projects_created: dict[str, int] | None = None class HistoryChangeModel(BaseModel): @@ -253,7 +252,7 @@ class HistoryAnnotationModel(BaseModel): namespace: str name: str tag: str = DEFAULT_TAG - history: List[HistoryChangeModel] + history: list[HistoryChangeModel] class SchemaVersionAnnotation(BaseModel): @@ -264,9 +263,9 @@ class SchemaVersionAnnotation(BaseModel): namespace: str schema_name: str version: str - contributors: Optional[Union[str, None]] = "" - release_notes: Optional[Union[str, None]] = "" - tags: Dict[str, Union[str, None]] = {} + contributors: str | None = "" + release_notes: str | None = "" + tags: dict[str, str | None] = {} release_date: datetime.datetime last_update_date: datetime.datetime @@ -278,11 +277,11 @@ class SchemaRecordAnnotation(BaseModel): namespace: str schema_name: str - description: Optional[Union[str, None]] = "" - maintainers: Optional[Union[str, None]] = "" - lifecycle_stage: Optional[Union[str, None]] = "" - latest_released_version: Optional[Union[str, None]] - private: Optional[bool] = False + description: str | None = "" + maintainers: str | None = "" + lifecycle_stage: str | None = "" + latest_released_version: str | None = None + private: bool | None = False last_update_date: datetime.datetime @@ -292,7 +291,7 @@ class SchemaSearchResult(BaseModel): """ pagination: PaginationResult - results: List[SchemaRecordAnnotation] + results: list[SchemaRecordAnnotation] class SchemaVersionSearchResult(BaseModel): @@ -301,21 +300,21 @@ class SchemaVersionSearchResult(BaseModel): """ pagination: PaginationResult - results: List[SchemaVersionAnnotation] + results: list[SchemaVersionAnnotation] class UpdateSchemaRecordFields(BaseModel): - maintainers: Optional[Union[str, None]] = None - lifecycle_stage: Optional[Union[str, None]] = None - private: Optional[bool] = False - name: Optional[Union[str, None]] = None - description: Optional[Union[str, None]] = None + maintainers: str | None = None + lifecycle_stage: str | None = None + private: bool | None = False + name: str | None = None + description: str | None = None class UpdateSchemaVersionFields(BaseModel): - contributors: Optional[Union[str, None]] = None - schema_value: Optional[dict] = None - release_notes: Optional[Union[str, None]] = None + contributors: str | None = None + schema_value: dict | None = None + release_notes: str | None = None class TarNamespaceModel(BaseModel): @@ -323,10 +322,10 @@ class TarNamespaceModel(BaseModel): Namespace archive model """ - identifier: int = None + identifier: int | None = None namespace: str file_path: str - creation_date: datetime.datetime = None + creation_date: datetime.datetime | None = None number_of_projects: int = 0 file_size: int = 0 @@ -337,4 +336,4 @@ class TarNamespaceModelReturn(BaseModel): """ count: int - results: List[TarNamespaceModel] + results: list[TarNamespaceModel] diff --git a/pepdbagent/modules/annotation.py b/pepdbagent/modules/annotation.py index 7159ece..ee7c6f2 100644 --- a/pepdbagent/modules/annotation.py +++ b/pepdbagent/modules/annotation.py @@ -1,6 +1,6 @@ import logging from datetime import datetime -from typing import List, Literal, Optional, Union +from typing import Literal from sqlalchemy import and_, func, or_, select from sqlalchemy.orm import Session @@ -17,7 +17,11 @@ from pepdbagent.db_utils import BaseEngine, Projects from pepdbagent.exceptions import FilterError, ProjectNotFoundError, RegistryPathError from pepdbagent.models import AnnotationList, AnnotationModel, RegistryPath -from pepdbagent.utils import convert_date_string_to_date, registry_path_converter, tuple_converter +from pepdbagent.utils import ( + convert_date_string_to_date, + registry_path_converter, + tuple_converter, +) _LOGGER = logging.getLogger(PKG_NAME) @@ -31,7 +35,8 @@ class PEPDatabaseAnnotation: def __init__(self, pep_db_engine: BaseEngine): """ - :param pep_db_engine: pepdbengine object with sa engine + Args: + pep_db_engine: PEPDatabaseAgent engine object. """ self._sa_engine = pep_db_engine.engine self._pep_db_engine = pep_db_engine @@ -42,44 +47,42 @@ def get( name: str = None, tag: str = None, query: str = None, - admin: Union[List[str], str] = None, + admin: list[str] | str | None = None, limit: int = DEFAULT_LIMIT, offset: int = DEFAULT_OFFSET, order_by: str = "update_date", order_desc: bool = False, - filter_by: Optional[Literal["submission_date", "last_update_date"]] = None, - filter_start_date: Optional[str] = None, - filter_end_date: Optional[str] = None, - pep_type: Optional[Literal["pep", "pop"]] = None, + filter_by: Literal["submission_date", "last_update_date"] | None = None, + filter_start_date: str | None = None, + filter_end_date: str | None = None, + pep_type: Literal["pep", "pop"] | None = None, ) -> AnnotationList: - """ - Get project annotations. - - There is 5 scenarios how to get project or projects annotations: - - provide name, namespace and tag. Return: project annotations of exact provided PK(namespace, name, tag) - - provide only namespace. Return: list of projects annotations in specified namespace - - Nothing is provided. Return: list of projects annotations in all database - - provide query. Return: list of projects annotations find in database that have query pattern. - - provide query and namespace. Return: list of projects annotations find in specific namespace - that have query pattern. - :param namespace: Namespace - :param name: Project name - :param tag: tag - :param query: query (search string): Pattern of name, tag or description - :param admin: admin name (namespace), or list of namespaces, where user is admin - :param limit: return limit - :param offset: return offset - :param order_by: sort the result-set by the information - Options: ["name", "update_date", "submission_date"] - [Default: update_date] - :param order_desc: Sort the records in descending order. [Default: False] - :param filter_by: data to use filter on. - Options: ["submission_date", "last_update_date"] - [Default: filter won't be used] - :param filter_start_date: Filter start date. Format: "YYYY/MM/DD" - :param filter_end_date: Filter end date. Format: "YYYY/MM/DD". if None: present date will be used - :param pep_type: Get pep with specified type. Options: ["pep", "pop"]. Default: None, get all peps - :return: pydantic model: AnnotationList + """Get project annotations. + + Five retrieval scenarios: + - namespace + name + tag: exact match. + - namespace only: all projects in that namespace. + - nothing: all projects in the database. + - query: full-text search across name, tag, description. + - query + namespace: full-text search within a namespace. + + Args: + namespace: Namespace to filter by. + name: Project name (use with namespace and tag for exact lookup). + tag: Project tag. + query: Search string matched against name, tag, and description. + admin: Namespace(s) where the caller has admin rights. + limit: Maximum number of results. + offset: Number of results to skip. + order_by: Sort field — "name", "update_date", or "submission_date". + order_desc: Sort in descending order if True. + filter_by: Date field to apply range filter on — "submission_date" or "last_update_date". + filter_start_date: Range start in YYYY/MM/DD format. + filter_end_date: Range end in YYYY/MM/DD format (default: today). + pep_type: Restrict to "pep" or "pop" (default: all). + + Returns: + AnnotationList with count, limit, offset, and results. """ if all([namespace, name, tag]): found_annotation = [ @@ -98,7 +101,9 @@ def get( ) if pep_type not in [None, "pep", "pop"]: - raise ValueError(f"pep_type should be one of ['pep', 'pop'], got {pep_type}") + raise ValueError( + f"pep_type should be one of ['pep', 'pop'], got {pep_type}" + ) return AnnotationList( limit=limit, @@ -131,18 +136,17 @@ def get( def get_by_rp( self, - registry_paths: Union[List[str], str], - admin: Union[List[str], str] = None, + registry_paths: list[str] | str, + admin: list[str] | str | None = None, ) -> AnnotationList: - """ - Get project annotations by providing registry_path or list of registry paths. - :param registry_paths: registry path string or list of registry paths - :param admin: list of namespaces where user is admin - :return: pydantic model: AnnotationReturnModel( - limit: - offset: - count: - result: List [AnnotationModel]) + """Get project annotations by registry path or list of registry paths. + + Args: + registry_paths: Single registry path or list of paths ("namespace/name:tag"). + admin: Namespace(s) where the caller has admin rights. + + Returns: + AnnotationList with count, limit, offset, and results. """ if isinstance(registry_paths, list): anno_results = [] @@ -153,7 +157,9 @@ def get_by_rp( _LOGGER.error(str(err), registry_paths) continue try: - single_return = self._get_single_annotation(namespace, name, tag, admin) + single_return = self._get_single_annotation( + namespace, name, tag, admin + ) if single_return: anno_results.append(single_return) except ProjectNotFoundError: @@ -175,15 +181,21 @@ def _get_single_annotation( namespace: str, name: str, tag: str = DEFAULT_TAG, - admin: Union[List[str], str] = None, - ) -> Union[AnnotationModel, None]: - """ - Retrieving project annotation dict by specifying project name - :param namespace: project registry_path - will return dict of project annotations - :param name: project name in database - :param tag: tag of the projects - :param admin: string or list of admins [e.g. "Khoroshevskyi", or ["doc_adin","Khoroshevskyi"]] - :return: pydantic Annotation Model of annotations of current project + admin: list[str] | str | None = None, + ) -> AnnotationModel | None: + """Retrieve annotation for a single project. + + Args: + namespace: Project namespace. + name: Project name. + tag: Project tag. + admin: Namespace(s) with admin/private access. + + Returns: + AnnotationModel for the project, or None if not found. + + Raises: + ProjectNotFoundError: If the project does not exist or is not accessible. """ _LOGGER.info(f"Getting annotation of the project: '{namespace}/{name}:{tag}'") admin_tuple = tuple_converter(admin) @@ -231,34 +243,35 @@ def _get_single_annotation( ) return annot else: - raise ProjectNotFoundError(f"Project '{namespace}/{name}:{tag}' was not found.") + raise ProjectNotFoundError( + f"Project '{namespace}/{name}:{tag}' was not found." + ) def _count_projects( self, namespace: str = None, search_str: str = None, tag: str = None, - admin: Union[str, List[str]] = None, - filter_by: Optional[Literal["submission_date", "last_update_date"]] = None, - filter_start_date: Optional[str] = None, - filter_end_date: Optional[str] = None, - pep_type: Optional[Literal["pep", "pop"]] = None, + admin: str | list[str] | None = None, + filter_by: Literal["submission_date", "last_update_date"] | None = None, + filter_start_date: str | None = None, + filter_end_date: str | None = None, + pep_type: Literal["pep", "pop"] | None = None, ) -> int: - """ - Count projects. [This function is related to _find_projects] - - :param namespace: namespace where to search for a project - :param search_str: search string. will be searched in name, tag and description information - :param tag: tag of the projects (find projects with specific tag) - :param admin: string or list of admins [e.g. "Khoroshevskyi", or ["doc_adin","Khoroshevskyi"]] - :param filter_by: data to use filter on. - Options: ["submission_date", "last_update_date"] - [Default: filter won't be used] - :param filter_start_date: Filter start date. Format: "YYYY:MM:DD" - :param filter_end_date: Filter end date. Format: "YYYY:MM:DD". if None: present date will be used - :param pep_type: Get pep with specified type. Options: ["pep", "pop"]. Default: None, get all peps - - :return: number of found project in specified namespace + """Count projects matching the given filters. + + Args: + namespace: Namespace to restrict the search to. + search_str: String to search in name, tag, and description. + tag: Exact tag to match. + admin: Namespace(s) with admin/private access. + filter_by: Date field for range filter — "submission_date" or "last_update_date". + filter_start_date: Range start in YYYY/MM/DD format. + filter_end_date: Range end in YYYY/MM/DD format (default: today). + pep_type: Restrict to "pep" or "pop" (default: all). + + Returns: + Number of matching projects. """ if admin is None: admin = [] @@ -288,38 +301,38 @@ def _get_projects( namespace: str = None, tag: str = None, search_str: str = None, - admin: Union[str, List[str]] = None, + admin: str | list[str] | None = None, limit: int = DEFAULT_LIMIT, offset: int = DEFAULT_OFFSET, order_by: str = "update_date", order_desc: bool = False, - filter_by: Optional[Literal["submission_date", "last_update_date"]] = None, - filter_start_date: Optional[str] = None, - filter_end_date: Optional[str] = None, - pep_type: Optional[Literal["pep", "pop"]] = None, - ) -> List[AnnotationModel]: + filter_by: Literal["submission_date", "last_update_date"] | None = None, + filter_start_date: str | None = None, + filter_end_date: str | None = None, + pep_type: Literal["pep", "pop"] | None = None, + ) -> list[AnnotationModel]: + """Get projects matching the given filters. + + Args: + namespace: Namespace to restrict the search to. + tag: Exact tag to match. + search_str: String to search in name, tag, and description. + admin: Namespace(s) with admin/private access. + limit: Maximum number of results. + offset: Number of results to skip. + order_by: Sort field — "update_date", "name", "submission_date", or "stars". + order_desc: Sort in descending order if True. + filter_by: Date field for range filter — "submission_date" or "last_update_date". + filter_start_date: Range start in YYYY/MM/DD format. + filter_end_date: Range end in YYYY/MM/DD format (default: today). + pep_type: Restrict to "pep" or "pop" (default: all). + + Returns: + List of AnnotationModel objects. """ - Get projects by providing search string. - - :param namespace: namespace where to search for a project - :param tag: tag of the projects (find projects with specific tag) - :param search_str: search string that has to be found in the name or tag - :param admin: True, if user is admin of the namespace [Default: False] - :param limit: limit of return results - :param offset: number of results off set (that were already showed) - :param order_by: sort the result-set by the information - Options: ["update_date", "name", "submission_date", "stars"] - [Default: "update_date"] - :param order_desc: Sort the records in descending order. [Default: False] - :param filter_by: data to use filter on. - Options: ["submission_date", "last_update_date"] - [Default: filter won't be used] - :param filter_start_date: Filter start date. Format: "YYYY:MM:DD" - :param filter_end_date: Filter end date. Format: "YYYY:MM:DD". if None: present date will be used - :param pep_type: Get pep with specified type. Options: ["pep", "pop"]. Default: None, get all peps - :return: list of found projects with their annotations. - """ - _LOGGER.info(f"Running annotation search: (namespace: {namespace}, query: {search_str}.") + _LOGGER.info( + f"Running annotation search: (namespace: {namespace}, query: {search_str}." + ) if admin is None: admin = [] @@ -377,15 +390,15 @@ def _get_projects( def _add_order_by_keyword( statement: Select, by: str = "update_date", desc: bool = False ) -> Select: - """ - Add order by clause to sqlalchemy statement - - :param statement: sqlalchemy representation of a SELECT statement. - :param by: sort the result-set by the information - Options: ["name", "update_date", "submission_date", "stars"] - [Default: "update_date"] - :param desc: Sort the records in descending order. [Default: False] - :return: sqlalchemy representation of a SELECT statement with order by keyword + """Add an ORDER BY clause to a SELECT statement. + + Args: + statement: SQLAlchemy SELECT statement to augment. + by: Sort field — "name", "update_date", "submission_date", or "stars". + desc: Sort in descending order if True. + + Returns: + Statement with ORDER BY applied. """ if by == "update_date": order_by_obj = Projects.last_update_date @@ -414,18 +427,20 @@ def _add_condition( statement: Select, namespace: str = None, search_str: str = None, - admin_list: Union[str, List[str]] = None, + admin_list: str | list[str] | None = None, tag: str = None, ) -> Select: - """ - Add where clause to sqlalchemy statement (in project search) - - :param statement: sqlalchemy representation of a SELECT statement. - :param namespace: project namespace sql:(where namespace = "") - :param search_str: search string that has to be found in the name or tag - :param admin_list: list or string of admin rights to namespace - :param tag: tag of the projects (find projects with specific tag) - :return: sqlalchemy representation of a SELECT statement with where clause. + """Add a WHERE clause to a project search statement. + + Args: + statement: SQLAlchemy SELECT statement to augment. + namespace: Filter to this namespace. + search_str: String to search in name, tag, and description. + admin_list: Namespace(s) with admin/private access. + tag: Exact tag to match. + + Returns: + Statement with WHERE clause applied. """ admin_list = tuple_converter(admin_list) if search_str: @@ -451,19 +466,20 @@ def _add_condition( @staticmethod def _add_date_filter_if_provided( statement: Select, - filter_by: Optional[Literal["submission_date", "last_update_date"]], - filter_start_date: Optional[str], - filter_end_date: Optional[str] = None, - ): - """ - Add filter to where clause to sqlalchemy statement (in project search) - - :param statement: sqlalchemy representation of a SELECT statement with where clause - :param filter_by: data to use filter on. - Options: ["submission_date", "last_update_date"] - :param filter_start_date: Filter start date. Format: "YYYY:MM:DD" - :param filter_end_date: Filter end date. Format: "YYYY:MM:DD". if None: present date will be used - :return: sqlalchemy representation of a SELECT statement with where clause with added filter + filter_by: Literal["submission_date", "last_update_date"] | None, + filter_start_date: str | None, + filter_end_date: str | None = None, + ) -> Select: + """Add a date range filter to a SELECT statement. + + Args: + statement: SQLAlchemy SELECT statement to augment. + filter_by: Date field to filter on — "submission_date" or "last_update_date". + filter_start_date: Range start in YYYY/MM/DD format. + filter_end_date: Range end in YYYY/MM/DD format (default: today). + + Returns: + Statement with date filter applied. """ if filter_by and filter_start_date: start_date = convert_date_string_to_date(filter_start_date) @@ -484,25 +500,31 @@ def _add_date_filter_if_provided( return statement else: if filter_by: - _LOGGER.warning("filter_start_date was not provided, skipping filter...") + _LOGGER.warning( + "filter_start_date was not provided, skipping filter..." + ) return statement def get_project_number_in_namespace( self, namespace: str, - admin: Union[str, List[str]] = None, + admin: str | list[str] | None = None, ) -> int: - """ - Get number of found projects by providing search string. + """Get the number of projects in a namespace. - :param namespace: namespace where to search for a project - :param admin: True, if user is admin of the namespace [Default: False] - :return Integer: number of projects in the namepsace + Args: + namespace: Namespace to count projects in. + admin: Namespace(s) with admin/private access. + + Returns: + Number of accessible projects in the namespace. """ if admin is None: admin = [] statement = ( - select(func.count()).select_from(Projects).where(Projects.namespace == namespace) + select(func.count()) + .select_from(Projects) + .where(Projects.namespace == namespace) ) statement = statement.where( or_(Projects.private.is_(False), Projects.namespace.in_(admin)) @@ -517,19 +539,17 @@ def get_project_number_in_namespace( def get_by_rp_list( self, - registry_paths: List[str], - admin: Union[str, List[str]] = None, + registry_paths: list[str], + admin: str | list[str] | None = None, ) -> AnnotationList: - """ - Get project annotations by providing list of registry paths. - - :param registry_paths: registry path string or list of registry paths - :param admin: list of namespaces where user is admin - :return: pydantic model: AnnotationReturnModel( - limit: - offset: - count: - result: List [AnnotationModel]) + """Get project annotations for a list of registry paths in a single query. + + Args: + registry_paths: List of registry paths ("namespace/name:tag"). + admin: Namespace(s) where the caller has admin rights. + + Returns: + AnnotationList preserving the input order (missing projects returned as None). """ admin_tuple = tuple_converter(admin) @@ -610,40 +630,39 @@ def get_projects_list( self, namespace: str = None, search_str: str = None, - admin: Union[str, List[str]] = None, + admin: str | list[str] | None = None, limit: int = DEFAULT_LIMIT, offset: int = DEFAULT_OFFSET, order_by: str = "update_date", order_desc: bool = False, - filter_by: Optional[Literal["submission_date", "last_update_date"]] = None, - filter_start_date: Optional[str] = None, - filter_end_date: Optional[str] = None, - pep_type: Optional[Literal["pep", "pop"]] = None, - ) -> List[RegistryPath]: + filter_by: Literal["submission_date", "last_update_date"] | None = None, + filter_start_date: str | None = None, + filter_end_date: str | None = None, + pep_type: Literal["pep", "pop"] | None = None, + ) -> list[RegistryPath]: + """Retrieve a list of registry paths matching the given filters. + + Lightweight alternative to get() — returns only registry paths, no annotation data. + + Args: + namespace: Namespace to restrict the search to. + search_str: String to search in name, tag, and description. + admin: Namespace(s) with admin/private access. + limit: Maximum number of results. + offset: Number of results to skip. + order_by: Sort field — "name", "update_date", "submission_date", or "stars". + order_desc: Sort in descending order if True. + filter_by: Date field for range filter — "submission_date" or "last_update_date". + filter_start_date: Range start in YYYY/MM/DD format. + filter_end_date: Range end in YYYY/MM/DD format (default: today). + pep_type: Restrict to "pep" or "pop" (default: all). + + Returns: + List of RegistryPath objects. """ - Retrieve a list of projects by providing a search string. - This function serves as a lightweight version of the full 'get' function, - returning only a list of registry paths without annotations. - It is designed for use cases where a large list of projects is needed with minimal processing time. - - :param namespace: namespace where to search for a project - :param search_str: search string that has to be found in the name or tag - :param admin: True, if user is admin of the namespace [Default: False] - :param limit: limit of return results - :param offset: number of results off set (that were already showed) - :param order_by: sort the result-set by the information - Options: ["name", "update_date", "submission_date", "stars"] - [Default: "update_date"] - :param order_desc: Sort the records in descending order. [Default: False] - :param filter_by: data to use filter on. - Options: ["submission_date", "last_update_date"] - [Default: filter won't be used] - :param filter_start_date: Filter start date. Format: "YYYY:MM:DD" - :param filter_end_date: Filter end date. Format: "YYYY:MM:DD". if None: present date will be used - :param pep_type: Get pep with specified type. Options: ["pep", "pop"]. Default: None, get all peps - :return: list of found projects with their annotations. - """ - _LOGGER.info(f"Running project search: (namespace: {namespace}, query: {search_str}.") + _LOGGER.info( + f"Running project search: (namespace: {namespace}, query: {search_str}." + ) if admin is None: admin = [] diff --git a/pepdbagent/modules/namespace.py b/pepdbagent/modules/namespace.py index 7766fb6..8e4a750 100644 --- a/pepdbagent/modules/namespace.py +++ b/pepdbagent/modules/namespace.py @@ -1,14 +1,13 @@ import logging from collections import Counter from datetime import datetime, timedelta -from typing import List, Tuple, Union -from sqlalchemy import distinct, func, or_, select, delete +from sqlalchemy import delete, distinct, func, or_, select from sqlalchemy.orm import Session from sqlalchemy.sql.selectable import Select from pepdbagent.const import DEFAULT_LIMIT, DEFAULT_LIMIT_INFO, DEFAULT_OFFSET, PKG_NAME -from pepdbagent.db_utils import BaseEngine, Projects, User, TarNamespace +from pepdbagent.db_utils import BaseEngine, Projects, TarNamespace, User from pepdbagent.exceptions import NamespaceNotFoundError from pepdbagent.models import ( ListOfNamespaceInfo, @@ -16,9 +15,9 @@ NamespaceInfo, NamespaceList, NamespaceStats, + PaginationResult, TarNamespaceModel, TarNamespaceModelReturn, - PaginationResult, ) from pepdbagent.utils import tuple_converter @@ -34,7 +33,8 @@ class PEPDatabaseNamespace: def __init__(self, pep_db_engine: BaseEngine): """ - :param pep_db_engine: pepdbengine object with sa engine + Args: + pep_db_engine: PEPDatabaseAgent engine object. """ self._sa_engine = pep_db_engine.engine self._pep_db_engine = pep_db_engine @@ -42,25 +42,24 @@ def __init__(self, pep_db_engine: BaseEngine): def get( self, query: str = "", - admin: Union[List[str], str] = None, + admin: list[str] | str | None = None, limit: int = DEFAULT_LIMIT, offset: int = DEFAULT_OFFSET, ) -> NamespaceList: + """Search available namespaces in the database. + + Args: + query: Search string. + admin: Namespaces where the user has admin rights. + limit: Maximum number of results to return. + offset: Number of results to skip. + + Returns: + NamespaceList with count, limit, offset, and results. """ - Search available namespaces in the database - :param query: search string - :param admin: list of namespaces where user is admin - :param offset: offset of the search - :param limit: limit of the search - :return: Search result: - { - total number of results - search limit - search offset - search results - } - """ - _LOGGER.info(f"Getting namespaces annotation with provided info: (query: {query})") + _LOGGER.info( + f"Getting namespaces annotation with provided info: (query: {query})" + ) admin_tuple = tuple_converter(admin) return NamespaceList( count=self._count_namespace(search_str=query, admin_nsp=admin_tuple), @@ -80,20 +79,17 @@ def _get_namespace( admin_nsp: tuple = None, limit: int = DEFAULT_LIMIT, offset: int = DEFAULT_OFFSET, - ) -> List[Namespace]: - """ - Search for namespace by providing search string. - - :param search_str: string of symbols, words, keywords to search in the - namespace name. - :param admin_nsp: tuple of namespaces where project can be retrieved if they are privet - :param limit: limit of return results - :param offset: number of results off set (that were already showed) - :return: list of dict with structure { - namespace, - number_of_projects, - number_of_samples, - } + ) -> list[Namespace]: + """Search for namespaces matching a search string. + + Args: + search_str: Keywords to search in namespace names. + admin_nsp: Namespaces accessible even when private. + limit: Maximum number of results. + offset: Number of results to skip. + + Returns: + List of Namespace objects. """ statement = ( select( @@ -127,14 +123,17 @@ def _get_namespace( ) return results_list - def _count_namespace(self, search_str: str = None, admin_nsp: tuple = tuple()) -> int: - """ - Get number of found namespace. [This function is related to _get_namespaces] + def _count_namespace( + self, search_str: str = None, admin_nsp: tuple = tuple() + ) -> int: + """Count namespaces matching a search string. - :param search_str: string of symbols, words, keywords to search in the - namespace name. - :param admin_nsp: tuple of namespaces where project can be retrieved if they are privet - :return: number of found namespaces + Args: + search_str: Keywords to search in namespace names. + admin_nsp: Namespaces accessible even when private. + + Returns: + Number of matching namespaces. """ statement = select( func.count(distinct(Projects.namespace)).label("number_of_namespaces") @@ -153,15 +152,17 @@ def _count_namespace(self, search_str: str = None, admin_nsp: tuple = tuple()) - def _add_condition( statement: Select, search_str: str = None, - admin_list: Union[Tuple[str], List[str], str] = None, + admin_list: tuple[str, ...] | list[str] | str | None = None, ) -> Select: - """ - Add where clause to sqlalchemy statement (in namespace search) + """Add a WHERE clause to a namespace search statement. - :param statement: sqlalchemy representation of a SELECT statement. - :param search_str: search string that has to be found namespace - :param admin_list: list or string of admin rights to namespace - :return: sqlalchemy representation of a SELECT statement with where clause. + Args: + statement: SQLAlchemy SELECT statement to augment. + search_str: String to search in namespace names. + admin_list: Namespaces with admin/private access. + + Returns: + Statement with WHERE clause applied. """ if search_str: sql_search_str = f"%{search_str}%" @@ -181,21 +182,17 @@ def info( page_size: int = DEFAULT_LIMIT_INFO, order_by: str = "number_of_projects", ) -> ListOfNamespaceInfo: - """ - Get list of top n namespaces in the database - ! Warning: this function counts number of all projects in namespaces. - ! it does not filter private projects (It was done for efficiency reasons) - - :param page: page number - :param page_size: number of namespaces to show - :param order_by: order by field. Options: number_of_projects, number_of_schemas [Default: number_of_projects] - - :return: number_of_namespaces: int - limit: int - results: { namespace: str - number_of_projects: int - number_of_schemas: int - } + """Get a paginated list of top namespaces. + + Warning: counts all projects including private ones (by design, for efficiency). + + Args: + page: Page number (zero-based). + page_size: Number of namespaces per page. + order_by: Sort field — "number_of_projects" or "number_of_schemas". + + Returns: + ListOfNamespaceInfo with pagination metadata and results. """ statement = select(User) @@ -206,8 +203,12 @@ def info( statement = statement.order_by(User.number_of_schemas.desc()) with Session(self._sa_engine) as session: - results = session.scalars(statement.limit(page_size).offset(page_size * page)) - total_number_of_namespaces = session.execute(select(func.count(User.id))).one()[0] + results = session.scalars( + statement.limit(page_size).offset(page_size * page) + ) + total_number_of_namespaces = session.execute( + select(func.count(User.id)) + ).one()[0] list_of_results = [] for result in results: @@ -229,11 +230,14 @@ def info( ) def stats(self, namespace: str = None, monthly: bool = False) -> NamespaceStats: - """ - Get statistics for project in the namespace or for all projects in the database. + """Get submission/update statistics for a namespace or the whole database. - :param namespace: namespace name [Default: None (all projects)] - :param monthly: if True, get statistics for the last 3 years monthly, else for the last 3 months daily. + Args: + namespace: Namespace to filter by (default: all namespaces). + monthly: Return monthly stats for 3 years if True, daily stats for 3 months if False. + + Returns: + NamespaceStats with projects_updated and projects_created histograms. """ if monthly: number_of_month = 12 * 3 @@ -248,15 +252,21 @@ def stats(self, namespace: str = None, monthly: bool = False) -> NamespaceStats: Projects.submission_date.between(three_month_ago, today_date) ) if namespace: - statement_last_update = statement_last_update.where(Projects.namespace == namespace) - statement_create_date = statement_create_date.where(Projects.namespace == namespace) + statement_last_update = statement_last_update.where( + Projects.namespace == namespace + ) + statement_create_date = statement_create_date.where( + Projects.namespace == namespace + ) with Session(self._sa_engine) as session: update_results = session.execute(statement_last_update).all() create_results = session.execute(statement_create_date).all() if not update_results: - raise NamespaceNotFoundError(f"Namespace {namespace} not found in the database") + raise NamespaceNotFoundError( + f"Namespace {namespace} not found in the database" + ) if monthly: year_month_str_submission = [ @@ -283,11 +293,10 @@ def stats(self, namespace: str = None, monthly: bool = False) -> NamespaceStats: ) def upload_tar_info(self, tar_info: TarNamespaceModel) -> None: - """ - Upload metadata of tar GEO files + """Upload metadata for a namespace tar archive. - tar_info: TarNamespaceModel - :return: None + Args: + tar_info: Tar archive metadata. """ with Session(self._sa_engine) as session: @@ -304,12 +313,13 @@ def upload_tar_info(self, tar_info: TarNamespaceModel) -> None: _LOGGER.info("Geo tar info was uploaded successfully!") def get_tar_info(self, namespace: str) -> TarNamespaceModelReturn: - """ - Get metadata of tar GEO files + """Get metadata for namespace tar archives. - :param namespace: namespace of the tar files + Args: + namespace: Namespace of the tar files. - :return: list with geo data + Returns: + TarNamespaceModelReturn with count and list of archive metadata. """ with Session(self._sa_engine) as session: @@ -335,19 +345,18 @@ def get_tar_info(self, namespace: str) -> TarNamespaceModelReturn: return TarNamespaceModelReturn(count=len(results), results=results) def delete_tar_info(self, namespace: str = None) -> None: - """ - Delete all metadata of tar GEO files + """Delete tar archive metadata for a namespace. - :param namespace: namespace of the tar files - - :return: None + Args: + namespace: Namespace to delete archives for (default: all namespaces). """ with Session(self._sa_engine) as session: - delete_statement = delete(TarNamespace) if namespace: - delete_statement = delete_statement.where(TarNamespace.namespace == namespace) + delete_statement = delete_statement.where( + TarNamespace.namespace == namespace + ) session.execute(delete_statement) session.commit() _LOGGER.info("Geo tar info was deleted successfully!") diff --git a/pepdbagent/modules/project.py b/pepdbagent/modules/project.py index bf87e78..54ccaae 100644 --- a/pepdbagent/modules/project.py +++ b/pepdbagent/modules/project.py @@ -1,16 +1,12 @@ import datetime -import json import logging -from typing import Dict, List, NoReturn, Union import numpy as np -import peppy -from peppy.const import ( +import peprs +from peprs.const import ( CONFIG_KEY, - SAMPLE_NAME_ATTR, SAMPLE_RAW_DICT_KEY, - SAMPLE_TABLE_INDEX_KEY, - SUBSAMPLE_RAW_LIST_KEY, + SUBSAMPLE_RAW_DICT_KEY, ) from sqlalchemy import Select, and_, delete, select from sqlalchemy.exc import IntegrityError, NoResultFound @@ -20,11 +16,13 @@ from pepdbagent.const import ( DEFAULT_TAG, DESCRIPTION_KEY, + LATEST_SCHEMA_VERSION, MAX_HISTORY_SAMPLES_NUMBER, NAME_KEY, PEPHUB_SAMPLE_ID_KEY, PKG_NAME, - LATEST_SCHEMA_VERSION, + SAMPLE_NAME_ATTR, + SAMPLE_TABLE_INDEX_KEY, ) from pepdbagent.db_utils import ( BaseEngine, @@ -74,7 +72,8 @@ class PEPDatabaseProject: def __init__(self, pep_db_engine: BaseEngine): """ - :param pep_db_engine: pepdbengine object with sa engine + Args: + pep_db_engine: PEPDatabaseAgent engine object. """ self._sa_engine = pep_db_engine.engine self._pep_db_engine = pep_db_engine @@ -86,23 +85,21 @@ def get( tag: str = DEFAULT_TAG, raw: bool = True, with_id: bool = False, - ) -> Union[peppy.Project, dict, None]: - """ - Retrieve project from database by specifying namespace, name and tag - - :param namespace: namespace of the project - :param name: name of the project (Default: name is taken from the project object) - :param tag: tag (or version) of the project. - :param raw: retrieve unprocessed (raw) PEP dict. - :param with_id: retrieve project with id [default: False] - :return: peppy.Project object with found project or dict with unprocessed - PEP elements: { - name: str - description: str - _config: dict - _sample_dict: dict - _subsample_dict: dict - } + ) -> peprs.Project | dict | None: + """Retrieve a project from the database. + + Args: + namespace: Namespace of the project. + name: Name of the project. + tag: Tag of the project. + raw: Return raw dict if True, peprs.Project object if False. + with_id: Include pephub_sample_id in each sample dict if True. + + Returns: + Raw dict or peprs.Project object. + + Raises: + ProjectNotFoundError: If the project does not exist. """ # name = name.lower() namespace = namespace.lower() @@ -121,7 +118,9 @@ def get( for subsample in found_prj.subsamples_mapping: if subsample.subsample_number not in subsample_dict.keys(): subsample_dict[subsample.subsample_number] = [] - subsample_dict[subsample.subsample_number].append(subsample.subsample) + subsample_dict[subsample.subsample_number].append( + subsample.subsample + ) subsample_list = list(subsample_dict.values()) else: subsample_list = [] @@ -133,13 +132,13 @@ def get( project_value = { CONFIG_KEY: found_prj.config, SAMPLE_RAW_DICT_KEY: sample_list, - SUBSAMPLE_RAW_LIST_KEY: subsample_list, + SUBSAMPLE_RAW_DICT_KEY: subsample_list, } if raw: return project_value else: - project_obj = peppy.Project().from_dict(project_value) + project_obj = peprs.Project.from_dict(project_value) return project_obj else: @@ -151,14 +150,16 @@ def get( except NoResultFound: raise ProjectNotFoundError - def _get_samples(self, session: Session, prj_id: int, with_id: bool) -> List[Dict]: - """ - Get samples from the project. This method is used to retrieve samples from the project, - with open session object. + def _get_samples(self, session: Session, prj_id: int, with_id: bool) -> list[dict]: + """Get ordered samples from a project using an open session. + + Args: + session: Open SQLAlchemy session. + prj_id: Project id. + with_id: Include pephub_sample_id in each sample dict if True. - :param session: open session object - :param prj_id: project id - :param with_id: retrieve sample with id + Returns: + Ordered list of sample dicts. """ result_dict = self._get_samples_dict(prj_id, session, with_id) @@ -168,24 +169,20 @@ def _get_samples(self, session: Session, prj_id: int, with_id: bool) -> List[Dic return ordered_samples_list @staticmethod - def _get_samples_dict(prj_id: int, session: Session, with_id: bool) -> Dict: - """ - Get not ordered samples from the project. This method is used to retrieve samples from the project - - :param prj_id: project id - :param session: open session object - :param with_id: retrieve sample with id - - :return: dictionary with samples: - {guid: - { - "sample": sample_dict, - "guid": guid, - "parent_guid": parent_guid - } - } + def _get_samples_dict(prj_id: int, session: Session, with_id: bool) -> dict: + """Get unordered samples from a project keyed by guid. + + Args: + prj_id: Project id. + session: Open SQLAlchemy session. + with_id: Include pephub_sample_id in each sample dict if True. + + Returns: + Dict mapping guid → {sample, guid, parent_guid}. """ - samples_results = session.scalars(select(Samples).where(Samples.project_id == prj_id)) + samples_results = session.scalars( + select(Samples).where(Samples.project_id == prj_id) + ) result_dict = {} for sample in samples_results: sample_dict = sample.sample @@ -201,14 +198,18 @@ def _get_samples_dict(prj_id: int, session: Session, with_id: bool) -> Dict: return result_dict @staticmethod - def _create_select_statement(name: str, namespace: str, tag: str = DEFAULT_TAG) -> Select: - """ - Create simple select statement for retrieving project from database - - :param name: name of the project - :param namespace: namespace of the project - :param tag: tag of the project - :return: select statement + def _create_select_statement( + name: str, namespace: str, tag: str = DEFAULT_TAG + ) -> Select: + """Create a SELECT statement for a single project. + + Args: + name: Project name. + namespace: Project namespace. + tag: Project tag. + + Returns: + SQLAlchemy SELECT statement for the Projects table. """ statement = select(Projects) statement = statement.where( @@ -224,20 +225,18 @@ def get_by_rp( self, registry_path: str, raw: bool = False, - ) -> Union[peppy.Project, dict, None]: - """ - Retrieve project from database by specifying project registry_path - - :param registry_path: project registry_path [e.g. namespace/name:tag] - :param raw: retrieve unprocessed (raw) PEP dict. - :return: peppy.Project object with found project or dict with unprocessed - PEP elements: { - name: str - description: str - _config: dict - _sample_dict: dict - _subsample_dict: dict - } + ) -> peprs.Project | dict | None: + """Retrieve a project from the database by registry path. + + Args: + registry_path: Registry path, e.g., "namespace/name:tag". + raw: Return raw dict if True, peprs.Project object if False. + + Returns: + Raw dict or peprs.Project object. + + Raises: + ProjectNotFoundError: If the project does not exist. """ namespace, name, tag = registry_path_converter(registry_path) return self.get(namespace=namespace, name=name, tag=tag, raw=raw) @@ -248,13 +247,15 @@ def delete( name: str = None, tag: str = None, ) -> None: - """ - Delete record from database + """Delete a project from the database. + + Args: + namespace: Project namespace. + name: Project name. + tag: Project tag. - :param namespace: Namespace - :param name: Name - :param tag: Tag - :return: None + Raises: + ProjectNotFoundError: If the project does not exist. """ # name = name.lower() namespace = namespace.lower() @@ -285,18 +286,20 @@ def delete_by_rp( self, registry_path: str, ) -> None: - """ - Delete record from database by using registry_path + """Delete a project from the database by registry path. + + Args: + registry_path: Registry path of the project, e.g., "namespace/name:tag". - :param registry_path: Registry path of the project ('namespace/name:tag') - :return: None + Raises: + ProjectNotFoundError: If the project does not exist. """ namespace, name, tag = registry_path_converter(registry_path) return self.delete(namespace=namespace, name=name, tag=tag) def create( self, - project: Union[peppy.Project, dict], + project: peprs.Project | dict, namespace: str, name: str = None, tag: str = DEFAULT_TAG, @@ -307,33 +310,26 @@ def create( overwrite: bool = False, update_only: bool = False, ) -> None: + """Upload a project to the database. + + Args: + project: peprs.Project or dict with config, samples, and subsamples keys. + namespace: Namespace to upload to. + name: Project name (default: taken from project config). + tag: Project tag. + description: Project description. + is_private: Mark project as private if True. + pop: Mark as a PEP-of-PEPs (POP) if True. + pep_schema: Schema to associate, e.g., "namespace/name". + overwrite: Overwrite the project if it already exists. + update_only: Update the project if it exists; do nothing otherwise. + + Raises: + ProjectUniqueNameError: If the project already exists and overwrite is False. + SchemaDoesNotExistError: If pep_schema is provided but does not exist. """ - Upload project to the database. - Project with the key, that already exists won't be uploaded(but case, when argument - update is set True) - - :param peppy.Project project: Project object that has to be uploaded to the DB - danger zone: - optionally, project can be a dictionary with PEP elements - ({ - _config: dict, - _sample_dict: Union[list, dict], - _subsample_list: list - }) - :param namespace: namespace of the project (Default: 'other') - :param name: name of the project (Default: name is taken from the project object) - :param tag: tag (or version) of the project. - :param is_private: boolean value if the project should be visible just for user that creates it. - :param pep_schema: assign PEP to a specific schema. Example: 'namespace/name' [Default: None] - :param pop: if project is a pep of peps (POP) [Default: False] - :param overwrite: if project exists overwrite the project, otherwise upload it. - [Default: False - project won't be overwritten if it exists in db] - :param update_only: if project exists overwrite it, otherwise do nothing. [Default: False] - :param description: description of the project - :return: None - """ - if isinstance(project, peppy.Project): - proj_dict = project.to_dict(extended=True, orient="records") + if isinstance(project, peprs.Project): + proj_dict = project.to_dict(raw=True, by_sample=True) elif isinstance(project, dict): # verify if the dictionary has all necessary elements. # samples should be always presented as list of dicts (orient="records")) @@ -343,11 +339,14 @@ def create( proj_dict = ProjectDict(**project).model_dump(by_alias=True) else: raise PEPDatabaseAgentError( - "Project has to be peppy.Project object or dictionary with PEP elements" + "Project has to be peprs.Project object or dictionary with PEP elements" ) if not description: - description = project.get(description, "") + if isinstance(project, peprs.Project): + description = project.description or "" + else: + description = proj_dict.get(CONFIG_KEY, {}).get(DESCRIPTION_KEY, "") proj_dict[CONFIG_KEY][DESCRIPTION_KEY] = description namespace = namespace.lower() @@ -356,7 +355,9 @@ def create( elif proj_dict[CONFIG_KEY][NAME_KEY]: proj_name = proj_dict[CONFIG_KEY][NAME_KEY].lower() else: - raise ValueError("Name of the project wasn't provided. Project will not be uploaded.") + raise ValueError( + "Name of the project wasn't provided. Project will not be uploaded." + ) proj_dict[CONFIG_KEY][NAME_KEY] = proj_name @@ -367,13 +368,16 @@ def create( number_of_samples = len(proj_dict[SAMPLE_RAW_DICT_KEY]) if pep_schema: - schema_namespace, schema_name, schema_version = schema_path_converter(pep_schema) + schema_namespace, schema_name, schema_version = schema_path_converter( + pep_schema + ) with Session(self._sa_engine) as session: - if schema_version == LATEST_SCHEMA_VERSION: schema_mapping = session.scalar( select(SchemaVersions) - .join(SchemaRecords, SchemaRecords.id == SchemaVersions.schema_id) + .join( + SchemaRecords, SchemaRecords.id == SchemaVersions.schema_id + ) .where( and_( SchemaRecords.namespace == schema_namespace, @@ -401,7 +405,9 @@ def create( pep_schema = schema_mapping.id if update_only: - _LOGGER.info(f"Update_only argument is set True. Updating project {proj_name} ...") + _LOGGER.info( + f"Update_only argument is set True. Updating project {proj_name} ..." + ) self._overwrite( project_dict=proj_dict, namespace=namespace, @@ -442,12 +448,14 @@ def create( ), ) - if proj_dict[SUBSAMPLE_RAW_LIST_KEY]: - subsamples = proj_dict[SUBSAMPLE_RAW_LIST_KEY] + if proj_dict.get(SUBSAMPLE_RAW_DICT_KEY): + subsamples = proj_dict[SUBSAMPLE_RAW_DICT_KEY] self._add_subsamples_to_project(new_prj, subsamples) with Session(self._sa_engine) as session: - user = session.scalar(select(User).where(User.namespace == namespace)) + user = session.scalar( + select(User).where(User.namespace == namespace) + ) if not user: user = User(namespace=namespace) @@ -485,7 +493,7 @@ def create( def _overwrite( self, - project_dict: json, + project_dict: dict, namespace: str, proj_name: str, tag: str, @@ -496,20 +504,22 @@ def _overwrite( description: str = "", pop: bool = False, ) -> None: - """ - Update existing project by providing all necessary information. - - :param project_dict: project dictionary in json format - :param namespace: project namespace - :param proj_name: project name - :param tag: project tag - :param project_digest: project digest - :param number_of_samples: number of samples in project - :param private: boolean value if the project should be visible just for user that creates it. - :param pep_schema: assign PEP to a specific schema. [DefaultL: None] - :param description: project description - :param pop: if project is a pep of peps, simply POP [Default: False] - :return: None + """Overwrite an existing project with new content. + + Args: + project_dict: Full project dict (config, samples, subsamples). + namespace: Project namespace. + proj_name: Project name. + tag: Project tag. + project_digest: Pre-computed digest. + number_of_samples: Sample count. + private: Mark as private if True. + pep_schema: Schema id to associate (integer FK). + description: Project description. + pop: Mark as POP if True. + + Raises: + ProjectNotFoundError: If the project does not exist. """ proj_name = proj_name.lower() namespace = namespace.lower() @@ -532,7 +542,9 @@ def _overwrite( found_prj.schema_id = pep_schema found_prj.config = project_dict[CONFIG_KEY] found_prj.description = description - found_prj.last_update_date = datetime.datetime.now(datetime.timezone.utc) + found_prj.last_update_date = datetime.datetime.now( + datetime.timezone.utc + ) found_prj.pop = pop # Deleting old samples and subsamples @@ -550,52 +562,48 @@ def _overwrite( self._add_samples_to_project( found_prj, project_dict[SAMPLE_RAW_DICT_KEY], - sample_table_index=project_dict[CONFIG_KEY].get(SAMPLE_TABLE_INDEX_KEY), + sample_table_index=project_dict[CONFIG_KEY].get( + SAMPLE_TABLE_INDEX_KEY + ), ) - if project_dict[SUBSAMPLE_RAW_LIST_KEY]: + if project_dict.get(SUBSAMPLE_RAW_DICT_KEY): self._add_subsamples_to_project( - found_prj, project_dict[SUBSAMPLE_RAW_LIST_KEY] + found_prj, project_dict[SUBSAMPLE_RAW_DICT_KEY] ) session.commit() - _LOGGER.info(f"Project '{namespace}/{proj_name}:{tag}' has been successfully updated!") + _LOGGER.info( + f"Project '{namespace}/{proj_name}:{tag}' has been successfully updated!" + ) return None else: - raise ProjectNotFoundError("Project does not exist! No project will be updated!") + raise ProjectNotFoundError( + "Project does not exist! No project will be updated!" + ) def update( self, - update_dict: Union[dict, UpdateItems], + update_dict: dict | UpdateItems, namespace: str, name: str, tag: str = DEFAULT_TAG, user: str = None, ) -> None: - """ - Update partial parts of the record in db - - :param update_dict: dict with update key->values. Dict structure: - { - project: Optional[peppy.Project] - is_private: Optional[bool] - tag: Optional[str] - name: Optional[str] - description: Optional[str] - is_private: Optional[bool] - pep_schema: Optional[str] - config: Optional[dict] - samples: Optional[List[dict]] - subsamples: Optional[List[List[dict]]] - pop: Optional[bool] - } - :param namespace: project namespace - :param name: project name - :param tag: project tag - :param user: user that updates the project if user is not provided, user will be set as Namespace - :return: None + """Update specific fields of a project. + + Args: + update_dict: Fields to update — any combination of project, is_private, tag, name, + description, pep_schema, config, samples, subsamples, pop. + namespace: Project namespace. + name: Project name. + tag: Project tag. + user: User performing the update (default: namespace). + + Raises: + ProjectNotFoundError: If the project does not exist. """ if self.exists(namespace=namespace, name=name, tag=tag): if isinstance(update_dict, UpdateItems): @@ -603,11 +611,13 @@ def update( else: if "project" in update_dict: project_dict = update_dict.pop("project").to_dict( - extended=True, orient="records" + raw=True, by_sample=True ) update_dict["config"] = project_dict[CONFIG_KEY] update_dict["samples"] = project_dict[SAMPLE_RAW_DICT_KEY] - update_dict["subsamples"] = project_dict[SUBSAMPLE_RAW_LIST_KEY] + update_dict["subsamples"] = project_dict.get( + SUBSAMPLE_RAW_DICT_KEY, [] + ) update_values = UpdateItems(**update_dict) @@ -647,7 +657,6 @@ def update( flag_modified(found_prj, "config") if "samples" in update_dict: - if PEPHUB_SAMPLE_ID_KEY not in update_dict["samples"][0]: raise SampleTableUpdateError( f"pephub_sample_id '{PEPHUB_SAMPLE_ID_KEY}' is missing in samples." @@ -685,9 +694,13 @@ def update( # Adding new subsamples if update_dict["subsamples"]: - self._add_subsamples_to_project(found_prj, update_dict["subsamples"]) + self._add_subsamples_to_project( + found_prj, update_dict["subsamples"] + ) - found_prj.last_update_date = datetime.datetime.now(datetime.timezone.utc) + found_prj.last_update_date = datetime.datetime.now( + datetime.timezone.utc + ) session.commit() @@ -697,15 +710,15 @@ def update( raise ProjectNotFoundError("No items will be updated!") @staticmethod - def _convert_update_schema_id(session: Session, update_values: dict): - """ - Convert schema path to schema_id in update_values and update it in update dict + def _convert_update_schema_id(session: Session, update_values: dict) -> None: + """Resolve pep_schema path to a schema_id in the update dict. + Args: + session: Open SQLAlchemy session. + update_values: Update dict that may contain a "pep_schema" key. - :param session: open session object - :param update_values: dict with update key->values - - return None + Raises: + SchemaDoesNotExistError: If the schema path does not resolve to an existing schema. """ if "pep_schema" in update_values: schema_namespace, schema_name, schema_version = schema_path_converter( @@ -745,30 +758,36 @@ def _convert_update_schema_id(session: Session, update_values: dict): def _update_samples( self, project_id: int, - samples_list: List[Dict[str, str]], + samples_list: list[dict[str, str]], sample_name_key: str = "sample_name", - history_sa_model: Union[HistoryProjects, None] = None, + history_sa_model: HistoryProjects | None = None, ) -> None: - """ - Update samples in the project - This is linked list method, that first finds differences in old and new samples list - and then updates, adds, inserts, deletes, or changes the order. - - :param project_id: project id in PEPhub database - :param samples_list: list of samples to be updated - :param sample_name_key: key of the sample name - :param history_sa_model: HistoryProjects object, to write to the history table - :return: None + """Update samples in a project using a linked-list diff approach. + + Finds differences between old and new sample lists, then applies inserts, updates, + deletes, and order changes. + + Args: + project_id: Project id in the database. + samples_list: New list of sample dicts (must include pephub_sample_id). + sample_name_key: Sample table index key. + history_sa_model: HistoryProjects row to append change records to. + + Raises: + ProjectDuplicatedSampleGUIDsError: If any pephub_sample_id is duplicated. """ with Session(self._sa_engine) as session: - old_samples = session.scalars(select(Samples).where(Samples.project_id == project_id)) + old_samples = session.scalars( + select(Samples).where(Samples.project_id == project_id) + ) old_samples_mapping: dict = {sample.guid: sample for sample in old_samples} # old_child_parent_id needed because of the parent_guid is sometimes set to none in sqlalchemy mapping :( bug - old_child_parent_id: Dict[str, str] = { - child: mapping.parent_guid for child, mapping in old_samples_mapping.items() + old_child_parent_id: dict[str, str] = { + child: mapping.parent_guid + for child, mapping in old_samples_mapping.items() } old_samples_ids_set: set = set(old_samples_mapping.keys()) @@ -796,7 +815,6 @@ def _update_samples( del new_samples_ids_list, new_samples_ids_set for remove_id in deleted_ids: - if history_sa_model: history_sa_model.sample_changes_mapping.append( HistorySamples( @@ -839,7 +857,6 @@ def _update_samples( else: current_history = None if old_samples_mapping[current_id].sample != sample_value: - if history_sa_model: current_history = HistorySamples( guid=old_samples_mapping[current_id].guid, @@ -849,7 +866,9 @@ def _update_samples( ) old_samples_mapping[current_id].sample = sample_value - old_samples_mapping[current_id].sample_name = sample_value[sample_name_key] + old_samples_mapping[current_id].sample_name = sample_value[ + sample_name_key + ] # !bug workaround: if project was deleted and sometimes old_samples_mapping[current_id].parent_guid # and it can cause an error in history. For this we have `old_child_parent_id` dict @@ -876,13 +895,13 @@ def _update_samples( @staticmethod def __create_update_dict(update_values: UpdateItems) -> dict: - """ - Modify keys and values that set for update and create unified - dictionary of the values that have to be updated + """Build a normalised update dict from an UpdateItems model. + + Args: + update_values: UpdateItems with fields to apply. - :param update_values: UpdateItems (pydantic class) with - updating values - :return: unified update dict + Returns: + Dict of field→value pairs ready for setattr on the Projects row. """ update_final = UpdateModel.model_construct() @@ -903,7 +922,8 @@ def __create_update_dict(update_values: UpdateItems) -> dict: ) if update_values.config is not None: update_final = UpdateModel( - config=update_values.config, **update_final.model_dump(exclude_unset=True) + config=update_values.config, + **update_final.model_dump(exclude_unset=True), ) name = update_values.config.get(NAME_KEY) description = update_values.config.get(DESCRIPTION_KEY) @@ -915,7 +935,9 @@ def __create_update_dict(update_values: UpdateItems) -> dict: if description: update_final = UpdateModel( description=description, - **update_final.model_dump(exclude_unset=True, exclude={DESCRIPTION_KEY}), + **update_final.model_dump( + exclude_unset=True, exclude={DESCRIPTION_KEY} + ), ) if update_values.tag is not None: @@ -954,12 +976,15 @@ def exists( name: str, tag: str = DEFAULT_TAG, ) -> bool: - """ - Check if project exists in the database. - :param namespace: project namespace - :param name: project name - :param tag: project tag - :return: Returning True if project exist + """Check if a project exists in the database. + + Args: + namespace: Project namespace. + name: Project name. + tag: Project tag. + + Returns: + True if the project exists. """ statement = select(Projects.id) @@ -979,18 +1004,19 @@ def exists( @staticmethod def _add_samples_to_project( - projects_sa: Projects, samples: List[dict], sample_table_index: str = "sample_name" + projects_sa: Projects, + samples: list[dict], + sample_table_index: str = "sample_name", ) -> None: - """ - Add samples to the project sa object. (With commit this samples will be added to the 'samples table') - :param projects_sa: Projects sa object, in open session - :param samples: list of samles to be added to the database - :param sample_table_index: index of the sample table - :return: NoReturn + """Append sample rows to a Projects ORM object. + + Args: + projects_sa: Projects ORM object in an open session. + samples: List of sample dicts to add. + sample_table_index: Column name to use as the sample identifier. """ previous_sample_guid = None for sample in samples: - sample = Samples( sample=sample, sample_name=sample.get(sample_table_index), @@ -1004,32 +1030,39 @@ def _add_samples_to_project( @staticmethod def _add_subsamples_to_project( - projects_sa: Projects, subsamples: List[List[dict]] - ) -> NoReturn: - """ - Add subsamples to the project sa object. (With commit this samples will be added to the 'subsamples table') + projects_sa: Projects, subsamples: list[list[dict]] + ) -> None: + """Append subsample rows to a Projects ORM object. - :param projects_sa: Projects sa object, in open session - :param subsamples: list of subsamles to be added to the database - :return: NoReturn + Args: + projects_sa: Projects ORM object in an open session. + subsamples: List of subsample groups (list of list of dicts). """ for i, subs in enumerate(subsamples): for row_number, sub_item in enumerate(subs): projects_sa.subsamples_mapping.append( - Subsamples(subsample=sub_item, subsample_number=i, row_number=row_number) + Subsamples( + subsample=sub_item, subsample_number=i, row_number=row_number + ) ) - def get_project_id(self, namespace: str, name: str, tag: str) -> Union[int, None]: - """ - Get Project id by providing namespace, name, and tag + def get_project_id(self, namespace: str, name: str, tag: str) -> int | None: + """Get the database id of a project. + + Args: + namespace: Project namespace. + name: Project name. + tag: Project tag. - :param namespace: project namespace - :param name: project name - :param tag: project tag - :return: projects id + Returns: + Project id, or None if the project does not exist. """ statement = select(Projects.id).where( - and_(Projects.namespace == namespace, Projects.name == name, Projects.tag == tag) + and_( + Projects.namespace == namespace, + Projects.name == name, + Projects.tag == tag, + ) ) with Session(self._sa_engine) as session: result = session.execute(statement).one_or_none() @@ -1049,18 +1082,20 @@ def fork( description: str = None, private: bool = False, ) -> None: - """ - Fork project from one namespace to another - - :param original_namespace: namespace of the project to be forked - :param original_name: name of the project to be forked - :param original_tag: tag of the project to be forked - :param fork_namespace: namespace of the forked project - :param fork_name: name of the forked project - :param fork_tag: tag of the forked project - :param description: description of the forked project - :param private: boolean value if the project should be visible just for user that creates it. - :return: None + """Fork a project into a new namespace. + + Args: + original_namespace: Namespace of the source project. + original_name: Name of the source project. + original_tag: Tag of the source project. + fork_namespace: Target namespace. + fork_name: Name for the fork (default: same as original). + fork_tag: Tag for the fork (default: same as original). + description: Description for the fork. + private: Mark fork as private if True. + + Raises: + ProjectNotFoundError: If the source project does not exist. """ self.create( @@ -1103,17 +1138,23 @@ def fork( session.commit() - def get_config(self, namespace: str, name: str, tag: str) -> Union[dict, None]: - """ - Get project configuration by providing namespace, name, and tag + def get_config(self, namespace: str, name: str, tag: str) -> dict | None: + """Get the config dict for a project. - :param namespace: project namespace - :param name: project name - :param tag: project tag - :return: project configuration + Args: + namespace: Project namespace. + name: Project name. + tag: Project tag. + + Returns: + Config dict, or None if the project does not exist. """ statement = select(Projects.config).where( - and_(Projects.namespace == namespace, Projects.name == name, Projects.tag == tag) + and_( + Projects.namespace == namespace, + Projects.name == name, + Projects.tag == tag, + ) ) with Session(self._sa_engine) as session: result = session.execute(statement).one_or_none() @@ -1122,29 +1163,37 @@ def get_config(self, namespace: str, name: str, tag: str) -> Union[dict, None]: return result[0] return None - def get_subsamples(self, namespace: str, name: str, tag: str) -> Union[list, None]: - """ - Get project subsamples by providing namespace, name, and tag + def get_subsamples(self, namespace: str, name: str, tag: str) -> list | None: + """Get the subsample list for a project. - :param namespace: project namespace - :param name: project name - :param tag: project tag - :return: list with project subsamples + Args: + namespace: Project namespace. + name: Project name. + tag: Project tag. + + Returns: + List of subsample groups, or an empty list if there are no subsamples. + + Raises: + ProjectNotFoundError: If the project does not exist. """ statement = self._create_select_statement(name, namespace, tag) with Session(self._sa_engine) as session: - found_prj = session.scalar(statement) if found_prj: - _LOGGER.info(f"Project has been found: {found_prj.namespace}, {found_prj.name}") + _LOGGER.info( + f"Project has been found: {found_prj.namespace}, {found_prj.name}" + ) subsample_dict = {} if found_prj.subsamples_mapping: for subsample in found_prj.subsamples_mapping: if subsample.subsample_number not in subsample_dict.keys(): subsample_dict[subsample.subsample_number] = [] - subsample_dict[subsample.subsample_number].append(subsample.subsample) + subsample_dict[subsample.subsample_number].append( + subsample.subsample + ) return list(subsample_dict.values()) else: return [] @@ -1155,38 +1204,50 @@ def get_subsamples(self, namespace: str, name: str, tag: str) -> Union[list, Non ) def get_samples( - self, namespace: str, name: str, tag: str, raw: bool = True, with_ids: bool = False + self, + namespace: str, + name: str, + tag: str, + raw: bool = True, + with_ids: bool = False, ) -> list: - """ - Get project samples by providing namespace, name, and tag + """Get samples for a project. - :param namespace: project namespace - :param name: project name - :param tag: project tag - :param raw: if True, retrieve unprocessed (raw) PEP dict. [Default: True] - :param with_ids: if True, retrieve samples with ids. [Default: False] + Args: + namespace: Project namespace. + name: Project name. + tag: Project tag. + raw: Return raw dicts if True, records-orient pandas dicts if False. + with_ids: Include pephub_sample_id in each sample if True. - :return: list with project samples + Returns: + List of sample dicts. """ if raw: return self.get( namespace=namespace, name=name, tag=tag, raw=True, with_id=with_ids ).get(SAMPLE_RAW_DICT_KEY) return ( - self.get(namespace=namespace, name=name, tag=tag, raw=False, with_id=with_ids) - .sample_table.replace({np.nan: None}) + self.get( + namespace=namespace, name=name, tag=tag, raw=False, with_id=with_ids + ) + .to_pandas() + .replace({np.nan: None}) .to_dict(orient="records") ) - def get_history(self, namespace: str, name: str, tag: str) -> HistoryAnnotationModel: - """ - Get project history annotation by providing namespace, name, and tag + def get_history( + self, namespace: str, name: str, tag: str + ) -> HistoryAnnotationModel: + """Get the list of history entries for a project. - :param namespace: project namespace - :param name: project name - :param tag: project tag + Args: + namespace: Project namespace. + name: Project name. + tag: Project tag. - :return: project history annotation + Returns: + HistoryAnnotationModel with a list of HistoryChangeModel entries. """ with Session(self._sa_engine) as session: @@ -1207,7 +1268,7 @@ def get_history(self, namespace: str, name: str, tag: str) -> HistoryAnnotationM .order_by(HistoryProjects.update_time.desc()) ) results = session.scalars(statement) - return_results: List = [] + return_results: list = [] if results: for result in results: @@ -1233,18 +1294,23 @@ def get_project_from_history( history_id: int, raw: bool = True, with_id: bool = False, - ) -> Union[dict, peppy.Project]: - """ - Get project sample history annotation by providing namespace, name, and tag - - :param namespace: project namespace - :param name: project name - :param tag: project tag - :param history_id: history id - :param raw: if True, retrieve unprocessed (raw) PEP dict. [Default: True] - :param with_id: if True, retrieve samples with ids. [Default: False] - - :return: project sample history annotation + ) -> dict | peprs.Project: + """Reconstruct a project at a past history checkpoint. + + Args: + namespace: Project namespace. + name: Project name. + tag: Project tag. + history_id: Id of the history entry to restore to. + raw: Return raw dict if True, peprs.Project object if False. + with_id: Include pephub_sample_id in each sample if True. + + Returns: + Raw dict or peprs.Project at the requested history state. + + Raises: + ProjectNotFoundError: If the project does not exist. + HistoryNotFoundError: If the history entry does not exist. """ with Session(self._sa_engine) as session: @@ -1319,31 +1385,35 @@ def get_project_from_history( return { CONFIG_KEY: project_config or project_mapping.config, SAMPLE_RAW_DICT_KEY: ordered_samples_list, - SUBSAMPLE_RAW_LIST_KEY: self.get_subsamples(namespace, name, tag), + SUBSAMPLE_RAW_DICT_KEY: self.get_subsamples(namespace, name, tag), } - return peppy.Project.from_dict( + return peprs.Project.from_dict( pep_dictionary={ CONFIG_KEY: project_config or project_mapping.config, SAMPLE_RAW_DICT_KEY: ordered_samples_list, - SUBSAMPLE_RAW_LIST_KEY: self.get_subsamples(namespace, name, tag), + SUBSAMPLE_RAW_DICT_KEY: self.get_subsamples(namespace, name, tag), } ) @staticmethod def _apply_history_changes(sample_dict: dict, change: HistoryProjects) -> dict: - """ - Apply changes from the history to the sample list + """Apply a history change record to a sample dict. + + Args: + sample_dict: Current sample dict keyed by guid. + change: HistoryProjects entry whose sample_changes_mapping to apply. - :param sample_dict: dictionary with samples - :param change: history change - :return: updated sample list + Returns: + Updated sample dict. """ for sample_change in change.sample_changes_mapping: sample_id = sample_change.guid if sample_change.change_type == UpdateTypes.UPDATE: sample_dict[sample_id]["sample"] = sample_change.sample_json - sample_dict[sample_id]["sample"][PEPHUB_SAMPLE_ID_KEY] = sample_change.guid + sample_dict[sample_id]["sample"][PEPHUB_SAMPLE_ID_KEY] = ( + sample_change.guid + ) sample_dict[sample_id]["parent_guid"] = sample_change.parent_guid elif sample_change.change_type == UpdateTypes.DELETE: @@ -1359,17 +1429,19 @@ def _apply_history_changes(sample_dict: dict, change: HistoryProjects) -> dict: return sample_dict def delete_history( - self, namespace: str, name: str, tag: str, history_id: Union[int, None] = None + self, namespace: str, name: str, tag: str, history_id: int | None = None ) -> None: - """ - Delete history from the project + """Delete one or all history entries for a project. - :param namespace: project namespace - :param name: project name - :param tag: project tag - :param history_id: history id. If none is provided, all history will be deleted + Args: + namespace: Project namespace. + name: Project name. + tag: Project tag. + history_id: Id of the entry to delete; deletes all history if None. - :return: None + Raises: + ProjectNotFoundError: If the project does not exist. + HistoryNotFoundError: If the specified history entry does not exist. """ with Session(self._sa_engine) as session: project_mapping = session.scalar( @@ -1389,7 +1461,9 @@ def delete_history( if history_id is None: session.execute( - delete(HistoryProjects).where(HistoryProjects.project_id == project_mapping.id) + delete(HistoryProjects).where( + HistoryProjects.project_id == project_mapping.id + ) ) session.commit() return None @@ -1419,16 +1493,14 @@ def restore( history_id: int, user: str = None, ) -> None: - """ - Restore project to the specific history state - - :param namespace: project namespace - :param name: project name - :param tag: project tag - :param history_id: history id - :param user: user that restores the project if user is not provided, user will be set as Namespace - - :return: None + """Restore a project to a past history state. + + Args: + namespace: Project namespace. + name: Project name. + tag: Project tag. + history_id: Id of the history entry to restore to. + user: User performing the restore (default: namespace). """ restore_project = self.get_project_from_history( @@ -1440,7 +1512,7 @@ def restore( with_id=True, ) self.update( - update_dict={"project": peppy.Project.from_dict(restore_project)}, + update_dict={"project": peprs.Project.from_dict(restore_project)}, namespace=namespace, name=name, tag=tag, @@ -1448,11 +1520,10 @@ def restore( ) def clean_history(self, days: int = 90) -> None: - """ - Delete all history data that is older than 3 month, or specific number of days + """Delete history entries older than a given number of days. - :param days: number of days to keep history data - :return: None + Args: + days: Retain history newer than this many days (default: 90). """ with Session(self._sa_engine) as session: diff --git a/pepdbagent/modules/sample.py b/pepdbagent/modules/sample.py index 90e216b..cc5afe4 100644 --- a/pepdbagent/modules/sample.py +++ b/pepdbagent/modules/sample.py @@ -1,14 +1,12 @@ import datetime import logging -from typing import Union -import peppy -from peppy.const import SAMPLE_TABLE_INDEX_KEY +import peprs from sqlalchemy import and_, select from sqlalchemy.orm import Session from sqlalchemy.orm.attributes import flag_modified -from pepdbagent.const import DEFAULT_TAG, PKG_NAME +from pepdbagent.const import DEFAULT_TAG, PKG_NAME, SAMPLE_TABLE_INDEX_KEY from pepdbagent.db_utils import BaseEngine, Projects, Samples from pepdbagent.exceptions import SampleAlreadyExistsError, SampleNotFoundError from pepdbagent.utils import generate_guid, order_samples @@ -25,7 +23,8 @@ class PEPDatabaseSample: def __init__(self, pep_db_engine: BaseEngine): """ - :param pep_db_engine: pepdbengine object with sa engine + Args: + pep_db_engine: PEPDatabaseAgent engine object. """ self._sa_engine = pep_db_engine.engine self._pep_db_engine = pep_db_engine @@ -37,23 +36,21 @@ def get( sample_name: str, tag: str = DEFAULT_TAG, raw: bool = True, - ) -> Union[peppy.Sample, dict, None]: - """ - Retrieve sample from the database using namespace, name, tag, and sample_name - - :param namespace: namespace of the project - :param name: name of the project (Default: name is taken from the project object) - :param tag: tag (or version) of the project. - :param sample_name: sample_name of the sample - :param raw: return raw dict or peppy.Sample object [Default: True] - :return: peppy.Project object with found project or dict with unprocessed - PEP elements: { - name: str - description: str - _config: dict - _sample_dict: dict - _subsample_dict: dict - } + ) -> peprs.Sample | dict | None: + """Retrieve a sample from the database. + + Args: + namespace: Namespace of the project. + name: Name of the project. + sample_name: Name of the sample. + tag: Tag of the project. + raw: Return raw dict if True, peprs.Sample object if False. + + Returns: + Raw sample dict or peprs.Sample object. + + Raises: + SampleNotFoundError: If the sample does not exist. """ statement_sample = select(Samples).where( and_( @@ -83,13 +80,10 @@ def get( if result: if not raw: config = session.execute(project_config_statement).one_or_none()[0] - project = peppy.Project().from_dict( + project = peprs.Project.from_dict( pep_dictionary={ - "name": name, - "description": config.get("description"), - "_config": config, - "_sample_dict": [result.sample], - "_subsample_dict": None, + "config": config, + "samples": [result.sample], } ) return project.samples[0] @@ -109,18 +103,18 @@ def update( update_dict: dict, full_update: bool = False, ) -> None: - """ - Update one sample in the database - - :param namespace: namespace of the project - :param name: name of the project (Default: name is taken from the project object) - :param tag: tag (or version) of the project. - :param sample_name: sample_name of the sample - :param update_dict: dictionary with sample data (key: value pairs). e.g. - {"sample_name": "sample1", - "sample_protocol": "sample1 protocol"} - :param full_update: if True, update all sample fields, if False, update only fields from update_dict - :return: None + """Update a sample in the database. + + Args: + namespace: Namespace of the project. + name: Name of the project. + tag: Tag of the project. + sample_name: Name of the sample. + update_dict: Dict of fields to update, e.g., {"sample_name": "s1", "protocol": "rna"}. + full_update: Replace all sample fields if True, merge if False. + + Raises: + SampleNotFoundError: If the sample does not exist. """ statement = select(Samples).where( and_( @@ -155,17 +149,21 @@ def update( sample_mapping.sample.update(update_dict) try: sample_mapping.sample_name = sample_mapping.sample[ - project_mapping.config.get(SAMPLE_TABLE_INDEX_KEY, "sample_name") + project_mapping.config.get( + SAMPLE_TABLE_INDEX_KEY, "sample_name" + ) ] except KeyError: raise KeyError( - f"Sample index key {project_mapping.config.get(SAMPLE_TABLE_INDEX_KEY, 'sample_name')} not found in sample dict" + f"Sample index key {project_mapping.config.get('sample_table_index', 'sample_name')} not found in sample dict" ) # This line needed due to: https://github.com/sqlalchemy/sqlalchemy/issues/5218 flag_modified(sample_mapping, "sample") - project_mapping.last_update_date = datetime.datetime.now(datetime.timezone.utc) + project_mapping.last_update_date = datetime.datetime.now( + datetime.timezone.utc + ) session.commit() else: @@ -181,17 +179,17 @@ def add( sample_dict: dict, overwrite: bool = False, ) -> None: - """ - Add one sample to the project in the database - - :param namespace: namespace of the project - :param name: name of the project - :param tag: tag (or version) of the project. - :param overwrite: overwrite sample if it already exists - :param sample_dict: dictionary with sample data (key: value pairs). e.g. - {"sample_name": "sample1", - "sample_protocol": "sample1 protocol"} - :return: None + """Add a sample to a project in the database. + + Args: + namespace: Namespace of the project. + name: Name of the project. + tag: Tag of the project. + sample_dict: Sample data, e.g., {"sample_name": "s1", "protocol": "rna"}. + overwrite: Overwrite the sample if it already exists. + + Raises: + SampleAlreadyExistsError: If the sample exists and overwrite is False. """ with Session(self._sa_engine) as session: @@ -210,10 +208,13 @@ def add( ] except KeyError: raise KeyError( - f"Sample index key {project_mapping.config.get(SAMPLE_TABLE_INDEX_KEY, 'sample_name')} not found in sample dict" + f"Sample index key {project_mapping.config.get('sample_table_index', 'sample_name')} not found in sample dict" ) statement = select(Samples).where( - and_(Samples.project_id == project_mapping.id, Samples.sample_name == sample_name) + and_( + Samples.project_id == project_mapping.id, + Samples.sample_name == sample_name, + ) ) sample_mapping = session.scalar(statement) @@ -240,17 +241,21 @@ def add( parent_guid=self._get_last_sample_guid(project_mapping.id), ) project_mapping.number_of_samples += 1 - project_mapping.last_update_date = datetime.datetime.now(datetime.timezone.utc) + project_mapping.last_update_date = datetime.datetime.now( + datetime.timezone.utc + ) session.add(sample_mapping) session.commit() def _get_last_sample_guid(self, project_id: int) -> str: - """ - Get last sample guid from the project + """Get the guid of the last sample in the project chain. - :param project_id: project_id of the project - :return: guid of the last sample + Args: + project_id: Database id of the project. + + Returns: + GUID of the last sample. """ statement = select(Samples).where(Samples.project_id == project_id) with Session(self._sa_engine) as session: @@ -274,14 +279,16 @@ def delete( tag: str, sample_name: str, ) -> None: - """ - Delete one sample from the database + """Delete a sample from the database. - :param namespace: namespace of the project - :param name: name of the project - :param tag: tag (or version) of the project. - :param sample_name: sample_name of the sample - :return: None + Args: + namespace: Namespace of the project. + name: Name of the project. + tag: Tag of the project. + sample_name: Name of the sample. + + Raises: + SampleNotFoundError: If the sample does not exist. """ statement = select(Samples).where( and_( @@ -316,7 +323,9 @@ def delete( if child_mapping: child_mapping.parent_mapping = parent_mapping project_mapping.number_of_samples -= 1 - project_mapping.last_update_date = datetime.datetime.now(datetime.timezone.utc) + project_mapping.last_update_date = datetime.datetime.now( + datetime.timezone.utc + ) session.commit() else: raise SampleNotFoundError( diff --git a/pepdbagent/modules/schema.py b/pepdbagent/modules/schema.py index fe216ad..51421ee 100644 --- a/pepdbagent/modules/schema.py +++ b/pepdbagent/modules/schema.py @@ -1,27 +1,31 @@ import logging -from typing import List, Optional, Union, Dict - from sqlalchemy import Select, and_, func, or_, select from sqlalchemy.orm import Session from sqlalchemy.orm.attributes import flag_modified -from pepdbagent.const import PKG_NAME, DEFAULT_TAG_VERSION, LATEST_SCHEMA_VERSION -from pepdbagent.db_utils import BaseEngine, SchemaRecords, SchemaTags, SchemaVersions, User +from pepdbagent.const import DEFAULT_TAG_VERSION, LATEST_SCHEMA_VERSION, PKG_NAME +from pepdbagent.db_utils import ( + BaseEngine, + SchemaRecords, + SchemaTags, + SchemaVersions, + User, +) from pepdbagent.exceptions import ( SchemaAlreadyExistsError, - SchemaVersionDoesNotExistError, SchemaDoesNotExistError, SchemaTagAlreadyExistsError, SchemaTagDoesNotExistError, SchemaVersionAlreadyExistsError, + SchemaVersionDoesNotExistError, ) from pepdbagent.models import ( + PaginationResult, SchemaRecordAnnotation, + SchemaSearchResult, SchemaVersionAnnotation, - PaginationResult, SchemaVersionSearchResult, - SchemaSearchResult, UpdateSchemaRecordFields, UpdateSchemaVersionFields, ) @@ -38,20 +42,25 @@ class PEPDatabaseSchema: def __init__(self, pep_db_engine: BaseEngine): """ - :param pep_db_engine: pepdbengine object with sa engine + Args: + pep_db_engine: PEPDatabaseAgent engine object. """ self._sa_engine = pep_db_engine.engine self._pep_db_engine = pep_db_engine def get(self, namespace: str, name: str, version: str) -> dict: - """ - Get schema from the database. + """Get a schema value from the database. + + Args: + namespace: User namespace. + name: Schema name. + version: Schema version (use "latest" for the most recent). - :param namespace: user namespace - :param name: schema name - :param version: schema version + Returns: + Schema value dict. - :return: schema dict + Raises: + SchemaVersionDoesNotExistError: If the schema or version does not exist. """ with Session(self._sa_engine) as session: @@ -69,7 +78,6 @@ def get(self, namespace: str, name: str, version: str) -> dict: ) else: - schema_obj = session.scalar( select(SchemaVersions) .join(SchemaRecords, SchemaRecords.id == SchemaVersions.schema_id) @@ -100,25 +108,26 @@ def create( maintainers: str = "", contributors: str = "", release_notes: str = "", - tags: Optional[Union[List[str], str, Dict[str, str], List[Dict[str, str]]]] = None, + tags: list[str] | str | dict[str, str] | list[dict[str, str]] | None = None, private: bool = False, # TODO: for simplicity was not implemented yet ) -> None: - """ - Create or update schema in the database. - - :param namespace: user namespace - :param name: schema name - :param schema_value: schema dict - :param version: schema version [Default: "1.0.0"] - :param description: schema description [Default: ""] - :param lifecycle_stage: schema lifecycle stage [Default: ""] - :param maintainers: schema maintainers [Default: ""] - :param contributors: schema contributors [Default: ""] - :param release_notes: schema release notes [Default: ""] - :param tags: schema tags [Default: None] - :param private: schema privacy [Default: False] - - :return: None + """Create a new schema record in the database. + + Args: + namespace: User namespace. + name: Schema name. + schema_value: Schema content as a dict. + version: Initial version string. + description: Schema description. + lifecycle_stage: Lifecycle stage, e.g., "stable" or "deprecated". + maintainers: Comma-separated maintainer names. + contributors: Comma-separated contributor names. + release_notes: Release notes for this version. + tags: Tags to associate with the schema version. + private: Mark schema as private if True. + + Raises: + SchemaAlreadyExistsError: If a schema with the same name exists in the namespace. """ tags = self._unify_tags(tags) @@ -126,12 +135,16 @@ def create( with Session(self._sa_engine) as session: schema_obj = session.scalar( select(SchemaRecords).where( - and_(SchemaRecords.namespace == namespace, SchemaRecords.name == name) + and_( + SchemaRecords.namespace == namespace, SchemaRecords.name == name + ) ) ) if schema_obj: - raise SchemaAlreadyExistsError(f"Schema '{name}' already exists in the database") + raise SchemaAlreadyExistsError( + f"Schema '{name}' already exists in the database" + ) user = session.scalar(select(User).where(User.namespace == namespace)) @@ -162,10 +175,14 @@ def create( ) for tag_name, tag_value in tags.items(): - tag_obj = session.scalar(select(SchemaTags).where(SchemaTags.tag_name == tag_name)) + tag_obj = session.scalar( + select(SchemaTags).where(SchemaTags.tag_name == tag_name) + ) if not tag_obj: tag_obj = SchemaTags( - tag_name=tag_name, tag_value=tag_value, schema_mapping=schema_version_obj + tag_name=tag_name, + tag_value=tag_value, + schema_mapping=schema_version_obj, ) session.add(tag_obj) @@ -183,7 +200,7 @@ def add_version( release_notes: str = "", contributors: str = "", overwrite: bool = False, - tags: Optional[Union[List[str], str, Dict[str, str], List[Dict[str, str]]]] = None, + tags: list[str] | str | dict[str, str] | list[dict[str, str]] | None = None, ) -> None: tags = self._unify_tags(tags) @@ -191,7 +208,9 @@ def add_version( with Session(self._sa_engine) as session: schema_obj = session.scalar( select(SchemaRecords).where( - and_(SchemaRecords.namespace == namespace, SchemaRecords.name == name) + and_( + SchemaRecords.namespace == namespace, SchemaRecords.name == name + ) ) ) if not schema_obj: @@ -240,7 +259,9 @@ def add_version( for tag_name, tag_value in tags.items(): tag_obj = SchemaTags( - tag_name=tag_name, tag_value=tag_value, schema_mapping=schema_version_obj + tag_name=tag_name, + tag_value=tag_value, + schema_mapping=schema_version_obj, ) session.add(tag_obj) @@ -254,22 +275,24 @@ def update_schema_version( namespace: str, name: str, version: str, - update_fields: Union[UpdateSchemaVersionFields, dict], + update_fields: UpdateSchemaVersionFields | dict, ) -> None: - """ - Update schema version in the database. - - :param namespace: user namespace - :param name: schema name - :param version: schema version - :param update_fields: fields to be updated. Fields are optional, and include: - - contributors: str - - schema_value: dict - - release_notes: str + """Update fields of an existing schema version. + + Args: + namespace: User namespace. + name: Schema name. + version: Schema version to update. + update_fields: Fields to update — contributors, schema_value, and/or release_notes. + + Raises: + SchemaVersionDoesNotExistError: If the version does not exist. """ if isinstance(update_fields, dict): update_fields = UpdateSchemaVersionFields(**update_fields) - update_fields = update_fields.model_dump(exclude_unset=True, exclude_defaults=True) + update_fields = update_fields.model_dump( + exclude_unset=True, exclude_defaults=True + ) with Session(self._sa_engine) as session: schema_obj = session.scalar( @@ -301,34 +324,39 @@ def update_schema_record( self, namespace: str, name: str, - update_fields: Union[UpdateSchemaRecordFields, dict], + update_fields: UpdateSchemaRecordFields | dict, ) -> None: - """ - Update schema record in the database. - - :param namespace: user namespace - :param name: schema name - :param update_fields: fields to be updated. Fields are optional, and include: - - maintainers: str - - lifecycle_stage: str - - private: bool - - name: str + """Update metadata fields of a schema record. + + Args: + namespace: User namespace. + name: Schema name. + update_fields: Fields to update — maintainers, lifecycle_stage, private, and/or name. + + Raises: + SchemaDoesNotExistError: If the schema does not exist. """ if isinstance(update_fields, dict): update_fields = UpdateSchemaRecordFields(**update_fields) - update_fields = update_fields.model_dump(exclude_unset=True, exclude_defaults=True) + update_fields = update_fields.model_dump( + exclude_unset=True, exclude_defaults=True + ) with Session(self._sa_engine) as session: schema_obj = session.scalar( select(SchemaRecords).where( - and_(SchemaRecords.namespace == namespace, SchemaRecords.name == name) + and_( + SchemaRecords.namespace == namespace, SchemaRecords.name == name + ) ) ) if not schema_obj: - raise SchemaDoesNotExistError(f"Schema '{name}' does not exist in the database") + raise SchemaDoesNotExistError( + f"Schema '{name}' does not exist in the database" + ) for field, value in update_fields.items(): setattr(schema_obj, field, value) @@ -336,32 +364,36 @@ def update_schema_record( session.commit() def schema_exist(self, namespace: str, name: str) -> bool: - """ - Check if schema exists in the database. + """Check whether a schema exists in the database. - :param namespace: user namespace - :param name: schema name + Args: + namespace: User namespace. + name: Schema name. - :return: True if schema exists, False otherwise + Returns: + True if the schema exists. """ with Session(self._sa_engine) as session: schema_obj = session.scalar( select(SchemaRecords).where( - and_(SchemaRecords.namespace == namespace, SchemaRecords.name == name) + and_( + SchemaRecords.namespace == namespace, SchemaRecords.name == name + ) ) ) return True if schema_obj else False def version_exist(self, namespace: str, name: str, version: str) -> bool: - """ - Check if schema version exists in the database. + """Check whether a specific schema version exists. - :param namespace: user namespace - :param name: schema name - :param version: schema version + Args: + namespace: User namespace. + name: Schema name. + version: Schema version. - :return: True if schema version exists, False otherwise + Returns: + True if the version exists. """ with Session(self._sa_engine) as session: @@ -379,24 +411,32 @@ def version_exist(self, namespace: str, name: str, version: str) -> bool: return True if schema_obj else False def get_schema_info(self, namespace: str, name: str) -> SchemaRecordAnnotation: - """ - Get schema information from the database. + """Get metadata for a schema record. + + Args: + namespace: User namespace. + name: Schema name. - :param namespace: user namespace - :param name: schema name + Returns: + SchemaRecordAnnotation with schema metadata. - :return: SchemaRecordAnnotation + Raises: + SchemaDoesNotExistError: If the schema does not exist. """ with Session(self._sa_engine) as session: schema_obj = session.scalar( select(SchemaRecords).where( - and_(SchemaRecords.namespace == namespace, SchemaRecords.name == name) + and_( + SchemaRecords.namespace == namespace, SchemaRecords.name == name + ) ) ) if not schema_obj: - raise SchemaDoesNotExistError(f"Schema '{name}' does not exist in the database") + raise SchemaDoesNotExistError( + f"Schema '{name}' does not exist in the database" + ) return SchemaRecordAnnotation( namespace=schema_obj.namespace, @@ -409,25 +449,35 @@ def get_schema_info(self, namespace: str, name: str) -> SchemaRecordAnnotation: lifecycle_stage=schema_obj.lifecycle_stage, ) - def get_version_info(self, namespace: str, name: str, version: str) -> SchemaVersionAnnotation: - """ - Get schema version information from the database. + def get_version_info( + self, namespace: str, name: str, version: str + ) -> SchemaVersionAnnotation: + """Get metadata for a specific schema version. + + Args: + namespace: User namespace. + name: Schema name. + version: Schema version (use "latest" for the most recent). - :param namespace: user namespace - :param name: schema name - :param version: schema version + Returns: + SchemaVersionAnnotation with version metadata and tags. - :return: SchemaVersionAnnotation + Raises: + SchemaVersionDoesNotExistError: If the version does not exist. """ with Session(self._sa_engine) as session: - # if user provided "latest" version if version == LATEST_SCHEMA_VERSION: version_obj = session.scalar( select(SchemaVersions) .join(SchemaRecords, SchemaRecords.id == SchemaVersions.schema_id) - .where(and_(SchemaRecords.namespace == namespace, SchemaRecords.name == name)) + .where( + and_( + SchemaRecords.namespace == namespace, + SchemaRecords.name == name, + ) + ) .order_by(SchemaVersions.version.desc()) .limit(1) ) @@ -472,28 +522,23 @@ def fetch_schemas( order_by: str = "update_date", order_desc: bool = False, ) -> SchemaSearchResult: - """ - Get schemas with providing filters. - If not filters provided, return all schemas. - - :param namespace: user namespace [Default: None]. If None, search in all namespaces - :param name: schema name [Default: None] - :param maintainer: schema maintainer [Default: None] - :param lifecycle_stage: schema lifecycle stage [Default: None] - :param latest_version: schema latest version [Default: None] - - :param page: page number [Default: 0] - :param page_size: number of schemas per page [Default: 0] - :param order_by: sort the result-set by the information - Options: ["name", "update_date"] - [Default: update_date] - :param order_desc: Sort the records in descending order. [Default: False] - - :return: { - pagination: {page: int, - page_size: int, - total: int}, - results: [SchemaRecordAnnotation] + """Get schemas matching the given filters. + + Returns all schemas if no filters are provided. + + Args: + namespace: Restrict to this namespace (default: all namespaces). + name: Partial name match. + maintainer: Partial maintainer match. + lifecycle_stage: Partial lifecycle stage match. + latest_version: Partial latest version match. + page: Page number (zero-based). + page_size: Number of results per page. + order_by: Sort field — "name" or "update_date". + order_desc: Sort in descending order if True. + + Returns: + SchemaSearchResult with pagination and list of SchemaRecordAnnotation objects. """ # filters = [ @@ -517,7 +562,9 @@ def fetch_schemas( conditions = [f for f in filters if f is not None] statement = ( - select(SchemaRecords).where(and_(*conditions)) if conditions else select(SchemaRecords) + select(SchemaRecords).where(and_(*conditions)) + if conditions + else select(SchemaRecords) ) statement_count = ( select(func.count(SchemaRecords.id)).where(and_(*conditions)) @@ -528,9 +575,13 @@ def fetch_schemas( with Session(self._sa_engine) as session: total = session.scalar(statement_count) - statement = self._add_order_by_schemas_keyword(statement, by=order_by, desc=order_desc) + statement = self._add_order_by_schemas_keyword( + statement, by=order_by, desc=order_desc + ) - results_objects = session.scalars(statement.limit(page_size).offset(page * page_size)) + results_objects = session.scalars( + statement.limit(page_size).offset(page * page_size) + ) return SchemaSearchResult( pagination=PaginationResult( page=page, @@ -560,23 +611,18 @@ def query_schemas( order_by: str = "update_date", order_desc: bool = False, ) -> SchemaSearchResult: - """ - Search schemas in the database with pagination. - - :param namespace: user namespace [Default: None]. If None, search in all namespaces - :param search_str: query string. [Default: ""]. If empty, return all schemas - :param page: page number [Default: 0] - :param page_size: number of schemas per page [Default: 0] - :param order_by: sort the result-set by the information - Options: ["name", "update_date"] - [Default: update_date] - :param order_desc: Sort the records in descending order. [Default: False] - - :return: { - pagination: {page: int, - page_size: int, - total: int}, - results: [SchemaRecordAnnotation] + """Search schemas by name and description with pagination. + + Args: + namespace: Restrict to this namespace (default: all namespaces). + search_str: Text to search in name and description (default: all schemas). + page: Page number (zero-based). + page_size: Number of results per page. + order_by: Sort field — "name" or "update_date". + order_desc: Sort in descending order if True. + + Returns: + SchemaSearchResult with pagination and list of SchemaRecordAnnotation objects. """ search_str = search_str.lower() if search_str else "" @@ -586,17 +632,23 @@ def query_schemas( SchemaRecords.description.ilike(f"%{search_str}%"), ) if namespace: - where_statement = and_(where_statement, SchemaRecords.namespace == namespace) + where_statement = and_( + where_statement, SchemaRecords.namespace == namespace + ) with Session(self._sa_engine) as session: - total = session.scalar(select(func.count(SchemaRecords.id)).where(where_statement)) + total = session.scalar( + select(func.count(SchemaRecords.id)).where(where_statement) + ) statement = ( select(SchemaRecords) .where(where_statement) .limit(page_size) .offset(page * page_size) ) - statement = self._add_order_by_schemas_keyword(statement, by=order_by, desc=order_desc) + statement = self._add_order_by_schemas_keyword( + statement, by=order_by, desc=order_desc + ) results_objects = session.scalars(statement) return SchemaSearchResult( @@ -628,21 +680,21 @@ def query_schema_version( page: int = 0, page_size: int = 10, ) -> SchemaVersionSearchResult: - """ - Search schema versions in the database with pagination. - - :param namespace: user namespace - :param name: schema name - :param tag: tag name. [Default: None]. If None, return versions with all tags - :param search_str: query string. [Default: ""]. If empty, return all schemas - :param page: result page number [Default: 10] - :param page_size: number of schemas per page [Default: 10] - - :return: { - pagination: {page: int, - page_size: int, - total: int}, - results: [SchemaVersionAnnotation] + """Search versions of a schema with pagination. + + Args: + namespace: User namespace. + name: Schema name. + tag: Filter by tag name (default: all tags). + search_str: Text to search in version and release_notes. + page: Page number (zero-based). + page_size: Number of results per page. + + Returns: + SchemaVersionSearchResult with pagination and list of SchemaVersionAnnotation objects. + + Raises: + SchemaDoesNotExistError: If the schema does not exist. """ search_str = search_str.lower() if search_str else "" @@ -650,12 +702,16 @@ def query_schema_version( with Session(self._sa_engine) as session: schema_obj = session.scalar( select(SchemaRecords).where( - and_(SchemaRecords.namespace == namespace, SchemaRecords.name == name) + and_( + SchemaRecords.namespace == namespace, SchemaRecords.name == name + ) ) ) if not schema_obj: - raise SchemaDoesNotExistError(f"Schema '{name}' does not exist in the database") + raise SchemaDoesNotExistError( + f"Schema '{name}' does not exist in the database" + ) where_statement = and_( SchemaRecords.namespace == namespace, @@ -687,7 +743,9 @@ def query_schema_version( .join(SchemaRecords) .where(where_statement) ) - find_statement = select(SchemaVersions).join(SchemaRecords).where(where_statement) + find_statement = ( + select(SchemaVersions).join(SchemaRecords).where(where_statement) + ) total = session.scalar(total_statement) @@ -710,7 +768,9 @@ def query_schema_version( version=result.version, contributors=result.contributors, release_notes=result.release_notes, - tags={tag.tag_name: tag.tag_value for tag in result.tags_mapping}, + tags={ + tag.tag_name: tag.tag_value for tag in result.tags_mapping + }, release_date=result.release_date, last_update_date=result.last_update_date, ) @@ -719,23 +779,29 @@ def query_schema_version( ) def delete_schema(self, namespace: str, name: str) -> None: - """ - Delete schema from the database. + """Delete a schema record and all its versions from the database. + + Args: + namespace: User namespace. + name: Schema name. - :param namespace: user namespace - :param name: schema name - :return: None + Raises: + SchemaDoesNotExistError: If the schema does not exist. """ with Session(self._sa_engine) as session: schema_obj = session.scalar( select(SchemaRecords).where( - and_(SchemaRecords.namespace == namespace, SchemaRecords.name == name) + and_( + SchemaRecords.namespace == namespace, SchemaRecords.name == name + ) ) ) if not schema_obj: - raise SchemaDoesNotExistError(f"Schema '{name}' does not exist in the database") + raise SchemaDoesNotExistError( + f"Schema '{name}' does not exist in the database" + ) statement = select(User).where(User.namespace == namespace) user = session.scalar(statement) @@ -747,15 +813,15 @@ def delete_schema(self, namespace: str, name: str) -> None: session.commit() def delete_version(self, namespace: str, name: str, version: str) -> None: - """ - Delete version of the schema + """Delete a specific schema version. - :param namespace: Namespace of the schema - :param name: Name of the schema - :param version: Version of the Schema + Args: + namespace: Namespace of the schema. + name: Name of the schema. + version: Version to delete. - :raise: SchemaVersionDoesNotExistError if version doesn't exist - :return: None + Raises: + SchemaVersionDoesNotExistError: If the version does not exist. """ with Session(self._sa_engine) as session: schema_obj = session.scalar( @@ -782,18 +848,19 @@ def add_tag_to_schema( namespace: str, name: str, version: str, - tag: Optional[Union[List[str], str, Dict[str, str]]], + tag: list[str] | str | dict[str, str] | None, ) -> None: - """ - Add tag to the schema + """Add a tag to a schema version. - :param namespace: Namespace of the schema - :param name: Name of the schema - :param version: Version of the Schema - :param tag: Tag to be added. Can be a string, list of strings or dictionaries + Args: + namespace: Namespace of the schema. + name: Name of the schema. + version: Schema version. + tag: Tag to add — string, list of strings, or dict of name→value pairs. - :raise: SchemaVersionDoesNotExistError if version doesn't exist - :return: None + Raises: + SchemaVersionDoesNotExistError: If the version does not exist. + SchemaTagAlreadyExistsError: If the tag already exists on the version. """ tag = self._unify_tags(tag) @@ -818,10 +885,14 @@ def add_tag_to_schema( tag = [tag] for tag_name, tag_value in tag.items(): - tag_obj = session.scalar(select(SchemaTags).where(SchemaTags.tag_name == tag_name)) + tag_obj = session.scalar( + select(SchemaTags).where(SchemaTags.tag_name == tag_name) + ) if not tag_obj: tag_obj = SchemaTags( - tag_name=tag_name, tag_value=tag_value, schema_mapping=schema_obj + tag_name=tag_name, + tag_value=tag_value, + schema_mapping=schema_obj, ) session.add(tag_obj) else: @@ -831,17 +902,20 @@ def add_tag_to_schema( session.commit() - def remove_tag_from_schema(self, namespace: str, name: str, version: str, tag: str) -> None: - """ - Remove tag from the schema + def remove_tag_from_schema( + self, namespace: str, name: str, version: str, tag: str + ) -> None: + """Remove a tag from a schema version. - :param namespace: Namespace of the schema - :param name: Name of the schema - :param version: Version of the Schema - :param tag: Tag to be removed + Args: + namespace: Namespace of the schema. + name: Name of the schema. + version: Schema version. + tag: Name of the tag to remove. - :raise: SchemaVersionDoesNotExistError if version doesn't exist - :return: None + Raises: + SchemaVersionDoesNotExistError: If the version does not exist. + SchemaTagDoesNotExistError: If the tag does not exist on the version. """ with Session(self._sa_engine) as session: schema_obj = session.scalar( @@ -862,11 +936,14 @@ def remove_tag_from_schema(self, namespace: str, name: str, version: str, tag: s tag_obj = session.scalar( select(SchemaTags).where( - SchemaTags.tag_name == tag, SchemaTags.schema_version_id == schema_obj.id + SchemaTags.tag_name == tag, + SchemaTags.schema_version_id == schema_obj.id, ) ) if not tag_obj: - raise SchemaTagDoesNotExistError(f"Tag '{tag}' does not exist in the schema") + raise SchemaTagDoesNotExistError( + f"Tag '{tag}' does not exist in the schema" + ) session.delete(tag_obj) session.commit() @@ -875,15 +952,15 @@ def remove_tag_from_schema(self, namespace: str, name: str, version: str, tag: s def _add_order_by_schemas_keyword( statement: Select, by: str = "update_date", desc: bool = False ) -> Select: - """ - Add order by clause to sqlalchemy statement - - :param statement: sqlalchemy representation of a SELECT statement. - :param by: sort the result-set by the information - Options: ["name", "update_date"] - [Default: "update_date"] - :param desc: Sort the records in descending order. [Default: False] - :return: sqlalchemy representation of a SELECT statement with order by keyword + """Add an ORDER BY clause for schema queries. + + Args: + statement: SQLAlchemy SELECT statement to augment. + by: Sort field — "name" or "update_date". + desc: Sort in descending order if True. + + Returns: + Statement with ORDER BY applied. """ if by == "update_date": order_by_obj = SchemaRecords.last_update_date @@ -904,15 +981,18 @@ def _add_order_by_schemas_keyword( return statement.order_by(order_by_obj) def _unify_tags( - self, tags: Optional[Union[List[str], str, Dict[str, str], List[Dict[str, str]]]] - ) -> [Dict[str, str]]: - """ - Convert provided tags to one standard + self, tags: list[str] | str | dict[str, str] | list[dict[str, str]] | None + ) -> dict[str, str]: + """Normalise tags to a dict[str, str] representation. + + Args: + tags: Tags as a string, list of strings, dict, or list of dicts. - :param tags: tags to be converted from types: str, dict, list of str, list of dict + Returns: + Dict mapping tag names to tag values. - :raise: ValueError if tags are not in the correct format - :return: dictionary of tags + Raises: + ValueError: If tags are in an unsupported format. """ if not tags: tags = {} diff --git a/pepdbagent/modules/user.py b/pepdbagent/modules/user.py index 416df88..be334fd 100644 --- a/pepdbagent/modules/user.py +++ b/pepdbagent/modules/user.py @@ -1,5 +1,4 @@ import logging -from typing import Union from sqlalchemy import and_, delete, select from sqlalchemy.exc import IntegrityError @@ -26,17 +25,20 @@ class PEPDatabaseUser: def __init__(self, pep_db_engine: BaseEngine): """ - :param pep_db_engine: pepdbengine object with sa engine + Args: + pep_db_engine: PEPDatabaseAgent engine object. """ self._sa_engine = pep_db_engine.engine self._pep_db_engine = pep_db_engine def create_user(self, namespace: str) -> int: - """ - Create new user + """Create a new user. + + Args: + namespace: User namespace. - :param namespace: user namespace - :return: user id + Returns: + New user id. """ new_user_raw = User(namespace=namespace) @@ -46,12 +48,14 @@ def create_user(self, namespace: str) -> int: user_id = new_user_raw.id return user_id - def get_user_id(self, namespace: str) -> Union[int, None]: - """ - Get user id using username + def get_user_id(self, namespace: str) -> int | None: + """Get user id by namespace. - :param namespace: user namespace - :return: user id + Args: + namespace: User namespace. + + Returns: + User id, or None if the user does not exist. """ statement = select(User.id).where(User.namespace == namespace) with Session(self._sa_engine) as session: @@ -62,16 +66,19 @@ def get_user_id(self, namespace: str) -> Union[int, None]: return None def add_project_to_favorites( - self, namespace: str, project_namespace: str, project_name: str, project_tag: str + self, + namespace: str, + project_namespace: str, + project_name: str, + project_tag: str, ) -> None: - """ - Add project to favorites + """Add a project to a user's favorites. - :param namespace: namespace of the user - :param project_namespace: namespace of the project - :param project_name: name of the project - :param project_tag: tag of the project - :return: None + Args: + namespace: Namespace of the user. + project_namespace: Namespace of the project. + project_name: Name of the project. + project_tag: Tag of the project. """ user_id = self.get_user_id(namespace) @@ -91,7 +98,9 @@ def add_project_to_favorites( ) ) - new_favorites_raw = Stars(user_id=user_id, project_id=project_mapping.id) + new_favorites_raw = Stars( + user_id=user_id, project_id=project_mapping.id + ) session.add(new_favorites_raw) project_mapping.number_of_stars += 1 @@ -101,16 +110,19 @@ def add_project_to_favorites( return None def remove_project_from_favorites( - self, namespace: str, project_namespace: str, project_name: str, project_tag: str + self, + namespace: str, + project_namespace: str, + project_name: str, + project_tag: str, ) -> None: - """ - Remove project from favorites + """Remove a project from a user's favorites. - :param namespace: namespace of the user - :param project_namespace: namespace of the project - :param project_name: name of the project - :param project_tag: tag of the project - :return: None + Args: + namespace: Namespace of the user. + project_namespace: Namespace of the project. + project_name: Name of the project. + project_tag: Tag of the project. """ _LOGGER.debug( f"Removing project {project_namespace}/{project_name}:{project_tag} from favorites in {namespace}" @@ -145,11 +157,13 @@ def remove_project_from_favorites( return None def get_favorites(self, namespace: str) -> AnnotationList: - """ - Get list of favorites for user + """Get list of favorite projects for a user. + + Args: + namespace: Namespace of the user. - :param namespace: namespace of the user - :return: list of favorite projects with annotations + Returns: + AnnotationList of favorite projects. """ _LOGGER.debug(f"Getting favorites for user {namespace}") if not self.exists(namespace): @@ -162,7 +176,9 @@ def get_favorites(self, namespace: str) -> AnnotationList: statement = select(User).where(User.namespace == namespace) with Session(self._sa_engine) as session: query_result = session.scalar(statement) - number_of_projects = len([kk.project_mapping for kk in query_result.stars_mapping]) + number_of_projects = len( + [kk.project_mapping for kk in query_result.stars_mapping] + ) project_list = [] for prj_list in query_result.stars_mapping: prj = prj_list.project_mapping @@ -203,11 +219,13 @@ def exists( self, namespace: str, ) -> bool: - """ - Check if user exists in the database. + """Check if a user exists in the database. - :param namespace: project namespace - :return: Returning True if project exist + Args: + namespace: User namespace. + + Returns: + True if the user exists. """ statement = select(User.id) @@ -224,11 +242,10 @@ def exists( return False def delete(self, namespace: str) -> None: - """ - Delete user from the database with all related data + """Delete a user and all related data from the database. - :param namespace: user namespace - :return: None + Args: + namespace: User namespace. """ if not self.exists(namespace): raise UserNotFoundError diff --git a/pepdbagent/modules/view.py b/pepdbagent/modules/view.py index 8704c97..6530116 100644 --- a/pepdbagent/modules/view.py +++ b/pepdbagent/modules/view.py @@ -1,15 +1,20 @@ # View of the PEP. In other words, it is a part of the PEP, or subset of the samples in the PEP. import logging -from typing import List, Union -import peppy +import peprs from sqlalchemy import and_, delete, select from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session from pepdbagent.const import DEFAULT_TAG, PKG_NAME -from pepdbagent.db_utils import BaseEngine, Projects, Samples, Views, ViewSampleAssociation +from pepdbagent.db_utils import ( + BaseEngine, + Projects, + Samples, + Views, + ViewSampleAssociation, +) from pepdbagent.exceptions import ( ProjectNotFoundError, SampleAlreadyInView, @@ -32,7 +37,8 @@ class PEPDatabaseView: def __init__(self, pep_db_engine: BaseEngine): """ - :param pep_db_engine: pepdbengine object with sa engine + Args: + pep_db_engine: PEPDatabaseAgent engine object. """ self._sa_engine = pep_db_engine.engine self._pep_db_engine = pep_db_engine @@ -44,25 +50,23 @@ def get( tag: str = DEFAULT_TAG, view_name: str = None, raw: bool = True, - ) -> Union[peppy.Project, dict, None]: - """ - Retrieve view of the project from the database. - View is a subset of the samples in the project. e.g. bed-db project has all the samples in bedbase, - bedset is a view of the bedbase project with only the samples in the bedset. - - :param namespace: namespace of the project - :param name: name of the project (Default: name is taken from the project object) - :param tag: tag of the project (Default: tag is taken from the project object) - :param view_name: name of the view - :param raw: retrieve unprocessed (raw) PEP dict. [Default: True] - :return: peppy.Project object with found project or dict with unprocessed - PEP elements: { - name: str - description: str - _config: dict - _sample_dict: dict - _subsample_dict: dict - } + ) -> peprs.Project | dict | None: + """Retrieve a view of the project from the database. + + A view is a subset of samples in the project. + + Args: + namespace: Namespace of the project. + name: Name of the project. + tag: Tag of the project. + view_name: Name of the view. + raw: Return raw dict if True, peprs.Project object if False. + + Returns: + Raw dict or peprs.Project with only the view's samples. + + Raises: + ViewNotFoundError: If the view does not exist. """ _LOGGER.debug(f"Get view {view_name} from {namespace}/{name}:{tag}") view_statement = select(Views).where( @@ -80,31 +84,32 @@ def get( ) samples = [sample.sample.sample for sample in view.samples] config = view.project_mapping.config - sub_project_dict = {"_config": config, "_sample_dict": samples, "_subsample_dict": None} + sub_project_dict = {"config": config, "samples": samples} if raw: return sub_project_dict else: - return peppy.Project.from_dict(sub_project_dict) + return peprs.Project.from_dict(sub_project_dict) def get_annotation( self, namespace: str, name: str, tag: str = DEFAULT_TAG, view_name: str = None ) -> ViewAnnotation: + """Get annotation for a view. + + Args: + namespace: Namespace of the project. + name: Name of the project. + tag: Tag of the project. + view_name: Name of the view. + + Returns: + ViewAnnotation with project coordinates, name, description, and sample count. + + Raises: + ViewNotFoundError: If the view does not exist. """ - Get annotation of the view. - - :param namespace: namespace of the project - :param name: name of the project - :param tag: tag of the project - :param view_name: name of the sample - :return: ViewAnnotation object: - {project_namespace: str, - project_name: str, - project_tag: str, - name: str, - description: str, - number_of_samples: int} - """ - _LOGGER.debug(f"Get annotation for view {view_name} in {namespace}/{name}:{tag}") + _LOGGER.debug( + f"Get annotation for view {view_name} in {namespace}/{name}:{tag}" + ) view_statement = select(Views).where( and_( Views.project_mapping.has(namespace=namespace, name=name, tag=tag), @@ -130,27 +135,27 @@ def get_annotation( def create( self, view_name: str, - view_dict: Union[dict, CreateViewDictModel], + view_dict: dict | CreateViewDictModel, description: str = None, no_fail: bool = False, ) -> None: + """Create a view of a project in the database. + + Args: + view_name: Name for the new view. + view_dict: View definition with project_namespace, project_name, project_tag, + and sample_list keys (or a CreateViewDictModel). + description: Optional description of the view. + no_fail: Skip samples that do not exist instead of raising an error. + + Raises: + ProjectNotFoundError: If the project does not exist. + SampleNotFoundError: If a sample does not exist and no_fail is False. + ViewAlreadyExistsError: If a view with the same name already exists. """ - Create a view of the project in the database. - - :param view_name: namespace of the project - :param view_dict: dict or CreateViewDictModel object with view samples. - Dict should have the following structure: - { - project_namespace: str - project_name: str - project_tag: str - sample_list: List[str] # list of sample names - } - :param description: description of the view - :param no_fail: if True, skip samples that doesn't exist in the project - retrun: None - """ - _LOGGER.debug(f"Creating view {view_name} with provided info: (view_dict: {view_dict})") + _LOGGER.debug( + f"Creating view {view_name} with provided info: (view_dict: {view_dict})" + ) if isinstance(view_dict, dict): view_dict = CreateViewDictModel(**view_dict) @@ -192,7 +197,9 @@ def create( else: continue - sa_session.add(ViewSampleAssociation(sample_id=sample_id, view=view)) + sa_session.add( + ViewSampleAssociation(sample_id=sample_id, view=view) + ) sa_session.commit() except IntegrityError: @@ -207,14 +214,16 @@ def delete( project_tag: str = DEFAULT_TAG, view_name: str = None, ) -> None: - """ - Delete a view of the project in the database. + """Delete a view from the database. - :param project_namespace: namespace of the project - :param project_name: name of the project - :param project_tag: tag of the project - :param view_name: name of the view - :return: None + Args: + project_namespace: Namespace of the project. + project_name: Name of the project. + project_tag: Tag of the project. + view_name: Name of the view to delete. + + Raises: + ViewNotFoundError: If the view does not exist. """ _LOGGER.debug( f"Deleting view {view_name} from {project_namespace}/{project_name}:{project_tag}" @@ -243,17 +252,21 @@ def add_sample( name: str, tag: str, view_name: str, - sample_name: Union[str, List[str]], - ): - """ - Add sample to the view. - - :param namespace: namespace of the project - :param name: name of the project - :param tag: tag of the project - :param view_name: name of the view - :param sample_name: sample name - :return: None + sample_name: str | list[str], + ) -> None: + """Add one or more samples to a view. + + Args: + namespace: Namespace of the project. + name: Name of the project. + tag: Tag of the project. + view_name: Name of the view. + sample_name: Sample name or list of sample names. + + Raises: + ViewNotFoundError: If the view does not exist. + SampleNotFoundError: If a sample does not exist. + SampleAlreadyInView: If a sample is already in the view. """ _LOGGER.debug( f"Adding sample {sample_name} to view {view_name} in {namespace}/{name}:{tag}" @@ -301,15 +314,18 @@ def remove_sample( view_name: str, sample_name: str, ) -> None: - """ - Remove sample from the view. - - :param namespace: namespace of the project - :param name: name of the project - :param tag: tag of the project - :param view_name: name of the view - :param sample_name: sample name - :return: None + """Remove a sample from a view. + + Args: + namespace: Namespace of the project. + name: Name of the project. + tag: Tag of the project. + view_name: Name of the view. + sample_name: Name of the sample to remove. + + Raises: + ViewNotFoundError: If the view does not exist. + SampleNotInViewError: If the sample is not in the view. """ _LOGGER.debug( f"Removing sample {sample_name} from view {view_name} in {namespace}/{name}:{tag}" @@ -348,18 +364,30 @@ def remove_sample( sa_session.commit() def get_snap_view( - self, namespace: str, name: str, tag: str, sample_name_list: List[str], raw: bool = False - ) -> Union[peppy.Project, dict]: - """ - Get a snap view of the project. Snap view is a view of the project - with only the samples in the list. This view won't be saved in the database. - - :param namespace: project namespace - :param name: name of the project - :param tag: tag of the project - :param sample_name_list: list of sample names e.g. ["sample1", "sample2"] - :param raw: retrieve unprocessed (raw) PEP dict. - :return: peppy.Project object + self, + namespace: str, + name: str, + tag: str, + sample_name_list: list[str], + raw: bool = False, + ) -> peprs.Project | dict: + """Get an ephemeral view of a project limited to a set of samples. + + The snap view is not persisted in the database. + + Args: + namespace: Namespace of the project. + name: Name of the project. + tag: Tag of the project. + sample_name_list: Sample names to include, e.g., ["sample1", "sample2"]. + raw: Return raw dict if True, peprs.Project object if False. + + Returns: + Raw dict or peprs.Project containing only the requested samples. + + Raises: + ProjectNotFoundError: If the project does not exist. + SampleNotFoundError: If any sample does not exist. """ _LOGGER.debug(f"Creating snap view for {namespace}/{name}:{tag}") project_statement = select(Projects).where( @@ -372,7 +400,9 @@ def get_snap_view( with Session(self._sa_engine) as sa_session: project = sa_session.scalar(project_statement) if not project: - raise ProjectNotFoundError(f"Project {namespace}/{name}:{tag} does not exist") + raise ProjectNotFoundError( + f"Project {namespace}/{name}:{tag} does not exist" + ) samples = [] for sample_name in sample_name_list: sample_statement = select(Samples).where( @@ -390,22 +420,22 @@ def get_snap_view( config = project.config if raw: - return {"_config": config, "_sample_dict": samples, "_subsample_dict": None} + return {"config": config, "samples": samples} else: - return peppy.Project.from_dict( - {"_config": config, "_sample_dict": samples, "_subsample_dict": None} - ) + return peprs.Project.from_dict({"config": config, "samples": samples}) def get_views_annotation( self, namespace: str, name: str, tag: str = DEFAULT_TAG - ) -> Union[ProjectViews, None]: - """ - Get list of views of the project + ) -> ProjectViews | None: + """Get annotation for all views of a project. + + Args: + namespace: Namespace of the project. + name: Name of the project. + tag: Tag of the project. - :param namespace: namespace of the project - :param name: name of the project - :param tag: tag of the project - :return: list of views of the project + Returns: + ProjectViews with a list of ViewAnnotation objects. """ _LOGGER.debug(f"Get views annotation for {namespace}/{name}:{tag}") statement = select(Views).where( diff --git a/pepdbagent/pepdbagent.py b/pepdbagent/pepdbagent.py index 07d66af..6fc82ad 100644 --- a/pepdbagent/pepdbagent.py +++ b/pepdbagent/pepdbagent.py @@ -12,29 +12,28 @@ class PEPDatabaseAgent(object): def __init__( self, - host="localhost", - port=5432, - database="pep-db", - user=None, - password=None, - drivername=POSTGRES_DIALECT, - dsn=None, - echo=False, - run_migrations=False, + host: str = "localhost", + port: int = 5432, + database: str = "pep-db", + user: str | None = None, + password: str | None = None, + drivername: str = POSTGRES_DIALECT, + dsn: str | None = None, + echo: bool = False, + run_migrations: bool = False, ): - """ - Initialize connection to the pep_db database. You can use The basic connection parameters - or libpq connection string. - :param host: database server address e.g., localhost or an IP address. - :param port: the port number that defaults to 5432 if it is not provided. - :param database: the name of the database that you want to connect. - :param user: the username used to authenticate. - :param password: password used to authenticate. - :param drivername: driver of the database [Default: postgresql] - :param dsn: libpq connection string using the dsn parameter - (e.g. "localhost://username:password@pdp_db:5432") - - :param run_migrations: run migrations on the database + """Initialize connection to the pep_db database. + + Args: + host: Database server address, e.g., localhost or an IP address. + port: Port number (default: 5432). + database: Name of the database to connect to. + user: Username for authentication. + password: Password for authentication. + drivername: Database driver (default: postgresql). + dsn: libpq connection string, e.g., "localhost://username:password@pdp_db:5432". + echo: Log all SQL statements if True. + run_migrations: Run migrations on the database if True. """ pep_db_engine = BaseEngine( @@ -98,5 +97,5 @@ def __exit__(self): self._sa_engine.__exit__() @property - def connection(self): + def connection(self) -> BaseEngine: return self._sa_engine diff --git a/pepdbagent/utils.py b/pepdbagent/utils.py index fc96684..87b5e7a 100644 --- a/pepdbagent/utils.py +++ b/pepdbagent/utils.py @@ -3,22 +3,23 @@ import uuid from collections.abc import Iterable from hashlib import md5 -from typing import List, Tuple, Union import ubiquerg -from peppy.const import SAMPLE_RAW_DICT_KEY +from peprs.const import SAMPLE_RAW_DICT_KEY from pepdbagent.exceptions import RegistryPathError def is_valid_registry_path(rpath: str) -> bool: - """ - Verify that a registry path is valid. Checks for two things: - 1. Contains forward slash ("/"), and - 2. Forward slash divides two strings + """Verify that a registry path is valid. + + Checks that the path contains a forward slash dividing two non-empty strings. + + Args: + rpath: Registry path to test. - :param str rpath: registry path to test - :return bool: Is it a valid registry or not. + Returns: + True if the path is a valid registry path. """ # check for string if not isinstance(rpath, str): @@ -33,11 +34,13 @@ def is_valid_registry_path(rpath: str) -> bool: def all_elements_are_strings(iterable: Iterable) -> bool: - """ - Helper method to determine if an iterable only contains `str` objects. + """Check if every element of an iterable is a str. + + Args: + iterable: An iterable item. - :param Iterable iterable: An iterable item - :returns bool: Boolean value indicating if the iterable only contains strings. + Returns: + True if every element is a string. """ if not isinstance(iterable, Iterable): return False @@ -45,11 +48,13 @@ def all_elements_are_strings(iterable: Iterable) -> bool: def create_digest(project_dict: dict) -> str: - """ - Create digest for PEP project + """Create an MD5 digest for a PEP project. + + Args: + project_dict: Project dictionary. - :param project_dict: project dict - :return: digest string + Returns: + MD5 hex digest of the sample table. """ sample_digest = md5( json.dumps( @@ -63,12 +68,17 @@ def create_digest(project_dict: dict) -> str: return sample_digest -def registry_path_converter(registry_path: str) -> Tuple[str, str, str]: - """ - Convert registry path to namespace, name, tag +def registry_path_converter(registry_path: str) -> tuple[str, str, str]: + """Convert a registry path to (namespace, name, tag). + + Args: + registry_path: Registry path with structure "namespace/name:tag". + + Returns: + Tuple of (namespace, name, tag). - :param registry_path: registry path that has structure: "namespace/name:tag" - :return: tuple(namespace, name, tag) + Raises: + RegistryPathError: If the path is not a valid registry path. """ if is_valid_registry_path(registry_path): reg = ubiquerg.parse_registry_path(registry_path) @@ -80,12 +90,17 @@ def registry_path_converter(registry_path: str) -> Tuple[str, str, str]: raise RegistryPathError(f"Error in: '{registry_path}'") -def schema_path_converter(schema_path: str) -> Tuple[str, str, str]: - """ - Convert schema path to namespace, name +def schema_path_converter(schema_path: str) -> tuple[str, str, str]: + """Convert a schema path to (namespace, name, version). + + Args: + schema_path: Schema path with structure "namespace/name:version". + + Returns: + Tuple of (namespace, name, version). - :param schema_path: schema path that has structure: "namespace/name.yaml" - :return: tuple(namespace, name, version) + Raises: + RegistryPathError: If the path is invalid. """ if "/" in schema_path: namespace, name_tag = schema_path.split("/") @@ -97,13 +112,14 @@ def schema_path_converter(schema_path: str) -> Tuple[str, str, str]: raise RegistryPathError(f"Error in: '{schema_path}'") -def tuple_converter(value: Union[tuple, list, str, None]) -> tuple: - """ - Convert string list or tuple to tuple. - # is used to create admin tuple. +def tuple_converter(value: tuple | list | str | None) -> tuple: + """Convert a string, list, or tuple to a tuple. + + Args: + value: Value to convert. - :param value: Any value that has to be converted to tuple - :return: tuple of strings + Returns: + Tuple of strings. """ if isinstance(value, str): value = [value] @@ -115,28 +131,30 @@ def tuple_converter(value: Union[tuple, list, str, None]) -> tuple: def convert_date_string_to_date(date_string: str) -> datetime.datetime: - """ - Convert string into datetime format + """Convert a date string to a datetime. + + Args: + date_string: Date string in format YYYY/MM/DD, e.g., "2022/02/22". - :param date_string: date string in format [YYYY/MM/DD]. e.g. 2022/02/22 - :return: datetime format + Returns: + Parsed datetime offset by one day. """ - return datetime.datetime.strptime(date_string, "%Y/%m/%d") + datetime.timedelta(days=1) + return datetime.datetime.strptime(date_string, "%Y/%m/%d") + datetime.timedelta( + days=1 + ) -def order_samples(results: dict) -> List[dict]: - """ - Order samples by their parent_guid +def order_samples(results: dict) -> list[dict]: + """Order samples by their parent_guid chain. # TODO: To make this function more efficient, we should write it in Rust! - :param results: dictionary of samples. Structure: { - "sample": sample_dict, - "guid": sample.guid, - "parent_guid": sample.parent_guid, - } + Args: + results: Dict of samples keyed by guid, each value a dict with + "sample", "guid", and "parent_guid" keys. - :return: ordered list of samples + Returns: + Ordered list of samples from root to leaf. """ # Find the Root Node # Create a lookup dictionary for nodes by their GUIDs diff --git a/pyproject.toml b/pyproject.toml index 9348158..f13d871 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,4 +1,65 @@ -[tool.black] -line-length = 99 -target-version = ['py38', 'py311'] -include = '\.pyi?$' +[project] +name = "pepdbagent" +version = "0.13.0" +description = "A python-based database manager for portable encapsulated projects" +readme = "README.md" +license = "BSD-2-Clause" +requires-python = ">=3.10" +authors = [ + { name = "Oleksandr Khoroshevskyi" }, +] +keywords = ["project", "metadata", "bioinformatics", "database"] +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Scientific/Engineering :: Bio-Informatics", +] +dependencies = [ + "sqlalchemy>=2.0.0", + "logmuse>=0.2.7", + "peprs>=0.2.1", + "ubiquerg>=0.6.2", + "coloredlogs>=15.0.1", + "pydantic>=2.0", + "psycopg>=3.1.15", + "numpy>=1.24.4", + "alembic>=1.15.1", +] + +[project.urls] +Homepage = "https://github.com/pepkit/pepdbagent/" + +[project.optional-dependencies] +test = [ + "pytest", + "pytest-mock", + "python-dotenv", + "coverage", + "smokeshow", + "pyyaml", + "pandas", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.pytest.ini_options] +addopts = "-rfE" +testpaths = ["tests"] + +[tool.ruff] +line-length = 88 + +[tool.ruff.lint] +select = ["E", "F", "I"] +ignore = ["F403", "F405", "E501"] + +[tool.ruff.lint.isort] +known-first-party = ["pepdbagent"] + +[tool.ruff.lint.per-file-ignores] +"pepdbagent/alembic/env.py" = ["E402"] diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt deleted file mode 100644 index 786a29d..0000000 --- a/requirements/requirements-all.txt +++ /dev/null @@ -1,10 +0,0 @@ -sqlalchemy>=2.0.0 -logmuse>=0.2.7 -peppy>=0.40.6 -ubiquerg>=0.6.2 -coloredlogs>=15.0.1 -pytest-mock -pydantic>=2.0 -psycopg>=3.1.15 -numpy>=1.24.4 -alembic>=1.15.1 diff --git a/requirements/requirements-dev.txt b/requirements/requirements-dev.txt deleted file mode 100644 index 9b1f849..0000000 --- a/requirements/requirements-dev.txt +++ /dev/null @@ -1,5 +0,0 @@ -black -pytest -python-dotenv -coverage -smokeshow \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index f53601e..0000000 --- a/setup.py +++ /dev/null @@ -1,64 +0,0 @@ -import os -import sys - -from setuptools import find_packages, setup - -PACKAGE_NAME = "pepdbagent" - -# Ordinary dependencies -DEPENDENCIES = [] -with open("requirements/requirements-all.txt", "r") as reqs_file: - for line in reqs_file: - if not line.strip(): - continue - # DEPENDENCIES.append(line.split("=")[0].rstrip("<>")) - DEPENDENCIES.append(line) - -# Additional keyword arguments for setup(). -extra = {"install_requires": DEPENDENCIES} - - -# Additional files to include with package -def get_static(name, condition=None): - static = [ - os.path.join(name, f) - for f in os.listdir(os.path.join(os.path.dirname(os.path.realpath(__file__)), name)) - ] - if condition is None: - return static - else: - return [i for i in filter(lambda x: eval(condition), static)] - - -with open(f"{PACKAGE_NAME}/_version.py", "r") as versionfile: - version = versionfile.readline().split()[-1].strip("\"'\n") - -with open("README.md") as f: - long_description = f.read() - -setup( - name=PACKAGE_NAME, - packages=find_packages(), - version=version, - description="A python-based database manager for portable encapsulated projects", - long_description=long_description, - long_description_content_type="text/markdown", - classifiers=[ - "Development Status :: 4 - Beta", - "License :: OSI Approved :: BSD License", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Topic :: Scientific/Engineering :: Bio-Informatics", - ], - keywords="project, metadata, bioinformatics, database", - url="https://github.com/pepkit/pepdbagent/", - author="Oleksandr Khoroshevskyi ", - license="BSD2", - include_package_data=True, - # tests_require=(["pytest"]), - setup_requires=(["pytest-runner"] if {"test", "pytest", "ptr"} & set(sys.argv) else []), - **extra, -) diff --git a/tests/data/namespace1/amendments1/project_config.yaml b/tests/data/namespace1/amendments1/project_config.yaml index 79da9ab..86c8fd8 100644 --- a/tests/data/namespace1/amendments1/project_config.yaml +++ b/tests/data/namespace1/amendments1/project_config.yaml @@ -7,7 +7,6 @@ sample_modifiers: attributes: [file_path] sources: source1: /data/lab/project/{organism}_{time}h.fastq - source2: /path/from/collaborator/weirdNamingScheme_{external_id}.fastq project_modifiers: amend: newLib: diff --git a/tests/data/namespace2/derive/project_config.yaml b/tests/data/namespace2/derive/project_config.yaml index 445929d..2e23feb 100644 --- a/tests/data/namespace2/derive/project_config.yaml +++ b/tests/data/namespace2/derive/project_config.yaml @@ -7,4 +7,4 @@ sample_modifiers: attributes: [file_path] sources: source1: $HOME/data/lab/project/{organism}_{time}h.fastq - source2: /path/from/collaborator/weirdNamingScheme_{external_id}.fastq + diff --git a/tests/data/namespace3/piface/project_config.yaml b/tests/data/namespace3/piface/project_config.yaml index f808189..7326eaa 100644 --- a/tests/data/namespace3/piface/project_config.yaml +++ b/tests/data/namespace3/piface/project_config.yaml @@ -11,7 +11,7 @@ looper: sample_modifiers: append: attr: "val" - pipeline_interfaces: ["pipeline_interface1_sample.yaml", "pipeline_interface2_sample.yaml"] + pipeline_interfaces: "pipeline_interface1_sample.yaml" derive: attributes: [read1, read2] sources: diff --git a/tests/data/namespace3/remove/project_config.yaml b/tests/data/namespace3/remove/project_config.yaml index 7821eba..8a789eb 100644 --- a/tests/data/namespace3/remove/project_config.yaml +++ b/tests/data/namespace3/remove/project_config.yaml @@ -7,6 +7,5 @@ sample_modifiers: attributes: [file_path] sources: source1: /data/lab/project/{organism}_{time}h.fastq - source2: /path/from/collaborator/weirdNamingScheme_{external_id}.fastq remove: - protocol diff --git a/tests/data/private_test/amendments1/project_config.yaml b/tests/data/private_test/amendments1/project_config.yaml index 79da9ab..86c8fd8 100644 --- a/tests/data/private_test/amendments1/project_config.yaml +++ b/tests/data/private_test/amendments1/project_config.yaml @@ -7,7 +7,6 @@ sample_modifiers: attributes: [file_path] sources: source1: /data/lab/project/{organism}_{time}h.fastq - source2: /path/from/collaborator/weirdNamingScheme_{external_id}.fastq project_modifiers: amend: newLib: diff --git a/tests/data/private_test/derive/project_config.yaml b/tests/data/private_test/derive/project_config.yaml index 445929d..6f24c70 100644 --- a/tests/data/private_test/derive/project_config.yaml +++ b/tests/data/private_test/derive/project_config.yaml @@ -7,4 +7,3 @@ sample_modifiers: attributes: [file_path] sources: source1: $HOME/data/lab/project/{organism}_{time}h.fastq - source2: /path/from/collaborator/weirdNamingScheme_{external_id}.fastq diff --git a/tests/data/private_test/remove/project_config.yaml b/tests/data/private_test/remove/project_config.yaml index 7821eba..8a789eb 100644 --- a/tests/data/private_test/remove/project_config.yaml +++ b/tests/data/private_test/remove/project_config.yaml @@ -7,6 +7,5 @@ sample_modifiers: attributes: [file_path] sources: source1: /data/lab/project/{organism}_{time}h.fastq - source2: /path/from/collaborator/weirdNamingScheme_{external_id}.fastq remove: - protocol diff --git a/tests/test_annotation.py b/tests/test_annotation.py index 9e5603e..07d1466 100644 --- a/tests/test_annotation.py +++ b/tests/test_annotation.py @@ -114,7 +114,9 @@ def test_annotation_limit(self, namespace, limit, admin, n_projects): @pytest.mark.parametrize("admin", ["private_test"]) def test_order_by(self, namespace, admin, order_by, first_name): with PEPDBAgentContextManager(add_data=True) as agent: - result = agent.annotation.get(namespace=namespace, admin=admin, order_by=order_by) + result = agent.annotation.get( + namespace=namespace, admin=admin, order_by=order_by + ) assert result.results[0].name == first_name @pytest.mark.parametrize( @@ -164,7 +166,9 @@ def test_name_search(self, namespace, query, found_number): ) def test_name_search_private(self, namespace, query, found_number): with PEPDBAgentContextManager(add_data=True) as agent: - result = agent.annotation.get(namespace=namespace, query=query, admin="private_test") + result = agent.annotation.get( + namespace=namespace, query=query, admin="private_test" + ) assert len(result.results) == found_number @pytest.mark.parametrize( @@ -289,7 +293,6 @@ def test_search_incorrect_filter_by_string(self, namespace, query, found_number) ) def test_get_annotation_by_rp_list(self, rp_list, admin, found_number): with PEPDBAgentContextManager(add_data=True) as agent: - result = agent.annotation.get_by_rp_list(rp_list) assert len(result.results) == found_number @@ -306,7 +309,6 @@ def test_get_annotation_by_rp_enpty_list(self): ) def test_search_incorrect_incorrect_pep_type(self, namespace, query, found_number): with PEPDBAgentContextManager(add_data=True) as agent: - with pytest.raises(ValueError): agent.annotation.get(namespace=namespace, pep_type="incorrect") diff --git a/tests/test_namespace.py b/tests/test_namespace.py index d4fd309..de36e2d 100644 --- a/tests/test_namespace.py +++ b/tests/test_namespace.py @@ -109,11 +109,17 @@ def test_count_project_one(self, namespace, name): ) def test_remove_from_favorite(self, namespace, name): with PEPDBAgentContextManager(add_data=True) as agent: - agent.user.add_project_to_favorites("namespace1", namespace, name, "default") - agent.user.add_project_to_favorites("namespace1", namespace, "amendments2", "default") + agent.user.add_project_to_favorites( + "namespace1", namespace, name, "default" + ) + agent.user.add_project_to_favorites( + "namespace1", namespace, "amendments2", "default" + ) result = agent.user.get_favorites("namespace1") assert result.count == len(result.results) == 2 - agent.user.remove_project_from_favorites("namespace1", namespace, name, "default") + agent.user.remove_project_from_favorites( + "namespace1", namespace, name, "default" + ) result = agent.user.get_favorites("namespace1") assert result.count == len(result.results) == 1 @@ -126,7 +132,9 @@ def test_remove_from_favorite(self, namespace, name): def test_remove_from_favorite_error(self, namespace, name): with PEPDBAgentContextManager(add_data=True) as agent: with pytest.raises(ProjectNotInFavorites): - agent.user.remove_project_from_favorites("namespace1", namespace, name, "default") + agent.user.remove_project_from_favorites( + "namespace1", namespace, name, "default" + ) @pytest.mark.parametrize( "namespace, name", @@ -136,9 +144,13 @@ def test_remove_from_favorite_error(self, namespace, name): ) def test_favorites_duplication_error(self, namespace, name): with PEPDBAgentContextManager(add_data=True) as agent: - agent.user.add_project_to_favorites("namespace1", namespace, name, "default") + agent.user.add_project_to_favorites( + "namespace1", namespace, name, "default" + ) with pytest.raises(ProjectAlreadyInFavorites): - agent.user.add_project_to_favorites("namespace1", namespace, name, "default") + agent.user.add_project_to_favorites( + "namespace1", namespace, name, "default" + ) @pytest.mark.parametrize( "namespace, name", @@ -148,7 +160,9 @@ def test_favorites_duplication_error(self, namespace, name): ) def test_annotation_favorite_number(self, namespace, name): with PEPDBAgentContextManager(add_data=True) as agent: - agent.user.add_project_to_favorites("namespace1", namespace, name, "default") + agent.user.add_project_to_favorites( + "namespace1", namespace, name, "default" + ) annotations_in_namespace = agent.annotation.get("namespace1") for prj_annot in annotations_in_namespace.results: @@ -169,14 +183,12 @@ class TestUser: def test_create_user(self): with PEPDBAgentContextManager(add_data=True) as agent: - agent.user.create_user("test_user") assert agent.user.exists("test_user") def test_delete_user(self): with PEPDBAgentContextManager(add_data=True) as agent: - test_user = "test_user" agent.user.create_user(test_user) assert agent.user.exists(test_user) @@ -185,7 +197,6 @@ def test_delete_user(self): def test_delete_user_deletes_projects(self): with PEPDBAgentContextManager(add_data=True) as agent: - test_user = "namespace1" assert agent.user.exists(test_user) diff --git a/tests/test_project.py b/tests/test_project.py index 426f026..ea3df5e 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -1,10 +1,14 @@ import numpy as np -import peppy +import peprs import pytest from pepdbagent.exceptions import ProjectNotFoundError -from .utils import PEPDBAgentContextManager, get_path_to_example_file, list_of_available_peps +from .utils import ( + PEPDBAgentContextManager, + get_path_to_example_file, + list_of_available_peps, +) @pytest.mark.skipif( @@ -18,15 +22,15 @@ class TestProject: def test_create_project(self): with PEPDBAgentContextManager(add_data=False) as agent: - prj = peppy.Project(list_of_available_peps()["namespace3"]["subtables"]) + prj = peprs.Project(list_of_available_peps()["namespace3"]["subtables"]) agent.project.create(prj, namespace="test", name="imply", overwrite=False) assert True def test_create_project_from_dict(self): with PEPDBAgentContextManager(add_data=False) as agent: - prj = peppy.Project(list_of_available_peps()["namespace3"]["subtables"]) + prj = peprs.Project(list_of_available_peps()["namespace3"]["subtables"]) agent.project.create( - prj.to_dict(extended=True, orient="records"), + prj.to_dict(raw=True, by_sample=True), namespace="test", name="imply", overwrite=True, @@ -47,9 +51,18 @@ def test_create_project_from_dict(self): ) def test_get_project(self, namespace, name): with PEPDBAgentContextManager(add_data=True) as agent: - kk = agent.project.get(namespace=namespace, name=name, tag="default", raw=False) - ff = peppy.Project(get_path_to_example_file(namespace, name)) - assert kk == ff + kk = agent.project.get( + namespace=namespace, name=name, tag="default", raw=True + ) + ff = peprs.Project(get_path_to_example_file(namespace, name)).to_dict( + raw=True, by_sample=True + ) + # pepdbagent always sets the registry name on the stored config, + # overriding any name in the file (which may be empty or different). + ff["config"]["name"] = name + assert kk["config"] == ff["config"] + assert kk["samples"] == ff["samples"] + assert kk.get("subsamples", []) == ff.get("subsamples", []) @pytest.mark.parametrize( "namespace, name", @@ -65,10 +78,11 @@ def test_get_config(self, namespace, name): name=name, tag="default", ) - ff = peppy.Project(get_path_to_example_file(namespace, name)) - ff["_original_config"]["description"] = description - ff["_original_config"]["name"] = name - assert kk == ff["_original_config"] + ff = peprs.Project(get_path_to_example_file(namespace, name)) + expected_config = ff.config.copy() + expected_config["description"] = description + expected_config["name"] = name + assert kk == expected_config @pytest.mark.parametrize( "namespace, name", @@ -83,11 +97,11 @@ def test_get_subsamples(self, namespace, name): name=name, tag="default", ) - orgiginal_prj = peppy.Project(get_path_to_example_file(namespace, name)) + orgiginal_prj = peprs.Project(get_path_to_example_file(namespace, name)) assert ( prj_subtables - == orgiginal_prj.to_dict(extended=True, orient="records")["_subsample_list"] + == orgiginal_prj.to_dict(raw=True, by_sample=True)["subsamples"] ) @pytest.mark.parametrize( @@ -101,11 +115,11 @@ def test_get_samples_raw(self, namespace, name): prj_samples = agent.project.get_samples( namespace=namespace, name=name, tag="default", raw=True ) - orgiginal_prj = peppy.Project(get_path_to_example_file(namespace, name)) + orgiginal_prj = peprs.Project(get_path_to_example_file(namespace, name)) assert ( prj_samples - == orgiginal_prj.to_dict(extended=True, orient="records")["_sample_dict"] + == orgiginal_prj.to_dict(raw=True, by_sample=True)["samples"] ) @pytest.mark.parametrize( @@ -122,12 +136,26 @@ def test_get_samples_processed(self, namespace, name): tag="default", raw=False, ) - orgiginal_prj = peppy.Project(get_path_to_example_file(namespace, name)) - - assert prj_samples == orgiginal_prj.sample_table.replace({np.nan: None}).to_dict( - orient="records" + orgiginal_prj = peprs.Project(get_path_to_example_file(namespace, name)) + expected = ( + orgiginal_prj.to_pandas() + .replace({np.nan: None}) + .to_dict(orient="records") ) + # Normalize numpy arrays (used for subsample list columns) to plain lists + # so dict equality works without raising "truth value is ambiguous". + def _normalize(samples): + return [ + { + k: (v.tolist() if isinstance(v, np.ndarray) else v) + for k, v in s.items() + } + for s in samples + ] + + assert _normalize(prj_samples) == _normalize(expected) + @pytest.mark.parametrize( "namespace, name,tag", [ @@ -164,7 +192,9 @@ def test_overwrite_project(self, namespace, name): overwrite=True, ) - assert agent.project.get(namespace=namespace, name=name, raw=False) == new_prj + assert ( + agent.project.get(namespace=namespace, name=name, raw=False) == new_prj + ) @pytest.mark.parametrize( "namespace, name", @@ -183,7 +213,9 @@ def test_delete_project(self, namespace, name): def test_delete_not_existing_project(self): with PEPDBAgentContextManager(add_data=True) as agent: with pytest.raises(ProjectNotFoundError, match="Project does not exist."): - agent.project.delete(namespace="namespace1", name="nothing", tag="default") + agent.project.delete( + namespace="namespace1", name="nothing", tag="default" + ) @pytest.mark.parametrize( "namespace, name", @@ -204,7 +236,9 @@ def test_fork_projects(self, namespace, name): fork_tag="new_tag", ) - assert agent.project.exists(namespace="new_namespace", name="new_name", tag="new_tag") + assert agent.project.exists( + namespace="new_namespace", name="new_name", tag="new_tag" + ) result = agent.annotation.get( namespace="new_namespace", name="new_name", tag="new_tag" ) @@ -231,9 +265,13 @@ def test_parent_project_delete(self, namespace, name): fork_tag="new_tag", ) - assert agent.project.exists(namespace="new_namespace", name="new_name", tag="new_tag") + assert agent.project.exists( + namespace="new_namespace", name="new_name", tag="new_tag" + ) agent.project.delete(namespace=namespace, name=name, tag="default") - assert agent.project.exists(namespace="new_namespace", name="new_name", tag="new_tag") + assert agent.project.exists( + namespace="new_namespace", name="new_name", tag="new_tag" + ) @pytest.mark.parametrize( "namespace, name", @@ -256,9 +294,13 @@ def test_child_project_delete(self, namespace, name): fork_tag="new_tag", ) - assert agent.project.exists(namespace="new_namespace", name="new_name", tag="new_tag") + assert agent.project.exists( + namespace="new_namespace", name="new_name", tag="new_tag" + ) assert agent.project.exists(namespace=namespace, name=name, tag="default") - agent.project.delete(namespace="new_namespace", name="new_name", tag="new_tag") + agent.project.delete( + namespace="new_namespace", name="new_name", tag="new_tag" + ) assert agent.project.exists(namespace=namespace, name=name, tag="default") @pytest.mark.parametrize( diff --git a/tests/test_project_history.py b/tests/test_project_history.py index 332e1ca..8bd38af 100644 --- a/tests/test_project_history.py +++ b/tests/test_project_history.py @@ -1,4 +1,4 @@ -import peppy +import peprs import pytest from pepdbagent.const import PEPHUB_SAMPLE_ID_KEY @@ -26,10 +26,10 @@ def test_get_add_history_all_annotation(self, namespace, name, sample_name): with PEPDBAgentContextManager(add_data=True) as agent: prj = agent.project.get(namespace, name, tag="default", with_id=True) - prj["_sample_dict"][0]["sample_name"] = "new_sample_name" + prj["samples"][0]["sample_name"] = "new_sample_name" - del prj["_sample_dict"][1] - del prj["_sample_dict"][2] + del prj["samples"][1] + del prj["samples"][2] new_sample1 = { "sample_name": "new_sample", "protocol": "new_protocol", @@ -41,14 +41,14 @@ def test_get_add_history_all_annotation(self, namespace, name, sample_name): PEPHUB_SAMPLE_ID_KEY: None, } - prj["_sample_dict"].append(new_sample1.copy()) - prj["_sample_dict"].append(new_sample2.copy()) + prj["samples"].append(new_sample1.copy()) + prj["samples"].append(new_sample2.copy()) agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) project_history = agent.project.get_history(namespace, name, tag="default") @@ -66,10 +66,10 @@ def test_get_add_history_all_project(self, namespace, name, sample_name): prj_init = agent.project.get(namespace, name, tag="default", raw=False) prj = agent.project.get(namespace, name, tag="default", with_id=True) - # prj["_sample_dict"][0]["sample_name"] = "new_sample_name" + # prj["samples"][0]["sample_name"] = "new_sample_name" - del prj["_sample_dict"][1] - del prj["_sample_dict"][2] + del prj["samples"][1] + del prj["samples"][2] new_sample1 = { "sample_name": "new_sample", "protocol": "new_protocol", @@ -81,14 +81,14 @@ def test_get_add_history_all_project(self, namespace, name, sample_name): PEPHUB_SAMPLE_ID_KEY: None, } - prj["_sample_dict"].append(new_sample1.copy()) - prj["_sample_dict"].append(new_sample2.copy()) + prj["samples"].append(new_sample1.copy()) + prj["samples"].append(new_sample2.copy()) agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) history_prj = agent.project.get_project_from_history( @@ -106,13 +106,13 @@ def test_get_history_multiple_changes(self, namespace, name, sample_name): with PEPDBAgentContextManager(add_data=True) as agent: prj = agent.project.get(namespace, name, tag="default", with_id=True) - del prj["_sample_dict"][1] + del prj["samples"][1] agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) prj = agent.project.get(namespace, name, tag="default", with_id=True) @@ -122,13 +122,13 @@ def test_get_history_multiple_changes(self, namespace, name, sample_name): "protocol": "new_protocol", PEPHUB_SAMPLE_ID_KEY: None, } - prj["_sample_dict"].append(new_sample1.copy()) + prj["samples"].append(new_sample1.copy()) agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) history = agent.project.get_history(namespace, name, tag="default") @@ -144,13 +144,13 @@ def test_get_history_multiple_changes(self, namespace, name, sample_name): def test_get_project_incorrect_history_id(self, namespace, name, sample_name): with PEPDBAgentContextManager(add_data=True) as agent: prj = agent.project.get(namespace, name, tag="default", with_id=True) - del prj["_sample_dict"][1] + del prj["samples"][1] agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) with pytest.raises(HistoryNotFoundError): @@ -179,13 +179,13 @@ def test_delete_all_history(self, namespace, name, sample_name): with PEPDBAgentContextManager(add_data=True) as agent: prj = agent.project.get(namespace, name, tag="default", with_id=True) - del prj["_sample_dict"][1] + del prj["samples"][1] agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) prj = agent.project.get(namespace, name, tag="default", with_id=True) @@ -195,20 +195,22 @@ def test_delete_all_history(self, namespace, name, sample_name): "protocol": "new_protocol", PEPHUB_SAMPLE_ID_KEY: None, } - prj["_sample_dict"].append(new_sample1.copy()) + prj["samples"].append(new_sample1.copy()) agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) history = agent.project.get_history(namespace, name, tag="default") assert len(history.history) == 2 - agent.project.delete_history(namespace, name, tag="default", history_id=None) + agent.project.delete_history( + namespace, name, tag="default", history_id=None + ) history = agent.project.get_history(namespace, name, tag="default") assert len(history.history) == 0 @@ -226,13 +228,13 @@ def test_delete_one_history(self, namespace, name, sample_name): with PEPDBAgentContextManager(add_data=True) as agent: prj = agent.project.get(namespace, name, tag="default", with_id=True) - del prj["_sample_dict"][1] + del prj["samples"][1] agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) prj = agent.project.get(namespace, name, tag="default", with_id=True) @@ -242,13 +244,13 @@ def test_delete_one_history(self, namespace, name, sample_name): "protocol": "new_protocol", PEPHUB_SAMPLE_ID_KEY: None, } - prj["_sample_dict"].append(new_sample1.copy()) + prj["samples"].append(new_sample1.copy()) agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) history = agent.project.get_history(namespace, name, tag="default") @@ -273,13 +275,13 @@ def test_restore_project(self, namespace, name, sample_name): prj_org = agent.project.get(namespace, name, tag="default", with_id=False) prj = agent.project.get(namespace, name, tag="default", with_id=True) - del prj["_sample_dict"][1] + del prj["samples"][1] agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) prj = agent.project.get(namespace, name, tag="default", with_id=True) @@ -289,17 +291,19 @@ def test_restore_project(self, namespace, name, sample_name): "protocol": "new_protocol", PEPHUB_SAMPLE_ID_KEY: None, } - prj["_sample_dict"].append(new_sample1.copy()) + prj["samples"].append(new_sample1.copy()) agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) agent.project.restore(namespace, name, tag="default", history_id=1) - restored_project = agent.project.get(namespace, name, tag="default", with_id=False) + restored_project = agent.project.get( + namespace, name, tag="default", with_id=False + ) assert prj_org == restored_project diff --git a/tests/test_samples.py b/tests/test_samples.py index e8a6862..d0b0d6e 100644 --- a/tests/test_samples.py +++ b/tests/test_samples.py @@ -1,4 +1,4 @@ -import peppy +import peprs import pytest from pepdbagent.exceptions import SampleNotFoundError @@ -20,7 +20,7 @@ class TestSamples: def test_retrieve_one_sample(self, namespace, name, sample_name): with PEPDBAgentContextManager(add_data=True) as agent: one_sample = agent.sample.get(namespace, name, sample_name, raw=False) - assert isinstance(one_sample, peppy.Sample) + assert isinstance(one_sample, peprs.Sample) assert one_sample.sample_name == sample_name @pytest.mark.parametrize( @@ -41,10 +41,12 @@ def test_retrieve_raw_sample(self, namespace, name, sample_name): ["namespace2", "custom_index", "frog_1"], ], ) - def test_retrieve_sample_with_modified_sample_id(self, namespace, name, sample_name): + def test_retrieve_sample_with_modified_sample_id( + self, namespace, name, sample_name + ): with PEPDBAgentContextManager(add_data=True) as agent: one_sample = agent.sample.get(namespace, name, sample_name, raw=False) - assert isinstance(one_sample, peppy.Sample) + assert isinstance(one_sample, peprs.Sample) assert one_sample.sample_id == "frog_1" @pytest.mark.parametrize( @@ -142,7 +144,8 @@ def test_project_timestamp_was_changed(self, namespace, name, sample_name): annotation2 = agent.annotation.get(namespace, name, "default") assert ( - annotation1.results[0].last_update_date != annotation2.results[0].last_update_date + annotation1.results[0].last_update_date + != annotation2.results[0].last_update_date ) @pytest.mark.parametrize( @@ -154,7 +157,7 @@ def test_project_timestamp_was_changed(self, namespace, name, sample_name): def test_delete_sample(self, namespace, name, sample_name): with PEPDBAgentContextManager(add_data=True) as agent: one_sample = agent.sample.get(namespace, name, sample_name, raw=False) - assert isinstance(one_sample, peppy.Sample) + assert isinstance(one_sample, peprs.Sample) agent.sample.delete(namespace, name, tag="default", sample_name=sample_name) @@ -203,7 +206,12 @@ def test_add_sample(self, namespace, name, tag, sample_dict): ) def test_overwrite_sample(self, namespace, name, tag, sample_dict): with PEPDBAgentContextManager(add_data=True) as agent: - assert agent.project.get(namespace, name, raw=False).get_sample("pig_0h").time == "0" + # peprs/polars infers numeric values from the sample table, so the + # original time is loaded as int (not string). + assert ( + agent.project.get(namespace, name, raw=False).get_sample("pig_0h").time + == 0 + ) agent.sample.add(namespace, name, tag, sample_dict, overwrite=True) assert ( @@ -232,4 +240,7 @@ def test_delete_and_add(self, namespace, name, tag, sample_dict): agent.sample.delete(namespace, name, tag, "pig_0h") agent.sample.add(namespace, name, tag, sample_dict) prj2 = agent.project.get(namespace, name, raw=False) - assert prj.get_sample("pig_0h").to_dict() == prj2.get_sample("pig_0h").to_dict() + assert ( + prj.get_sample("pig_0h").to_dict() + == prj2.get_sample("pig_0h").to_dict() + ) diff --git a/tests/test_schema.py b/tests/test_schema.py index d4b8ecb..e8e1171 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -1,8 +1,8 @@ import pytest -from .utils import PEPDBAgentContextManager +from pepdbagent.models import UpdateSchemaRecordFields, UpdateSchemaVersionFields -from pepdbagent.models import UpdateSchemaVersionFields, UpdateSchemaRecordFields +from .utils import PEPDBAgentContextManager DEFAULT_SCHEMA_VERSION = "1.0.0" @@ -12,7 +12,6 @@ reason="DB is not setup", ) class TestSchemas: - @pytest.mark.parametrize( "namespace, name", [ @@ -64,7 +63,9 @@ def test_update_schema_update_date(self): }, } - first_time = agent.schema.get_schema_info("namespace1", "2.0.0").last_update_date + first_time = agent.schema.get_schema_info( + "namespace1", "2.0.0" + ).last_update_date agent.schema.add_version( "namespace1", @@ -98,10 +99,15 @@ def test_add_schema_version(self): release_notes=new_release_notes, ), ) - result = agent.schema.get_version_info("namespace1", "2.0.0", DEFAULT_SCHEMA_VERSION) + result = agent.schema.get_version_info( + "namespace1", "2.0.0", DEFAULT_SCHEMA_VERSION + ) assert result.contributors == new_contributors assert result.release_notes == new_release_notes - assert agent.schema.get("namespace1", "2.0.0", DEFAULT_SCHEMA_VERSION) == new_schema + assert ( + agent.schema.get("namespace1", "2.0.0", DEFAULT_SCHEMA_VERSION) + == new_schema + ) def test_search_schema_version(self): with PEPDBAgentContextManager(add_schemas=True) as agent: @@ -143,12 +149,16 @@ def test_search_schema_version_with_tags(self): release_notes="language", ) - result = agent.schema.query_schema_version("namespace1", "2.0.0", tag="tag1") + result = agent.schema.query_schema_version( + "namespace1", "2.0.0", tag="tag1" + ) assert result.pagination.total == 1 assert len(result.results) == 1 - result = agent.schema.query_schema_version("namespace1", "2.0.0", tag="bioinfo") + result = agent.schema.query_schema_version( + "namespace1", "2.0.0", tag="bioinfo" + ) assert result.pagination.total == 2 assert len(result.results) == 2 @@ -226,7 +236,9 @@ def test_insert_tags(self): "namespace1", "2.0.0", DEFAULT_SCHEMA_VERSION, tag=[new_tag1, new_tag2] ) - result = agent.schema.get_version_info("namespace1", "2.0.0", DEFAULT_SCHEMA_VERSION) + result = agent.schema.get_version_info( + "namespace1", "2.0.0", DEFAULT_SCHEMA_VERSION + ) assert new_tag1 in result.tags assert new_tag2 in result.tags @@ -237,7 +249,9 @@ def test_insert_one_tag(self): agent.schema.add_tag_to_schema( "namespace1", "2.0.0", DEFAULT_SCHEMA_VERSION, tag=new_tag1 ) - result = agent.schema.get_version_info("namespace1", "2.0.0", DEFAULT_SCHEMA_VERSION) + result = agent.schema.get_version_info( + "namespace1", "2.0.0", DEFAULT_SCHEMA_VERSION + ) assert new_tag1 in result.tags @pytest.mark.parametrize( @@ -249,11 +263,17 @@ def test_insert_one_tag(self): def test_delete_tag(self, namespace, name): with PEPDBAgentContextManager(add_schemas=True) as agent: new_tag1 = "new_tag" - agent.schema.add_tag_to_schema(namespace, name, DEFAULT_SCHEMA_VERSION, tag=new_tag1) - result = agent.schema.get_version_info(namespace, name, DEFAULT_SCHEMA_VERSION) + agent.schema.add_tag_to_schema( + namespace, name, DEFAULT_SCHEMA_VERSION, tag=new_tag1 + ) + result = agent.schema.get_version_info( + namespace, name, DEFAULT_SCHEMA_VERSION + ) assert new_tag1 in result.tags agent.schema.remove_tag_from_schema( namespace, name, DEFAULT_SCHEMA_VERSION, tag=new_tag1 ) - result = agent.schema.get_version_info(namespace, name, DEFAULT_SCHEMA_VERSION) + result = agent.schema.get_version_info( + namespace, name, DEFAULT_SCHEMA_VERSION + ) assert new_tag1 not in result.tags diff --git a/tests/test_tar_meta.py b/tests/test_tar_meta.py index 2488a4b..99d2caa 100644 --- a/tests/test_tar_meta.py +++ b/tests/test_tar_meta.py @@ -29,7 +29,6 @@ class TestGeoTar: def test_create_meta_tar(self): with PEPDBAgentContextManager(add_data=True) as agent: - agent.namespace.upload_tar_info(tar_info=self.tar_info) result = agent.namespace.get_tar_info(namespace=self.test_namespace) diff --git a/tests/test_updates.py b/tests/test_updates.py index 628772e..d762c12 100644 --- a/tests/test_updates.py +++ b/tests/test_updates.py @@ -1,9 +1,11 @@ -import peppy +import peprs import pytest -from peppy.exceptions import IllegalStateException from pepdbagent.const import PEPHUB_SAMPLE_ID_KEY -from pepdbagent.exceptions import ProjectDuplicatedSampleGUIDsError, SampleTableUpdateError +from pepdbagent.exceptions import ( + ProjectDuplicatedSampleGUIDsError, + SampleTableUpdateError, +) from .utils import PEPDBAgentContextManager @@ -28,7 +30,9 @@ def test_update_project_name(self, namespace, name, new_name): tag="default", update_dict={"name": new_name}, ) - assert agent.project.exists(namespace=namespace, name=new_name, tag="default") + assert agent.project.exists( + namespace=namespace, name=new_name, tag="default" + ) @pytest.mark.parametrize( "namespace, name,new_name", @@ -39,7 +43,9 @@ def test_update_project_name(self, namespace, name, new_name): ) def test_update_project_name_in_config(self, namespace, name, new_name): with PEPDBAgentContextManager(add_data=True) as agent: - prj = agent.project.get(namespace=namespace, name=name, raw=False, with_id=True) + prj = agent.project.get( + namespace=namespace, name=name, raw=False, with_id=True + ) prj.name = new_name agent.project.update( namespace=namespace, @@ -47,7 +53,9 @@ def test_update_project_name_in_config(self, namespace, name, new_name): tag="default", update_dict={"project": prj}, ) - assert agent.project.exists(namespace=namespace, name=new_name, tag="default") + assert agent.project.exists( + namespace=namespace, name=new_name, tag="default" + ) @pytest.mark.parametrize( "namespace, name, new_tag", @@ -116,9 +124,13 @@ def test_update_project_schema(self, namespace, name, new_schema): ["namespace1", "amendments1", "desc1 f"], ], ) - def test_update_project_description_in_config(self, namespace, name, new_description): + def test_update_project_description_in_config( + self, namespace, name, new_description + ): with PEPDBAgentContextManager(add_data=True) as agent: - prj = agent.project.get(namespace=namespace, name=name, raw=False, with_id=True) + prj = agent.project.get( + namespace=namespace, name=name, raw=False, with_id=True + ) prj.description = new_description agent.project.update( namespace=namespace, @@ -247,17 +259,19 @@ def test_project_can_have_2_sample_names(self, namespace, name): ensure that update works correctly """ with PEPDBAgentContextManager(add_data=True) as agent: - new_prj = agent.project.get(namespace=namespace, name=name, raw=False, with_id=True) - prj_dict = new_prj.to_dict(extended=True, orient="records") + new_prj = agent.project.get( + namespace=namespace, name=name, raw=False, with_id=True + ) + prj_dict = new_prj.to_dict(raw=True, by_sample=True) - prj_dict["_sample_dict"].append( + prj_dict["samples"].append( { "file": "data/frog23_data.txt", "protocol": "anySample3Type", "sample_name": "frog_2", } ) - prj_dict["_sample_dict"].append( + prj_dict["samples"].append( { "file": "data/frog23_data.txt4", "protocol": "anySample3Type4", @@ -270,12 +284,12 @@ def test_project_can_have_2_sample_names(self, namespace, name): namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj_dict)}, + update_dict={"project": peprs.Project.from_dict(prj_dict)}, ) prj = agent.project.get(namespace=namespace, name=name, raw=True) - assert len(prj["_sample_dict"]) == 4 + assert len(prj["samples"]) == 4 @pytest.mark.skipif( @@ -296,7 +310,9 @@ def test_update_whole_project_with_id(self, namespace, name): """ with PEPDBAgentContextManager(add_data=True) as agent: peppy_prj = agent.project.get(namespace=namespace, name=name, raw=True) - prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) + prj = agent.project.get( + namespace=namespace, name=name, raw=True, with_id=True + ) new_sample = { "sample_name": "new_sample", @@ -304,23 +320,23 @@ def test_update_whole_project_with_id(self, namespace, name): PEPHUB_SAMPLE_ID_KEY: None, } - prj["_sample_dict"].append(new_sample.copy()) - prj["_sample_dict"][0]["sample_name"] = "new_sample_name2" - del prj["_sample_dict"][1] + prj["samples"].append(new_sample.copy()) + prj["samples"][0]["sample_name"] = "new_sample_name2" + del prj["samples"][1] agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) del new_sample[PEPHUB_SAMPLE_ID_KEY] - peppy_prj["_sample_dict"].append(new_sample.copy()) # add sample without id - peppy_prj["_sample_dict"][0]["sample_name"] = "new_sample_name2" # modify sample - del peppy_prj["_sample_dict"][1] # delete sample + peppy_prj["samples"].append(new_sample.copy()) # add sample without id + peppy_prj["samples"][0]["sample_name"] = "new_sample_name2" # modify sample + del peppy_prj["samples"][1] # delete sample - assert peppy.Project.from_dict(peppy_prj) == agent.project.get( + assert peprs.Project.from_dict(peppy_prj) == agent.project.get( namespace=namespace, name=name, raw=False ) @@ -334,7 +350,9 @@ def test_update_whole_project_with_id(self, namespace, name): def test_insert_new_row(self, namespace, name): with PEPDBAgentContextManager(add_data=True) as agent: peppy_prj = agent.project.get(namespace=namespace, name=name, raw=True) - prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) + prj = agent.project.get( + namespace=namespace, name=name, raw=True, with_id=True + ) new_sample = { "sample_name": "new_sample", @@ -342,19 +360,19 @@ def test_insert_new_row(self, namespace, name): PEPHUB_SAMPLE_ID_KEY: None, } - prj["_sample_dict"].append(new_sample.copy()) + prj["samples"].append(new_sample.copy()) agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) del new_sample[PEPHUB_SAMPLE_ID_KEY] - peppy_prj["_sample_dict"].append(new_sample.copy()) # add sample without id + peppy_prj["samples"].append(new_sample.copy()) # add sample without id - assert peppy.Project.from_dict(peppy_prj) == agent.project.get( + assert peprs.Project.from_dict(peppy_prj) == agent.project.get( namespace=namespace, name=name, raw=False ) @@ -368,7 +386,9 @@ def test_insert_new_row(self, namespace, name): def test_insert_new_multiple_rows(self, namespace, name): with PEPDBAgentContextManager(add_data=True) as agent: peppy_prj = agent.project.get(namespace=namespace, name=name, raw=True) - prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) + prj = agent.project.get( + namespace=namespace, name=name, raw=True, with_id=True + ) new_sample1 = { "sample_name": "new_sample", @@ -381,22 +401,22 @@ def test_insert_new_multiple_rows(self, namespace, name): PEPHUB_SAMPLE_ID_KEY: None, } - prj["_sample_dict"].append(new_sample1.copy()) - prj["_sample_dict"].append(new_sample2.copy()) + prj["samples"].append(new_sample1.copy()) + prj["samples"].append(new_sample2.copy()) agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) del new_sample1[PEPHUB_SAMPLE_ID_KEY] del new_sample2[PEPHUB_SAMPLE_ID_KEY] - peppy_prj["_sample_dict"].append(new_sample1.copy()) # add sample without id - peppy_prj["_sample_dict"].append(new_sample2.copy()) # add sample without id + peppy_prj["samples"].append(new_sample1.copy()) # add sample without id + peppy_prj["samples"].append(new_sample2.copy()) # add sample without id - assert peppy.Project.from_dict(peppy_prj) == agent.project.get( + assert peprs.Project.from_dict(peppy_prj) == agent.project.get( namespace=namespace, name=name, raw=False ) @@ -407,8 +427,14 @@ def test_insert_new_multiple_rows(self, namespace, name): ], ) def test_insert_new_multiple_rows_duplicated_samples(self, namespace, name): + """PEP 2.1.0 allows duplicate sample names, so this should succeed.""" with PEPDBAgentContextManager(add_data=True) as agent: - prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) + original_count = len( + agent.project.get(namespace=namespace, name=name, raw=True)["samples"] + ) + prj = agent.project.get( + namespace=namespace, name=name, raw=True, with_id=True + ) new_sample1 = { "sample_name": "new_sample", @@ -421,16 +447,17 @@ def test_insert_new_multiple_rows_duplicated_samples(self, namespace, name): PEPHUB_SAMPLE_ID_KEY: None, } - prj["_sample_dict"].append(new_sample1.copy()) - prj["_sample_dict"].append(new_sample2.copy()) + prj["samples"].append(new_sample1.copy()) + prj["samples"].append(new_sample2.copy()) - with pytest.raises(IllegalStateException): - agent.project.update( - namespace=namespace, - name=name, - tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, - ) + agent.project.update( + namespace=namespace, + name=name, + tag="default", + update_dict={"project": peprs.Project.from_dict(prj)}, + ) + updated = agent.project.get(namespace=namespace, name=name, raw=True) + assert len(updated["samples"]) == original_count + 2 @pytest.mark.parametrize( "namespace, name", @@ -442,22 +469,24 @@ def test_insert_new_multiple_rows_duplicated_samples(self, namespace, name): def test_delete_multiple_rows(self, namespace, name): with PEPDBAgentContextManager(add_data=True) as agent: peppy_prj = agent.project.get(namespace=namespace, name=name, raw=True) - prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) + prj = agent.project.get( + namespace=namespace, name=name, raw=True, with_id=True + ) - del prj["_sample_dict"][1] - del prj["_sample_dict"][2] + del prj["samples"][1] + del prj["samples"][2] agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) - del peppy_prj["_sample_dict"][1] # delete sample - del peppy_prj["_sample_dict"][2] # delete sample + del peppy_prj["samples"][1] # delete sample + del peppy_prj["samples"][2] # delete sample - assert peppy.Project.from_dict(peppy_prj) == agent.project.get( + assert peprs.Project.from_dict(peppy_prj) == agent.project.get( namespace=namespace, name=name, raw=False ) @@ -471,20 +500,22 @@ def test_delete_multiple_rows(self, namespace, name): def test_modify_one_row(self, namespace, name): with PEPDBAgentContextManager(add_data=True) as agent: peppy_prj = agent.project.get(namespace=namespace, name=name, raw=True) - prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) + prj = agent.project.get( + namespace=namespace, name=name, raw=True, with_id=True + ) - prj["_sample_dict"][0]["sample_name"] = "new_sample_name2" + prj["samples"][0]["sample_name"] = "new_sample_name2" agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) - peppy_prj["_sample_dict"][0]["sample_name"] = "new_sample_name2" # modify sample + peppy_prj["samples"][0]["sample_name"] = "new_sample_name2" # modify sample - assert peppy.Project.from_dict(peppy_prj) == agent.project.get( + assert peprs.Project.from_dict(peppy_prj) == agent.project.get( namespace=namespace, name=name, raw=False ) @@ -498,22 +529,24 @@ def test_modify_one_row(self, namespace, name): def test_modify_multiple_rows(self, namespace, name): with PEPDBAgentContextManager(add_data=True) as agent: peppy_prj = agent.project.get(namespace=namespace, name=name, raw=True) - prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) + prj = agent.project.get( + namespace=namespace, name=name, raw=True, with_id=True + ) - prj["_sample_dict"][0]["sample_name"] = "new_sample_name2" - prj["_sample_dict"][1]["sample_name"] = "new_sample_name3" + prj["samples"][0]["sample_name"] = "new_sample_name2" + prj["samples"][1]["sample_name"] = "new_sample_name3" agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) - peppy_prj["_sample_dict"][0]["sample_name"] = "new_sample_name2" # modify sample - peppy_prj["_sample_dict"][1]["sample_name"] = "new_sample_name3" # modify sample + peppy_prj["samples"][0]["sample_name"] = "new_sample_name2" # modify sample + peppy_prj["samples"][1]["sample_name"] = "new_sample_name3" # modify sample - assert peppy.Project.from_dict(peppy_prj) == agent.project.get( + assert peprs.Project.from_dict(peppy_prj) == agent.project.get( namespace=namespace, name=name, raw=False ) @@ -527,7 +560,9 @@ def test_modify_multiple_rows(self, namespace, name): def test_add_new_first_sample(self, namespace, name): with PEPDBAgentContextManager(add_data=True) as agent: peppy_prj = agent.project.get(namespace=namespace, name=name, raw=True) - prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) + prj = agent.project.get( + namespace=namespace, name=name, raw=True, with_id=True + ) new_sample = { "sample_name": "new_sample", @@ -535,19 +570,19 @@ def test_add_new_first_sample(self, namespace, name): PEPHUB_SAMPLE_ID_KEY: None, } - prj["_sample_dict"].insert(0, new_sample.copy()) + prj["samples"].insert(0, new_sample.copy()) agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) del new_sample[PEPHUB_SAMPLE_ID_KEY] - peppy_prj["_sample_dict"].insert(0, new_sample.copy()) # add sample without id + peppy_prj["samples"].insert(0, new_sample.copy()) # add sample without id - assert peppy.Project.from_dict(peppy_prj) == agent.project.get( + assert peprs.Project.from_dict(peppy_prj) == agent.project.get( namespace=namespace, name=name, raw=False ) @@ -561,28 +596,30 @@ def test_add_new_first_sample(self, namespace, name): def test_change_sample_order(self, namespace, name): with PEPDBAgentContextManager(add_data=True) as agent: peppy_prj = agent.project.get(namespace=namespace, name=name, raw=True) - prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) + prj = agent.project.get( + namespace=namespace, name=name, raw=True, with_id=True + ) - sample1 = prj["_sample_dict"][0].copy() - sample2 = prj["_sample_dict"][1].copy() + sample1 = prj["samples"][0].copy() + sample2 = prj["samples"][1].copy() - prj["_sample_dict"][0] = sample2 - prj["_sample_dict"][1] = sample1 + prj["samples"][0] = sample2 + prj["samples"][1] = sample1 agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) - peppy_prj["_sample_dict"][0] = sample2 - peppy_prj["_sample_dict"][1] = sample1 + peppy_prj["samples"][0] = sample2 + peppy_prj["samples"][1] = sample1 - del peppy_prj["_sample_dict"][0][PEPHUB_SAMPLE_ID_KEY] - del peppy_prj["_sample_dict"][1][PEPHUB_SAMPLE_ID_KEY] + del peppy_prj["samples"][0][PEPHUB_SAMPLE_ID_KEY] + del peppy_prj["samples"][1][PEPHUB_SAMPLE_ID_KEY] - assert peppy.Project.from_dict(peppy_prj) == agent.project.get( + assert peprs.Project.from_dict(peppy_prj) == agent.project.get( namespace=namespace, name=name, raw=False ) @@ -595,17 +632,18 @@ def test_change_sample_order(self, namespace, name): ) def test_update_porject_without_ids(self, namespace, name): with PEPDBAgentContextManager(add_data=True) as agent: - prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=False) + prj = agent.project.get( + namespace=namespace, name=name, raw=True, with_id=False + ) - prj["_sample_dict"][0]["sample_name"] = "new_sample_name2" + prj["samples"][0]["sample_name"] = "new_sample_name2" with pytest.raises(SampleTableUpdateError): - agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(prj)}, + update_dict={"project": peprs.Project.from_dict(prj)}, ) @pytest.mark.parametrize( @@ -616,13 +654,15 @@ def test_update_porject_without_ids(self, namespace, name): ) def test_update_project_with_duplicated_sample_guids(self, namespace, name): with PEPDBAgentContextManager(add_data=True) as agent: - new_prj = agent.project.get(namespace=namespace, name=name, raw=True, with_id=True) - new_prj["_sample_dict"].append(new_prj["_sample_dict"][0]) + new_prj = agent.project.get( + namespace=namespace, name=name, raw=True, with_id=True + ) + new_prj["samples"].append(new_prj["samples"][0]) with pytest.raises(ProjectDuplicatedSampleGUIDsError): agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"project": peppy.Project.from_dict(new_prj)}, + update_dict={"project": peprs.Project.from_dict(new_prj)}, ) diff --git a/tests/test_views.py b/tests/test_views.py index 475936b..85ee0de 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -38,7 +38,9 @@ def test_create_view(self, namespace, name, sample_name, view_name): ) project = agent.project.get(namespace, name, raw=False) - view_project = agent.view.get(namespace, name, "default", view_name, raw=False) + view_project = agent.view.get( + namespace, name, "default", view_name, raw=False + ) assert len(view_project.samples) == 2 assert view_project != project @@ -48,7 +50,9 @@ def test_create_view(self, namespace, name, sample_name, view_name): ["namespace1", "amendments1", "pig_0h", "view1"], ], ) - def test_create_view_with_incorrect_sample(self, namespace, name, sample_name, view_name): + def test_create_view_with_incorrect_sample( + self, namespace, name, sample_name, view_name + ): with PEPDBAgentContextManager(add_data=True) as agent: with pytest.raises(SampleNotFoundError): agent.view.create( @@ -82,7 +86,9 @@ def test_create_view_with_incorrect_sample_no_fail( no_fail=True, ) project = agent.project.get(namespace, name, raw=False) - view_project = agent.view.get(namespace, name, "default", view_name, raw=False) + view_project = agent.view.get( + namespace, name, "default", view_name, raw=False + ) assert len(view_project.samples) == 2 assert view_project != project @@ -103,7 +109,14 @@ def test_delete_view(self, namespace, name, sample_name): "sample_list": [sample_name, "pig_1h"], }, ) - assert len(agent.view.get(namespace, name, "default", "view1", raw=False).samples) == 2 + assert ( + len( + agent.view.get( + namespace, name, "default", "view1", raw=False + ).samples + ) + == 2 + ) agent.view.delete(namespace, name, "default", "view1") with pytest.raises(ViewNotFoundError): agent.view.get(namespace, name, "default", "view1", raw=False) @@ -127,7 +140,14 @@ def test_add_sample_to_view(self, namespace, name, sample_name): }, ) agent.view.add_sample(namespace, name, "default", "view1", "pig_1h") - assert len(agent.view.get(namespace, name, "default", "view1", raw=False).samples) == 2 + assert ( + len( + agent.view.get( + namespace, name, "default", "view1", raw=False + ).samples + ) + == 2 + ) @pytest.mark.parametrize( "namespace, name, sample_name", @@ -146,8 +166,17 @@ def test_add_multiple_samples_to_view(self, namespace, name, sample_name): "sample_list": [sample_name], }, ) - agent.view.add_sample(namespace, name, "default", "view1", ["pig_1h", "frog_0h"]) - assert len(agent.view.get(namespace, name, "default", "view1", raw=False).samples) == 3 + agent.view.add_sample( + namespace, name, "default", "view1", ["pig_1h", "frog_0h"] + ) + assert ( + len( + agent.view.get( + namespace, name, "default", "view1", raw=False + ).samples + ) + == 3 + ) @pytest.mark.parametrize( "namespace, name, sample_name", @@ -167,11 +196,20 @@ def test_remove_sample_from_view(self, namespace, name, sample_name): }, ) agent.view.remove_sample(namespace, name, "default", "view1", sample_name) - assert len(agent.view.get(namespace, name, "default", "view1", raw=False).samples) == 1 + assert ( + len( + agent.view.get( + namespace, name, "default", "view1", raw=False + ).samples + ) + == 1 + ) assert len(agent.project.get(namespace, name, raw=False).samples) == 4 with pytest.raises(SampleNotInViewError): - agent.view.remove_sample(namespace, name, "default", "view1", sample_name) + agent.view.remove_sample( + namespace, name, "default", "view1", sample_name + ) @pytest.mark.parametrize( "namespace, name, sample_name", @@ -218,7 +256,10 @@ def test_get_snap_view(self, namespace, name, sample_name, view_name): ) def test_get_view_list_from_project(self, namespace, name, sample_name, view_name): with PEPDBAgentContextManager(add_data=True) as agent: - assert len(agent.view.get_views_annotation(namespace, name, "default").views) == 0 + assert ( + len(agent.view.get_views_annotation(namespace, name, "default").views) + == 0 + ) agent.view.create( "view1", { @@ -228,4 +269,7 @@ def test_get_view_list_from_project(self, namespace, name, sample_name, view_nam "sample_list": [sample_name, "pig_1h"], }, ) - assert len(agent.view.get_views_annotation(namespace, name, "default").views) == 1 + assert ( + len(agent.view.get_views_annotation(namespace, name, "default").views) + == 1 + ) diff --git a/tests/utils.py b/tests/utils.py index fc9cc89..26c321e 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,7 +1,7 @@ import os import warnings -import peppy +import peprs import yaml from sqlalchemy.exc import OperationalError @@ -70,7 +70,9 @@ class PEPDBAgentContextManager: Class with context manager to connect to database. Adds data and drops everything from the database upon exit to ensure. """ - def __init__(self, url: str = DSN, add_data: bool = False, add_schemas=True, echo=False): + def __init__( + self, url: str = DSN, add_data: bool = False, add_schemas=True, echo=False + ): """ :param url: database url e.g. "postgresql+psycopg://postgres:docker@localhost:5432/pep-db" :param add_data: add data to the database @@ -106,7 +108,12 @@ def _insert_data(self): else: private = False for name, path in item.items(): - prj = peppy.Project(path) + try: + prj = peprs.Project(path) + except Exception as e: + raise RuntimeError( + f"Failed to load test project {namespace}/{name} from {path}" + ) from e pepdb_con.project.create( namespace=namespace, name=name,