From cf5c223a8800ad55d6ec99d2b8655c1d53cf964e Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 12:33:43 +0000 Subject: [PATCH 01/32] feat(samples): add `samples import` command for public-repository accessions Adds `client.samples.import_samples`/`get_import` (POST/GET `/v2/sample-imports`) and the `flowbio samples import` CLI verb, mirroring `upload-batch`'s validate-up-front / report structure for a CSV accession sheet (an `accession` column plus optional name/organism and metadata columns, in place of reads1/reads2/project). Unlike upload-batch, the server tracks every accession submitted together as a single job (one status/error, accessions/sample_ids returned positionally) rather than a per-row upload. The CLI submits all valid rows in one call and polls that one job to completion (--poll-interval/--timeout), per the ticket amendment to poll per job rather than per accession. A completed job reports every row imported; a failed or timed-out job reports every row failed with the job's message, since the job has one outcome for the whole batch. _sheet.py's per-row metadata validation (required/closed-option/annotation rules) is extracted into a shared `metadata_errors` helper so the new accession-sheet validation in _accession_sheet.py reuses it instead of duplicating it; behaviour of validate_row is unchanged. Verified with respx-mocked unit tests (client methods, sheet parsing, and the CLI command including polling/timeout/failure paths) and manual CLI --help smoke checks; full suite passes (343 tests). Refs FLOW-689. Acceptance-test driver's import_by_accession is not routed through flowbio in this PR - see PR description for why. --- flowbio/cli/_accession_sheet.py | 158 +++++++++++++ flowbio/cli/_samples.py | 232 ++++++++++++++++++- flowbio/cli/_sheet.py | 32 ++- flowbio/v2/samples.py | 102 ++++++++- source/cli.rst | 63 +++++ source/v2/samples.rst | 45 +++- tests/unit/cli/test_accession_sheet.py | 227 ++++++++++++++++++ tests/unit/cli/test_samples.py | 305 +++++++++++++++++++++++++ tests/unit/v2/test_samples.py | 186 +++++++++++++++ 9 files changed, 1338 insertions(+), 12 deletions(-) create mode 100644 flowbio/cli/_accession_sheet.py create mode 100644 tests/unit/cli/test_accession_sheet.py diff --git a/flowbio/cli/_accession_sheet.py b/flowbio/cli/_accession_sheet.py new file mode 100644 index 0000000..8446923 --- /dev/null +++ b/flowbio/cli/_accession_sheet.py @@ -0,0 +1,158 @@ +"""CSV accession-sheet parsing and pre-flight validation for ``samples import``. + +An accession sheet is a CSV with one row per accession to import: an +``accession`` column (a public-repository run or experiment accession) plus +optional ``name``/``organism`` and per-accession metadata columns. This +mirrors ``_sheet.py``'s reads-based sample sheet, but the reserved columns +differ — there is nothing to upload (no ``reads1``/``reads2``) and the import +API has no project field, so ``RESERVED_COLUMNS`` is ``accession``, ``name``, +``organism`` instead. +""" +from __future__ import annotations + +import csv +import re +from dataclasses import dataclass +from pathlib import Path + +from flowbio.cli._exit_codes import CliUsageError +from flowbio.cli._files import existing_file +from flowbio.cli._sheet import metadata_errors +from flowbio.v2.samples import MetadataAttribute, SampleTypeId + +RESERVED_COLUMNS = ("accession", "name", "organism") + +# Mirrors the API's own accession format rule (one run or experiment accession +# per entry), so a malformed accession is reported up front like every other +# validation problem instead of failing the whole batch request server-side. +_SUPPORTED_ACCESSION = re.compile(r"^[SED]R[RX]\d+$") + + +@dataclass(frozen=True) +class AccessionSheetRow: + """One data row of an accession sheet. + + ``accession`` may be empty (empty cell) — that is reported by + :func:`validate_accession_row` rather than rejected here, so all errors + surface together. + """ + + row_number: int + accession: str + name: str | None + organism: str | None + metadata: dict[str, str] + + +@dataclass(frozen=True) +class AccessionSheet: + """A parsed accession sheet.""" + + path: Path + metadata_columns: list[str] + rows: list[AccessionSheetRow] + + +def parse_accession_sheet(path: Path) -> AccessionSheet: + """Parse a CSV accession sheet into an :class:`AccessionSheet`. + + :param path: The accession-sheet file. Must be a ``.csv`` — an ``.xlsx`` or + ``.tsv`` is a usage error directing the user to export to CSV. + :returns: The parsed sheet with reserved/metadata columns separated, + accessions normalised to upper case, and empty cells dropped. + :raises CliUsageError: If the file is not a readable ``.csv``. + """ + if path.suffix.lower() != ".csv": + raise CliUsageError( + f"Accession sheet must be a .csv file: {path}. " + f"Export your spreadsheet to CSV first.", + ) + existing_file(path) + with path.open(newline="", encoding="utf-8-sig") as handle: + reader = csv.DictReader(handle) + headers = reader.fieldnames or [] + metadata_columns = [ + header for header in headers if header not in RESERVED_COLUMNS + ] + rows = [ + _build_row(record, row_number, metadata_columns) + for row_number, record in enumerate(reader, start=1) + ] + return AccessionSheet(path=path, metadata_columns=metadata_columns, rows=rows) + + +def validate_accession_row( + row: AccessionSheetRow, + attributes: list[MetadataAttribute], + sample_type: SampleTypeId, +) -> list[str]: + """Return every validation problem on ``row`` (empty when the row is valid). + + :param row: The parsed row to validate. + :param attributes: The server's metadata attributes, deciding required and + closed-option columns. + :param sample_type: The sample type applied to the whole import; an + attribute required for it must be present. + :returns: One human-readable message per problem, collected so the caller + can report them all at once. + """ + errors: list[str] = [] + if not row.accession: + errors.append("missing required value: accession") + elif not _SUPPORTED_ACCESSION.match(row.accession): + errors.append( + f"'{row.accession}' is not a supported accession — import one run " + f"(SRR/ERR/DRR) or experiment (SRX/ERX/DRX) accession per row", + ) + errors.extend(metadata_errors(row.metadata, attributes, sample_type)) + return errors + + +def duplicate_accession_errors(rows: list[AccessionSheetRow]) -> dict[int, list[str]]: + """Return extra errors for rows whose accession repeats an earlier row. + + Mirrors the API's own duplicate-accession rejection so a sheet with + repeats is reported up front, in the same per-row shape as every other + validation problem, rather than surfacing as one opaque batch failure. + + :param rows: Every row in the sheet, including ones already found invalid. + :returns: A mapping of ``row_number`` to the duplicate-accession messages + for rows after the first occurrence of a repeated accession. Rows with + a blank accession (already reported by :func:`validate_accession_row`) + are never flagged as duplicates of each other. + """ + first_seen: dict[str, int] = {} + errors: dict[int, list[str]] = {} + for row in rows: + if not row.accession: + continue + if row.accession in first_seen: + errors.setdefault(row.row_number, []).append( + f"duplicate accession '{row.accession}' " + f"(first seen at row {first_seen[row.accession]})", + ) + else: + first_seen[row.accession] = row.row_number + return errors + + +def _build_row( + record: dict[str, str], row_number: int, metadata_columns: list[str], +) -> AccessionSheetRow: + def cell(column: str) -> str | None: + value = (record.get(column) or "").strip() + return value or None + + metadata = { + column: value + for column in metadata_columns + if (value := (record.get(column) or "").strip()) + } + accession = cell("accession") + return AccessionSheetRow( + row_number=row_number, + accession=accession.upper() if accession else "", + name=cell("name"), + organism=cell("organism"), + metadata=metadata, + ) diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index abd1896..afd1f3e 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -10,10 +10,18 @@ import argparse import json +import math +import time from dataclasses import dataclass from pathlib import Path from typing import Literal +from flowbio.cli._accession_sheet import ( + AccessionSheetRow, + duplicate_accession_errors, + parse_accession_sheet, + validate_accession_row, +) from flowbio.cli._exit_codes import CliUsageError, ExitCode from flowbio.cli._files import existing_file from flowbio.cli._output import Output, format_issue @@ -26,7 +34,12 @@ from flowbio.cli._types import JsonValue from flowbio.v2.client import Client from flowbio.v2.exceptions import FlowApiError -from flowbio.v2.samples import MetadataAttribute, SampleTypeId +from flowbio.v2.samples import ( + MetadataAttribute, + SampleImportJob, + SampleImportSpec, + SampleTypeId, +) def register( @@ -76,6 +89,16 @@ def register( "valid rows sequentially, reporting each row's outcome." ), )) + _configure_import(verbs.add_parser( + "import", + parents=[global_parent], + help="Import samples from public-repository accessions.", + description=( + "Validate every row of a CSV accession sheet up front, kick off a " + "single import job for the valid rows, poll it to completion, and " + "report each accession's outcome." + ), + )) def _configure_upload(upload: argparse.ArgumentParser) -> None: @@ -233,6 +256,47 @@ def _configure_upload_batch(upload_batch: argparse.ArgumentParser) -> None: ) +_DEFAULT_POLL_INTERVAL = 5.0 +_DEFAULT_TIMEOUT = 1800.0 + + +def _configure_import(import_parser: argparse.ArgumentParser) -> None: + import_parser.set_defaults(command_parser=import_parser, handler=_import_command) + import_parser.add_argument( + "--sheet", + required=True, + metavar="PATH", + type=Path, + help="CSV accession sheet (accession, optional name/organism, plus metadata columns).", + ) + import_parser.add_argument( + "--sample-type", + required=True, + metavar="TYPE", + type=SampleTypeId, + help="Sample type applied to every accession (sent as-is; validated server-side).", + ) + import_parser.add_argument( + "--skip-invalid", + action="store_true", + help="Skip invalid rows (reporting why) instead of aborting the import.", + ) + import_parser.add_argument( + "--poll-interval", + type=float, + default=_DEFAULT_POLL_INTERVAL, + metavar="SECONDS", + help=f"Seconds to wait between checks of the import job's status (default: {_DEFAULT_POLL_INTERVAL:g}).", + ) + import_parser.add_argument( + "--timeout", + type=float, + default=_DEFAULT_TIMEOUT, + metavar="SECONDS", + help=f"Maximum seconds to wait for the import job to finish (default: {_DEFAULT_TIMEOUT:g}).", + ) + + def _upload_command(args: argparse.Namespace, client: Client, output: Output) -> ExitCode: """Upload a single sample and report its identifier. @@ -558,6 +622,172 @@ def _invalid_line(row: SheetRow, reasons: list[str]) -> str: return f"Row {row.row_number} ({row.name}): {'; '.join(reasons)}" +@dataclass(frozen=True) +class _ImportResult: + """The outcome of an ``import`` run, rendered to text or JSON. + + Unlike :class:`_BatchResult`, every accession shares one server-side job: + all rows land in ``imported`` together on a completed job, or all land in + ``failed`` together (carrying the job's one error message) otherwise. + """ + + imported: list[dict[str, JsonValue]] + failed: list[dict[str, JsonValue]] + skipped: list[dict[str, JsonValue]] + + @property + def counts(self) -> dict[str, int]: + return { + "imported": len(self.imported), + "failed": len(self.failed), + "skipped": len(self.skipped), + } + + @property + def document(self) -> dict[str, JsonValue]: + return { + "imported": self.imported, + "failed": self.failed, + "skipped": self.skipped, + "counts": self.counts, + } + + @property + def summary(self) -> str: + counts = self.counts + return ( + f"Imported {counts['imported']}, failed {counts['failed']}, " + f"skipped {counts['skipped']}." + ) + + @property + def exit_code(self) -> ExitCode: + return ExitCode.RUNTIME if self.failed else ExitCode.SUCCESS + + +def _import_command( + args: argparse.Namespace, client: Client, output: Output, +) -> ExitCode: + """Validate an accession sheet up front, then run one import job for it. + + :param args: Parsed command-line arguments. + :param client: The authenticated Flow client. + :param output: The result/error renderer. + :returns: :attr:`ExitCode.SUCCESS` when the import job completes, + :attr:`ExitCode.USAGE` on a pre-flight validation failure without + ``--skip-invalid``, or :attr:`ExitCode.RUNTIME` if the import job + fails or does not finish within ``--timeout``. + """ + sheet = parse_accession_sheet(args.sheet) + attributes = client.samples.get_metadata_attributes() + duplicates = duplicate_accession_errors(sheet.rows) + classified = [ + ( + row, + validate_accession_row(row, attributes, args.sample_type) + + duplicates.get(row.row_number, []), + ) + for row in sheet.rows + ] + invalid = [(row, reasons) for row, reasons in classified if reasons] + valid = [row for row, reasons in classified if not reasons] + + if invalid and not args.skip_invalid: + output.emit_error( + "Accession sheet has invalid rows; nothing was imported.", + details=[_invalid_accession_line(row, reasons) for row, reasons in invalid], + ) + return ExitCode.USAGE + + for row, reasons in invalid: + output.emit_advisory(f"Skipped {_invalid_accession_line(row, reasons)}") + skipped = [ + {"row_number": row.row_number, "accession": row.accession, "reasons": reasons} + for row, reasons in invalid + ] + + result = _run_import(valid, args, client, output, skipped) + output.emit_result(result.summary, result.document) + return result.exit_code + + +def _run_import( + rows: list[AccessionSheetRow], + args: argparse.Namespace, + client: Client, + output: Output, + skipped: list[dict[str, JsonValue]], +) -> _ImportResult: + if not rows: + return _ImportResult(imported=[], failed=[], skipped=skipped) + + specs = [ + SampleImportSpec( + accession=row.accession, + sample_type=args.sample_type, + name=row.name, + organism_id=row.organism, + metadata=row.metadata or None, + ) + for row in rows + ] + job = client.samples.import_samples(specs) + output.emit_advisory(f"Import job {job.id} started for {len(rows)} accession(s); polling...") + job = _poll_job(client, job, args.poll_interval, args.timeout) + return _import_result(job, rows, skipped, output) + + +def _poll_job( + client: Client, job: SampleImportJob, poll_interval: float, timeout: float, +) -> SampleImportJob: + max_attempts = max(1, math.ceil(timeout / poll_interval)) + for _ in range(max_attempts): + if job.status != "RUNNING": + break + time.sleep(poll_interval) + job = client.samples.get_import(job.id) + return job + + +def _import_result( + job: SampleImportJob, + rows: list[AccessionSheetRow], + skipped: list[dict[str, JsonValue]], + output: Output, +) -> _ImportResult: + if job.status == "COMPLETED": + accession_to_sample_id = dict(zip(job.accessions, job.sample_ids)) + imported = [ + { + "row_number": row.row_number, + "accession": row.accession, + "sample_id": accession_to_sample_id.get(row.accession), + } + for row in rows + ] + for entry in imported: + output.emit_advisory( + f"Row {entry['row_number']} ({entry['accession']}): " + f"imported sample {entry['sample_id']}", + ) + return _ImportResult(imported=imported, failed=[], skipped=skipped) + + message = job.error or f"import job {job.id} ended with status {job.status}" + failed = [ + {"row_number": row.row_number, "accession": row.accession, "message": message} + for row in rows + ] + for entry in failed: + output.emit_advisory( + f"Row {entry['row_number']} ({entry['accession']}): import failed — {message}", + ) + return _ImportResult(imported=[], failed=failed, skipped=skipped) + + +def _invalid_accession_line(row: AccessionSheetRow, reasons: list[str]) -> str: + return f"Row {row.row_number} ({row.accession or ''}): {'; '.join(reasons)}" + + def _merge_metadata( pairs: list[str] | None, json_text: str | None, ) -> dict[str, str]: diff --git a/flowbio/cli/_sheet.py b/flowbio/cli/_sheet.py index e956ccb..2e70a4f 100644 --- a/flowbio/cli/_sheet.py +++ b/flowbio/cli/_sheet.py @@ -96,7 +96,6 @@ def validate_row( :returns: One human-readable message per problem, collected so the caller can report them all at once. """ - by_identifier = {attribute.identifier: attribute for attribute in attributes} errors: list[str] = [] if not row.name: errors.append("missing required value: name") @@ -107,7 +106,7 @@ def validate_row( for label, reads in (("reads1", row.reads1), ("reads2", row.reads2)): if reads is not None and not reads.is_file(): errors.append(f"{label} file not found: {reads}") - errors.extend(_metadata_errors(row, by_identifier, attributes, sample_type)) + errors.extend(metadata_errors(row.metadata, attributes, sample_type)) return errors @@ -144,20 +143,35 @@ def _resolve(value: str | None, base_dir: Path) -> Path | None: return path if path.is_absolute() else base_dir / path -def _metadata_errors( - row: SheetRow, - by_identifier: dict[str, MetadataAttribute], +def metadata_errors( + metadata: dict[str, str], attributes: list[MetadataAttribute], sample_type: SampleTypeId, ) -> list[str]: + """Return every metadata validation problem for ``metadata`` (empty when valid). + + Shared by :func:`validate_row` and the accession-sheet equivalent in + :mod:`flowbio.cli._accession_sheet`, since required/closed-option/annotation + rules apply identically regardless of what the rest of the row looks like. + + :param metadata: The row's metadata, keyed by attribute identifier (or + ``__annotation`` for a free-text companion). + :param attributes: The server's metadata attributes, deciding required and + closed-option columns. + :param sample_type: The sample type applied to the whole batch; an attribute + required for it must be present. + :returns: One human-readable message per problem, collected so the caller can + report them all at once. + """ + by_identifier = {attribute.identifier: attribute for attribute in attributes} errors: list[str] = [] for attribute in attributes: required = ( attribute.required or sample_type in attribute.required_for_sample_types ) - if required and not row.metadata.get(attribute.identifier): + if required and not metadata.get(attribute.identifier): errors.append(f"missing required metadata: {attribute.identifier}") - for key, value in row.metadata.items(): + for key, value in metadata.items(): if key.endswith(ANNOTATION_SUFFIX): continue attribute = by_identifier.get(key) @@ -170,11 +184,11 @@ def _metadata_errors( f"value '{value}' for {key} is not one of: " f"{', '.join(attribute.options)}", ) - for key in row.metadata: + for key in metadata: if not key.endswith(ANNOTATION_SUFFIX): continue base = key[: -len(ANNOTATION_SUFFIX)] - if not row.metadata.get(base): + if not metadata.get(base): errors.append(f"{key} set without a value for {base}") continue attribute = by_identifier.get(base) diff --git a/flowbio/v2/samples.py b/flowbio/v2/samples.py index fd888ed..8dada4f 100644 --- a/flowbio/v2/samples.py +++ b/flowbio/v2/samples.py @@ -27,8 +27,9 @@ from __future__ import annotations from collections.abc import Sequence +from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, NewType +from typing import TYPE_CHECKING, Literal, NewType from pydantic import BaseModel, Field @@ -139,6 +140,58 @@ class MultiplexedUpload(BaseModel, frozen=True): ) +SampleImportJobId = NewType("SampleImportJobId", int) +"""The identifier of a sample-import job, as returned by +:meth:`SampleResource.import_samples`.""" + + +SampleImportStatus = Literal["RUNNING", "COMPLETED", "FAILED"] +"""The lifecycle state of a :class:`SampleImportJob`.""" + + +@dataclass(frozen=True) +class SampleImportSpec: + """One accession to import, with its per-accession identity and metadata. + + Example:: + + specs = [ + SampleImportSpec(accession="ERR1160845", sample_type="RNA-Seq"), + SampleImportSpec( + accession="ERR10677146", + sample_type="RNA-Seq", + organism_id="Hs", + metadata={"strandedness": "reverse"}, + ), + ] + """ + + accession: str + sample_type: SampleTypeId + name: str | None = None + organism_id: str | None = None + metadata: dict[str, str] | None = None + + +class SampleImportJob(BaseModel, frozen=True): + """A batch job that imports one or more accessions into samples. + + All accessions submitted in one :meth:`SampleResource.import_samples` call + share a single job: ``status`` and ``error`` describe the whole batch, and + ``accessions``/``sample_ids`` correspond positionally once ``status`` is + ``"COMPLETED"``. + """ + + id: SampleImportJobId = Field(description="Unique identifier for this import job.") + status: SampleImportStatus = Field(description="The job's current lifecycle state.") + accessions: list[str] = Field(description="The accessions submitted with this job, in submission order.") + sample_ids: list[int] = Field( + description="The created samples' ids, corresponding to `accessions` once the job has completed.", + ) + execution_id: int | None = Field(description="The pipeline execution backing this job, if one was created.") + error: str | None = Field(description='The failure reason, set only when status is "FAILED".') + + class SampleResource: """Provides access to sample-related API endpoints. @@ -399,6 +452,53 @@ def get_metadata_attributes(self) -> list[MetadataAttribute]: """ return [self._create_metadata_attribute(item) for item in (self._transport.get("/samples/metadata"))] + def import_samples(self, imports: Sequence[SampleImportSpec]) -> SampleImportJob: + """Kick off a batch import of samples from public-repository accessions. + + Every accession is submitted together and tracked as a single job — poll + it with :meth:`get_import` until its status leaves ``"RUNNING"``. + + Requires authentication. + + Example:: + + job = client.samples.import_samples([ + SampleImportSpec(accession="ERR1160845", sample_type="RNA-Seq"), + ]) + print(f"Import job: {job.id}") + + :param imports: The accessions to import, one :class:`SampleImportSpec` each. + :raises FlowApiError: If any entry is invalid, e.g. an unsupported + accession format, unknown sample type, or missing required metadata. + """ + payload = {"imports": [self._import_spec_fields(spec) for spec in imports]} + return SampleImportJob(**self._transport.post("/v2/sample-imports", json=payload)) + + def get_import(self, job_id: SampleImportJobId) -> SampleImportJob: + """Fetch the current state of an import job. + + Example:: + + job = client.samples.get_import(job.id) + if job.status == "COMPLETED": + print(f"Imported samples: {job.sample_ids}") + + :param job_id: The job id returned by :meth:`import_samples`. + :raises NotFoundError: If no import job with that id exists. + """ + return SampleImportJob(**self._transport.get(f"/v2/sample-imports/{job_id}")) + + @staticmethod + def _import_spec_fields(spec: SampleImportSpec) -> dict: + fields: dict = {"accession": spec.accession, "sample_type": spec.sample_type} + if spec.name is not None: + fields["name"] = spec.name + if spec.organism_id is not None: + fields["organism"] = spec.organism_id + if spec.metadata: + fields["metadata"] = spec.metadata + return fields + def _create_metadata_attribute(self, item: dict) -> MetadataAttribute: item["required_for_sample_types"] = [ SampleTypeId(link["sample_type_identifier"]) diff --git a/source/cli.rst b/source/cli.rst index 029c86a..e26fcf1 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -367,6 +367,69 @@ authentication failure; otherwise the standard mapping above. $ flowbio samples upload-batch --sheet ./samples.csv --sample-type RNA-Seq --json {"uploaded": [{"row_number": 1, "name": "liver_r1", "sample_id": "samp_1"}], "failed": [], "skipped": [], "counts": {"uploaded": 1, "failed": 0, "skipped": 0}} +``samples import`` +~~~~~~~~~~~~~~~~~~~ + +Import samples from public-repository accessions (SRR/ERR/DRR run or +SRX/ERX/DRX experiment accessions), applying one sample type to every row — +no files to upload yourself. + +:: + + flowbio samples import --sheet PATH --sample-type TYPE + [--skip-invalid] [--poll-interval SECONDS] [--timeout SECONDS] + +Run ``flowbio samples import --help`` for the full option list. The sheet is +a CSV with an ``accession`` column plus optional ``name``/``organism`` and +metadata columns (there is no ``batch-template`` equivalent for it, since it +has no reads files or project field). ``name`` defaults to the accession when +omitted. The sample type is sent as-is and validated server-side. + +**Validation is up front**, mirroring ``upload-batch``: a missing or +malformed accession, a duplicate accession within the sheet, a value outside +a closed-option attribute's allowed values, or missing metadata required for +the chosen type — every problem is collected before anything is submitted. +By default any invalid row aborts the whole run (exit ``2``); +``--skip-invalid`` skips them (reporting why) and imports the rest. + +Unlike ``upload-batch``, every valid row is submitted **together as one +server-side job** — there is no per-row upload loop. The command polls that +job (every ``--poll-interval`` seconds, default 5, for up to ``--timeout`` +seconds, default 1800) until it leaves ``"RUNNING"``. Once the job completes, +every submitted row is reported as imported; if it fails or times out, every +row is reported as failed with the same message, since the job has one +outcome for the whole batch. + +**Output** — human: each row's outcome on stderr, then a final counts summary +on stdout. ``--json``: a single document on stdout with ``imported``, +``failed``, and ``skipped`` lists plus a ``counts`` summary: + +.. code-block:: json + + { + "imported": [{"row_number": 1, "accession": "ERR1160845", "sample_id": 101}], + "failed": [], + "skipped": [{"row_number": 2, "accession": "bogus", "reasons": ["..."]}], + "counts": {"imported": 1, "failed": 0, "skipped": 1} + } + +**Exit codes** — ``0`` the import job completed; ``2`` a pre-flight +validation failure (without ``--skip-invalid``) or a non-CSV sheet; ``1`` the +import job failed or did not finish within ``--timeout``; ``3`` +authentication failure; otherwise the standard mapping above. + +**Example** + +.. code-block:: bash + + $ flowbio samples import --sheet ./accessions.csv --sample-type RNA-Seq + Row 1 (ERR1160845): imported sample 101 + Row 2 (ERR10677146): imported sample 102 + Imported 2, failed 0, skipped 0. + + $ flowbio samples import --sheet ./accessions.csv --sample-type RNA-Seq --json + {"imported": [{"row_number": 1, "accession": "ERR1160845", "sample_id": 101}], "failed": [], "skipped": [], "counts": {"imported": 1, "failed": 0, "skipped": 0}} + ``api get`` ~~~~~~~~~~~ diff --git a/source/v2/samples.rst b/source/v2/samples.rst index 6fb5dd2..d78855b 100644 --- a/source/v2/samples.rst +++ b/source/v2/samples.rst @@ -166,6 +166,45 @@ To reject the upload when warnings are present, set This raises :class:`~flowbio.v2.exceptions.AnnotationValidationError` if the annotation has any warnings. +.. _sample-imports: + +Importing samples from public repositories +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use :meth:`~flowbio.v2.samples.SampleResource.import_samples` to create +samples directly from public-repository run or experiment accessions +(SRR/ERR/DRR or SRX/ERX/DRX), instead of uploading files yourself. Every +accession submitted together is tracked as a single job:: + + from flowbio.v2.samples import SampleImportSpec + + job = client.samples.import_samples([ + SampleImportSpec(accession="ERR1160845", sample_type="RNA-Seq"), + SampleImportSpec( + accession="ERR10677146", + sample_type="RNA-Seq", + metadata={"strandedness": "reverse"}, + ), + ]) + +The job starts out ``"RUNNING"``. Poll it with +:meth:`~flowbio.v2.samples.SampleResource.get_import` until its status +leaves ``"RUNNING"``:: + + import time + + while job.status == "RUNNING": + time.sleep(5) + job = client.samples.get_import(job.id) + + if job.status == "COMPLETED": + print(f"Imported samples: {job.sample_ids}") + else: + print(f"Import failed: {job.error}") + +``job.accessions`` and ``job.sample_ids`` correspond positionally once the +job has completed. + API Reference ------------- @@ -185,4 +224,8 @@ Models .. autopydantic_model:: flowbio.v2.samples.Organism -.. autopydantic_model:: flowbio.v2.samples.MultiplexedUpload \ No newline at end of file +.. autopydantic_model:: flowbio.v2.samples.MultiplexedUpload + +.. autoclass:: flowbio.v2.samples.SampleImportSpec + +.. autopydantic_model:: flowbio.v2.samples.SampleImportJob \ No newline at end of file diff --git a/tests/unit/cli/test_accession_sheet.py b/tests/unit/cli/test_accession_sheet.py new file mode 100644 index 0000000..f7edca8 --- /dev/null +++ b/tests/unit/cli/test_accession_sheet.py @@ -0,0 +1,227 @@ +import csv +from pathlib import Path + +import pytest + +from flowbio.cli._accession_sheet import ( + duplicate_accession_errors, + parse_accession_sheet, + validate_accession_row, +) +from flowbio.cli._exit_codes import CliUsageError +from flowbio.v2.samples import MetadataAttribute, SampleTypeId + +SAMPLE_TYPE = SampleTypeId("rna_seq") +HEADERS = ["accession", "name", "organism", "cell_type", "source", "source__annotation"] + + +def _attributes() -> list[MetadataAttribute]: + return [ + MetadataAttribute( + identifier="cell_type", + name="Cell Type", + description="The cell type", + required=False, + required_for_sample_types=[SAMPLE_TYPE], + options=["Neuron", "Fibroblast"], + allow_annotation=False, + ), + MetadataAttribute( + identifier="source", + name="Source", + description="Sample source", + required=False, + required_for_sample_types=[], + options=None, + allow_annotation=True, + ), + ] + + +def _write_sheet( + directory: Path, *records: dict[str, str], headers: list[str] = HEADERS, +) -> Path: + path = directory / "sheet.csv" + with path.open("w", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=headers) + writer.writeheader() + writer.writerows(records) + return path + + +class TestParseAccessionSheet: + + def test_separates_reserved_and_metadata_columns(self, tmp_path: Path) -> None: + sheet = parse_accession_sheet( + _write_sheet(tmp_path, {"accession": "ERR1160845"}), + ) + + assert sheet.metadata_columns == ["cell_type", "source", "source__annotation"] + + def test_accession_is_normalised_to_upper_case(self, tmp_path: Path) -> None: + sheet = parse_accession_sheet( + _write_sheet(tmp_path, {"accession": "err1160845"}), + ) + + assert sheet.rows[0].accession == "ERR1160845" + + def test_empty_cells_omitted_from_metadata(self, tmp_path: Path) -> None: + sheet = parse_accession_sheet(_write_sheet( + tmp_path, + {"accession": "ERR1160845", "cell_type": "", "source": "blood"}, + )) + + assert sheet.rows[0].metadata == {"source": "blood"} + + def test_name_and_organism_are_optional(self, tmp_path: Path) -> None: + sheet = parse_accession_sheet( + _write_sheet(tmp_path, {"accession": "ERR1160845"}), + ) + + assert sheet.rows[0].name is None + assert sheet.rows[0].organism is None + + def test_name_and_organism_are_parsed_when_present(self, tmp_path: Path) -> None: + sheet = parse_accession_sheet(_write_sheet( + tmp_path, + {"accession": "ERR1160845", "name": "liver_r1", "organism": "Hs"}, + )) + + assert sheet.rows[0].name == "liver_r1" + assert sheet.rows[0].organism == "Hs" + + def test_utf8_bom_is_stripped_from_first_header(self, tmp_path: Path) -> None: + path = tmp_path / "sheet.csv" + with path.open("w", newline="", encoding="utf-8-sig") as handle: + writer = csv.DictWriter(handle, fieldnames=HEADERS) + writer.writeheader() + writer.writerow({"accession": "ERR1160845"}) + + sheet = parse_accession_sheet(path) + + assert sheet.metadata_columns == ["cell_type", "source", "source__annotation"] + assert sheet.rows[0].accession == "ERR1160845" + + def test_row_numbers_are_one_based(self, tmp_path: Path) -> None: + sheet = parse_accession_sheet(_write_sheet( + tmp_path, + {"accession": "ERR1160845"}, + {"accession": "ERR10677146"}, + )) + + assert [row.row_number for row in sheet.rows] == [1, 2] + + def test_non_csv_xlsx_rejected_with_export_message(self, tmp_path: Path) -> None: + xlsx = tmp_path / "sheet.xlsx" + xlsx.write_bytes(b"PK") + + with pytest.raises(CliUsageError, match="CSV"): + parse_accession_sheet(xlsx) + + def test_tsv_sheet_rejected(self, tmp_path: Path) -> None: + tsv = tmp_path / "sheet.tsv" + tsv.write_text("accession\nERR1160845\n") + + with pytest.raises(CliUsageError, match="CSV"): + parse_accession_sheet(tsv) + + def test_missing_file_is_usage_error(self, tmp_path: Path) -> None: + with pytest.raises(CliUsageError): + parse_accession_sheet(tmp_path / "absent.csv") + + +class TestValidateAccessionRow: + + def _row(self, directory: Path, **overrides: str): + record = {"accession": "ERR1160845"} + record.update(overrides) + sheet = parse_accession_sheet(_write_sheet(directory, record)) + return sheet.rows[0] + + def test_valid_row_has_no_errors(self, tmp_path: Path) -> None: + row = self._row(tmp_path, cell_type="Neuron") + + assert validate_accession_row(row, _attributes(), SAMPLE_TYPE) == [] + + def test_missing_accession_reports_error(self, tmp_path: Path) -> None: + row = self._row(tmp_path, accession="", cell_type="Neuron") + + errors = validate_accession_row(row, _attributes(), SAMPLE_TYPE) + + assert any("accession" in error for error in errors) + + def test_unsupported_accession_format_reports_error(self, tmp_path: Path) -> None: + row = self._row(tmp_path, accession="GSE12345", cell_type="Neuron") + + errors = validate_accession_row(row, _attributes(), SAMPLE_TYPE) + + assert any("GSE12345" in error for error in errors) + + def test_run_and_experiment_accessions_are_supported(self, tmp_path: Path) -> None: + for accession in ("ERR1160845", "SRR1234567", "DRR7654321", "SRX111", "ERX222", "DRX333"): + row = self._row(tmp_path, accession=accession, cell_type="Neuron") + + assert validate_accession_row(row, _attributes(), SAMPLE_TYPE) == [] + + def test_value_outside_options_reports_error(self, tmp_path: Path) -> None: + row = self._row(tmp_path, cell_type="Alien") + + errors = validate_accession_row(row, _attributes(), SAMPLE_TYPE) + + assert any("Alien" in error for error in errors) + + def test_required_for_type_metadata_missing_reports_error( + self, tmp_path: Path, + ) -> None: + row = self._row(tmp_path) + + errors = validate_accession_row(row, _attributes(), SAMPLE_TYPE) + + assert any("cell_type" in error for error in errors) + + def test_annotation_set_without_its_value_reports_error( + self, tmp_path: Path, + ) -> None: + row = self._row( + tmp_path, cell_type="Neuron", **{"source__annotation": "qPCR"}, + ) + + errors = validate_accession_row(row, _attributes(), SAMPLE_TYPE) + + assert any("source__annotation" in error for error in errors) + + def test_all_per_row_errors_collected(self, tmp_path: Path) -> None: + sheet = parse_accession_sheet(_write_sheet( + tmp_path, {"accession": "", "cell_type": "Alien"}, + )) + + errors = validate_accession_row(sheet.rows[0], _attributes(), SAMPLE_TYPE) + + assert len(errors) >= 2 + + +class TestDuplicateAccessionErrors: + + def test_no_errors_when_all_unique(self, tmp_path: Path) -> None: + sheet = parse_accession_sheet(_write_sheet( + tmp_path, {"accession": "ERR1"}, {"accession": "ERR2"}, + )) + + assert duplicate_accession_errors(sheet.rows) == {} + + def test_repeated_accession_reports_error_on_later_row(self, tmp_path: Path) -> None: + sheet = parse_accession_sheet(_write_sheet( + tmp_path, {"accession": "ERR1"}, {"accession": "err1"}, + )) + + errors = duplicate_accession_errors(sheet.rows) + + assert 1 not in errors + assert any("ERR1" in message for message in errors[2]) + + def test_blank_accessions_are_not_treated_as_duplicates(self, tmp_path: Path) -> None: + sheet = parse_accession_sheet(_write_sheet( + tmp_path, {"accession": ""}, {"accession": ""}, + )) + + assert duplicate_accession_errors(sheet.rows) == {} diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index 5c461c1..3d2bf6a 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -2,6 +2,7 @@ import json from http import HTTPStatus from pathlib import Path +from unittest.mock import patch import httpx import respx @@ -13,6 +14,7 @@ ANNOTATION_TEMPLATE_URL = f"{DEFAULT_BASE_URL}/annotation" ANNOTATION_UPLOAD_URL = f"{DEFAULT_BASE_URL}/upload/annotation" MULTIPLEXED_UPLOAD_URL = f"{DEFAULT_BASE_URL}/upload/multiplexed" +SAMPLE_IMPORTS_URL = f"{DEFAULT_BASE_URL}/v2/sample-imports" TOKEN = "test.token" @@ -923,3 +925,306 @@ def test_non_csv_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None: assert result.exit_code == 2 assert upload.call_count == 0 assert "CSV" in result.stderr + + +IMPORT_HEADERS = ["accession", "name", "organism", "cell_type", "source", "source__annotation"] + + +def _write_import_sheet(directory: Path, *records: dict[str, str]) -> Path: + path = directory / "accessions.csv" + with path.open("w", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=IMPORT_HEADERS) + writer.writeheader() + writer.writerows(records) + return path + + +def _job_json( + job_id: int, + status: str, + accessions: list[str], + sample_ids: list[int] | None = None, + error: str | None = None, +) -> dict: + return { + "id": job_id, + "status": status, + "accessions": accessions, + "sample_ids": sample_ids or [], + "execution_id": 7, + "error": error, + } + + +class TestSamplesImport: + + @respx.mock + def test_completed_job_reports_imported_samples( + self, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "COMPLETED", ["ERR1", "ERR2"], [101, 102], + )), + ) + sheet = _write_import_sheet( + tmp_path, + {"accession": "ERR1", "cell_type": "Neuron"}, + {"accession": "ERR2", "cell_type": "Fibroblast"}, + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + ) + + assert result.exit_code == 0 + assert "101" in result.stderr + assert "102" in result.stderr + assert "Imported 2, failed 0, skipped 0." in result.stdout + + @respx.mock + def test_json_document_reports_outcomes_and_counts( + self, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "COMPLETED", ["ERR1"], [101], + )), + ) + sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", "--json", + ) + + assert result.exit_code == 0 + document = json.loads(result.stdout) + assert result.stdout.count("\n") == 1 + assert document["counts"] == {"imported": 1, "failed": 0, "skipped": 0} + assert document["imported"][0] == { + "row_number": 1, "accession": "ERR1", "sample_id": 101, + } + + @respx.mock + def test_sends_sample_type_and_metadata_in_kickoff_payload( + self, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + route = respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "COMPLETED", ["ERR1"], [101], + )), + ) + sheet = _write_import_sheet( + tmp_path, {"accession": "err1", "cell_type": "Neuron", "organism": "Hs"}, + ) + + run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + ) + + payload = json.loads(route.calls[0].request.content) + assert payload == { + "imports": [{ + "accession": "ERR1", + "sample_type": "rna_seq", + "organism": "Hs", + "metadata": {"cell_type": "Neuron"}, + }], + } + + @respx.mock + @patch("time.sleep") + def test_polls_until_job_completes( + self, _mock_sleep, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "RUNNING", ["ERR1"], + )), + ) + poll_route = respx.get(f"{SAMPLE_IMPORTS_URL}/1") + poll_route.side_effect = [ + httpx.Response(HTTPStatus.OK, json=_job_json(1, "RUNNING", ["ERR1"])), + httpx.Response(HTTPStatus.OK, json=_job_json(1, "COMPLETED", ["ERR1"], [101])), + ] + sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + "--poll-interval", "1", + ) + + assert result.exit_code == 0 + assert poll_route.call_count == 2 + assert "101" in result.stderr + + @respx.mock + @patch("time.sleep") + def test_failed_job_reports_error_message_per_row( + self, _mock_sleep, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + error_message = "download failed: connection reset" + respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "RUNNING", ["ERR1", "ERR2"], + )), + ) + respx.get(f"{SAMPLE_IMPORTS_URL}/1").mock( + return_value=httpx.Response(HTTPStatus.OK, json=_job_json( + 1, "FAILED", ["ERR1", "ERR2"], error=error_message, + )), + ) + sheet = _write_import_sheet( + tmp_path, + {"accession": "ERR1", "cell_type": "Neuron"}, + {"accession": "ERR2", "cell_type": "Neuron"}, + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + "--poll-interval", "1", + ) + + assert result.exit_code == 1 + assert error_message in result.stderr + assert "Row 1 (ERR1)" in result.stderr + assert "Row 2 (ERR2)" in result.stderr + + @respx.mock + @patch("time.sleep") + def test_timeout_reports_failure_without_hanging( + self, _mock_sleep, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "RUNNING", ["ERR1"], + )), + ) + poll_route = respx.get(f"{SAMPLE_IMPORTS_URL}/1").mock( + return_value=httpx.Response(HTTPStatus.OK, json=_job_json(1, "RUNNING", ["ERR1"])), + ) + sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + "--poll-interval", "1", "--timeout", "2", + ) + + assert result.exit_code == 1 + assert poll_route.call_count == 2 + assert "Row 1 (ERR1)" in result.stderr + + @respx.mock + def test_invalid_row_aborts_and_imports_nothing( + self, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + route = respx.post(SAMPLE_IMPORTS_URL) + sheet = _write_import_sheet( + tmp_path, + {"accession": "ERR1", "cell_type": "Neuron"}, + {"accession": "not-an-accession", "cell_type": "Neuron"}, + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + ) + + assert result.exit_code == 2 + assert route.call_count == 0 + assert "Row 2" in result.stderr + assert "NOT-AN-ACCESSION" in result.stderr + + @respx.mock + def test_skip_invalid_imports_valid_rows( + self, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + route = respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "COMPLETED", ["ERR1"], [101], + )), + ) + sheet = _write_import_sheet( + tmp_path, + {"accession": "ERR1", "cell_type": "Neuron"}, + {"accession": "bogus", "cell_type": "Neuron"}, + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + "--skip-invalid", + ) + + assert result.exit_code == 0 + assert route.call_count == 1 + payload = json.loads(route.calls[0].request.content) + assert len(payload["imports"]) == 1 + assert "BOGUS" in result.stderr + + @respx.mock + def test_duplicate_accession_is_usage_error( + self, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + route = respx.post(SAMPLE_IMPORTS_URL) + sheet = _write_import_sheet( + tmp_path, + {"accession": "ERR1", "cell_type": "Neuron"}, + {"accession": "err1", "cell_type": "Neuron"}, + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + ) + + assert result.exit_code == 2 + assert route.call_count == 0 + assert "duplicate" in result.stderr.lower() + + @respx.mock + def test_non_csv_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None: + _mock_metadata() + route = respx.post(SAMPLE_IMPORTS_URL) + xlsx = tmp_path / "sheet.xlsx" + xlsx.write_bytes(b"PK\x03\x04") + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(xlsx), "--sample-type", "rna_seq", + ) + + assert result.exit_code == 2 + assert route.call_count == 0 + assert "CSV" in result.stderr + + def test_missing_sheet_is_usage_error(self, run_cli) -> None: + result = run_cli( + "--token", TOKEN, "samples", "import", "--sample-type", "rna_seq", + ) + + assert result.exit_code == 2 + + def test_missing_sample_type_is_usage_error(self, run_cli, tmp_path: Path) -> None: + sheet = _write_import_sheet(tmp_path, {"accession": "ERR1"}) + + result = run_cli( + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), + ) + + assert result.exit_code == 2 diff --git a/tests/unit/v2/test_samples.py b/tests/unit/v2/test_samples.py index cc42ad7..1f62a46 100644 --- a/tests/unit/v2/test_samples.py +++ b/tests/unit/v2/test_samples.py @@ -1,3 +1,4 @@ +import json from http import HTTPStatus from pathlib import Path from unittest.mock import ANY, patch @@ -10,6 +11,7 @@ from flowbio.v2.exceptions import ( AnnotationValidationError, BadRequestError, + FlowApiError, NotFoundError, ) from flowbio.v2.samples import ( @@ -17,6 +19,9 @@ MultiplexedUpload, Organism, Project, + SampleImportJob, + SampleImportJobId, + SampleImportSpec, SampleResource, SampleType, Sample, @@ -1049,3 +1054,184 @@ def test_chunked_multiplexed_upload(self, tmp_path: Path) -> None: assert mux_route.call_count == 3 assert result.data_ids == ["mux_1"] + +class TestImportSamples: + + @respx.mock + def test_posts_imports_and_parses_job(self) -> None: + route = respx.post(f"{DEFAULT_BASE_URL}/v2/sample-imports").mock( + return_value=httpx.Response(HTTPStatus.CREATED, json={ + "id": 42, + "status": "RUNNING", + "created": 1700000000, + "started": None, + "finished": None, + "accessions": ["ERR1160845"], + "sample_ids": [], + "execution_id": None, + "error": None, + }), + ) + + client = Client() + result = client.samples.import_samples([ + SampleImportSpec(accession="ERR1160845", sample_type="rna_seq"), + ]) + + assert result == SampleImportJob( + id=SampleImportJobId(42), + status="RUNNING", + accessions=["ERR1160845"], + sample_ids=[], + execution_id=None, + error=None, + ) + assert route.call_count == 1 + + @respx.mock + def test_sends_accession_and_sample_type(self) -> None: + route = respx.post(f"{DEFAULT_BASE_URL}/v2/sample-imports").mock( + return_value=httpx.Response(HTTPStatus.CREATED, json={ + "id": 1, "status": "RUNNING", "accessions": ["ERR1"], + "sample_ids": [], "execution_id": None, "error": None, + }), + ) + + client = Client() + client.samples.import_samples([ + SampleImportSpec(accession="ERR1", sample_type="rna_seq"), + ]) + + payload = json.loads(route.calls[0].request.content) + assert payload == { + "imports": [{"accession": "ERR1", "sample_type": "rna_seq"}], + } + + @respx.mock + def test_sends_optional_fields_when_present(self) -> None: + route = respx.post(f"{DEFAULT_BASE_URL}/v2/sample-imports").mock( + return_value=httpx.Response(HTTPStatus.CREATED, json={ + "id": 1, "status": "RUNNING", "accessions": ["ERR1"], + "sample_ids": [], "execution_id": None, "error": None, + }), + ) + + client = Client() + client.samples.import_samples([ + SampleImportSpec( + accession="ERR1", + sample_type="rna_seq", + name="my_sample", + organism_id="Hs", + metadata={"strandedness": "reverse"}, + ), + ]) + + payload = json.loads(route.calls[0].request.content) + assert payload == { + "imports": [{ + "accession": "ERR1", + "sample_type": "rna_seq", + "name": "my_sample", + "organism": "Hs", + "metadata": {"strandedness": "reverse"}, + }], + } + + @respx.mock + def test_sends_multiple_imports_in_one_request(self) -> None: + route = respx.post(f"{DEFAULT_BASE_URL}/v2/sample-imports").mock( + return_value=httpx.Response(HTTPStatus.CREATED, json={ + "id": 1, "status": "RUNNING", "accessions": ["ERR1", "ERR2"], + "sample_ids": [], "execution_id": None, "error": None, + }), + ) + + client = Client() + client.samples.import_samples([ + SampleImportSpec(accession="ERR1", sample_type="rna_seq"), + SampleImportSpec(accession="ERR2", sample_type="rna_seq"), + ]) + + payload = json.loads(route.calls[0].request.content) + assert [entry["accession"] for entry in payload["imports"]] == ["ERR1", "ERR2"] + assert route.call_count == 1 + + @respx.mock + def test_raises_flow_api_error_on_validation_failure(self) -> None: + respx.post(f"{DEFAULT_BASE_URL}/v2/sample-imports").mock( + return_value=httpx.Response( + HTTPStatus.UNPROCESSABLE_ENTITY, + json={"error": "at least one accession is required"}, + ), + ) + + client = Client() + + with pytest.raises(FlowApiError) as exc_info: + client.samples.import_samples([]) + + assert exc_info.value.status_code == HTTPStatus.UNPROCESSABLE_ENTITY + + +class TestGetImport: + + @respx.mock + def test_parses_completed_job(self) -> None: + respx.get(f"{DEFAULT_BASE_URL}/v2/sample-imports/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json={ + "id": 42, + "status": "COMPLETED", + "created": 1700000000, + "started": 1700000001, + "finished": 1700000002, + "accessions": ["ERR1160845", "ERR10677146"], + "sample_ids": [101, 102], + "execution_id": 7, + "error": None, + }), + ) + + client = Client() + result = client.samples.get_import(SampleImportJobId(42)) + + assert result == SampleImportJob( + id=SampleImportJobId(42), + status="COMPLETED", + accessions=["ERR1160845", "ERR10677146"], + sample_ids=[101, 102], + execution_id=7, + error=None, + ) + + @respx.mock + def test_parses_failed_job_with_error(self) -> None: + respx.get(f"{DEFAULT_BASE_URL}/v2/sample-imports/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json={ + "id": 42, + "status": "FAILED", + "accessions": ["ERR1160845"], + "sample_ids": [], + "execution_id": 7, + "error": "download failed: connection reset", + }), + ) + + client = Client() + result = client.samples.get_import(SampleImportJobId(42)) + + assert result.status == "FAILED" + assert result.error == "download failed: connection reset" + + @respx.mock + def test_raises_not_found_for_unknown_job(self) -> None: + respx.get(f"{DEFAULT_BASE_URL}/v2/sample-imports/999").mock( + return_value=httpx.Response( + HTTPStatus.NOT_FOUND, json={"error": "sample import 999 does not exist"}, + ), + ) + + client = Client() + + with pytest.raises(NotFoundError): + client.samples.get_import(SampleImportJobId(999)) From aec9c961647043396dac71f09470a066e40f1a1b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 13:58:53 +0000 Subject: [PATCH 02/32] fix(samples): address import-command review findings - Surface job_id/job_status/execution_id in the JSON document (null when no job was created), so a timed-out or interrupted import can be resumed with client.samples.get_import instead of leaving an untrackable server-side job. - Reject non-positive --poll-interval/--timeout as a usage error (exit 2) instead of crashing with ZeroDivisionError/ValueError. - Switch _poll_job from an attempt-count budget to a time.monotonic() deadline, so --timeout is actually a wall-clock bound rather than poll_interval * attempts (which silently added unbounded request latency and ignored a --timeout smaller than --poll-interval). - A COMPLETED job whose sample_ids don't cover every submitted accession no longer silently reports those rows as imported with sample_id: null; they are now reported failed with an explanatory message. - A failed/timed-out job now still reports the sample id for any accession the job did manage to create before failing, so a retry doesn't have to guess what to skip. - The timeout message no longer reads as "ended with status RUNNING"; it now says the job didn't finish in time and points at get_import to resume. - Reordered _sheet.py so the public metadata_errors sits above the private _build_row/_resolve helpers. Addresses Claude's review on flowbio#18; see PR comments for the findings left as deliberate, explained trade-offs (SampleImportSpec as a dataclass, sample_ids staying int, the accession-format regex, and not adding poll-error retry tolerance). Refs FLOW-689. --- flowbio/cli/_samples.py | 121 +++++++++++++++++------- flowbio/cli/_sheet.py | 66 ++++++------- source/cli.rst | 27 ++++-- source/v2/samples.rst | 2 +- tests/unit/cli/test_samples.py | 164 ++++++++++++++++++++++++++++++++- 5 files changed, 302 insertions(+), 78 deletions(-) diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index afd1f3e..2bfb371 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -10,7 +10,6 @@ import argparse import json -import math import time from dataclasses import dataclass from pathlib import Path @@ -286,14 +285,20 @@ def _configure_import(import_parser: argparse.ArgumentParser) -> None: type=float, default=_DEFAULT_POLL_INTERVAL, metavar="SECONDS", - help=f"Seconds to wait between checks of the import job's status (default: {_DEFAULT_POLL_INTERVAL:g}).", + help=( + "Seconds to wait between checks of the import job's status; must be " + f"positive (default: {_DEFAULT_POLL_INTERVAL:g})." + ), ) import_parser.add_argument( "--timeout", type=float, default=_DEFAULT_TIMEOUT, metavar="SECONDS", - help=f"Maximum seconds to wait for the import job to finish (default: {_DEFAULT_TIMEOUT:g}).", + help=( + "Maximum seconds to wait for the import job to finish; must be " + f"positive (default: {_DEFAULT_TIMEOUT:g})." + ), ) @@ -629,11 +634,17 @@ class _ImportResult: Unlike :class:`_BatchResult`, every accession shares one server-side job: all rows land in ``imported`` together on a completed job, or all land in ``failed`` together (carrying the job's one error message) otherwise. + ``job_id``/``job_status``/``execution_id`` are ``None`` only when no job + was ever created (every row was invalid or skipped), so a timed-out or + interrupted run can still be resumed with :meth:`~flowbio.v2.samples.SampleResource.get_import`. """ imported: list[dict[str, JsonValue]] failed: list[dict[str, JsonValue]] skipped: list[dict[str, JsonValue]] + job_id: int | None = None + job_status: str | None = None + execution_id: int | None = None @property def counts(self) -> dict[str, int]: @@ -650,6 +661,9 @@ def document(self) -> dict[str, JsonValue]: "failed": self.failed, "skipped": self.skipped, "counts": self.counts, + "job_id": self.job_id, + "job_status": self.job_status, + "execution_id": self.execution_id, } @property @@ -675,9 +689,11 @@ def _import_command( :param output: The result/error renderer. :returns: :attr:`ExitCode.SUCCESS` when the import job completes, :attr:`ExitCode.USAGE` on a pre-flight validation failure without - ``--skip-invalid``, or :attr:`ExitCode.RUNTIME` if the import job - fails or does not finish within ``--timeout``. + ``--skip-invalid`` or a non-positive ``--poll-interval``/``--timeout``, + or :attr:`ExitCode.RUNTIME` if the import job fails or does not + finish within ``--timeout``. """ + _validate_poll_options(args.poll_interval, args.timeout) sheet = parse_accession_sheet(args.sheet) attributes = client.samples.get_metadata_attributes() duplicates = duplicate_accession_errors(sheet.rows) @@ -711,6 +727,13 @@ def _import_command( return result.exit_code +def _validate_poll_options(poll_interval: float, timeout: float) -> None: + if poll_interval <= 0: + raise CliUsageError(f"--poll-interval must be positive, got {poll_interval!r}.") + if timeout <= 0: + raise CliUsageError(f"--timeout must be positive, got {timeout!r}.") + + def _run_import( rows: list[AccessionSheetRow], args: argparse.Namespace, @@ -740,10 +763,8 @@ def _run_import( def _poll_job( client: Client, job: SampleImportJob, poll_interval: float, timeout: float, ) -> SampleImportJob: - max_attempts = max(1, math.ceil(timeout / poll_interval)) - for _ in range(max_attempts): - if job.status != "RUNNING": - break + deadline = time.monotonic() + timeout + while job.status == "RUNNING" and time.monotonic() < deadline: time.sleep(poll_interval) job = client.samples.get_import(job.id) return job @@ -755,33 +776,67 @@ def _import_result( skipped: list[dict[str, JsonValue]], output: Output, ) -> _ImportResult: + accession_to_sample_id = dict(zip(job.accessions, job.sample_ids)) if job.status == "COMPLETED": - accession_to_sample_id = dict(zip(job.accessions, job.sample_ids)) - imported = [ - { - "row_number": row.row_number, - "accession": row.accession, - "sample_id": accession_to_sample_id.get(row.accession), - } - for row in rows - ] - for entry in imported: - output.emit_advisory( - f"Row {entry['row_number']} ({entry['accession']}): " - f"imported sample {entry['sample_id']}", - ) - return _ImportResult(imported=imported, failed=[], skipped=skipped) + imported, failed = _completed_outcomes(job, rows, accession_to_sample_id, output) + else: + imported, failed = [], _failed_outcomes(job, rows, accession_to_sample_id, output) + return _ImportResult( + imported=imported, + failed=failed, + skipped=skipped, + job_id=job.id, + job_status=job.status, + execution_id=job.execution_id, + ) - message = job.error or f"import job {job.id} ended with status {job.status}" - failed = [ - {"row_number": row.row_number, "accession": row.accession, "message": message} - for row in rows - ] - for entry in failed: - output.emit_advisory( - f"Row {entry['row_number']} ({entry['accession']}): import failed — {message}", + +def _completed_outcomes( + job: SampleImportJob, + rows: list[AccessionSheetRow], + accession_to_sample_id: dict[str, int], + output: Output, +) -> tuple[list[dict[str, JsonValue]], list[dict[str, JsonValue]]]: + imported: list[dict[str, JsonValue]] = [] + failed: list[dict[str, JsonValue]] = [] + for row in rows: + sample_id = accession_to_sample_id.get(row.accession) + if sample_id is None: + message = f"import job {job.id} completed but returned no sample id for this accession" + failed.append({"row_number": row.row_number, "accession": row.accession, "message": message}) + output.emit_advisory(f"Row {row.row_number} ({row.accession}): import failed — {message}") + continue + imported.append({"row_number": row.row_number, "accession": row.accession, "sample_id": sample_id}) + output.emit_advisory(f"Row {row.row_number} ({row.accession}): imported sample {sample_id}") + return imported, failed + + +def _failed_outcomes( + job: SampleImportJob, + rows: list[AccessionSheetRow], + accession_to_sample_id: dict[str, int], + output: Output, +) -> list[dict[str, JsonValue]]: + if job.status == "RUNNING": + message = ( + f"import job {job.id} did not finish within the configured timeout " + f"(still RUNNING) — check it later with client.samples.get_import({job.id})" ) - return _ImportResult(imported=[], failed=failed, skipped=skipped) + else: + message = job.error or f"import job {job.id} ended with status {job.status}" + failed: list[dict[str, JsonValue]] = [] + for row in rows: + entry: dict[str, JsonValue] = { + "row_number": row.row_number, "accession": row.accession, "message": message, + } + sample_id = accession_to_sample_id.get(row.accession) + advisory = f"Row {row.row_number} ({row.accession}): import failed — {message}" + if sample_id is not None: + entry["sample_id"] = sample_id + advisory += f" (sample {sample_id} was created)" + failed.append(entry) + output.emit_advisory(advisory) + return failed def _invalid_accession_line(row: AccessionSheetRow, reasons: list[str]) -> str: diff --git a/flowbio/cli/_sheet.py b/flowbio/cli/_sheet.py index 2e70a4f..a5effa1 100644 --- a/flowbio/cli/_sheet.py +++ b/flowbio/cli/_sheet.py @@ -110,39 +110,6 @@ def validate_row( return errors -def _build_row( - record: dict[str, str], - row_number: int, - base_dir: Path, - metadata_columns: list[str], -) -> SheetRow: - def cell(column: str) -> str | None: - value = (record.get(column) or "").strip() - return value or None - - metadata = { - column: value - for column in metadata_columns - if (value := (record.get(column) or "").strip()) - } - return SheetRow( - row_number=row_number, - name=cell("name") or "", - reads1=_resolve(cell("reads1"), base_dir), - reads2=_resolve(cell("reads2"), base_dir), - project=cell("project"), - organism=cell("organism"), - metadata=metadata, - ) - - -def _resolve(value: str | None, base_dir: Path) -> Path | None: - if value is None: - return None - path = Path(value) - return path if path.is_absolute() else base_dir / path - - def metadata_errors( metadata: dict[str, str], attributes: list[MetadataAttribute], @@ -195,3 +162,36 @@ def metadata_errors( if attribute is not None and not attribute.allow_annotation: errors.append(f"{base} does not allow an annotation") return errors + + +def _build_row( + record: dict[str, str], + row_number: int, + base_dir: Path, + metadata_columns: list[str], +) -> SheetRow: + def cell(column: str) -> str | None: + value = (record.get(column) or "").strip() + return value or None + + metadata = { + column: value + for column in metadata_columns + if (value := (record.get(column) or "").strip()) + } + return SheetRow( + row_number=row_number, + name=cell("name") or "", + reads1=_resolve(cell("reads1"), base_dir), + reads2=_resolve(cell("reads2"), base_dir), + project=cell("project"), + organism=cell("organism"), + metadata=metadata, + ) + + +def _resolve(value: str | None, base_dir: Path) -> Path | None: + if value is None: + return None + path = Path(value) + return path if path.is_absolute() else base_dir / path diff --git a/source/cli.rst b/source/cli.rst index e26fcf1..5d3762f 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -395,14 +395,19 @@ By default any invalid row aborts the whole run (exit ``2``); Unlike ``upload-batch``, every valid row is submitted **together as one server-side job** — there is no per-row upload loop. The command polls that job (every ``--poll-interval`` seconds, default 5, for up to ``--timeout`` -seconds, default 1800) until it leaves ``"RUNNING"``. Once the job completes, -every submitted row is reported as imported; if it fails or times out, every -row is reported as failed with the same message, since the job has one -outcome for the whole batch. +seconds, default 1800; both must be positive) until it leaves ``"RUNNING"``. +Once the job completes, every submitted row is reported as imported; if it +fails or times out, every row is reported as failed with the same message +(and the sample id, if the job did partially create one before failing), +since the job has one outcome for the whole batch. Either way, the job's +``job_id`` is included in the output so a timed-out or interrupted run can be +resumed with ``client.samples.get_import(job_id)`` from the library. **Output** — human: each row's outcome on stderr, then a final counts summary on stdout. ``--json``: a single document on stdout with ``imported``, -``failed``, and ``skipped`` lists plus a ``counts`` summary: +``failed``, and ``skipped`` lists, a ``counts`` summary, and the job's +``job_id``/``job_status``/``execution_id`` (``null`` when no job was created, +e.g. every row was invalid or skipped): .. code-block:: json @@ -410,13 +415,15 @@ on stdout. ``--json``: a single document on stdout with ``imported``, "imported": [{"row_number": 1, "accession": "ERR1160845", "sample_id": 101}], "failed": [], "skipped": [{"row_number": 2, "accession": "bogus", "reasons": ["..."]}], - "counts": {"imported": 1, "failed": 0, "skipped": 1} + "counts": {"imported": 1, "failed": 0, "skipped": 1}, + "job_id": 42, "job_status": "COMPLETED", "execution_id": 7 } **Exit codes** — ``0`` the import job completed; ``2`` a pre-flight -validation failure (without ``--skip-invalid``) or a non-CSV sheet; ``1`` the -import job failed or did not finish within ``--timeout``; ``3`` -authentication failure; otherwise the standard mapping above. +validation failure (without ``--skip-invalid``), a non-CSV sheet, or a +non-positive ``--poll-interval``/``--timeout``; ``1`` the import job failed +or did not finish within ``--timeout``; ``3`` authentication failure; +otherwise the standard mapping above. **Example** @@ -428,7 +435,7 @@ authentication failure; otherwise the standard mapping above. Imported 2, failed 0, skipped 0. $ flowbio samples import --sheet ./accessions.csv --sample-type RNA-Seq --json - {"imported": [{"row_number": 1, "accession": "ERR1160845", "sample_id": 101}], "failed": [], "skipped": [], "counts": {"imported": 1, "failed": 0, "skipped": 0}} + {"imported": [{"row_number": 1, "accession": "ERR1160845", "sample_id": 101}], "failed": [], "skipped": [], "counts": {"imported": 1, "failed": 0, "skipped": 0}, "job_id": 42, "job_status": "COMPLETED", "execution_id": 7} ``api get`` ~~~~~~~~~~~ diff --git a/source/v2/samples.rst b/source/v2/samples.rst index d78855b..db5726d 100644 --- a/source/v2/samples.rst +++ b/source/v2/samples.rst @@ -228,4 +228,4 @@ Models .. autoclass:: flowbio.v2.samples.SampleImportSpec -.. autopydantic_model:: flowbio.v2.samples.SampleImportJob \ No newline at end of file +.. autopydantic_model:: flowbio.v2.samples.SampleImportJob diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index 3d2bf6a..3d4471b 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -1008,6 +1008,50 @@ def test_json_document_reports_outcomes_and_counts( assert document["imported"][0] == { "row_number": 1, "accession": "ERR1", "sample_id": 101, } + assert document["job_id"] == 1 + assert document["job_status"] == "COMPLETED" + assert document["execution_id"] == 7 + + @respx.mock + def test_json_document_has_null_job_fields_when_nothing_submitted( + self, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + route = respx.post(SAMPLE_IMPORTS_URL) + sheet = _write_import_sheet( + tmp_path, {"accession": "bogus", "cell_type": "Neuron"}, + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + "--skip-invalid", "--json", + ) + + assert result.exit_code == 0 + assert route.call_count == 0 + document = json.loads(result.stdout) + assert document["counts"] == {"imported": 0, "failed": 0, "skipped": 1} + assert document["job_id"] is None + assert document["job_status"] is None + assert document["execution_id"] is None + + @respx.mock + def test_header_only_sheet_imports_nothing_without_calling_api( + self, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + route = respx.post(SAMPLE_IMPORTS_URL) + sheet = _write_import_sheet(tmp_path) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + ) + + assert result.exit_code == 0 + assert route.call_count == 0 + assert "Imported 0, failed 0, skipped 0." in result.stdout @respx.mock def test_sends_sample_type_and_metadata_in_kickoff_payload( @@ -1101,11 +1145,13 @@ def test_failed_job_reports_error_message_per_row( assert "Row 2 (ERR2)" in result.stderr @respx.mock + @patch("time.monotonic") @patch("time.sleep") def test_timeout_reports_failure_without_hanging( - self, _mock_sleep, run_cli, tmp_path: Path, + self, _mock_sleep, mock_monotonic, run_cli, tmp_path: Path, ) -> None: _mock_metadata() + mock_monotonic.side_effect = [0.0, 0.5, 1.5, 2.5] respx.post(SAMPLE_IMPORTS_URL).mock( return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( 1, "RUNNING", ["ERR1"], @@ -1125,6 +1171,122 @@ def test_timeout_reports_failure_without_hanging( assert result.exit_code == 1 assert poll_route.call_count == 2 assert "Row 1 (ERR1)" in result.stderr + assert "did not finish within" in result.stderr + assert "still RUNNING" in result.stderr + assert "ended with status RUNNING" not in result.stderr + + @respx.mock + def test_zero_poll_interval_is_usage_error(self, run_cli, tmp_path: Path) -> None: + _mock_metadata() + route = respx.post(SAMPLE_IMPORTS_URL) + sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + "--poll-interval", "0", + ) + + assert result.exit_code == 2 + assert route.call_count == 0 + assert "--poll-interval" in result.stderr + + @respx.mock + def test_negative_timeout_is_usage_error(self, run_cli, tmp_path: Path) -> None: + _mock_metadata() + route = respx.post(SAMPLE_IMPORTS_URL) + sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + "--timeout", "-1", + ) + + assert result.exit_code == 2 + assert route.call_count == 0 + assert "--timeout" in result.stderr + + @respx.mock + def test_completed_job_missing_sample_id_reports_row_as_failed( + self, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "COMPLETED", ["ERR1"], [], + )), + ) + sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", "--json", + ) + + assert result.exit_code == 1 + document = json.loads(result.stdout) + assert document["imported"] == [] + assert document["failed"][0]["accession"] == "ERR1" + assert "no sample id" in document["failed"][0]["message"] + + @respx.mock + @patch("time.sleep") + def test_failed_job_reports_partial_sample_ids( + self, _mock_sleep, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "RUNNING", ["ERR1", "ERR2"], + )), + ) + respx.get(f"{SAMPLE_IMPORTS_URL}/1").mock( + return_value=httpx.Response(HTTPStatus.OK, json=_job_json( + 1, "FAILED", ["ERR1", "ERR2"], [101], error="download failed", + )), + ) + sheet = _write_import_sheet( + tmp_path, + {"accession": "ERR1", "cell_type": "Neuron"}, + {"accession": "ERR2", "cell_type": "Neuron"}, + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + "--poll-interval", "1", "--json", + ) + + assert result.exit_code == 1 + document = json.loads(result.stdout) + failed_by_accession = {entry["accession"]: entry for entry in document["failed"]} + assert failed_by_accession["ERR1"]["sample_id"] == 101 + assert "sample_id" not in failed_by_accession["ERR2"] + + @respx.mock + @patch("time.sleep") + def test_transient_poll_error_propagates( + self, _mock_sleep, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "RUNNING", ["ERR1"], + )), + ) + respx.get(f"{SAMPLE_IMPORTS_URL}/1").mock( + return_value=httpx.Response(HTTPStatus.NOT_FOUND, json={"error": "gone"}), + ) + sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + "--poll-interval", "1", + ) + + assert result.exit_code == 4 @respx.mock def test_invalid_row_aborts_and_imports_nothing( From 9688f9f464dbd24361c6100854bc98b685b5f653 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 14:12:53 +0000 Subject: [PATCH 03/32] fix(samples): address round-2 import-command review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Export SampleImportSpec/SampleImportJob/SampleImportJobId from flowbio.v2, matching every other public sample model; update the samples.rst example to import from the package root instead of the submodule. - Clamp _poll_job's sleep to the time remaining before the deadline, so --poll-interval can no longer make a single sleep overshoot --timeout (e.g. --timeout 10 --poll-interval 300 previously blocked for 300s). - Stop pairing job.accessions/job.sample_ids positionally via zip when their lengths disagree — that silently truncated (reporting a row failed even though the job created its sample) or mis-attributed (handing a row another row's sample id) instead of raising. Both directions are now reported as an explicit, unmatchable failure instead. - Give each failed-row entry a status: "failed" (safe to retry), "running" (timed out but the job may still finish — don't re-run) or "unknown" (accessions/sample_ids couldn't be matched), so an automated caller doesn't treat a timeout the same as a genuine failure. - Move --poll-interval/--timeout validation into an argparse type= callable, so a non-positive value fails at parse time (exit 2) before _dispatch resolves credentials, instead of after an interactive login. - Narrow _ImportResult.job_id/job_status to SampleImportJobId/ SampleImportStatus instead of bare int/str. - Replace the timeout test's hand-counted time.monotonic() side_effect list with itertools.count(...), an infinite fake clock that doesn't need re-counting whenever _poll_job's loop shape changes. Addresses Claude's round-2 review on flowbio#18. Finding 8 (sheet with no accession column degrading confusingly) is left for the same follow-up as round 1's unknown-column-warning deferral, as suggested. Refs FLOW-689. --- flowbio/cli/_samples.py | 100 +++++++++++++++------ flowbio/v2/__init__.py | 6 ++ source/cli.rst | 24 +++-- source/v2/samples.rst | 2 +- tests/unit/cli/test_samples.py | 123 ++++++++++++++++++++++++-- tests/unit/v2/test_package_exports.py | 12 +++ tests/unit/v2/test_samples.py | 1 + 7 files changed, 227 insertions(+), 41 deletions(-) diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index 2bfb371..fe859a5 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -36,7 +36,9 @@ from flowbio.v2.samples import ( MetadataAttribute, SampleImportJob, + SampleImportJobId, SampleImportSpec, + SampleImportStatus, SampleTypeId, ) @@ -259,6 +261,13 @@ def _configure_upload_batch(upload_batch: argparse.ArgumentParser) -> None: _DEFAULT_TIMEOUT = 1800.0 +def _positive_float(value: str) -> float: + parsed = float(value) + if parsed <= 0: + raise argparse.ArgumentTypeError(f"must be positive, got {value!r}") + return parsed + + def _configure_import(import_parser: argparse.ArgumentParser) -> None: import_parser.set_defaults(command_parser=import_parser, handler=_import_command) import_parser.add_argument( @@ -282,7 +291,7 @@ def _configure_import(import_parser: argparse.ArgumentParser) -> None: ) import_parser.add_argument( "--poll-interval", - type=float, + type=_positive_float, default=_DEFAULT_POLL_INTERVAL, metavar="SECONDS", help=( @@ -292,7 +301,7 @@ def _configure_import(import_parser: argparse.ArgumentParser) -> None: ) import_parser.add_argument( "--timeout", - type=float, + type=_positive_float, default=_DEFAULT_TIMEOUT, metavar="SECONDS", help=( @@ -633,17 +642,21 @@ class _ImportResult: Unlike :class:`_BatchResult`, every accession shares one server-side job: all rows land in ``imported`` together on a completed job, or all land in - ``failed`` together (carrying the job's one error message) otherwise. - ``job_id``/``job_status``/``execution_id`` are ``None`` only when no job - was ever created (every row was invalid or skipped), so a timed-out or - interrupted run can still be resumed with :meth:`~flowbio.v2.samples.SampleResource.get_import`. + ``failed`` together (carrying the job's one error message) otherwise. Each + ``failed`` entry carries a ``status`` of ``"failed"`` (the job genuinely + failed), ``"running"`` (the job timed out but may still complete), or + ``"unknown"`` (the job's accessions/sample_ids couldn't be matched) — only + ``"failed"`` is safe to blindly retry. ``job_id``/``job_status``/ + ``execution_id`` are ``None`` only when no job was ever created (every row + was invalid or skipped), so a timed-out or interrupted run can still be + resumed with :meth:`~flowbio.v2.samples.SampleResource.get_import`. """ imported: list[dict[str, JsonValue]] failed: list[dict[str, JsonValue]] skipped: list[dict[str, JsonValue]] - job_id: int | None = None - job_status: str | None = None + job_id: SampleImportJobId | None = None + job_status: SampleImportStatus | None = None execution_id: int | None = None @property @@ -689,11 +702,9 @@ def _import_command( :param output: The result/error renderer. :returns: :attr:`ExitCode.SUCCESS` when the import job completes, :attr:`ExitCode.USAGE` on a pre-flight validation failure without - ``--skip-invalid`` or a non-positive ``--poll-interval``/``--timeout``, - or :attr:`ExitCode.RUNTIME` if the import job fails or does not - finish within ``--timeout``. + ``--skip-invalid``, or :attr:`ExitCode.RUNTIME` if the import job + fails or does not finish within ``--timeout``. """ - _validate_poll_options(args.poll_interval, args.timeout) sheet = parse_accession_sheet(args.sheet) attributes = client.samples.get_metadata_attributes() duplicates = duplicate_accession_errors(sheet.rows) @@ -727,13 +738,6 @@ def _import_command( return result.exit_code -def _validate_poll_options(poll_interval: float, timeout: float) -> None: - if poll_interval <= 0: - raise CliUsageError(f"--poll-interval must be positive, got {poll_interval!r}.") - if timeout <= 0: - raise CliUsageError(f"--timeout must be positive, got {timeout!r}.") - - def _run_import( rows: list[AccessionSheetRow], args: argparse.Namespace, @@ -764,8 +768,11 @@ def _poll_job( client: Client, job: SampleImportJob, poll_interval: float, timeout: float, ) -> SampleImportJob: deadline = time.monotonic() + timeout - while job.status == "RUNNING" and time.monotonic() < deadline: - time.sleep(poll_interval) + while job.status == "RUNNING": + remaining = deadline - time.monotonic() + if remaining <= 0: + break + time.sleep(min(poll_interval, remaining)) job = client.samples.get_import(job.id) return job @@ -776,11 +783,15 @@ def _import_result( skipped: list[dict[str, JsonValue]], output: Output, ) -> _ImportResult: - accession_to_sample_id = dict(zip(job.accessions, job.sample_ids)) - if job.status == "COMPLETED": - imported, failed = _completed_outcomes(job, rows, accession_to_sample_id, output) + if _sample_ids_unmatchable(job): + imported: list[dict[str, JsonValue]] = [] + failed = _mismatched_outcome(job, rows, output) else: - imported, failed = [], _failed_outcomes(job, rows, accession_to_sample_id, output) + accession_to_sample_id = dict(zip(job.accessions, job.sample_ids)) + if job.status == "COMPLETED": + imported, failed = _completed_outcomes(job, rows, accession_to_sample_id, output) + else: + imported, failed = [], _failed_outcomes(job, rows, accession_to_sample_id, output) return _ImportResult( imported=imported, failed=failed, @@ -791,6 +802,33 @@ def _import_result( ) +def _sample_ids_unmatchable(job: SampleImportJob) -> bool: + # accessions/sample_ids are only guaranteed to correspond positionally when + # their lengths agree (an empty sample_ids on a RUNNING/FAILED job is + # expected, not a mismatch) — anything else means the job's response + # doesn't support the positional pairing the outcome mapping relies on. + return len(job.sample_ids) not in (0, len(job.accessions)) + + +def _mismatched_outcome( + job: SampleImportJob, rows: list[AccessionSheetRow], output: Output, +) -> list[dict[str, JsonValue]]: + message = ( + f"import job {job.id} returned {len(job.sample_ids)} sample id(s) for " + f"{len(job.accessions)} accession(s); cannot match them to rows" + ) + failed: list[dict[str, JsonValue]] = [] + for row in rows: + failed.append({ + "row_number": row.row_number, + "accession": row.accession, + "message": message, + "status": "unknown", + }) + output.emit_advisory(f"Row {row.row_number} ({row.accession}): import failed — {message}") + return failed + + def _completed_outcomes( job: SampleImportJob, rows: list[AccessionSheetRow], @@ -803,7 +841,10 @@ def _completed_outcomes( sample_id = accession_to_sample_id.get(row.accession) if sample_id is None: message = f"import job {job.id} completed but returned no sample id for this accession" - failed.append({"row_number": row.row_number, "accession": row.accession, "message": message}) + failed.append({ + "row_number": row.row_number, "accession": row.accession, + "message": message, "status": "failed", + }) output.emit_advisory(f"Row {row.row_number} ({row.accession}): import failed — {message}") continue imported.append({"row_number": row.row_number, "accession": row.accession, "sample_id": sample_id}) @@ -817,17 +858,20 @@ def _failed_outcomes( accession_to_sample_id: dict[str, int], output: Output, ) -> list[dict[str, JsonValue]]: - if job.status == "RUNNING": + timed_out = job.status == "RUNNING" + if timed_out: message = ( f"import job {job.id} did not finish within the configured timeout " f"(still RUNNING) — check it later with client.samples.get_import({job.id})" ) else: message = job.error or f"import job {job.id} ended with status {job.status}" + entry_status = "running" if timed_out else "failed" failed: list[dict[str, JsonValue]] = [] for row in rows: entry: dict[str, JsonValue] = { - "row_number": row.row_number, "accession": row.accession, "message": message, + "row_number": row.row_number, "accession": row.accession, + "message": message, "status": entry_status, } sample_id = accession_to_sample_id.get(row.accession) advisory = f"Row {row.row_number} ({row.accession}): import failed — {message}" diff --git a/flowbio/v2/__init__.py b/flowbio/v2/__init__.py index 1b239ee..99f2621 100644 --- a/flowbio/v2/__init__.py +++ b/flowbio/v2/__init__.py @@ -45,6 +45,9 @@ Organism, Project, Sample, + SampleImportJob, + SampleImportJobId, + SampleImportSpec, SampleType, SampleTypeId, ) @@ -59,6 +62,9 @@ "Organism", "Project", "Sample", + "SampleImportJob", + "SampleImportJobId", + "SampleImportSpec", "SampleType", "SampleTypeId", "TokenCredentials", diff --git a/source/cli.rst b/source/cli.rst index 5d3762f..a78e301 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -398,10 +398,18 @@ job (every ``--poll-interval`` seconds, default 5, for up to ``--timeout`` seconds, default 1800; both must be positive) until it leaves ``"RUNNING"``. Once the job completes, every submitted row is reported as imported; if it fails or times out, every row is reported as failed with the same message -(and the sample id, if the job did partially create one before failing), -since the job has one outcome for the whole batch. Either way, the job's -``job_id`` is included in the output so a timed-out or interrupted run can be -resumed with ``client.samples.get_import(job_id)`` from the library. +(and the sample id, if the job did create one before failing), since the job +has one outcome for the whole batch. Either way, the job's ``job_id`` is +included in the output so a timed-out or interrupted run can be resumed with +``client.samples.get_import(job_id)`` from the library. + +Each ``failed`` entry carries a ``status``: ``"failed"`` means the job +genuinely failed (or completed without creating that row's sample) — safe to +re-run once fixed. ``"running"`` means ``--timeout`` was reached while the job +was still in progress: it may yet succeed, so check ``job_status``/ +``get_import(job_id)`` rather than re-submitting, which would risk duplicate +samples. ``"unknown"`` means the job's ``accessions``/``sample_ids`` couldn't +be matched to rows at all (an unexpected shape from the server). **Output** — human: each row's outcome on stderr, then a final counts summary on stdout. ``--json``: a single document on stdout with ``imported``, @@ -421,9 +429,11 @@ e.g. every row was invalid or skipped): **Exit codes** — ``0`` the import job completed; ``2`` a pre-flight validation failure (without ``--skip-invalid``), a non-CSV sheet, or a -non-positive ``--poll-interval``/``--timeout``; ``1`` the import job failed -or did not finish within ``--timeout``; ``3`` authentication failure; -otherwise the standard mapping above. +non-positive ``--poll-interval``/``--timeout``; ``1`` the import job failed, +did not finish within ``--timeout`` (check ``job_status``/each row's +``status`` before re-running — see above), or returned a shape the rows +couldn't be matched against; ``3`` authentication failure; otherwise the +standard mapping above. **Example** diff --git a/source/v2/samples.rst b/source/v2/samples.rst index db5726d..eec9091 100644 --- a/source/v2/samples.rst +++ b/source/v2/samples.rst @@ -176,7 +176,7 @@ samples directly from public-repository run or experiment accessions (SRR/ERR/DRR or SRX/ERX/DRX), instead of uploading files yourself. Every accession submitted together is tracked as a single job:: - from flowbio.v2.samples import SampleImportSpec + from flowbio.v2 import SampleImportSpec job = client.samples.import_samples([ SampleImportSpec(accession="ERR1160845", sample_type="RNA-Seq"), diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index 3d4471b..97b0144 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -1,4 +1,5 @@ import csv +import itertools import json from http import HTTPStatus from pathlib import Path @@ -1151,7 +1152,7 @@ def test_timeout_reports_failure_without_hanging( self, _mock_sleep, mock_monotonic, run_cli, tmp_path: Path, ) -> None: _mock_metadata() - mock_monotonic.side_effect = [0.0, 0.5, 1.5, 2.5] + mock_monotonic.side_effect = itertools.count(0.0, 1.0) respx.post(SAMPLE_IMPORTS_URL).mock( return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( 1, "RUNNING", ["ERR1"], @@ -1165,7 +1166,7 @@ def test_timeout_reports_failure_without_hanging( result = run_cli( "--token", TOKEN, "samples", "import", "--sheet", str(sheet), "--sample-type", "rna_seq", - "--poll-interval", "1", "--timeout", "2", + "--poll-interval", "1", "--timeout", "3", ) assert result.exit_code == 1 @@ -1175,6 +1176,62 @@ def test_timeout_reports_failure_without_hanging( assert "still RUNNING" in result.stderr assert "ended with status RUNNING" not in result.stderr + @respx.mock + @patch("time.monotonic") + @patch("time.sleep") + def test_timeout_failed_rows_have_running_status( + self, _mock_sleep, mock_monotonic, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + mock_monotonic.side_effect = itertools.count(0.0, 1.0) + respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "RUNNING", ["ERR1"], + )), + ) + respx.get(f"{SAMPLE_IMPORTS_URL}/1").mock( + return_value=httpx.Response(HTTPStatus.OK, json=_job_json(1, "RUNNING", ["ERR1"])), + ) + sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + "--poll-interval", "1", "--timeout", "3", "--json", + ) + + assert result.exit_code == 1 + document = json.loads(result.stdout) + assert document["failed"][0]["status"] == "running" + assert document["job_status"] == "RUNNING" + + @respx.mock + @patch("time.monotonic") + @patch("time.sleep") + def test_poll_interval_exceeding_timeout_is_clamped( + self, mock_sleep, mock_monotonic, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + mock_monotonic.side_effect = [0.0, 0.0] + respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "RUNNING", ["ERR1"], + )), + ) + respx.get(f"{SAMPLE_IMPORTS_URL}/1").mock( + return_value=httpx.Response(HTTPStatus.OK, json=_job_json(1, "COMPLETED", ["ERR1"], [101])), + ) + sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + "--poll-interval", "300", "--timeout", "10", + ) + + assert result.exit_code == 0 + mock_sleep.assert_called_once_with(10.0) + @respx.mock def test_zero_poll_interval_is_usage_error(self, run_cli, tmp_path: Path) -> None: _mock_metadata() @@ -1228,11 +1285,66 @@ def test_completed_job_missing_sample_id_reports_row_as_failed( document = json.loads(result.stdout) assert document["imported"] == [] assert document["failed"][0]["accession"] == "ERR1" + assert document["failed"][0]["status"] == "failed" assert "no sample id" in document["failed"][0]["message"] + @respx.mock + def test_fewer_sample_ids_than_accessions_is_reported_as_unmatchable( + self, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "COMPLETED", ["ERR1", "ERR2"], [101], + )), + ) + sheet = _write_import_sheet( + tmp_path, + {"accession": "ERR1", "cell_type": "Neuron"}, + {"accession": "ERR2", "cell_type": "Neuron"}, + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", "--json", + ) + + assert result.exit_code == 1 + document = json.loads(result.stdout) + assert document["imported"] == [] + assert len(document["failed"]) == 2 + for entry in document["failed"]: + assert entry["status"] == "unknown" + assert "sample_id" not in entry + assert "cannot match" in entry["message"] + + @respx.mock + def test_more_sample_ids_than_accessions_is_reported_as_unmatchable( + self, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "COMPLETED", ["ERX1"], [101, 102], + )), + ) + sheet = _write_import_sheet(tmp_path, {"accession": "ERX1", "cell_type": "Neuron"}) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", "--json", + ) + + assert result.exit_code == 1 + document = json.loads(result.stdout) + assert document["imported"] == [] + assert document["failed"][0]["status"] == "unknown" + assert "sample_id" not in document["failed"][0] + assert "cannot match" in document["failed"][0]["message"] + @respx.mock @patch("time.sleep") - def test_failed_job_reports_partial_sample_ids( + def test_failed_job_with_all_sample_ids_created_reports_them_per_row( self, _mock_sleep, run_cli, tmp_path: Path, ) -> None: _mock_metadata() @@ -1243,7 +1355,7 @@ def test_failed_job_reports_partial_sample_ids( ) respx.get(f"{SAMPLE_IMPORTS_URL}/1").mock( return_value=httpx.Response(HTTPStatus.OK, json=_job_json( - 1, "FAILED", ["ERR1", "ERR2"], [101], error="download failed", + 1, "FAILED", ["ERR1", "ERR2"], [101, 102], error="post-processing failed", )), ) sheet = _write_import_sheet( @@ -1262,7 +1374,8 @@ def test_failed_job_reports_partial_sample_ids( document = json.loads(result.stdout) failed_by_accession = {entry["accession"]: entry for entry in document["failed"]} assert failed_by_accession["ERR1"]["sample_id"] == 101 - assert "sample_id" not in failed_by_accession["ERR2"] + assert failed_by_accession["ERR2"]["sample_id"] == 102 + assert failed_by_accession["ERR1"]["status"] == "failed" @respx.mock @patch("time.sleep") diff --git a/tests/unit/v2/test_package_exports.py b/tests/unit/v2/test_package_exports.py index 2bb0a12..d1bebbd 100644 --- a/tests/unit/v2/test_package_exports.py +++ b/tests/unit/v2/test_package_exports.py @@ -20,6 +20,18 @@ def test_username_password_credentials_is_exported(self) -> None: assert UsernamePasswordCredentials is DirectCredentials + def test_sample_import_types_are_exported(self) -> None: + from flowbio.v2 import SampleImportJob, SampleImportJobId, SampleImportSpec + from flowbio.v2.samples import ( + SampleImportJob as DirectJob, + SampleImportJobId as DirectJobId, + SampleImportSpec as DirectSpec, + ) + + assert SampleImportSpec is DirectSpec + assert SampleImportJob is DirectJob + assert SampleImportJobId is DirectJobId + class TestSubmoduleExports: diff --git a/tests/unit/v2/test_samples.py b/tests/unit/v2/test_samples.py index 1f62a46..c254c3e 100644 --- a/tests/unit/v2/test_samples.py +++ b/tests/unit/v2/test_samples.py @@ -1055,6 +1055,7 @@ def test_chunked_multiplexed_upload(self, tmp_path: Path) -> None: assert mux_route.call_count == 3 assert result.data_ids == ["mux_1"] + class TestImportSamples: @respx.mock From d7ed277e82362a9145eabc83d3d356d33d2add1f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 14:25:03 +0000 Subject: [PATCH 04/32] fix(samples): address round-3 import-command review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Stop discarding a job's sample_ids just because they can't be paired positionally: a length mismatch (the normal partial-failure shape, e.g. a FAILED job that created one of two samples before failing) now surfaces those ids verbatim as job_sample_ids in the document instead of silently dropping them, while still refusing to attribute them to specific rows. - Treat a COMPLETED job that returns zero sample ids for its accessions as unmatchable (status "unknown", don't re-run) instead of "failed" (safe to re-run) — a completed job reporting nothing created is exactly as broken an invariant as any other length mismatch, and re-running it risks duplicating whatever it did create. An empty sample_ids on a RUNNING/FAILED job is still the normal, expected shape. - Reject non-finite --poll-interval/--timeout values (nan, inf): nan previously disabled the poll deadline entirely (every comparison against nan is False) and reached time.sleep(nan), an uncaught ValueError. - Export SampleImportStatus from flowbio.v2 alongside the other three new names, so a caller never has to reach into flowbio.v2.samples. - Distinguish "import failed" (genuine failure) from "import did not finish" (timeout) in the human-readable advisory, matching the JSON status field's distinction. - Add a test asserting the sheet's name column reaches the kickoff payload. - Reword source/cli.rst's failed/running/unknown status guidance, document job_sample_ids, and make the human/--json worked examples agree (both import the same two accessions) instead of silently disagreeing. - Replace the last hand-counted time.monotonic() side_effect list with itertools.count(...), matching the round-2 fixture fix. Addresses Claude's round-3 review on flowbio#18. Refs FLOW-689. --- flowbio/cli/_samples.py | 45 +++++---- flowbio/v2/__init__.py | 2 + source/cli.rst | 56 ++++++----- tests/unit/cli/test_samples.py | 133 +++++++++++++++++++++++++- tests/unit/v2/test_package_exports.py | 9 +- 5 files changed, 198 insertions(+), 47 deletions(-) diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index fe859a5..ddb5974 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -10,8 +10,9 @@ import argparse import json +import math import time -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from typing import Literal @@ -263,8 +264,8 @@ def _configure_upload_batch(upload_batch: argparse.ArgumentParser) -> None: def _positive_float(value: str) -> float: parsed = float(value) - if parsed <= 0: - raise argparse.ArgumentTypeError(f"must be positive, got {value!r}") + if not math.isfinite(parsed) or parsed <= 0: + raise argparse.ArgumentTypeError(f"must be a positive, finite number, got {value!r}") return parsed @@ -644,12 +645,16 @@ class _ImportResult: all rows land in ``imported`` together on a completed job, or all land in ``failed`` together (carrying the job's one error message) otherwise. Each ``failed`` entry carries a ``status`` of ``"failed"`` (the job genuinely - failed), ``"running"`` (the job timed out but may still complete), or - ``"unknown"`` (the job's accessions/sample_ids couldn't be matched) — only - ``"failed"`` is safe to blindly retry. ``job_id``/``job_status``/ - ``execution_id`` are ``None`` only when no job was ever created (every row - was invalid or skipped), so a timed-out or interrupted run can still be - resumed with :meth:`~flowbio.v2.samples.SampleResource.get_import`. + failed, or completed without creating that row's sample), ``"running"`` + (the job timed out but may still complete), or ``"unknown"`` (the job's + accessions/sample_ids couldn't be matched to rows) — only ``"failed"`` is + safe to blindly retry. ``job_sample_ids`` carries whatever sample ids the + job did return, verbatim and unattributed to any row, so a ``"running"`` + or ``"unknown"`` outcome doesn't hide that the job may already have + created something. ``job_id``/``job_status``/``execution_id`` are + ``None`` only when no job was ever created (every row was invalid or + skipped), so a timed-out or interrupted run can still be resumed with + :meth:`~flowbio.v2.samples.SampleResource.get_import`. """ imported: list[dict[str, JsonValue]] @@ -658,6 +663,7 @@ class _ImportResult: job_id: SampleImportJobId | None = None job_status: SampleImportStatus | None = None execution_id: int | None = None + job_sample_ids: list[int] = field(default_factory=list) @property def counts(self) -> dict[str, int]: @@ -677,6 +683,7 @@ def document(self) -> dict[str, JsonValue]: "job_id": self.job_id, "job_status": self.job_status, "execution_id": self.execution_id, + "job_sample_ids": self.job_sample_ids, } @property @@ -799,15 +806,19 @@ def _import_result( job_id=job.id, job_status=job.status, execution_id=job.execution_id, + job_sample_ids=list(job.sample_ids), ) def _sample_ids_unmatchable(job: SampleImportJob) -> bool: - # accessions/sample_ids are only guaranteed to correspond positionally when - # their lengths agree (an empty sample_ids on a RUNNING/FAILED job is - # expected, not a mismatch) — anything else means the job's response - # doesn't support the positional pairing the outcome mapping relies on. - return len(job.sample_ids) not in (0, len(job.accessions)) + # An empty sample_ids is the normal, expected shape before anything has + # been created (RUNNING) or when nothing survived to be created (FAILED). + # A COMPLETED job is different: it has told us the batch is done, so + # returning zero sample ids for one or more accessions is exactly as + # broken an invariant as any other length mismatch, not a quiet success. + if not job.sample_ids: + return job.status == "COMPLETED" + return len(job.sample_ids) != len(job.accessions) def _mismatched_outcome( @@ -825,7 +836,7 @@ def _mismatched_outcome( "message": message, "status": "unknown", }) - output.emit_advisory(f"Row {row.row_number} ({row.accession}): import failed — {message}") + output.emit_advisory(f"Row {row.row_number} ({row.accession}): import outcome unknown — {message}") return failed @@ -864,8 +875,10 @@ def _failed_outcomes( f"import job {job.id} did not finish within the configured timeout " f"(still RUNNING) — check it later with client.samples.get_import({job.id})" ) + verb = "did not finish" else: message = job.error or f"import job {job.id} ended with status {job.status}" + verb = "failed" entry_status = "running" if timed_out else "failed" failed: list[dict[str, JsonValue]] = [] for row in rows: @@ -874,7 +887,7 @@ def _failed_outcomes( "message": message, "status": entry_status, } sample_id = accession_to_sample_id.get(row.accession) - advisory = f"Row {row.row_number} ({row.accession}): import failed — {message}" + advisory = f"Row {row.row_number} ({row.accession}): import {verb} — {message}" if sample_id is not None: entry["sample_id"] = sample_id advisory += f" (sample {sample_id} was created)" diff --git a/flowbio/v2/__init__.py b/flowbio/v2/__init__.py index 99f2621..d0d5f1f 100644 --- a/flowbio/v2/__init__.py +++ b/flowbio/v2/__init__.py @@ -48,6 +48,7 @@ SampleImportJob, SampleImportJobId, SampleImportSpec, + SampleImportStatus, SampleType, SampleTypeId, ) @@ -65,6 +66,7 @@ "SampleImportJob", "SampleImportJobId", "SampleImportSpec", + "SampleImportStatus", "SampleType", "SampleTypeId", "TokenCredentials", diff --git a/source/cli.rst b/source/cli.rst index a78e301..1fee17e 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -395,43 +395,49 @@ By default any invalid row aborts the whole run (exit ``2``); Unlike ``upload-batch``, every valid row is submitted **together as one server-side job** — there is no per-row upload loop. The command polls that job (every ``--poll-interval`` seconds, default 5, for up to ``--timeout`` -seconds, default 1800; both must be positive) until it leaves ``"RUNNING"``. -Once the job completes, every submitted row is reported as imported; if it -fails or times out, every row is reported as failed with the same message -(and the sample id, if the job did create one before failing), since the job -has one outcome for the whole batch. Either way, the job's ``job_id`` is -included in the output so a timed-out or interrupted run can be resumed with -``client.samples.get_import(job_id)`` from the library. +seconds, default 1800; both must be positive, finite numbers) until it leaves +``"RUNNING"``. Once the job completes, every submitted row is reported as +imported; if it fails or times out, every row is reported as failed, since +the job has one outcome for the whole batch. Either way, the job's ``job_id`` +is included in the output so a timed-out or interrupted run can be resumed +with ``client.samples.get_import(job_id)`` from the library. Each ``failed`` entry carries a ``status``: ``"failed"`` means the job -genuinely failed (or completed without creating that row's sample) — safe to -re-run once fixed. ``"running"`` means ``--timeout`` was reached while the job -was still in progress: it may yet succeed, so check ``job_status``/ -``get_import(job_id)`` rather than re-submitting, which would risk duplicate -samples. ``"unknown"`` means the job's ``accessions``/``sample_ids`` couldn't -be matched to rows at all (an unexpected shape from the server). - -**Output** — human: each row's outcome on stderr, then a final counts summary -on stdout. ``--json``: a single document on stdout with ``imported``, -``failed``, and ``skipped`` lists, a ``counts`` summary, and the job's -``job_id``/``job_status``/``execution_id`` (``null`` when no job was created, -e.g. every row was invalid or skipped): +genuinely failed, or completed but didn't create that specific row's sample — +safe to re-run once fixed. ``"running"`` means ``--timeout`` was reached +while the job was still in progress: it may yet succeed, so check +``job_status``/``get_import(job_id)`` rather than re-submitting, which would +risk duplicate samples. ``"unknown"`` means the job's ``accessions``/ +``sample_ids`` couldn't be matched to rows at all (e.g. a completed job that +returned no sample ids, or fewer/more than the accessions submitted) — an +unexpected shape from the server, so treat it like ``"running"`` and don't +blindly re-run. Whatever sample ids the job did return are always available, +unattributed, as ``job_sample_ids`` — useful to check a ``"running"``/ +``"unknown"`` outcome didn't already create something. + +**Output** — human: each row's outcome on stderr (``import failed`` for a +genuine failure, ``import did not finish`` for a timeout, ``import outcome +unknown`` for an unmatchable job), then a final counts summary on stdout. +``--json``: a single document on stdout with ``imported``, ``failed``, and +``skipped`` lists, a ``counts`` summary, and the job's ``job_id``/ +``job_status``/``execution_id``/``job_sample_ids`` (``null``/``[]`` when no +job was created, e.g. every row was invalid or skipped): .. code-block:: json { "imported": [{"row_number": 1, "accession": "ERR1160845", "sample_id": 101}], "failed": [], - "skipped": [{"row_number": 2, "accession": "bogus", "reasons": ["..."]}], + "skipped": [{"row_number": 2, "accession": "BOGUS", "reasons": ["..."]}], "counts": {"imported": 1, "failed": 0, "skipped": 1}, - "job_id": 42, "job_status": "COMPLETED", "execution_id": 7 + "job_id": 42, "job_status": "COMPLETED", "execution_id": 7, "job_sample_ids": [101] } **Exit codes** — ``0`` the import job completed; ``2`` a pre-flight validation failure (without ``--skip-invalid``), a non-CSV sheet, or a -non-positive ``--poll-interval``/``--timeout``; ``1`` the import job failed, -did not finish within ``--timeout`` (check ``job_status``/each row's -``status`` before re-running — see above), or returned a shape the rows +non-positive/non-finite ``--poll-interval``/``--timeout``; ``1`` the import +job failed, did not finish within ``--timeout`` (check ``job_status``/each +row's ``status`` before re-running — see above), or returned a shape the rows couldn't be matched against; ``3`` authentication failure; otherwise the standard mapping above. @@ -445,7 +451,7 @@ standard mapping above. Imported 2, failed 0, skipped 0. $ flowbio samples import --sheet ./accessions.csv --sample-type RNA-Seq --json - {"imported": [{"row_number": 1, "accession": "ERR1160845", "sample_id": 101}], "failed": [], "skipped": [], "counts": {"imported": 1, "failed": 0, "skipped": 0}, "job_id": 42, "job_status": "COMPLETED", "execution_id": 7} + {"imported": [{"row_number": 1, "accession": "ERR1160845", "sample_id": 101}, {"row_number": 2, "accession": "ERR10677146", "sample_id": 102}], "failed": [], "skipped": [], "counts": {"imported": 2, "failed": 0, "skipped": 0}, "job_id": 42, "job_status": "COMPLETED", "execution_id": 7, "job_sample_ids": [101, 102]} ``api get`` ~~~~~~~~~~~ diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index 97b0144..e2eebf3 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -1083,6 +1083,26 @@ def test_sends_sample_type_and_metadata_in_kickoff_payload( }], } + @respx.mock + def test_sends_name_in_kickoff_payload(self, run_cli, tmp_path: Path) -> None: + _mock_metadata() + route = respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "COMPLETED", ["ERR1"], [101], + )), + ) + sheet = _write_import_sheet( + tmp_path, {"accession": "ERR1", "name": "liver_r1", "cell_type": "Neuron"}, + ) + + run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + ) + + payload = json.loads(route.calls[0].request.content) + assert payload["imports"][0]["name"] == "liver_r1" + @respx.mock @patch("time.sleep") def test_polls_until_job_completes( @@ -1212,7 +1232,7 @@ def test_poll_interval_exceeding_timeout_is_clamped( self, mock_sleep, mock_monotonic, run_cli, tmp_path: Path, ) -> None: _mock_metadata() - mock_monotonic.side_effect = [0.0, 0.0] + mock_monotonic.side_effect = itertools.count(0.0, 0.5) respx.post(SAMPLE_IMPORTS_URL).mock( return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( 1, "RUNNING", ["ERR1"], @@ -1230,7 +1250,9 @@ def test_poll_interval_exceeding_timeout_is_clamped( ) assert result.exit_code == 0 - mock_sleep.assert_called_once_with(10.0) + mock_sleep.assert_called_once() + slept_for = mock_sleep.call_args[0][0] + assert 0 < slept_for < 300 @respx.mock def test_zero_poll_interval_is_usage_error(self, run_cli, tmp_path: Path) -> None: @@ -1265,7 +1287,39 @@ def test_negative_timeout_is_usage_error(self, run_cli, tmp_path: Path) -> None: assert "--timeout" in result.stderr @respx.mock - def test_completed_job_missing_sample_id_reports_row_as_failed( + def test_nan_poll_interval_is_usage_error(self, run_cli, tmp_path: Path) -> None: + _mock_metadata() + route = respx.post(SAMPLE_IMPORTS_URL) + sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + "--poll-interval", "nan", + ) + + assert result.exit_code == 2 + assert route.call_count == 0 + assert "--poll-interval" in result.stderr + + @respx.mock + def test_infinite_timeout_is_usage_error(self, run_cli, tmp_path: Path) -> None: + _mock_metadata() + route = respx.post(SAMPLE_IMPORTS_URL) + sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + "--timeout", "inf", + ) + + assert result.exit_code == 2 + assert route.call_count == 0 + assert "--timeout" in result.stderr + + @respx.mock + def test_completed_job_with_no_sample_ids_is_reported_as_unmatchable( self, run_cli, tmp_path: Path, ) -> None: _mock_metadata() @@ -1285,8 +1339,37 @@ def test_completed_job_missing_sample_id_reports_row_as_failed( document = json.loads(result.stdout) assert document["imported"] == [] assert document["failed"][0]["accession"] == "ERR1" - assert document["failed"][0]["status"] == "failed" - assert "no sample id" in document["failed"][0]["message"] + assert document["failed"][0]["status"] == "unknown" + assert "cannot match" in document["failed"][0]["message"] + assert document["job_sample_ids"] == [] + + @respx.mock + def test_completed_job_with_matching_sample_id_missing_reports_row_as_failed( + self, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "COMPLETED", ["ERR1", "err-other"], [101, 102], + )), + ) + sheet = _write_import_sheet( + tmp_path, + {"accession": "ERR1", "cell_type": "Neuron"}, + {"accession": "ERR2", "cell_type": "Neuron"}, + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", "--json", + ) + + assert result.exit_code == 1 + document = json.loads(result.stdout) + failed_by_accession = {entry["accession"]: entry for entry in document["failed"]} + assert document["imported"][0]["accession"] == "ERR1" + assert failed_by_accession["ERR2"]["status"] == "failed" + assert "no sample id" in failed_by_accession["ERR2"]["message"] @respx.mock def test_fewer_sample_ids_than_accessions_is_reported_as_unmatchable( @@ -1317,6 +1400,7 @@ def test_fewer_sample_ids_than_accessions_is_reported_as_unmatchable( assert entry["status"] == "unknown" assert "sample_id" not in entry assert "cannot match" in entry["message"] + assert document["job_sample_ids"] == [101] @respx.mock def test_more_sample_ids_than_accessions_is_reported_as_unmatchable( @@ -1341,6 +1425,45 @@ def test_more_sample_ids_than_accessions_is_reported_as_unmatchable( assert document["failed"][0]["status"] == "unknown" assert "sample_id" not in document["failed"][0] assert "cannot match" in document["failed"][0]["message"] + assert document["job_sample_ids"] == [101, 102] + + @respx.mock + @patch("time.sleep") + def test_failed_job_with_partial_sample_ids_still_surfaces_them( + self, _mock_sleep, run_cli, tmp_path: Path, + ) -> None: + _mock_metadata() + respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "RUNNING", ["ERR1", "ERR2"], + )), + ) + respx.get(f"{SAMPLE_IMPORTS_URL}/1").mock( + return_value=httpx.Response(HTTPStatus.OK, json=_job_json( + 1, "FAILED", ["ERR1", "ERR2"], [101], error="download failed", + )), + ) + sheet = _write_import_sheet( + tmp_path, + {"accession": "ERR1", "cell_type": "Neuron"}, + {"accession": "ERR2", "cell_type": "Neuron"}, + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + "--poll-interval", "1", "--json", + ) + + assert result.exit_code == 1 + document = json.loads(result.stdout) + assert document["imported"] == [] + for entry in document["failed"]: + assert entry["status"] == "unknown" + assert "sample_id" not in entry + # The job did create one sample before failing; that isn't attributable + # to a specific row, but it must not be silently dropped. + assert document["job_sample_ids"] == [101] @respx.mock @patch("time.sleep") diff --git a/tests/unit/v2/test_package_exports.py b/tests/unit/v2/test_package_exports.py index d1bebbd..06bff5a 100644 --- a/tests/unit/v2/test_package_exports.py +++ b/tests/unit/v2/test_package_exports.py @@ -21,16 +21,23 @@ def test_username_password_credentials_is_exported(self) -> None: assert UsernamePasswordCredentials is DirectCredentials def test_sample_import_types_are_exported(self) -> None: - from flowbio.v2 import SampleImportJob, SampleImportJobId, SampleImportSpec + from flowbio.v2 import ( + SampleImportJob, + SampleImportJobId, + SampleImportSpec, + SampleImportStatus, + ) from flowbio.v2.samples import ( SampleImportJob as DirectJob, SampleImportJobId as DirectJobId, SampleImportSpec as DirectSpec, + SampleImportStatus as DirectStatus, ) assert SampleImportSpec is DirectSpec assert SampleImportJob is DirectJob assert SampleImportJobId is DirectJobId + assert SampleImportStatus is DirectStatus class TestSubmoduleExports: From 98d130614ca2a18da990cb155d1ed31723693002 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 15:26:31 +0000 Subject: [PATCH 05/32] refactor(samples): make 'import' non-blocking and let the API validate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per operator direction on the PR: - Drop all client-side pre-validation of accession sheets (accession format, duplicates, required/closed-option metadata). The API already validates every one of these server-side; duplicating the rules client-side was a second, driftable copy of the same logic for no benefit now that the command doesn't need a local reject/skip decision. A malformed sheet surfaces as a normal FlowApiError instead of a local pre-flight rejection — 'samples import' makes exactly one API call. - Make 'samples import' non-blocking: it submits the whole sheet as one batch, reports the created job's id/status, and returns immediately instead of polling to completion. Add a new 'samples import-status --job-id ID' verb (a thin, one-shot wrapper over client.samples.get_import) so a user (or script) can check on a job whenever they choose; there is no built-in polling. This removes _poll_job, the failed/running/unknown outcome taxonomy, job_sample_ids, the positional accessions/sample_ids matching and its mismatch handling, --skip-invalid/--poll-interval/--timeout, and the accession-format regex and duplicate-detection helpers — all of which existed to support blocking-with-local-validation behaviour that no longer applies. flowbio/v2/samples.py (import_samples/get_import) is unchanged; only the CLI orchestration on top of it is simplified. Refs FLOW-689. --- flowbio/cli/_accession_sheet.py | 77 +-- flowbio/cli/_samples.py | 361 +++----------- source/cli.rst | 123 +++-- tests/unit/cli/test_accession_sheet.py | 128 +---- tests/unit/cli/test_samples.py | 622 +++++-------------------- 5 files changed, 251 insertions(+), 1060 deletions(-) diff --git a/flowbio/cli/_accession_sheet.py b/flowbio/cli/_accession_sheet.py index 8446923..b55659c 100644 --- a/flowbio/cli/_accession_sheet.py +++ b/flowbio/cli/_accession_sheet.py @@ -1,4 +1,4 @@ -"""CSV accession-sheet parsing and pre-flight validation for ``samples import``. +"""CSV accession-sheet parsing for ``samples import``. An accession sheet is a CSV with one row per accession to import: an ``accession`` column (a public-repository run or experiment accession) plus @@ -7,35 +7,27 @@ differ — there is nothing to upload (no ``reads1``/``reads2``) and the import API has no project field, so ``RESERVED_COLUMNS`` is ``accession``, ``name``, ``organism`` instead. + +Rows are not validated here: the accession format, duplicates, sample type, +organism, and metadata rules are all checked server-side when the sheet is +submitted — duplicating that locally would just be a second, driftable copy +of the same rules. """ from __future__ import annotations import csv -import re from dataclasses import dataclass from pathlib import Path from flowbio.cli._exit_codes import CliUsageError from flowbio.cli._files import existing_file -from flowbio.cli._sheet import metadata_errors -from flowbio.v2.samples import MetadataAttribute, SampleTypeId RESERVED_COLUMNS = ("accession", "name", "organism") -# Mirrors the API's own accession format rule (one run or experiment accession -# per entry), so a malformed accession is reported up front like every other -# validation problem instead of failing the whole batch request server-side. -_SUPPORTED_ACCESSION = re.compile(r"^[SED]R[RX]\d+$") - @dataclass(frozen=True) class AccessionSheetRow: - """One data row of an accession sheet. - - ``accession`` may be empty (empty cell) — that is reported by - :func:`validate_accession_row` rather than rejected here, so all errors - surface together. - """ + """One data row of an accession sheet.""" row_number: int accession: str @@ -81,61 +73,6 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: return AccessionSheet(path=path, metadata_columns=metadata_columns, rows=rows) -def validate_accession_row( - row: AccessionSheetRow, - attributes: list[MetadataAttribute], - sample_type: SampleTypeId, -) -> list[str]: - """Return every validation problem on ``row`` (empty when the row is valid). - - :param row: The parsed row to validate. - :param attributes: The server's metadata attributes, deciding required and - closed-option columns. - :param sample_type: The sample type applied to the whole import; an - attribute required for it must be present. - :returns: One human-readable message per problem, collected so the caller - can report them all at once. - """ - errors: list[str] = [] - if not row.accession: - errors.append("missing required value: accession") - elif not _SUPPORTED_ACCESSION.match(row.accession): - errors.append( - f"'{row.accession}' is not a supported accession — import one run " - f"(SRR/ERR/DRR) or experiment (SRX/ERX/DRX) accession per row", - ) - errors.extend(metadata_errors(row.metadata, attributes, sample_type)) - return errors - - -def duplicate_accession_errors(rows: list[AccessionSheetRow]) -> dict[int, list[str]]: - """Return extra errors for rows whose accession repeats an earlier row. - - Mirrors the API's own duplicate-accession rejection so a sheet with - repeats is reported up front, in the same per-row shape as every other - validation problem, rather than surfacing as one opaque batch failure. - - :param rows: Every row in the sheet, including ones already found invalid. - :returns: A mapping of ``row_number`` to the duplicate-accession messages - for rows after the first occurrence of a repeated accession. Rows with - a blank accession (already reported by :func:`validate_accession_row`) - are never flagged as duplicates of each other. - """ - first_seen: dict[str, int] = {} - errors: dict[int, list[str]] = {} - for row in rows: - if not row.accession: - continue - if row.accession in first_seen: - errors.setdefault(row.row_number, []).append( - f"duplicate accession '{row.accession}' " - f"(first seen at row {first_seen[row.accession]})", - ) - else: - first_seen[row.accession] = row.row_number - return errors - - def _build_row( record: dict[str, str], row_number: int, metadata_columns: list[str], ) -> AccessionSheetRow: diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index ddb5974..c13ae01 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -10,18 +10,11 @@ import argparse import json -import math -import time -from dataclasses import dataclass, field +from dataclasses import dataclass from pathlib import Path from typing import Literal -from flowbio.cli._accession_sheet import ( - AccessionSheetRow, - duplicate_accession_errors, - parse_accession_sheet, - validate_accession_row, -) +from flowbio.cli._accession_sheet import parse_accession_sheet from flowbio.cli._exit_codes import CliUsageError, ExitCode from flowbio.cli._files import existing_file from flowbio.cli._output import Output, format_issue @@ -39,7 +32,6 @@ SampleImportJob, SampleImportJobId, SampleImportSpec, - SampleImportStatus, SampleTypeId, ) @@ -96,11 +88,17 @@ def register( parents=[global_parent], help="Import samples from public-repository accessions.", description=( - "Validate every row of a CSV accession sheet up front, kick off a " - "single import job for the valid rows, poll it to completion, and " - "report each accession's outcome." + "Kick off a batch import job from a CSV accession sheet and report " + "its id. Does not wait for the job to finish — check its progress " + "with 'samples import-status'." ), )) + _configure_import_status(verbs.add_parser( + "import-status", + parents=[global_parent], + help="Check the status of an import job.", + description="Fetch and report the current state of a 'samples import' job.", + )) def _configure_upload(upload: argparse.ArgumentParser) -> None: @@ -258,17 +256,6 @@ def _configure_upload_batch(upload_batch: argparse.ArgumentParser) -> None: ) -_DEFAULT_POLL_INTERVAL = 5.0 -_DEFAULT_TIMEOUT = 1800.0 - - -def _positive_float(value: str) -> float: - parsed = float(value) - if not math.isfinite(parsed) or parsed <= 0: - raise argparse.ArgumentTypeError(f"must be a positive, finite number, got {value!r}") - return parsed - - def _configure_import(import_parser: argparse.ArgumentParser) -> None: import_parser.set_defaults(command_parser=import_parser, handler=_import_command) import_parser.add_argument( @@ -285,30 +272,20 @@ def _configure_import(import_parser: argparse.ArgumentParser) -> None: type=SampleTypeId, help="Sample type applied to every accession (sent as-is; validated server-side).", ) - import_parser.add_argument( - "--skip-invalid", - action="store_true", - help="Skip invalid rows (reporting why) instead of aborting the import.", - ) - import_parser.add_argument( - "--poll-interval", - type=_positive_float, - default=_DEFAULT_POLL_INTERVAL, - metavar="SECONDS", - help=( - "Seconds to wait between checks of the import job's status; must be " - f"positive (default: {_DEFAULT_POLL_INTERVAL:g})." - ), - ) - import_parser.add_argument( - "--timeout", - type=_positive_float, - default=_DEFAULT_TIMEOUT, - metavar="SECONDS", - help=( - "Maximum seconds to wait for the import job to finish; must be " - f"positive (default: {_DEFAULT_TIMEOUT:g})." - ), + + +def _job_id(value: str) -> SampleImportJobId: + return SampleImportJobId(int(value)) + + +def _configure_import_status(import_status: argparse.ArgumentParser) -> None: + import_status.set_defaults(command_parser=import_status, handler=_import_status_command) + import_status.add_argument( + "--job-id", + required=True, + metavar="ID", + type=_job_id, + help="Import job id, as reported by 'samples import'.", ) @@ -637,124 +614,23 @@ def _invalid_line(row: SheetRow, reasons: list[str]) -> str: return f"Row {row.row_number} ({row.name}): {'; '.join(reasons)}" -@dataclass(frozen=True) -class _ImportResult: - """The outcome of an ``import`` run, rendered to text or JSON. - - Unlike :class:`_BatchResult`, every accession shares one server-side job: - all rows land in ``imported`` together on a completed job, or all land in - ``failed`` together (carrying the job's one error message) otherwise. Each - ``failed`` entry carries a ``status`` of ``"failed"`` (the job genuinely - failed, or completed without creating that row's sample), ``"running"`` - (the job timed out but may still complete), or ``"unknown"`` (the job's - accessions/sample_ids couldn't be matched to rows) — only ``"failed"`` is - safe to blindly retry. ``job_sample_ids`` carries whatever sample ids the - job did return, verbatim and unattributed to any row, so a ``"running"`` - or ``"unknown"`` outcome doesn't hide that the job may already have - created something. ``job_id``/``job_status``/``execution_id`` are - ``None`` only when no job was ever created (every row was invalid or - skipped), so a timed-out or interrupted run can still be resumed with - :meth:`~flowbio.v2.samples.SampleResource.get_import`. - """ - - imported: list[dict[str, JsonValue]] - failed: list[dict[str, JsonValue]] - skipped: list[dict[str, JsonValue]] - job_id: SampleImportJobId | None = None - job_status: SampleImportStatus | None = None - execution_id: int | None = None - job_sample_ids: list[int] = field(default_factory=list) - - @property - def counts(self) -> dict[str, int]: - return { - "imported": len(self.imported), - "failed": len(self.failed), - "skipped": len(self.skipped), - } - - @property - def document(self) -> dict[str, JsonValue]: - return { - "imported": self.imported, - "failed": self.failed, - "skipped": self.skipped, - "counts": self.counts, - "job_id": self.job_id, - "job_status": self.job_status, - "execution_id": self.execution_id, - "job_sample_ids": self.job_sample_ids, - } - - @property - def summary(self) -> str: - counts = self.counts - return ( - f"Imported {counts['imported']}, failed {counts['failed']}, " - f"skipped {counts['skipped']}." - ) - - @property - def exit_code(self) -> ExitCode: - return ExitCode.RUNTIME if self.failed else ExitCode.SUCCESS - - def _import_command( args: argparse.Namespace, client: Client, output: Output, ) -> ExitCode: - """Validate an accession sheet up front, then run one import job for it. + """Kick off a batch import job from an accession sheet and report its id. + + Every row is submitted as-is: the accession format, duplicates, sample + type, organism, and metadata rules are all validated server-side, so a + malformed sheet surfaces as a normal :class:`FlowApiError` rather than a + local pre-flight rejection. This command does not wait for the job to + finish — poll it yourself with ``samples import-status``. :param args: Parsed command-line arguments. :param client: The authenticated Flow client. :param output: The result/error renderer. - :returns: :attr:`ExitCode.SUCCESS` when the import job completes, - :attr:`ExitCode.USAGE` on a pre-flight validation failure without - ``--skip-invalid``, or :attr:`ExitCode.RUNTIME` if the import job - fails or does not finish within ``--timeout``. + :returns: :attr:`ExitCode.SUCCESS` once the job has been kicked off. """ sheet = parse_accession_sheet(args.sheet) - attributes = client.samples.get_metadata_attributes() - duplicates = duplicate_accession_errors(sheet.rows) - classified = [ - ( - row, - validate_accession_row(row, attributes, args.sample_type) - + duplicates.get(row.row_number, []), - ) - for row in sheet.rows - ] - invalid = [(row, reasons) for row, reasons in classified if reasons] - valid = [row for row, reasons in classified if not reasons] - - if invalid and not args.skip_invalid: - output.emit_error( - "Accession sheet has invalid rows; nothing was imported.", - details=[_invalid_accession_line(row, reasons) for row, reasons in invalid], - ) - return ExitCode.USAGE - - for row, reasons in invalid: - output.emit_advisory(f"Skipped {_invalid_accession_line(row, reasons)}") - skipped = [ - {"row_number": row.row_number, "accession": row.accession, "reasons": reasons} - for row, reasons in invalid - ] - - result = _run_import(valid, args, client, output, skipped) - output.emit_result(result.summary, result.document) - return result.exit_code - - -def _run_import( - rows: list[AccessionSheetRow], - args: argparse.Namespace, - client: Client, - output: Output, - skipped: list[dict[str, JsonValue]], -) -> _ImportResult: - if not rows: - return _ImportResult(imported=[], failed=[], skipped=skipped) - specs = [ SampleImportSpec( accession=row.accession, @@ -763,141 +639,56 @@ def _run_import( organism_id=row.organism, metadata=row.metadata or None, ) - for row in rows + for row in sheet.rows ] job = client.samples.import_samples(specs) - output.emit_advisory(f"Import job {job.id} started for {len(rows)} accession(s); polling...") - job = _poll_job(client, job, args.poll_interval, args.timeout) - return _import_result(job, rows, skipped, output) - - -def _poll_job( - client: Client, job: SampleImportJob, poll_interval: float, timeout: float, -) -> SampleImportJob: - deadline = time.monotonic() + timeout - while job.status == "RUNNING": - remaining = deadline - time.monotonic() - if remaining <= 0: - break - time.sleep(min(poll_interval, remaining)) - job = client.samples.get_import(job.id) - return job - - -def _import_result( - job: SampleImportJob, - rows: list[AccessionSheetRow], - skipped: list[dict[str, JsonValue]], - output: Output, -) -> _ImportResult: - if _sample_ids_unmatchable(job): - imported: list[dict[str, JsonValue]] = [] - failed = _mismatched_outcome(job, rows, output) - else: - accession_to_sample_id = dict(zip(job.accessions, job.sample_ids)) - if job.status == "COMPLETED": - imported, failed = _completed_outcomes(job, rows, accession_to_sample_id, output) - else: - imported, failed = [], _failed_outcomes(job, rows, accession_to_sample_id, output) - return _ImportResult( - imported=imported, - failed=failed, - skipped=skipped, - job_id=job.id, - job_status=job.status, - execution_id=job.execution_id, - job_sample_ids=list(job.sample_ids), - ) - - -def _sample_ids_unmatchable(job: SampleImportJob) -> bool: - # An empty sample_ids is the normal, expected shape before anything has - # been created (RUNNING) or when nothing survived to be created (FAILED). - # A COMPLETED job is different: it has told us the batch is done, so - # returning zero sample ids for one or more accessions is exactly as - # broken an invariant as any other length mismatch, not a quiet success. - if not job.sample_ids: - return job.status == "COMPLETED" - return len(job.sample_ids) != len(job.accessions) - - -def _mismatched_outcome( - job: SampleImportJob, rows: list[AccessionSheetRow], output: Output, -) -> list[dict[str, JsonValue]]: - message = ( - f"import job {job.id} returned {len(job.sample_ids)} sample id(s) for " - f"{len(job.accessions)} accession(s); cannot match them to rows" + output.emit_result( + f"Started import job {job.id} for {len(job.accessions)} accession(s) " + f"(status: {job.status}). Check progress with " + f"'flowbio samples import-status --job-id {job.id}'.", + _job_document(job), ) - failed: list[dict[str, JsonValue]] = [] - for row in rows: - failed.append({ - "row_number": row.row_number, - "accession": row.accession, - "message": message, - "status": "unknown", - }) - output.emit_advisory(f"Row {row.row_number} ({row.accession}): import outcome unknown — {message}") - return failed + return ExitCode.SUCCESS -def _completed_outcomes( - job: SampleImportJob, - rows: list[AccessionSheetRow], - accession_to_sample_id: dict[str, int], - output: Output, -) -> tuple[list[dict[str, JsonValue]], list[dict[str, JsonValue]]]: - imported: list[dict[str, JsonValue]] = [] - failed: list[dict[str, JsonValue]] = [] - for row in rows: - sample_id = accession_to_sample_id.get(row.accession) - if sample_id is None: - message = f"import job {job.id} completed but returned no sample id for this accession" - failed.append({ - "row_number": row.row_number, "accession": row.accession, - "message": message, "status": "failed", - }) - output.emit_advisory(f"Row {row.row_number} ({row.accession}): import failed — {message}") - continue - imported.append({"row_number": row.row_number, "accession": row.accession, "sample_id": sample_id}) - output.emit_advisory(f"Row {row.row_number} ({row.accession}): imported sample {sample_id}") - return imported, failed +def _import_status_command( + args: argparse.Namespace, client: Client, output: Output, +) -> ExitCode: + """Fetch and report the current state of an import job. + + :param args: Parsed command-line arguments. + :param client: The authenticated Flow client. + :param output: The result/error renderer. + :returns: :attr:`ExitCode.SUCCESS` once the job's state has been fetched + (the job's own ``status`` — not this command's exit code — reflects + whether the import itself succeeded). + """ + job = client.samples.get_import(args.job_id) + output.emit_result(_job_summary(job), _job_document(job)) + return ExitCode.SUCCESS + + +def _job_document(job: SampleImportJob) -> dict[str, JsonValue]: + return { + "id": job.id, + "status": job.status, + "accessions": job.accessions, + "sample_ids": job.sample_ids, + "execution_id": job.execution_id, + "error": job.error, + } + + +def _job_summary(job: SampleImportJob) -> str: + if job.status == "COMPLETED": + ids = ", ".join(str(sample_id) for sample_id in job.sample_ids) + return f"Job {job.id}: COMPLETED. Sample ids: {ids}." + if job.status == "FAILED": + detail = f" {job.error}" if job.error else "" + return f"Job {job.id}: FAILED.{detail}" + return f"Job {job.id}: {job.status}." -def _failed_outcomes( - job: SampleImportJob, - rows: list[AccessionSheetRow], - accession_to_sample_id: dict[str, int], - output: Output, -) -> list[dict[str, JsonValue]]: - timed_out = job.status == "RUNNING" - if timed_out: - message = ( - f"import job {job.id} did not finish within the configured timeout " - f"(still RUNNING) — check it later with client.samples.get_import({job.id})" - ) - verb = "did not finish" - else: - message = job.error or f"import job {job.id} ended with status {job.status}" - verb = "failed" - entry_status = "running" if timed_out else "failed" - failed: list[dict[str, JsonValue]] = [] - for row in rows: - entry: dict[str, JsonValue] = { - "row_number": row.row_number, "accession": row.accession, - "message": message, "status": entry_status, - } - sample_id = accession_to_sample_id.get(row.accession) - advisory = f"Row {row.row_number} ({row.accession}): import {verb} — {message}" - if sample_id is not None: - entry["sample_id"] = sample_id - advisory += f" (sample {sample_id} was created)" - failed.append(entry) - output.emit_advisory(advisory) - return failed - - -def _invalid_accession_line(row: AccessionSheetRow, reasons: list[str]) -> str: - return f"Row {row.row_number} ({row.accession or ''}): {'; '.join(reasons)}" def _merge_metadata( diff --git a/source/cli.rst b/source/cli.rst index 1fee17e..32cc406 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -370,88 +370,83 @@ authentication failure; otherwise the standard mapping above. ``samples import`` ~~~~~~~~~~~~~~~~~~~ -Import samples from public-repository accessions (SRR/ERR/DRR run or -SRX/ERX/DRX experiment accessions), applying one sample type to every row — -no files to upload yourself. +Kick off a batch import of samples from public-repository accessions (SRR/ +ERR/DRR run or SRX/ERX/DRX experiment accessions), applying one sample type +to every row — no files to upload yourself. :: flowbio samples import --sheet PATH --sample-type TYPE - [--skip-invalid] [--poll-interval SECONDS] [--timeout SECONDS] Run ``flowbio samples import --help`` for the full option list. The sheet is a CSV with an ``accession`` column plus optional ``name``/``organism`` and metadata columns (there is no ``batch-template`` equivalent for it, since it has no reads files or project field). ``name`` defaults to the accession when -omitted. The sample type is sent as-is and validated server-side. - -**Validation is up front**, mirroring ``upload-batch``: a missing or -malformed accession, a duplicate accession within the sheet, a value outside -a closed-option attribute's allowed values, or missing metadata required for -the chosen type — every problem is collected before anything is submitted. -By default any invalid row aborts the whole run (exit ``2``); -``--skip-invalid`` skips them (reporting why) and imports the rest. - -Unlike ``upload-batch``, every valid row is submitted **together as one -server-side job** — there is no per-row upload loop. The command polls that -job (every ``--poll-interval`` seconds, default 5, for up to ``--timeout`` -seconds, default 1800; both must be positive, finite numbers) until it leaves -``"RUNNING"``. Once the job completes, every submitted row is reported as -imported; if it fails or times out, every row is reported as failed, since -the job has one outcome for the whole batch. Either way, the job's ``job_id`` -is included in the output so a timed-out or interrupted run can be resumed -with ``client.samples.get_import(job_id)`` from the library. - -Each ``failed`` entry carries a ``status``: ``"failed"`` means the job -genuinely failed, or completed but didn't create that specific row's sample — -safe to re-run once fixed. ``"running"`` means ``--timeout`` was reached -while the job was still in progress: it may yet succeed, so check -``job_status``/``get_import(job_id)`` rather than re-submitting, which would -risk duplicate samples. ``"unknown"`` means the job's ``accessions``/ -``sample_ids`` couldn't be matched to rows at all (e.g. a completed job that -returned no sample ids, or fewer/more than the accessions submitted) — an -unexpected shape from the server, so treat it like ``"running"`` and don't -blindly re-run. Whatever sample ids the job did return are always available, -unattributed, as ``job_sample_ids`` — useful to check a ``"running"``/ -``"unknown"`` outcome didn't already create something. - -**Output** — human: each row's outcome on stderr (``import failed`` for a -genuine failure, ``import did not finish`` for a timeout, ``import outcome -unknown`` for an unmatchable job), then a final counts summary on stdout. -``--json``: a single document on stdout with ``imported``, ``failed``, and -``skipped`` lists, a ``counts`` summary, and the job's ``job_id``/ -``job_status``/``execution_id``/``job_sample_ids`` (``null``/``[]`` when no -job was created, e.g. every row was invalid or skipped): +omitted. The sample type, accession format, duplicates, and metadata rules +are all sent as-is and validated **server-side** — this command does not +pre-validate rows itself; an invalid sheet surfaces as a normal API error +(exit ``5`` or ``1``), not a local rejection. + +Every row is submitted **together as one server-side job**. This command +does **not wait for it to finish** — it reports the job's id and initial +status (almost always ``"RUNNING"``) and returns immediately. Check on it +with ``samples import-status --job-id ID``; polling (if you want it) is up +to you, e.g. in a shell loop. + +**Output** — human: a confirmation line with the job id and a pointer to +``import-status``. ``--json``: the created job as a single document — +``id``, ``status``, ``accessions``, ``sample_ids`` (empty until the job +completes), ``execution_id``, ``error``. + +**Exit codes** — ``0`` the job was created (regardless of its eventual +outcome — check that with ``import-status``); ``2`` a non-CSV sheet; ``1`` +the API rejected the batch (e.g. unknown sample type, missing required +metadata, an unsupported accession format — the server returns these as a +``422``, which isn't one of the exit codes with its own mapping below); ``3`` +authentication failure; otherwise the standard mapping above. -.. code-block:: json +**Example** - { - "imported": [{"row_number": 1, "accession": "ERR1160845", "sample_id": 101}], - "failed": [], - "skipped": [{"row_number": 2, "accession": "BOGUS", "reasons": ["..."]}], - "counts": {"imported": 1, "failed": 0, "skipped": 1}, - "job_id": 42, "job_status": "COMPLETED", "execution_id": 7, "job_sample_ids": [101] - } +.. code-block:: bash -**Exit codes** — ``0`` the import job completed; ``2`` a pre-flight -validation failure (without ``--skip-invalid``), a non-CSV sheet, or a -non-positive/non-finite ``--poll-interval``/``--timeout``; ``1`` the import -job failed, did not finish within ``--timeout`` (check ``job_status``/each -row's ``status`` before re-running — see above), or returned a shape the rows -couldn't be matched against; ``3`` authentication failure; otherwise the -standard mapping above. + $ flowbio samples import --sheet ./accessions.csv --sample-type RNA-Seq + Started import job 42 for 2 accession(s) (status: RUNNING). Check progress with 'flowbio samples import-status --job-id 42'. + + $ flowbio samples import --sheet ./accessions.csv --sample-type RNA-Seq --json + {"id": 42, "status": "RUNNING", "accessions": ["ERR1160845", "ERR10677146"], "sample_ids": [], "execution_id": 7, "error": null} + +``samples import-status`` +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Fetch and report the current state of a ``samples import`` job. + +:: + + flowbio samples import-status --job-id ID + +Read-only — checking a job's status never changes it. There is no built-in +polling; run this again (or wrap it in your own loop, e.g. ``watch``) until +``status`` leaves ``"RUNNING"``. + +**Output** — human: a one-line summary including the sample ids on +``"COMPLETED"`` or the error on ``"FAILED"``. ``--json``: the job as a single +document — ``id``, ``status``, ``accessions``, ``sample_ids``, +``execution_id``, ``error``. + +**Exit codes** — ``0`` the job's state was fetched (the job's own ``status``, +not this command's exit code, reflects whether the import itself succeeded +or failed); ``4`` no job with that id exists; ``3`` authentication failure; +otherwise the standard mapping above. **Example** .. code-block:: bash - $ flowbio samples import --sheet ./accessions.csv --sample-type RNA-Seq - Row 1 (ERR1160845): imported sample 101 - Row 2 (ERR10677146): imported sample 102 - Imported 2, failed 0, skipped 0. + $ flowbio samples import-status --job-id 42 + Job 42: COMPLETED. Sample ids: 101, 102. - $ flowbio samples import --sheet ./accessions.csv --sample-type RNA-Seq --json - {"imported": [{"row_number": 1, "accession": "ERR1160845", "sample_id": 101}, {"row_number": 2, "accession": "ERR10677146", "sample_id": 102}], "failed": [], "skipped": [], "counts": {"imported": 2, "failed": 0, "skipped": 0}, "job_id": 42, "job_status": "COMPLETED", "execution_id": 7, "job_sample_ids": [101, 102]} + $ flowbio samples import-status --job-id 42 --json + {"id": 42, "status": "COMPLETED", "accessions": ["ERR1160845", "ERR10677146"], "sample_ids": [101, 102], "execution_id": 7, "error": null} ``api get`` ~~~~~~~~~~~ diff --git a/tests/unit/cli/test_accession_sheet.py b/tests/unit/cli/test_accession_sheet.py index f7edca8..57d0a81 100644 --- a/tests/unit/cli/test_accession_sheet.py +++ b/tests/unit/cli/test_accession_sheet.py @@ -3,41 +3,12 @@ import pytest -from flowbio.cli._accession_sheet import ( - duplicate_accession_errors, - parse_accession_sheet, - validate_accession_row, -) +from flowbio.cli._accession_sheet import parse_accession_sheet from flowbio.cli._exit_codes import CliUsageError -from flowbio.v2.samples import MetadataAttribute, SampleTypeId -SAMPLE_TYPE = SampleTypeId("rna_seq") HEADERS = ["accession", "name", "organism", "cell_type", "source", "source__annotation"] -def _attributes() -> list[MetadataAttribute]: - return [ - MetadataAttribute( - identifier="cell_type", - name="Cell Type", - description="The cell type", - required=False, - required_for_sample_types=[SAMPLE_TYPE], - options=["Neuron", "Fibroblast"], - allow_annotation=False, - ), - MetadataAttribute( - identifier="source", - name="Source", - description="Sample source", - required=False, - required_for_sample_types=[], - options=None, - allow_annotation=True, - ), - ] - - def _write_sheet( directory: Path, *records: dict[str, str], headers: list[str] = HEADERS, ) -> Path: @@ -128,100 +99,3 @@ def test_tsv_sheet_rejected(self, tmp_path: Path) -> None: def test_missing_file_is_usage_error(self, tmp_path: Path) -> None: with pytest.raises(CliUsageError): parse_accession_sheet(tmp_path / "absent.csv") - - -class TestValidateAccessionRow: - - def _row(self, directory: Path, **overrides: str): - record = {"accession": "ERR1160845"} - record.update(overrides) - sheet = parse_accession_sheet(_write_sheet(directory, record)) - return sheet.rows[0] - - def test_valid_row_has_no_errors(self, tmp_path: Path) -> None: - row = self._row(tmp_path, cell_type="Neuron") - - assert validate_accession_row(row, _attributes(), SAMPLE_TYPE) == [] - - def test_missing_accession_reports_error(self, tmp_path: Path) -> None: - row = self._row(tmp_path, accession="", cell_type="Neuron") - - errors = validate_accession_row(row, _attributes(), SAMPLE_TYPE) - - assert any("accession" in error for error in errors) - - def test_unsupported_accession_format_reports_error(self, tmp_path: Path) -> None: - row = self._row(tmp_path, accession="GSE12345", cell_type="Neuron") - - errors = validate_accession_row(row, _attributes(), SAMPLE_TYPE) - - assert any("GSE12345" in error for error in errors) - - def test_run_and_experiment_accessions_are_supported(self, tmp_path: Path) -> None: - for accession in ("ERR1160845", "SRR1234567", "DRR7654321", "SRX111", "ERX222", "DRX333"): - row = self._row(tmp_path, accession=accession, cell_type="Neuron") - - assert validate_accession_row(row, _attributes(), SAMPLE_TYPE) == [] - - def test_value_outside_options_reports_error(self, tmp_path: Path) -> None: - row = self._row(tmp_path, cell_type="Alien") - - errors = validate_accession_row(row, _attributes(), SAMPLE_TYPE) - - assert any("Alien" in error for error in errors) - - def test_required_for_type_metadata_missing_reports_error( - self, tmp_path: Path, - ) -> None: - row = self._row(tmp_path) - - errors = validate_accession_row(row, _attributes(), SAMPLE_TYPE) - - assert any("cell_type" in error for error in errors) - - def test_annotation_set_without_its_value_reports_error( - self, tmp_path: Path, - ) -> None: - row = self._row( - tmp_path, cell_type="Neuron", **{"source__annotation": "qPCR"}, - ) - - errors = validate_accession_row(row, _attributes(), SAMPLE_TYPE) - - assert any("source__annotation" in error for error in errors) - - def test_all_per_row_errors_collected(self, tmp_path: Path) -> None: - sheet = parse_accession_sheet(_write_sheet( - tmp_path, {"accession": "", "cell_type": "Alien"}, - )) - - errors = validate_accession_row(sheet.rows[0], _attributes(), SAMPLE_TYPE) - - assert len(errors) >= 2 - - -class TestDuplicateAccessionErrors: - - def test_no_errors_when_all_unique(self, tmp_path: Path) -> None: - sheet = parse_accession_sheet(_write_sheet( - tmp_path, {"accession": "ERR1"}, {"accession": "ERR2"}, - )) - - assert duplicate_accession_errors(sheet.rows) == {} - - def test_repeated_accession_reports_error_on_later_row(self, tmp_path: Path) -> None: - sheet = parse_accession_sheet(_write_sheet( - tmp_path, {"accession": "ERR1"}, {"accession": "err1"}, - )) - - errors = duplicate_accession_errors(sheet.rows) - - assert 1 not in errors - assert any("ERR1" in message for message in errors[2]) - - def test_blank_accessions_are_not_treated_as_duplicates(self, tmp_path: Path) -> None: - sheet = parse_accession_sheet(_write_sheet( - tmp_path, {"accession": ""}, {"accession": ""}, - )) - - assert duplicate_accession_errors(sheet.rows) == {} diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index e2eebf3..b7ca63b 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -1,9 +1,7 @@ import csv -import itertools import json from http import HTTPStatus from pathlib import Path -from unittest.mock import patch import httpx import respx @@ -929,8 +927,6 @@ def test_non_csv_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None: IMPORT_HEADERS = ["accession", "name", "organism", "cell_type", "source", "source__annotation"] - - def _write_import_sheet(directory: Path, *records: dict[str, str]) -> Path: path = directory / "accessions.csv" with path.open("w", newline="") as handle: @@ -960,13 +956,12 @@ def _job_json( class TestSamplesImport: @respx.mock - def test_completed_job_reports_imported_samples( + def test_kicks_off_job_and_reports_id_without_waiting( self, run_cli, tmp_path: Path, ) -> None: - _mock_metadata() - respx.post(SAMPLE_IMPORTS_URL).mock( + route = respx.post(SAMPLE_IMPORTS_URL).mock( return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "COMPLETED", ["ERR1", "ERR2"], [101, 102], + 42, "RUNNING", ["ERR1", "ERR2"], )), ) sheet = _write_import_sheet( @@ -981,18 +976,17 @@ def test_completed_job_reports_imported_samples( ) assert result.exit_code == 0 - assert "101" in result.stderr - assert "102" in result.stderr - assert "Imported 2, failed 0, skipped 0." in result.stdout + assert route.call_count == 1 + assert "42" in result.stdout + assert "import-status" in result.stdout @respx.mock - def test_json_document_reports_outcomes_and_counts( + def test_json_document_reports_job_fields( self, run_cli, tmp_path: Path, ) -> None: - _mock_metadata() respx.post(SAMPLE_IMPORTS_URL).mock( return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "COMPLETED", ["ERR1"], [101], + 42, "RUNNING", ["ERR1"], )), ) sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) @@ -1005,68 +999,58 @@ def test_json_document_reports_outcomes_and_counts( assert result.exit_code == 0 document = json.loads(result.stdout) assert result.stdout.count("\n") == 1 - assert document["counts"] == {"imported": 1, "failed": 0, "skipped": 0} - assert document["imported"][0] == { - "row_number": 1, "accession": "ERR1", "sample_id": 101, + assert document == { + "id": 42, + "status": "RUNNING", + "accessions": ["ERR1"], + "sample_ids": [], + "execution_id": 7, + "error": None, } - assert document["job_id"] == 1 - assert document["job_status"] == "COMPLETED" - assert document["execution_id"] == 7 @respx.mock - def test_json_document_has_null_job_fields_when_nothing_submitted( + def test_sends_every_row_without_local_validation( self, run_cli, tmp_path: Path, ) -> None: - _mock_metadata() - route = respx.post(SAMPLE_IMPORTS_URL) + # No metadata/sample-type endpoints are mocked: if the command called + # client.samples.get_metadata_attributes() or otherwise validated + # locally, respx would fail the test for an unmocked request. + route = respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "RUNNING", ["NOT-AN-ACCESSION", "ERR1", "ERR1"], + )), + ) sheet = _write_import_sheet( - tmp_path, {"accession": "bogus", "cell_type": "Neuron"}, + tmp_path, + {"accession": "not-an-accession"}, + {"accession": "ERR1"}, + {"accession": "ERR1"}, ) result = run_cli( "--token", TOKEN, "samples", "import", "--sheet", str(sheet), "--sample-type", "rna_seq", - "--skip-invalid", "--json", ) assert result.exit_code == 0 - assert route.call_count == 0 - document = json.loads(result.stdout) - assert document["counts"] == {"imported": 0, "failed": 0, "skipped": 1} - assert document["job_id"] is None - assert document["job_status"] is None - assert document["execution_id"] is None - - @respx.mock - def test_header_only_sheet_imports_nothing_without_calling_api( - self, run_cli, tmp_path: Path, - ) -> None: - _mock_metadata() - route = respx.post(SAMPLE_IMPORTS_URL) - sheet = _write_import_sheet(tmp_path) - - result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - ) - - assert result.exit_code == 0 - assert route.call_count == 0 - assert "Imported 0, failed 0, skipped 0." in result.stdout + payload = json.loads(route.calls[0].request.content) + assert [entry["accession"] for entry in payload["imports"]] == [ + "NOT-AN-ACCESSION", "ERR1", "ERR1", + ] @respx.mock - def test_sends_sample_type_and_metadata_in_kickoff_payload( + def test_sends_name_organism_and_metadata_in_payload( self, run_cli, tmp_path: Path, ) -> None: - _mock_metadata() route = respx.post(SAMPLE_IMPORTS_URL).mock( return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "COMPLETED", ["ERR1"], [101], + 1, "RUNNING", ["ERR1"], )), ) - sheet = _write_import_sheet( - tmp_path, {"accession": "err1", "cell_type": "Neuron", "organism": "Hs"}, - ) + sheet = _write_import_sheet(tmp_path, { + "accession": "err1", "name": "liver_r1", "organism": "Hs", + "cell_type": "Neuron", + }) run_cli( "--token", TOKEN, "samples", "import", @@ -1078,551 +1062,161 @@ def test_sends_sample_type_and_metadata_in_kickoff_payload( "imports": [{ "accession": "ERR1", "sample_type": "rna_seq", + "name": "liver_r1", "organism": "Hs", "metadata": {"cell_type": "Neuron"}, }], } @respx.mock - def test_sends_name_in_kickoff_payload(self, run_cli, tmp_path: Path) -> None: - _mock_metadata() - route = respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "COMPLETED", ["ERR1"], [101], - )), - ) - sheet = _write_import_sheet( - tmp_path, {"accession": "ERR1", "name": "liver_r1", "cell_type": "Neuron"}, - ) - - run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - ) - - payload = json.loads(route.calls[0].request.content) - assert payload["imports"][0]["name"] == "liver_r1" - - @respx.mock - @patch("time.sleep") - def test_polls_until_job_completes( - self, _mock_sleep, run_cli, tmp_path: Path, - ) -> None: - _mock_metadata() - respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "RUNNING", ["ERR1"], - )), - ) - poll_route = respx.get(f"{SAMPLE_IMPORTS_URL}/1") - poll_route.side_effect = [ - httpx.Response(HTTPStatus.OK, json=_job_json(1, "RUNNING", ["ERR1"])), - httpx.Response(HTTPStatus.OK, json=_job_json(1, "COMPLETED", ["ERR1"], [101])), - ] - sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) - - result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - "--poll-interval", "1", - ) - - assert result.exit_code == 0 - assert poll_route.call_count == 2 - assert "101" in result.stderr - - @respx.mock - @patch("time.sleep") - def test_failed_job_reports_error_message_per_row( - self, _mock_sleep, run_cli, tmp_path: Path, - ) -> None: - _mock_metadata() - error_message = "download failed: connection reset" - respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "RUNNING", ["ERR1", "ERR2"], - )), - ) - respx.get(f"{SAMPLE_IMPORTS_URL}/1").mock( - return_value=httpx.Response(HTTPStatus.OK, json=_job_json( - 1, "FAILED", ["ERR1", "ERR2"], error=error_message, - )), - ) - sheet = _write_import_sheet( - tmp_path, - {"accession": "ERR1", "cell_type": "Neuron"}, - {"accession": "ERR2", "cell_type": "Neuron"}, - ) - - result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - "--poll-interval", "1", - ) - - assert result.exit_code == 1 - assert error_message in result.stderr - assert "Row 1 (ERR1)" in result.stderr - assert "Row 2 (ERR2)" in result.stderr - - @respx.mock - @patch("time.monotonic") - @patch("time.sleep") - def test_timeout_reports_failure_without_hanging( - self, _mock_sleep, mock_monotonic, run_cli, tmp_path: Path, - ) -> None: - _mock_metadata() - mock_monotonic.side_effect = itertools.count(0.0, 1.0) - respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "RUNNING", ["ERR1"], - )), - ) - poll_route = respx.get(f"{SAMPLE_IMPORTS_URL}/1").mock( - return_value=httpx.Response(HTTPStatus.OK, json=_job_json(1, "RUNNING", ["ERR1"])), - ) - sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) - - result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - "--poll-interval", "1", "--timeout", "3", - ) - - assert result.exit_code == 1 - assert poll_route.call_count == 2 - assert "Row 1 (ERR1)" in result.stderr - assert "did not finish within" in result.stderr - assert "still RUNNING" in result.stderr - assert "ended with status RUNNING" not in result.stderr - - @respx.mock - @patch("time.monotonic") - @patch("time.sleep") - def test_timeout_failed_rows_have_running_status( - self, _mock_sleep, mock_monotonic, run_cli, tmp_path: Path, + def test_api_rejection_propagates_as_error( + self, run_cli, tmp_path: Path, ) -> None: - _mock_metadata() - mock_monotonic.side_effect = itertools.count(0.0, 1.0) - respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "RUNNING", ["ERR1"], - )), - ) - respx.get(f"{SAMPLE_IMPORTS_URL}/1").mock( - return_value=httpx.Response(HTTPStatus.OK, json=_job_json(1, "RUNNING", ["ERR1"])), + route = respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response( + HTTPStatus.UNPROCESSABLE_ENTITY, + json={"error": "sample type 'bogus' does not exist"}, + ), ) - sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) + sheet = _write_import_sheet(tmp_path, {"accession": "ERR1"}) result = run_cli( "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - "--poll-interval", "1", "--timeout", "3", "--json", + "--sheet", str(sheet), "--sample-type", "bogus", ) assert result.exit_code == 1 - document = json.loads(result.stdout) - assert document["failed"][0]["status"] == "running" - assert document["job_status"] == "RUNNING" - - @respx.mock - @patch("time.monotonic") - @patch("time.sleep") - def test_poll_interval_exceeding_timeout_is_clamped( - self, mock_sleep, mock_monotonic, run_cli, tmp_path: Path, - ) -> None: - _mock_metadata() - mock_monotonic.side_effect = itertools.count(0.0, 0.5) - respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "RUNNING", ["ERR1"], - )), - ) - respx.get(f"{SAMPLE_IMPORTS_URL}/1").mock( - return_value=httpx.Response(HTTPStatus.OK, json=_job_json(1, "COMPLETED", ["ERR1"], [101])), - ) - sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) - - result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - "--poll-interval", "300", "--timeout", "10", - ) - - assert result.exit_code == 0 - mock_sleep.assert_called_once() - slept_for = mock_sleep.call_args[0][0] - assert 0 < slept_for < 300 + assert route.call_count == 1 + assert "bogus" in result.stderr @respx.mock - def test_zero_poll_interval_is_usage_error(self, run_cli, tmp_path: Path) -> None: - _mock_metadata() + def test_non_csv_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None: route = respx.post(SAMPLE_IMPORTS_URL) - sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) + xlsx = tmp_path / "sheet.xlsx" + xlsx.write_bytes(b"PK\x03\x04") result = run_cli( "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - "--poll-interval", "0", + "--sheet", str(xlsx), "--sample-type", "rna_seq", ) assert result.exit_code == 2 assert route.call_count == 0 - assert "--poll-interval" in result.stderr - - @respx.mock - def test_negative_timeout_is_usage_error(self, run_cli, tmp_path: Path) -> None: - _mock_metadata() - route = respx.post(SAMPLE_IMPORTS_URL) - sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) + assert "CSV" in result.stderr + def test_missing_sheet_is_usage_error(self, run_cli) -> None: result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - "--timeout", "-1", + "--token", TOKEN, "samples", "import", "--sample-type", "rna_seq", ) assert result.exit_code == 2 - assert route.call_count == 0 - assert "--timeout" in result.stderr - @respx.mock - def test_nan_poll_interval_is_usage_error(self, run_cli, tmp_path: Path) -> None: - _mock_metadata() - route = respx.post(SAMPLE_IMPORTS_URL) - sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) + def test_missing_sample_type_is_usage_error(self, run_cli, tmp_path: Path) -> None: + sheet = _write_import_sheet(tmp_path, {"accession": "ERR1"}) result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - "--poll-interval", "nan", + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), ) assert result.exit_code == 2 - assert route.call_count == 0 - assert "--poll-interval" in result.stderr - @respx.mock - def test_infinite_timeout_is_usage_error(self, run_cli, tmp_path: Path) -> None: - _mock_metadata() - route = respx.post(SAMPLE_IMPORTS_URL) - sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) - result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - "--timeout", "inf", - ) - - assert result.exit_code == 2 - assert route.call_count == 0 - assert "--timeout" in result.stderr +class TestSamplesImportStatus: @respx.mock - def test_completed_job_with_no_sample_ids_is_reported_as_unmatchable( - self, run_cli, tmp_path: Path, - ) -> None: - _mock_metadata() - respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "COMPLETED", ["ERR1"], [], - )), - ) - sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) - - result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", "--json", - ) - - assert result.exit_code == 1 - document = json.loads(result.stdout) - assert document["imported"] == [] - assert document["failed"][0]["accession"] == "ERR1" - assert document["failed"][0]["status"] == "unknown" - assert "cannot match" in document["failed"][0]["message"] - assert document["job_sample_ids"] == [] - - @respx.mock - def test_completed_job_with_matching_sample_id_missing_reports_row_as_failed( - self, run_cli, tmp_path: Path, - ) -> None: - _mock_metadata() - respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "COMPLETED", ["ERR1", "err-other"], [101, 102], - )), - ) - sheet = _write_import_sheet( - tmp_path, - {"accession": "ERR1", "cell_type": "Neuron"}, - {"accession": "ERR2", "cell_type": "Neuron"}, - ) - - result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", "--json", - ) - - assert result.exit_code == 1 - document = json.loads(result.stdout) - failed_by_accession = {entry["accession"]: entry for entry in document["failed"]} - assert document["imported"][0]["accession"] == "ERR1" - assert failed_by_accession["ERR2"]["status"] == "failed" - assert "no sample id" in failed_by_accession["ERR2"]["message"] - - @respx.mock - def test_fewer_sample_ids_than_accessions_is_reported_as_unmatchable( - self, run_cli, tmp_path: Path, - ) -> None: - _mock_metadata() - respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "COMPLETED", ["ERR1", "ERR2"], [101], + def test_reports_running_job(self, run_cli) -> None: + respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json=_job_json( + 42, "RUNNING", ["ERR1", "ERR2"], )), ) - sheet = _write_import_sheet( - tmp_path, - {"accession": "ERR1", "cell_type": "Neuron"}, - {"accession": "ERR2", "cell_type": "Neuron"}, - ) result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", "--json", + "--token", TOKEN, "samples", "import-status", "--job-id", "42", ) - assert result.exit_code == 1 - document = json.loads(result.stdout) - assert document["imported"] == [] - assert len(document["failed"]) == 2 - for entry in document["failed"]: - assert entry["status"] == "unknown" - assert "sample_id" not in entry - assert "cannot match" in entry["message"] - assert document["job_sample_ids"] == [101] + assert result.exit_code == 0 + assert "42" in result.stdout + assert "RUNNING" in result.stdout @respx.mock - def test_more_sample_ids_than_accessions_is_reported_as_unmatchable( - self, run_cli, tmp_path: Path, - ) -> None: - _mock_metadata() - respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "COMPLETED", ["ERX1"], [101, 102], + def test_reports_completed_job_with_sample_ids(self, run_cli) -> None: + respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json=_job_json( + 42, "COMPLETED", ["ERR1", "ERR2"], [101, 102], )), ) - sheet = _write_import_sheet(tmp_path, {"accession": "ERX1", "cell_type": "Neuron"}) result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", "--json", + "--token", TOKEN, "samples", "import-status", "--job-id", "42", ) - assert result.exit_code == 1 - document = json.loads(result.stdout) - assert document["imported"] == [] - assert document["failed"][0]["status"] == "unknown" - assert "sample_id" not in document["failed"][0] - assert "cannot match" in document["failed"][0]["message"] - assert document["job_sample_ids"] == [101, 102] + assert result.exit_code == 0 + assert "101" in result.stdout + assert "102" in result.stdout @respx.mock - @patch("time.sleep") - def test_failed_job_with_partial_sample_ids_still_surfaces_them( - self, _mock_sleep, run_cli, tmp_path: Path, - ) -> None: - _mock_metadata() - respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "RUNNING", ["ERR1", "ERR2"], - )), - ) - respx.get(f"{SAMPLE_IMPORTS_URL}/1").mock( + def test_reports_failed_job_with_error(self, run_cli) -> None: + respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( return_value=httpx.Response(HTTPStatus.OK, json=_job_json( - 1, "FAILED", ["ERR1", "ERR2"], [101], error="download failed", + 42, "FAILED", ["ERR1"], error="download failed", )), ) - sheet = _write_import_sheet( - tmp_path, - {"accession": "ERR1", "cell_type": "Neuron"}, - {"accession": "ERR2", "cell_type": "Neuron"}, - ) result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - "--poll-interval", "1", "--json", + "--token", TOKEN, "samples", "import-status", "--job-id", "42", ) - assert result.exit_code == 1 - document = json.loads(result.stdout) - assert document["imported"] == [] - for entry in document["failed"]: - assert entry["status"] == "unknown" - assert "sample_id" not in entry - # The job did create one sample before failing; that isn't attributable - # to a specific row, but it must not be silently dropped. - assert document["job_sample_ids"] == [101] + assert result.exit_code == 0 + assert "FAILED" in result.stdout + assert "download failed" in result.stdout @respx.mock - @patch("time.sleep") - def test_failed_job_with_all_sample_ids_created_reports_them_per_row( - self, _mock_sleep, run_cli, tmp_path: Path, - ) -> None: - _mock_metadata() - respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "RUNNING", ["ERR1", "ERR2"], - )), - ) - respx.get(f"{SAMPLE_IMPORTS_URL}/1").mock( + def test_json_document_matches_job_shape(self, run_cli) -> None: + respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( return_value=httpx.Response(HTTPStatus.OK, json=_job_json( - 1, "FAILED", ["ERR1", "ERR2"], [101, 102], error="post-processing failed", + 42, "COMPLETED", ["ERR1"], [101], )), ) - sheet = _write_import_sheet( - tmp_path, - {"accession": "ERR1", "cell_type": "Neuron"}, - {"accession": "ERR2", "cell_type": "Neuron"}, - ) result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - "--poll-interval", "1", "--json", + "--token", TOKEN, "samples", "import-status", "--job-id", "42", "--json", ) - assert result.exit_code == 1 + assert result.exit_code == 0 document = json.loads(result.stdout) - failed_by_accession = {entry["accession"]: entry for entry in document["failed"]} - assert failed_by_accession["ERR1"]["sample_id"] == 101 - assert failed_by_accession["ERR2"]["sample_id"] == 102 - assert failed_by_accession["ERR1"]["status"] == "failed" + assert result.stdout.count("\n") == 1 + assert document == { + "id": 42, + "status": "COMPLETED", + "accessions": ["ERR1"], + "sample_ids": [101], + "execution_id": 7, + "error": None, + } @respx.mock - @patch("time.sleep") - def test_transient_poll_error_propagates( - self, _mock_sleep, run_cli, tmp_path: Path, - ) -> None: - _mock_metadata() - respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "RUNNING", ["ERR1"], - )), - ) - respx.get(f"{SAMPLE_IMPORTS_URL}/1").mock( - return_value=httpx.Response(HTTPStatus.NOT_FOUND, json={"error": "gone"}), + def test_unknown_job_id_is_not_found(self, run_cli) -> None: + respx.get(f"{SAMPLE_IMPORTS_URL}/999").mock( + return_value=httpx.Response( + HTTPStatus.NOT_FOUND, json={"error": "sample import 999 does not exist"}, + ), ) - sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - "--poll-interval", "1", + "--token", TOKEN, "samples", "import-status", "--job-id", "999", ) assert result.exit_code == 4 - @respx.mock - def test_invalid_row_aborts_and_imports_nothing( - self, run_cli, tmp_path: Path, - ) -> None: - _mock_metadata() - route = respx.post(SAMPLE_IMPORTS_URL) - sheet = _write_import_sheet( - tmp_path, - {"accession": "ERR1", "cell_type": "Neuron"}, - {"accession": "not-an-accession", "cell_type": "Neuron"}, - ) - - result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - ) + def test_missing_job_id_is_usage_error(self, run_cli) -> None: + result = run_cli("--token", TOKEN, "samples", "import-status") assert result.exit_code == 2 - assert route.call_count == 0 - assert "Row 2" in result.stderr - assert "NOT-AN-ACCESSION" in result.stderr - - @respx.mock - def test_skip_invalid_imports_valid_rows( - self, run_cli, tmp_path: Path, - ) -> None: - _mock_metadata() - route = respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "COMPLETED", ["ERR1"], [101], - )), - ) - sheet = _write_import_sheet( - tmp_path, - {"accession": "ERR1", "cell_type": "Neuron"}, - {"accession": "bogus", "cell_type": "Neuron"}, - ) + def test_non_numeric_job_id_is_usage_error(self, run_cli) -> None: result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - "--skip-invalid", - ) - - assert result.exit_code == 0 - assert route.call_count == 1 - payload = json.loads(route.calls[0].request.content) - assert len(payload["imports"]) == 1 - assert "BOGUS" in result.stderr - - @respx.mock - def test_duplicate_accession_is_usage_error( - self, run_cli, tmp_path: Path, - ) -> None: - _mock_metadata() - route = respx.post(SAMPLE_IMPORTS_URL) - sheet = _write_import_sheet( - tmp_path, - {"accession": "ERR1", "cell_type": "Neuron"}, - {"accession": "err1", "cell_type": "Neuron"}, - ) - - result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - ) - - assert result.exit_code == 2 - assert route.call_count == 0 - assert "duplicate" in result.stderr.lower() - - @respx.mock - def test_non_csv_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None: - _mock_metadata() - route = respx.post(SAMPLE_IMPORTS_URL) - xlsx = tmp_path / "sheet.xlsx" - xlsx.write_bytes(b"PK\x03\x04") - - result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(xlsx), "--sample-type", "rna_seq", - ) - - assert result.exit_code == 2 - assert route.call_count == 0 - assert "CSV" in result.stderr - - def test_missing_sheet_is_usage_error(self, run_cli) -> None: - result = run_cli( - "--token", TOKEN, "samples", "import", "--sample-type", "rna_seq", - ) - - assert result.exit_code == 2 - - def test_missing_sample_type_is_usage_error(self, run_cli, tmp_path: Path) -> None: - sheet = _write_import_sheet(tmp_path, {"accession": "ERR1"}) - - result = run_cli( - "--token", TOKEN, "samples", "import", "--sheet", str(sheet), + "--token", TOKEN, "samples", "import-status", "--job-id", "not-a-number", ) assert result.exit_code == 2 From 694198fa34a79d3b120c762e024c96b8f6e8d316 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 15:41:33 +0000 Subject: [PATCH 06/32] fix(samples): address round-5 findings on the non-blocking design - Revert flowbio/cli/_sheet.py to master: the metadata_errors extraction existed to be shared with accession-sheet validation, which the non-blocking rework removed entirely. Its only caller is validate_row again, so the helper goes back to being private (_metadata_errors), and the stale docstring claiming a shared caller that no longer exists is gone with it. - samples import-status now exits RUNTIME for a FAILED job (was always SUCCESS), so a caller polling it can branch on the exit code alone without parsing --json output. - Drop the client-side accession.upper() call, the only remaining rewrite of user input left over from the pre-rework design. The server already normalises case itself; letting it do so is more consistent with letting the API validate than silently uppercasing client-side first. - samples import now rejects a header-only sheet locally (CliUsageError, exit 2) instead of spending a round trip to be told by the API that at least one accession is required -- this is a structural check (does the sheet have anything to submit), not a validation-rule duplication. - Documented the utf-8-sig BOM-handling rationale in _accession_sheet.py (previously only in _sheet.py, now that the two parsers aren't sharing code). - Docs: replaced the ambiguous "exit 5 or 1" with the one code this endpoint's 422 responses actually map to; added a bounded polling example to the library docs; added API Reference entries for SampleImportStatus and SampleImportJobId; gave SampleImportSpec's fields rendered descriptions (:members: plus :param: docstring, matching ClientConfig's pattern); fixed a stray single/double-backtick inconsistency; documented import-status's new exit-code contract with a --json/jq polling example. Verified the Sphinx build renders all of this without new warnings (a pre-existing, unrelated legacy.rst directive error aside). - Polish: extra blank lines, and _job_summary no longer renders an empty 'Sample ids: .' for a completed job with none. Addresses Claude's round-5 review on flowbio#18. Refs FLOW-689. --- flowbio/cli/_accession_sheet.py | 11 +-- flowbio/cli/_samples.py | 17 +++-- flowbio/cli/_sheet.py | 96 +++++++++++--------------- flowbio/v2/samples.py | 13 +++- source/cli.rst | 35 ++++++---- source/v2/samples.rst | 12 +++- tests/unit/cli/test_accession_sheet.py | 4 +- tests/unit/cli/test_samples.py | 39 +++++++++-- 8 files changed, 138 insertions(+), 89 deletions(-) diff --git a/flowbio/cli/_accession_sheet.py b/flowbio/cli/_accession_sheet.py index b55659c..89923c8 100644 --- a/flowbio/cli/_accession_sheet.py +++ b/flowbio/cli/_accession_sheet.py @@ -50,8 +50,9 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: :param path: The accession-sheet file. Must be a ``.csv`` — an ``.xlsx`` or ``.tsv`` is a usage error directing the user to export to CSV. - :returns: The parsed sheet with reserved/metadata columns separated, - accessions normalised to upper case, and empty cells dropped. + :returns: The parsed sheet with reserved/metadata columns separated and + empty cells dropped. Values are otherwise passed through unchanged — + including ``accession``, sent to the server exactly as entered. :raises CliUsageError: If the file is not a readable ``.csv``. """ if path.suffix.lower() != ".csv": @@ -60,6 +61,9 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: f"Export your spreadsheet to CSV first.", ) existing_file(path) + # utf-8-sig transparently strips a leading BOM, which spreadsheet tools + # (notably Excel's "CSV UTF-8" export) prepend — otherwise the first header + # parses as "accession" and every row reports a missing accession. with path.open(newline="", encoding="utf-8-sig") as handle: reader = csv.DictReader(handle) headers = reader.fieldnames or [] @@ -85,10 +89,9 @@ def cell(column: str) -> str | None: for column in metadata_columns if (value := (record.get(column) or "").strip()) } - accession = cell("accession") return AccessionSheetRow( row_number=row_number, - accession=accession.upper() if accession else "", + accession=cell("accession") or "", name=cell("name"), organism=cell("organism"), metadata=metadata, diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index c13ae01..1e05e86 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -629,8 +629,12 @@ def _import_command( :param client: The authenticated Flow client. :param output: The result/error renderer. :returns: :attr:`ExitCode.SUCCESS` once the job has been kicked off. + :raises CliUsageError: If the sheet has no rows — there is nothing an + API call could tell us about an empty sheet that we can't see already. """ sheet = parse_accession_sheet(args.sheet) + if not sheet.rows: + raise CliUsageError(f"Accession sheet has no rows: {args.sheet}") specs = [ SampleImportSpec( accession=row.accession, @@ -659,13 +663,14 @@ def _import_status_command( :param args: Parsed command-line arguments. :param client: The authenticated Flow client. :param output: The result/error renderer. - :returns: :attr:`ExitCode.SUCCESS` once the job's state has been fetched - (the job's own ``status`` — not this command's exit code — reflects - whether the import itself succeeded). + :returns: :attr:`ExitCode.SUCCESS` once the job's state has been fetched, + unless the job itself is ``"FAILED"``, in which case + :attr:`ExitCode.RUNTIME` — so a caller polling this command can + branch on its exit code alone, without parsing ``--json`` output. """ job = client.samples.get_import(args.job_id) output.emit_result(_job_summary(job), _job_document(job)) - return ExitCode.SUCCESS + return ExitCode.RUNTIME if job.status == "FAILED" else ExitCode.SUCCESS def _job_document(job: SampleImportJob) -> dict[str, JsonValue]: @@ -681,7 +686,7 @@ def _job_document(job: SampleImportJob) -> dict[str, JsonValue]: def _job_summary(job: SampleImportJob) -> str: if job.status == "COMPLETED": - ids = ", ".join(str(sample_id) for sample_id in job.sample_ids) + ids = ", ".join(str(sample_id) for sample_id in job.sample_ids) or "none" return f"Job {job.id}: COMPLETED. Sample ids: {ids}." if job.status == "FAILED": detail = f" {job.error}" if job.error else "" @@ -689,8 +694,6 @@ def _job_summary(job: SampleImportJob) -> str: return f"Job {job.id}: {job.status}." - - def _merge_metadata( pairs: list[str] | None, json_text: str | None, ) -> dict[str, str]: diff --git a/flowbio/cli/_sheet.py b/flowbio/cli/_sheet.py index a5effa1..e956ccb 100644 --- a/flowbio/cli/_sheet.py +++ b/flowbio/cli/_sheet.py @@ -96,6 +96,7 @@ def validate_row( :returns: One human-readable message per problem, collected so the caller can report them all at once. """ + by_identifier = {attribute.identifier: attribute for attribute in attributes} errors: list[str] = [] if not row.name: errors.append("missing required value: name") @@ -106,61 +107,7 @@ def validate_row( for label, reads in (("reads1", row.reads1), ("reads2", row.reads2)): if reads is not None and not reads.is_file(): errors.append(f"{label} file not found: {reads}") - errors.extend(metadata_errors(row.metadata, attributes, sample_type)) - return errors - - -def metadata_errors( - metadata: dict[str, str], - attributes: list[MetadataAttribute], - sample_type: SampleTypeId, -) -> list[str]: - """Return every metadata validation problem for ``metadata`` (empty when valid). - - Shared by :func:`validate_row` and the accession-sheet equivalent in - :mod:`flowbio.cli._accession_sheet`, since required/closed-option/annotation - rules apply identically regardless of what the rest of the row looks like. - - :param metadata: The row's metadata, keyed by attribute identifier (or - ``__annotation`` for a free-text companion). - :param attributes: The server's metadata attributes, deciding required and - closed-option columns. - :param sample_type: The sample type applied to the whole batch; an attribute - required for it must be present. - :returns: One human-readable message per problem, collected so the caller can - report them all at once. - """ - by_identifier = {attribute.identifier: attribute for attribute in attributes} - errors: list[str] = [] - for attribute in attributes: - required = ( - attribute.required or sample_type in attribute.required_for_sample_types - ) - if required and not metadata.get(attribute.identifier): - errors.append(f"missing required metadata: {attribute.identifier}") - for key, value in metadata.items(): - if key.endswith(ANNOTATION_SUFFIX): - continue - attribute = by_identifier.get(key) - if ( - attribute is not None - and attribute.options is not None - and value not in attribute.options - ): - errors.append( - f"value '{value}' for {key} is not one of: " - f"{', '.join(attribute.options)}", - ) - for key in metadata: - if not key.endswith(ANNOTATION_SUFFIX): - continue - base = key[: -len(ANNOTATION_SUFFIX)] - if not metadata.get(base): - errors.append(f"{key} set without a value for {base}") - continue - attribute = by_identifier.get(base) - if attribute is not None and not attribute.allow_annotation: - errors.append(f"{base} does not allow an annotation") + errors.extend(_metadata_errors(row, by_identifier, attributes, sample_type)) return errors @@ -195,3 +142,42 @@ def _resolve(value: str | None, base_dir: Path) -> Path | None: return None path = Path(value) return path if path.is_absolute() else base_dir / path + + +def _metadata_errors( + row: SheetRow, + by_identifier: dict[str, MetadataAttribute], + attributes: list[MetadataAttribute], + sample_type: SampleTypeId, +) -> list[str]: + errors: list[str] = [] + for attribute in attributes: + required = ( + attribute.required or sample_type in attribute.required_for_sample_types + ) + if required and not row.metadata.get(attribute.identifier): + errors.append(f"missing required metadata: {attribute.identifier}") + for key, value in row.metadata.items(): + if key.endswith(ANNOTATION_SUFFIX): + continue + attribute = by_identifier.get(key) + if ( + attribute is not None + and attribute.options is not None + and value not in attribute.options + ): + errors.append( + f"value '{value}' for {key} is not one of: " + f"{', '.join(attribute.options)}", + ) + for key in row.metadata: + if not key.endswith(ANNOTATION_SUFFIX): + continue + base = key[: -len(ANNOTATION_SUFFIX)] + if not row.metadata.get(base): + errors.append(f"{key} set without a value for {base}") + continue + attribute = by_identifier.get(base) + if attribute is not None and not attribute.allow_annotation: + errors.append(f"{base} does not allow an annotation") + return errors diff --git a/flowbio/v2/samples.py b/flowbio/v2/samples.py index 8dada4f..398baf4 100644 --- a/flowbio/v2/samples.py +++ b/flowbio/v2/samples.py @@ -164,6 +164,17 @@ class SampleImportSpec: metadata={"strandedness": "reverse"}, ), ] + + :param accession: The public-repository run or experiment accession + (e.g. ``"ERR1160845"``), validated server-side. + :param sample_type: The sample type identifier (e.g. ``"RNA-Seq"``), + validated server-side. See :meth:`SampleResource.get_types`. + :param name: Optional sample name. Defaults to ``accession`` server-side + when omitted. + :param organism_id: Optional organism id (e.g. ``"Hs"``) to associate + with the sample, sent as ``organism``. + :param metadata: Optional metadata key-value pairs. See + :ref:`metadata-attributes` for details on required attributes. """ accession: str @@ -186,7 +197,7 @@ class SampleImportJob(BaseModel, frozen=True): status: SampleImportStatus = Field(description="The job's current lifecycle state.") accessions: list[str] = Field(description="The accessions submitted with this job, in submission order.") sample_ids: list[int] = Field( - description="The created samples' ids, corresponding to `accessions` once the job has completed.", + description="The created samples' ids, corresponding to ``accessions`` once the job has completed.", ) execution_id: int | None = Field(description="The pipeline execution backing this job, if one was created.") error: str | None = Field(description='The failure reason, set only when status is "FAILED".') diff --git a/source/cli.rst b/source/cli.rst index 32cc406..265120c 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -384,8 +384,10 @@ metadata columns (there is no ``batch-template`` equivalent for it, since it has no reads files or project field). ``name`` defaults to the accession when omitted. The sample type, accession format, duplicates, and metadata rules are all sent as-is and validated **server-side** — this command does not -pre-validate rows itself; an invalid sheet surfaces as a normal API error -(exit ``5`` or ``1``), not a local rejection. +pre-validate rows itself (beyond checking the sheet has at least one row); an +invalid sheet surfaces as a normal API error (exit ``1`` — the API returns +these as an HTTP ``422``, which isn't one of the codes with its own mapping +below), not a local rejection. Every row is submitted **together as one server-side job**. This command does **not wait for it to finish** — it reports the job's id and initial @@ -399,11 +401,10 @@ to you, e.g. in a shell loop. completes), ``execution_id``, ``error``. **Exit codes** — ``0`` the job was created (regardless of its eventual -outcome — check that with ``import-status``); ``2`` a non-CSV sheet; ``1`` -the API rejected the batch (e.g. unknown sample type, missing required -metadata, an unsupported accession format — the server returns these as a -``422``, which isn't one of the exit codes with its own mapping below); ``3`` -authentication failure; otherwise the standard mapping above. +outcome — check that with ``import-status``); ``2`` a non-CSV or empty sheet; +``1`` the API rejected the batch (e.g. unknown sample type, missing required +metadata, an unsupported accession format); ``3`` authentication failure; +otherwise the standard mapping above. **Example** @@ -425,18 +426,26 @@ Fetch and report the current state of a ``samples import`` job. flowbio samples import-status --job-id ID Read-only — checking a job's status never changes it. There is no built-in -polling; run this again (or wrap it in your own loop, e.g. ``watch``) until -``status`` leaves ``"RUNNING"``. +polling; run this again (or wrap it in your own loop) until ``status`` leaves +``"RUNNING"``: + +.. code-block:: bash + + until [ "$(flowbio samples import-status --job-id 42 --json | jq -r .status)" != "RUNNING" ]; do + sleep 30 + done **Output** — human: a one-line summary including the sample ids on ``"COMPLETED"`` or the error on ``"FAILED"``. ``--json``: the job as a single document — ``id``, ``status``, ``accessions``, ``sample_ids``, ``execution_id``, ``error``. -**Exit codes** — ``0`` the job's state was fetched (the job's own ``status``, -not this command's exit code, reflects whether the import itself succeeded -or failed); ``4`` no job with that id exists; ``3`` authentication failure; -otherwise the standard mapping above. +**Exit codes** — ``0`` the job was fetched and is ``"RUNNING"`` or +``"COMPLETED"``; ``1`` the job was fetched but is ``"FAILED"`` (so a caller +can branch on the exit code alone, without parsing ``--json`` output — the +loop above still needs ``--json``/``jq`` to tell ``"RUNNING"`` from +``"COMPLETED"``, since both exit ``0``); ``4`` no job with that id exists; +``3`` authentication failure; otherwise the standard mapping above. **Example** diff --git a/source/v2/samples.rst b/source/v2/samples.rst index eec9091..8098515 100644 --- a/source/v2/samples.rst +++ b/source/v2/samples.rst @@ -189,18 +189,19 @@ accession submitted together is tracked as a single job:: The job starts out ``"RUNNING"``. Poll it with :meth:`~flowbio.v2.samples.SampleResource.get_import` until its status -leaves ``"RUNNING"``:: +leaves ``"RUNNING"`` — bound the wait so a stuck job doesn't loop forever:: import time - while job.status == "RUNNING": + deadline = time.monotonic() + 1800 # 30 minutes + while job.status == "RUNNING" and time.monotonic() < deadline: time.sleep(5) job = client.samples.get_import(job.id) if job.status == "COMPLETED": print(f"Imported samples: {job.sample_ids}") else: - print(f"Import failed: {job.error}") + print(f"Import failed or still running: {job.error}") ``job.accessions`` and ``job.sample_ids`` correspond positionally once the job has completed. @@ -227,5 +228,10 @@ Models .. autopydantic_model:: flowbio.v2.samples.MultiplexedUpload .. autoclass:: flowbio.v2.samples.SampleImportSpec + :members: .. autopydantic_model:: flowbio.v2.samples.SampleImportJob + +.. autodata:: flowbio.v2.samples.SampleImportJobId + +.. autodata:: flowbio.v2.samples.SampleImportStatus diff --git a/tests/unit/cli/test_accession_sheet.py b/tests/unit/cli/test_accession_sheet.py index 57d0a81..2ccd8ec 100644 --- a/tests/unit/cli/test_accession_sheet.py +++ b/tests/unit/cli/test_accession_sheet.py @@ -29,12 +29,12 @@ def test_separates_reserved_and_metadata_columns(self, tmp_path: Path) -> None: assert sheet.metadata_columns == ["cell_type", "source", "source__annotation"] - def test_accession_is_normalised_to_upper_case(self, tmp_path: Path) -> None: + def test_accession_is_passed_through_unchanged(self, tmp_path: Path) -> None: sheet = parse_accession_sheet( _write_sheet(tmp_path, {"accession": "err1160845"}), ) - assert sheet.rows[0].accession == "ERR1160845" + assert sheet.rows[0].accession == "err1160845" def test_empty_cells_omitted_from_metadata(self, tmp_path: Path) -> None: sheet = parse_accession_sheet(_write_sheet( diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index b7ca63b..4ebae57 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -927,6 +927,8 @@ def test_non_csv_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None: IMPORT_HEADERS = ["accession", "name", "organism", "cell_type", "source", "source__annotation"] + + def _write_import_sheet(directory: Path, *records: dict[str, str]) -> Path: path = directory / "accessions.csv" with path.open("w", newline="") as handle: @@ -1017,7 +1019,7 @@ def test_sends_every_row_without_local_validation( # locally, respx would fail the test for an unmocked request. route = respx.post(SAMPLE_IMPORTS_URL).mock( return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "RUNNING", ["NOT-AN-ACCESSION", "ERR1", "ERR1"], + 1, "RUNNING", ["not-an-accession", "ERR1", "ERR1"], )), ) sheet = _write_import_sheet( @@ -1035,7 +1037,7 @@ def test_sends_every_row_without_local_validation( assert result.exit_code == 0 payload = json.loads(route.calls[0].request.content) assert [entry["accession"] for entry in payload["imports"]] == [ - "NOT-AN-ACCESSION", "ERR1", "ERR1", + "not-an-accession", "ERR1", "ERR1", ] @respx.mock @@ -1048,7 +1050,7 @@ def test_sends_name_organism_and_metadata_in_payload( )), ) sheet = _write_import_sheet(tmp_path, { - "accession": "err1", "name": "liver_r1", "organism": "Hs", + "accession": "ERR1", "name": "liver_r1", "organism": "Hs", "cell_type": "Neuron", }) @@ -1120,6 +1122,20 @@ def test_missing_sample_type_is_usage_error(self, run_cli, tmp_path: Path) -> No assert result.exit_code == 2 + @respx.mock + def test_header_only_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None: + route = respx.post(SAMPLE_IMPORTS_URL) + sheet = _write_import_sheet(tmp_path) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + ) + + assert result.exit_code == 2 + assert route.call_count == 0 + assert str(sheet) in result.stderr + class TestSamplesImportStatus: @@ -1167,10 +1183,25 @@ def test_reports_failed_job_with_error(self, run_cli) -> None: "--token", TOKEN, "samples", "import-status", "--job-id", "42", ) - assert result.exit_code == 0 + assert result.exit_code == 1 assert "FAILED" in result.stdout assert "download failed" in result.stdout + @respx.mock + def test_completed_job_with_no_sample_ids_reports_none(self, run_cli) -> None: + respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json=_job_json( + 42, "COMPLETED", [], + )), + ) + + result = run_cli( + "--token", TOKEN, "samples", "import-status", "--job-id", "42", + ) + + assert result.exit_code == 0 + assert "none" in result.stdout + @respx.mock def test_json_document_matches_job_shape(self, run_cli) -> None: respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( From 8f829b7b02a68ff918e34cbec99e5979bd137456 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 15:51:44 +0000 Subject: [PATCH 07/32] fix(samples): address round-6 findings on the non-blocking design - Filter out rows with no accession (blank cell, comma-only trailing row, or a sheet with no 'accession' column at all) before submitting, and raise CliUsageError naming the sheet if nothing is left. Previously an empty accession was shipped as accession: "", which the API can never accept, turning a locally-detectable structural issue into an opaque all-or-nothing rejection of the whole batch. - samples import-status now emits a stderr advisory naming the job and its error when the job is FAILED, matching every other non-zero exit path in the CLI (which always has something on stderr, not just stdout/--json). - --job-id's type callback now raises a clear ArgumentTypeError instead of letting argparse report the private helper's own name ("invalid _job_id value"). - samples import's confirmation line now reports len(specs) (what was actually submitted) rather than echoing job.accessions back. - Fixed a test that didn't exercise the branch its name claimed (an empty job vs. a completed job with accessions but no sample ids). - Docs: hedged the exit-code claim for a rejected batch (1 for the 422s this endpoint returns in practice, 5 if it ever answers 400 instead), updated the import-status polling example to check the command's own exit code so a transient failure isn't read as "still running", and documented the FAILED stderr advisory. Rebuilt the docs locally to confirm they still render cleanly. Addresses Claude's round-6 review on flowbio#18. Refs FLOW-689. --- flowbio/cli/_samples.py | 24 ++++++--- source/cli.rst | 31 ++++++----- tests/unit/cli/test_samples.py | 95 +++++++++++++++++++++++++++++++--- 3 files changed, 122 insertions(+), 28 deletions(-) diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index 1e05e86..4309452 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -275,7 +275,10 @@ def _configure_import(import_parser: argparse.ArgumentParser) -> None: def _job_id(value: str) -> SampleImportJobId: - return SampleImportJobId(int(value)) + try: + return SampleImportJobId(int(value)) + except ValueError: + raise argparse.ArgumentTypeError(f"job id must be an integer, got {value!r}") from None def _configure_import_status(import_status: argparse.ArgumentParser) -> None: @@ -629,12 +632,17 @@ def _import_command( :param client: The authenticated Flow client. :param output: The result/error renderer. :returns: :attr:`ExitCode.SUCCESS` once the job has been kicked off. - :raises CliUsageError: If the sheet has no rows — there is nothing an - API call could tell us about an empty sheet that we can't see already. + :raises CliUsageError: If the sheet is not a readable ``.csv``, or if it + has no row with an accession — there is nothing an API call could + tell us about either that we can't see already. """ sheet = parse_accession_sheet(args.sheet) - if not sheet.rows: - raise CliUsageError(f"Accession sheet has no rows: {args.sheet}") + rows = [row for row in sheet.rows if row.accession] + if not rows: + raise CliUsageError( + f"Accession sheet has no row with an accession: {args.sheet}. " + f"Check it has an 'accession' column and at least one filled-in row.", + ) specs = [ SampleImportSpec( accession=row.accession, @@ -643,11 +651,11 @@ def _import_command( organism_id=row.organism, metadata=row.metadata or None, ) - for row in sheet.rows + for row in rows ] job = client.samples.import_samples(specs) output.emit_result( - f"Started import job {job.id} for {len(job.accessions)} accession(s) " + f"Started import job {job.id} for {len(specs)} accession(s) " f"(status: {job.status}). Check progress with " f"'flowbio samples import-status --job-id {job.id}'.", _job_document(job), @@ -669,6 +677,8 @@ def _import_status_command( branch on its exit code alone, without parsing ``--json`` output. """ job = client.samples.get_import(args.job_id) + if job.status == "FAILED": + output.emit_advisory(f"Job {job.id} failed: {job.error or 'no error message returned'}") output.emit_result(_job_summary(job), _job_document(job)) return ExitCode.RUNTIME if job.status == "FAILED" else ExitCode.SUCCESS diff --git a/source/cli.rst b/source/cli.rst index 265120c..3b2a8e2 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -383,11 +383,11 @@ a CSV with an ``accession`` column plus optional ``name``/``organism`` and metadata columns (there is no ``batch-template`` equivalent for it, since it has no reads files or project field). ``name`` defaults to the accession when omitted. The sample type, accession format, duplicates, and metadata rules -are all sent as-is and validated **server-side** — this command does not -pre-validate rows itself (beyond checking the sheet has at least one row); an -invalid sheet surfaces as a normal API error (exit ``1`` — the API returns -these as an HTTP ``422``, which isn't one of the codes with its own mapping -below), not a local rejection. +are all sent as-is and validated **server-side** — this command only checks +that the sheet is a readable ``.csv`` with at least one row that has an +accession (rows without one are dropped rather than submitted as an empty +string); anything else invalid surfaces as a normal API error, not a local +rejection. Every row is submitted **together as one server-side job**. This command does **not wait for it to finish** — it reports the job's id and initial @@ -401,9 +401,11 @@ to you, e.g. in a shell loop. completes), ``execution_id``, ``error``. **Exit codes** — ``0`` the job was created (regardless of its eventual -outcome — check that with ``import-status``); ``2`` a non-CSV or empty sheet; -``1`` the API rejected the batch (e.g. unknown sample type, missing required -metadata, an unsupported accession format); ``3`` authentication failure; +outcome — check that with ``import-status``); ``2`` the sheet isn't a +readable ``.csv``, or has no row with an accession; ``1`` the API rejected +the batch (e.g. unknown sample type, missing required metadata, an +unsupported accession format — these come back as an HTTP ``422``; ``5`` in +the unlikely case it answers ``400`` instead); ``3`` authentication failure; otherwise the standard mapping above. **Example** @@ -427,18 +429,21 @@ Fetch and report the current state of a ``samples import`` job. Read-only — checking a job's status never changes it. There is no built-in polling; run this again (or wrap it in your own loop) until ``status`` leaves -``"RUNNING"``: +``"RUNNING"``. Check the command's own exit code alongside ``jq`` so a +transient failure (auth, network) doesn't get read as ``"RUNNING"``: .. code-block:: bash - until [ "$(flowbio samples import-status --job-id 42 --json | jq -r .status)" != "RUNNING" ]; do + while status=$(flowbio samples import-status --job-id 42 --json | jq -r .status); do + [ "$status" = "RUNNING" ] || break sleep 30 done **Output** — human: a one-line summary including the sample ids on -``"COMPLETED"`` or the error on ``"FAILED"``. ``--json``: the job as a single -document — ``id``, ``status``, ``accessions``, ``sample_ids``, -``execution_id``, ``error``. +``"COMPLETED"`` or the error on ``"FAILED"``, which is also reported as an +advisory on stderr in that case. ``--json``: the job as a single document — +``id``, ``status``, ``accessions``, ``sample_ids``, ``execution_id``, +``error``. **Exit codes** — ``0`` the job was fetched and is ``"RUNNING"`` or ``"COMPLETED"``; ``1`` the job was fetched but is ``"FAILED"`` (so a caller diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index 4ebae57..4ba66f2 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -1136,6 +1136,65 @@ def test_header_only_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None assert route.call_count == 0 assert str(sheet) in result.stderr + @respx.mock + def test_blank_accession_row_is_skipped(self, run_cli, tmp_path: Path) -> None: + route = respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "RUNNING", ["ERR1"], + )), + ) + sheet = _write_import_sheet( + tmp_path, + {"accession": "ERR1"}, + {"accession": ""}, + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + ) + + assert result.exit_code == 0 + payload = json.loads(route.calls[0].request.content) + assert len(payload["imports"]) == 1 + assert payload["imports"][0]["accession"] == "ERR1" + + @respx.mock + def test_sheet_with_only_blank_accessions_is_usage_error( + self, run_cli, tmp_path: Path, + ) -> None: + route = respx.post(SAMPLE_IMPORTS_URL) + sheet = _write_import_sheet(tmp_path, {"accession": ""}, {"accession": ""}) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + ) + + assert result.exit_code == 2 + assert route.call_count == 0 + assert str(sheet) in result.stderr + + @respx.mock + def test_sheet_with_no_accession_column_is_usage_error( + self, run_cli, tmp_path: Path, + ) -> None: + route = respx.post(SAMPLE_IMPORTS_URL) + sheet = tmp_path / "accessions.csv" + with sheet.open("w", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=["run", "name"]) + writer.writeheader() + writer.writerow({"run": "ERR1160845", "name": "liver_r1"}) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + ) + + assert result.exit_code == 2 + assert route.call_count == 0 + assert str(sheet) in result.stderr + class TestSamplesImportStatus: @@ -1186,12 +1245,39 @@ def test_reports_failed_job_with_error(self, run_cli) -> None: assert result.exit_code == 1 assert "FAILED" in result.stdout assert "download failed" in result.stdout + assert "download failed" in result.stderr + + @respx.mock + def test_failed_job_json_mode_has_error_on_stdout_only(self, run_cli) -> None: + respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json=_job_json( + 42, "FAILED", ["ERR1"], error="download failed", + )), + ) + + result = run_cli( + "--token", TOKEN, "samples", "import-status", "--job-id", "42", "--json", + ) + + assert result.exit_code == 1 + assert result.stderr == "" + document = json.loads(result.stdout) + assert document["error"] == "download failed" + + def test_non_numeric_job_id_reports_clear_message(self, run_cli) -> None: + result = run_cli( + "--token", TOKEN, "samples", "import-status", "--job-id", "not-a-number", + ) + + assert result.exit_code == 2 + assert "_job_id" not in result.stderr + assert "job id" in result.stderr.lower() @respx.mock def test_completed_job_with_no_sample_ids_reports_none(self, run_cli) -> None: respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( return_value=httpx.Response(HTTPStatus.OK, json=_job_json( - 42, "COMPLETED", [], + 42, "COMPLETED", ["ERR1"], [], )), ) @@ -1244,10 +1330,3 @@ def test_missing_job_id_is_usage_error(self, run_cli) -> None: result = run_cli("--token", TOKEN, "samples", "import-status") assert result.exit_code == 2 - - def test_non_numeric_job_id_is_usage_error(self, run_cli) -> None: - result = run_cli( - "--token", TOKEN, "samples", "import-status", "--job-id", "not-a-number", - ) - - assert result.exit_code == 2 From 54d6279294ccd2fbc43f93f3d48bb67217736640 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 16:00:56 +0000 Subject: [PATCH 08/32] fix(samples): address round-7 findings on the non-blocking design - samples import now reports each skipped (accession-less) row on stderr before submitting, instead of only a lower count in the confirmation line -- a 200-row sheet missing one accession previously looked like a clean success for 199. - AccessionSheetRow.accession is now str | None instead of using "" as an absent-value sentinel, matching name/organism/metadata (all of which already drop empty cells to None). The filter in _import_command reads as an explicit "is None" check instead of a bare truthiness test. - _job_summary no longer repeats a FAILED job's error text on stdout now that _import_status_command also puts it on stderr as an advisory -- a human running the command was seeing the same sentence twice. - Docs: rewrote the import-status polling example to gate the while loop on flowbio's own exit code (assigning the JSON document, not a value piped through jq, in the condition) -- the previous version's condition captured jq's exit status, not flowbio's, so the "checks the command's own exit code" claim it made was false. Documented that exit 1 covers both "the job failed" and "the request itself failed", and how to tell them apart in each output mode. Fixed two RST heading underlines that overshot their titles by one character. Rebuilt the docs locally to confirm everything still renders. Addresses Claude's round-7 review on flowbio#18. The remaining deferred item (no import-template verb for the accession sheet, so a batch-template sheet's reads1/reads2/project columns ship as bogus metadata) is unchanged from prior rounds' decision -- worth a follow-up ticket rather than folding into this one, as the review itself suggests. Refs FLOW-689. --- flowbio/cli/_accession_sheet.py | 11 +++++-- flowbio/cli/_samples.py | 15 ++++++---- source/cli.rst | 51 +++++++++++++++++++-------------- tests/unit/cli/test_samples.py | 3 +- 4 files changed, 50 insertions(+), 30 deletions(-) diff --git a/flowbio/cli/_accession_sheet.py b/flowbio/cli/_accession_sheet.py index 89923c8..af16bfd 100644 --- a/flowbio/cli/_accession_sheet.py +++ b/flowbio/cli/_accession_sheet.py @@ -27,10 +27,15 @@ @dataclass(frozen=True) class AccessionSheetRow: - """One data row of an accession sheet.""" + """One data row of an accession sheet. + + ``accession`` may be ``None`` (empty cell, or no ``accession`` column at + all) — reported by the caller rather than rejected here, so a missing + accession is one visible problem instead of an opaque server rejection. + """ row_number: int - accession: str + accession: str | None name: str | None organism: str | None metadata: dict[str, str] @@ -91,7 +96,7 @@ def cell(column: str) -> str | None: } return AccessionSheetRow( row_number=row_number, - accession=cell("accession") or "", + accession=cell("accession"), name=cell("name"), organism=cell("organism"), metadata=metadata, diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index 4309452..11d8913 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -637,12 +637,14 @@ def _import_command( tell us about either that we can't see already. """ sheet = parse_accession_sheet(args.sheet) - rows = [row for row in sheet.rows if row.accession] - if not rows: + skipped = [row for row in sheet.rows if row.accession is None] + if len(skipped) == len(sheet.rows): raise CliUsageError( f"Accession sheet has no row with an accession: {args.sheet}. " f"Check it has an 'accession' column and at least one filled-in row.", ) + for row in skipped: + output.emit_advisory(f"Skipped row {row.row_number}: no accession") specs = [ SampleImportSpec( accession=row.accession, @@ -651,7 +653,8 @@ def _import_command( organism_id=row.organism, metadata=row.metadata or None, ) - for row in rows + for row in sheet.rows + if row.accession is not None ] job = client.samples.import_samples(specs) output.emit_result( @@ -699,8 +702,10 @@ def _job_summary(job: SampleImportJob) -> str: ids = ", ".join(str(sample_id) for sample_id in job.sample_ids) or "none" return f"Job {job.id}: COMPLETED. Sample ids: {ids}." if job.status == "FAILED": - detail = f" {job.error}" if job.error else "" - return f"Job {job.id}: FAILED.{detail}" + # The error, if any, is on stderr as an advisory (see + # _import_status_command) rather than repeated here, so a human + # running this doesn't see the same sentence twice. + return f"Job {job.id}: FAILED." return f"Job {job.id}: {job.status}." diff --git a/source/cli.rst b/source/cli.rst index 3b2a8e2..e53e39b 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -368,7 +368,7 @@ authentication failure; otherwise the standard mapping above. {"uploaded": [{"row_number": 1, "name": "liver_r1", "sample_id": "samp_1"}], "failed": [], "skipped": [], "counts": {"uploaded": 1, "failed": 0, "skipped": 0}} ``samples import`` -~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~ Kick off a batch import of samples from public-repository accessions (SRR/ ERR/DRR run or SRX/ERX/DRX experiment accessions), applying one sample type @@ -385,9 +385,9 @@ has no reads files or project field). ``name`` defaults to the accession when omitted. The sample type, accession format, duplicates, and metadata rules are all sent as-is and validated **server-side** — this command only checks that the sheet is a readable ``.csv`` with at least one row that has an -accession (rows without one are dropped rather than submitted as an empty -string); anything else invalid surfaces as a normal API error, not a local -rejection. +accession; rows without one are dropped (reported by row number on stderr) +rather than submitted as an empty string. Anything else invalid surfaces as +a normal API error, not a local rejection. Every row is submitted **together as one server-side job**. This command does **not wait for it to finish** — it reports the job's id and initial @@ -396,9 +396,9 @@ with ``samples import-status --job-id ID``; polling (if you want it) is up to you, e.g. in a shell loop. **Output** — human: a confirmation line with the job id and a pointer to -``import-status``. ``--json``: the created job as a single document — -``id``, ``status``, ``accessions``, ``sample_ids`` (empty until the job -completes), ``execution_id``, ``error``. +``import-status``, plus one advisory per skipped row. ``--json``: the created +job as a single document — ``id``, ``status``, ``accessions``, ``sample_ids`` +(empty until the job completes), ``execution_id``, ``error``. **Exit codes** — ``0`` the job was created (regardless of its eventual outcome — check that with ``import-status``); ``2`` the sheet isn't a @@ -419,7 +419,7 @@ otherwise the standard mapping above. {"id": 42, "status": "RUNNING", "accessions": ["ERR1160845", "ERR10677146"], "sample_ids": [], "execution_id": 7, "error": null} ``samples import-status`` -~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~ Fetch and report the current state of a ``samples import`` job. @@ -429,28 +429,37 @@ Fetch and report the current state of a ``samples import`` job. Read-only — checking a job's status never changes it. There is no built-in polling; run this again (or wrap it in your own loop) until ``status`` leaves -``"RUNNING"``. Check the command's own exit code alongside ``jq`` so a -transient failure (auth, network) doesn't get read as ``"RUNNING"``: +``"RUNNING"``. Gate the loop on ``flowbio``'s own exit code (assigning the +document, not a value piped through ``jq``, in the ``while`` condition) so a +transient failure (auth, network) breaks the loop instead of being read as +``"RUNNING"``: .. code-block:: bash - while status=$(flowbio samples import-status --job-id 42 --json | jq -r .status); do - [ "$status" = "RUNNING" ] || break + while out=$(flowbio samples import-status --job-id 42 --json); do + [ "$(printf '%s' "$out" | jq -r .status)" = RUNNING ] || break sleep 30 done + printf '%s' "$out" | jq -r .status # COMPLETED / FAILED; empty if the command errored **Output** — human: a one-line summary including the sample ids on -``"COMPLETED"`` or the error on ``"FAILED"``, which is also reported as an -advisory on stderr in that case. ``--json``: the job as a single document — -``id``, ``status``, ``accessions``, ``sample_ids``, ``execution_id``, -``error``. +``"COMPLETED"``, or — on ``"FAILED"`` — a plain ``FAILED.`` summary plus the +error as a separate advisory on stderr (so it isn't printed twice). ``--json`` +never prints prose to stderr (or anywhere but the one stdout document); the +failure reason there is the document's ``error`` field. ``--json``: the job +as a single document — ``id``, ``status``, ``accessions``, ``sample_ids``, +``execution_id``, ``error``. **Exit codes** — ``0`` the job was fetched and is ``"RUNNING"`` or -``"COMPLETED"``; ``1`` the job was fetched but is ``"FAILED"`` (so a caller -can branch on the exit code alone, without parsing ``--json`` output — the -loop above still needs ``--json``/``jq`` to tell ``"RUNNING"`` from -``"COMPLETED"``, since both exit ``0``); ``4`` no job with that id exists; -``3`` authentication failure; otherwise the standard mapping above. +``"COMPLETED"``; ``1`` either the job was fetched but is ``"FAILED"``, or the +request itself failed (e.g. a transient server error) — human mode +distinguishes them (an ``Error:`` line means the request failed; a +``Job N: FAILED.`` line means the job did), and under ``--json`` a request +failure's document is on stderr while a ``FAILED`` job's is on stdout like +any other successful fetch. The loop above still needs ``--json``/``jq`` to +tell ``"RUNNING"`` from ``"COMPLETED"``, since both exit ``0``; ``4`` no job +with that id exists; ``3`` authentication failure; otherwise the standard +mapping above. **Example** diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index 4ba66f2..ea9497d 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -1158,6 +1158,7 @@ def test_blank_accession_row_is_skipped(self, run_cli, tmp_path: Path) -> None: payload = json.loads(route.calls[0].request.content) assert len(payload["imports"]) == 1 assert payload["imports"][0]["accession"] == "ERR1" + assert "Skipped row 2" in result.stderr @respx.mock def test_sheet_with_only_blank_accessions_is_usage_error( @@ -1244,7 +1245,7 @@ def test_reports_failed_job_with_error(self, run_cli) -> None: assert result.exit_code == 1 assert "FAILED" in result.stdout - assert "download failed" in result.stdout + assert "download failed" not in result.stdout assert "download failed" in result.stderr @respx.mock From 154e9bcdc0d0b2031cfceff3389a2e6bc1e24a56 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 16:21:27 +0000 Subject: [PATCH 09/32] fix(samples): address round-8 findings (counter reset per operator) - SampleImportJob now declares created/started/finished (the API already sends them, per the existing mocked test payloads -- pydantic's default extra="ignore" was silently dropping them). created is required (the server always sends it); started/finished default to None. Surfaced in both samples import's and samples import-status's JSON documents, so a caller resuming a poll from elsewhere can tell how long a job has been running without tracking wall-clock time itself. - samples import's JSON document now includes a skipped list (row numbers of any accession-less rows). Previously this was only visible as a stderr advisory in human mode, so an automated --json caller had no signal a row had been dropped at all. - _import_command now computes specs/skipped in a single pass over sheet.rows instead of two separate list comprehensions restating the same "does this row have an accession" predicate. Addresses Claude's round-8 review on flowbio#18. Findings left as-is per prior rounds' reasoning (a job's execution_id/error having no default=None, _job_document's list-in-JsonValue shape, sample_ids as list[int]) are noted on the PR rather than repeated here. Refs FLOW-689. --- flowbio/cli/_samples.py | 36 +++++++++++++---------- flowbio/v2/samples.py | 5 ++++ source/cli.rst | 21 ++++++++----- tests/unit/cli/test_samples.py | 54 +++++++++++++++++++++++++++++++++- tests/unit/v2/test_samples.py | 15 ++++++++-- 5 files changed, 104 insertions(+), 27 deletions(-) diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index 11d8913..ce3c8ca 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -14,7 +14,7 @@ from pathlib import Path from typing import Literal -from flowbio.cli._accession_sheet import parse_accession_sheet +from flowbio.cli._accession_sheet import AccessionSheetRow, parse_accession_sheet from flowbio.cli._exit_codes import CliUsageError, ExitCode from flowbio.cli._files import existing_file from flowbio.cli._output import Output, format_issue @@ -637,31 +637,34 @@ def _import_command( tell us about either that we can't see already. """ sheet = parse_accession_sheet(args.sheet) - skipped = [row for row in sheet.rows if row.accession is None] - if len(skipped) == len(sheet.rows): - raise CliUsageError( - f"Accession sheet has no row with an accession: {args.sheet}. " - f"Check it has an 'accession' column and at least one filled-in row.", - ) - for row in skipped: - output.emit_advisory(f"Skipped row {row.row_number}: no accession") - specs = [ - SampleImportSpec( + specs: list[SampleImportSpec] = [] + skipped: list[AccessionSheetRow] = [] + for row in sheet.rows: + if row.accession is None: + skipped.append(row) + continue + specs.append(SampleImportSpec( accession=row.accession, sample_type=args.sample_type, name=row.name, organism_id=row.organism, metadata=row.metadata or None, + )) + if not specs: + raise CliUsageError( + f"Accession sheet has no row with an accession: {args.sheet}. " + f"Check it has an 'accession' column and at least one filled-in row.", ) - for row in sheet.rows - if row.accession is not None - ] + for row in skipped: + output.emit_advisory(f"Skipped row {row.row_number}: no accession") job = client.samples.import_samples(specs) + document = _job_document(job) + document["skipped"] = [{"row_number": row.row_number} for row in skipped] output.emit_result( f"Started import job {job.id} for {len(specs)} accession(s) " f"(status: {job.status}). Check progress with " f"'flowbio samples import-status --job-id {job.id}'.", - _job_document(job), + document, ) return ExitCode.SUCCESS @@ -690,6 +693,9 @@ def _job_document(job: SampleImportJob) -> dict[str, JsonValue]: return { "id": job.id, "status": job.status, + "created": job.created, + "started": job.started, + "finished": job.finished, "accessions": job.accessions, "sample_ids": job.sample_ids, "execution_id": job.execution_id, diff --git a/flowbio/v2/samples.py b/flowbio/v2/samples.py index 398baf4..ef49f4c 100644 --- a/flowbio/v2/samples.py +++ b/flowbio/v2/samples.py @@ -195,6 +195,11 @@ class SampleImportJob(BaseModel, frozen=True): id: SampleImportJobId = Field(description="Unique identifier for this import job.") status: SampleImportStatus = Field(description="The job's current lifecycle state.") + created: int = Field(description="Unix timestamp when the job was created.") + started: int | None = Field(default=None, description="Unix timestamp when the job started running, if it has.") + finished: int | None = Field( + default=None, description="Unix timestamp when the job finished (completed or failed), if it has.", + ) accessions: list[str] = Field(description="The accessions submitted with this job, in submission order.") sample_ids: list[int] = Field( description="The created samples' ids, corresponding to ``accessions`` once the job has completed.", diff --git a/source/cli.rst b/source/cli.rst index e53e39b..0e68950 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -397,8 +397,11 @@ to you, e.g. in a shell loop. **Output** — human: a confirmation line with the job id and a pointer to ``import-status``, plus one advisory per skipped row. ``--json``: the created -job as a single document — ``id``, ``status``, ``accessions``, ``sample_ids`` -(empty until the job completes), ``execution_id``, ``error``. +job as a single document — ``id``, ``status``, ``created``/``started``/ +``finished`` (Unix timestamps; the latter two ``null`` until the job reaches +those stages), ``accessions``, ``sample_ids`` (empty until the job +completes), ``execution_id``, ``error``, and ``skipped`` (the row numbers of +any accession-less rows, empty if none). **Exit codes** — ``0`` the job was created (regardless of its eventual outcome — check that with ``import-status``); ``2`` the sheet isn't a @@ -416,7 +419,7 @@ otherwise the standard mapping above. Started import job 42 for 2 accession(s) (status: RUNNING). Check progress with 'flowbio samples import-status --job-id 42'. $ flowbio samples import --sheet ./accessions.csv --sample-type RNA-Seq --json - {"id": 42, "status": "RUNNING", "accessions": ["ERR1160845", "ERR10677146"], "sample_ids": [], "execution_id": 7, "error": null} + {"id": 42, "status": "RUNNING", "created": 1712345678, "started": null, "finished": null, "accessions": ["ERR1160845", "ERR10677146"], "sample_ids": [], "execution_id": 7, "error": null, "skipped": []} ``samples import-status`` ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -444,11 +447,13 @@ transient failure (auth, network) breaks the loop instead of being read as **Output** — human: a one-line summary including the sample ids on ``"COMPLETED"``, or — on ``"FAILED"`` — a plain ``FAILED.`` summary plus the -error as a separate advisory on stderr (so it isn't printed twice). ``--json`` -never prints prose to stderr (or anywhere but the one stdout document); the -failure reason there is the document's ``error`` field. ``--json``: the job -as a single document — ``id``, ``status``, ``accessions``, ``sample_ids``, -``execution_id``, ``error``. +error as a separate advisory on stderr (so it isn't printed twice). ``--json``: +the job as a single document — ``id``, ``status``, ``created``/``started``/ +``finished`` (Unix timestamps, useful for judging how long a job has been +running when you've resumed polling one from elsewhere), ``accessions``, +``sample_ids``, ``execution_id``, ``error``. ``--json`` never prints prose to +stderr (or anywhere but that one stdout document); the failure reason on a +``"FAILED"`` job is the document's ``error`` field, not a separate message. **Exit codes** — ``0`` the job was fetched and is ``"RUNNING"`` or ``"COMPLETED"``; ``1`` either the job was fetched but is ``"FAILED"``, or the diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index ea9497d..3362b47 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -944,13 +944,17 @@ def _job_json( accessions: list[str], sample_ids: list[int] | None = None, error: str | None = None, + execution_id: int | None = 7, ) -> dict: return { "id": job_id, "status": status, + "created": 1700000000, + "started": 1700000001 if status != "RUNNING" else None, + "finished": 1700000002 if status in ("COMPLETED", "FAILED") else None, "accessions": accessions, "sample_ids": sample_ids or [], - "execution_id": 7, + "execution_id": execution_id, "error": error, } @@ -1004,10 +1008,14 @@ def test_json_document_reports_job_fields( assert document == { "id": 42, "status": "RUNNING", + "created": 1700000000, + "started": None, + "finished": None, "accessions": ["ERR1"], "sample_ids": [], "execution_id": 7, "error": None, + "skipped": [], } @respx.mock @@ -1160,6 +1168,31 @@ def test_blank_accession_row_is_skipped(self, run_cli, tmp_path: Path) -> None: assert payload["imports"][0]["accession"] == "ERR1" assert "Skipped row 2" in result.stderr + @respx.mock + def test_blank_accession_row_is_reported_in_json_document( + self, run_cli, tmp_path: Path, + ) -> None: + respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "RUNNING", ["ERR1"], + )), + ) + sheet = _write_import_sheet( + tmp_path, + {"accession": "ERR1"}, + {"accession": ""}, + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", "--json", + ) + + assert result.exit_code == 0 + assert result.stderr == "" + document = json.loads(result.stdout) + assert document["skipped"] == [{"row_number": 2}] + @respx.mock def test_sheet_with_only_blank_accessions_is_usage_error( self, run_cli, tmp_path: Path, @@ -1307,12 +1340,31 @@ def test_json_document_matches_job_shape(self, run_cli) -> None: assert document == { "id": 42, "status": "COMPLETED", + "created": 1700000000, + "started": 1700000001, + "finished": 1700000002, "accessions": ["ERR1"], "sample_ids": [101], "execution_id": 7, "error": None, } + @respx.mock + def test_reports_job_with_no_execution_yet(self, run_cli) -> None: + respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json=_job_json( + 42, "RUNNING", ["ERR1"], execution_id=None, + )), + ) + + result = run_cli( + "--token", TOKEN, "samples", "import-status", "--job-id", "42", "--json", + ) + + assert result.exit_code == 0 + document = json.loads(result.stdout) + assert document["execution_id"] is None + @respx.mock def test_unknown_job_id_is_not_found(self, run_cli) -> None: respx.get(f"{SAMPLE_IMPORTS_URL}/999").mock( diff --git a/tests/unit/v2/test_samples.py b/tests/unit/v2/test_samples.py index c254c3e..9b49029 100644 --- a/tests/unit/v2/test_samples.py +++ b/tests/unit/v2/test_samples.py @@ -1082,6 +1082,9 @@ def test_posts_imports_and_parses_job(self) -> None: assert result == SampleImportJob( id=SampleImportJobId(42), status="RUNNING", + created=1700000000, + started=None, + finished=None, accessions=["ERR1160845"], sample_ids=[], execution_id=None, @@ -1093,7 +1096,7 @@ def test_posts_imports_and_parses_job(self) -> None: def test_sends_accession_and_sample_type(self) -> None: route = respx.post(f"{DEFAULT_BASE_URL}/v2/sample-imports").mock( return_value=httpx.Response(HTTPStatus.CREATED, json={ - "id": 1, "status": "RUNNING", "accessions": ["ERR1"], + "id": 1, "status": "RUNNING", "created": 1700000000, "accessions": ["ERR1"], "sample_ids": [], "execution_id": None, "error": None, }), ) @@ -1112,7 +1115,7 @@ def test_sends_accession_and_sample_type(self) -> None: def test_sends_optional_fields_when_present(self) -> None: route = respx.post(f"{DEFAULT_BASE_URL}/v2/sample-imports").mock( return_value=httpx.Response(HTTPStatus.CREATED, json={ - "id": 1, "status": "RUNNING", "accessions": ["ERR1"], + "id": 1, "status": "RUNNING", "created": 1700000000, "accessions": ["ERR1"], "sample_ids": [], "execution_id": None, "error": None, }), ) @@ -1143,7 +1146,7 @@ def test_sends_optional_fields_when_present(self) -> None: def test_sends_multiple_imports_in_one_request(self) -> None: route = respx.post(f"{DEFAULT_BASE_URL}/v2/sample-imports").mock( return_value=httpx.Response(HTTPStatus.CREATED, json={ - "id": 1, "status": "RUNNING", "accessions": ["ERR1", "ERR2"], + "id": 1, "status": "RUNNING", "created": 1700000000, "accessions": ["ERR1", "ERR2"], "sample_ids": [], "execution_id": None, "error": None, }), ) @@ -1199,6 +1202,9 @@ def test_parses_completed_job(self) -> None: assert result == SampleImportJob( id=SampleImportJobId(42), status="COMPLETED", + created=1700000000, + started=1700000001, + finished=1700000002, accessions=["ERR1160845", "ERR10677146"], sample_ids=[101, 102], execution_id=7, @@ -1211,6 +1217,9 @@ def test_parses_failed_job_with_error(self) -> None: return_value=httpx.Response(HTTPStatus.OK, json={ "id": 42, "status": "FAILED", + "created": 1700000000, + "started": 1700000001, + "finished": 1700000002, "accessions": ["ERR1160845"], "sample_ids": [], "execution_id": 7, From b3de2ff3a2c9865d18aef132ae1640d261475702 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 16:30:38 +0000 Subject: [PATCH 10/32] fix(samples): address round-9 findings - created is now int | None (default None), matching started/finished -- round 8 gave two of the three timestamps a default and left the third required on identical evidence, so a response omitting created would have raised an uncatchable ValidationError in both samples import and samples import-status where the previous model simply ignored the field. Added a test parsing a job payload with no timestamp keys at all. - samples import's JSON "skipped" entries now carry a reason ("no accession"), matching upload-batch's skipped-row shape (row_number/name/ reasons) instead of a bare row number with no indication why. - samples import-status now surfaces started in the human summary too ("Job 42: RUNNING (started 2024-...UTC).") -- round 8 added the timestamps to --json only, leaving the human mode (arguably the more likely place to ask "how long has this been running") unchanged. - Docs: updated the import-status --json example to include created/ started/finished (it had been left out while the import example got them), gave the previously-unreferenced sample-imports doc label a purpose via a :ref: from the CLI section instead of leaving it dangling, and fixed a heading underline overshoot in samples.rst. Rebuilt the docs locally to confirm everything still renders and the new cross-reference resolves. Addresses Claude's round-9 review on flowbio#18 (counter reset per operator request). Standing deferred items (execution_id/error with no default, _job_document's list-in-JsonValue shape, metadata_columns' one test-only consumer) are unchanged from prior rounds' reasoning. Refs FLOW-689. --- flowbio/cli/_samples.py | 11 ++++++++++- flowbio/v2/samples.py | 6 ++++-- source/cli.rst | 27 +++++++++++++++------------ source/v2/samples.rst | 2 +- tests/unit/cli/test_samples.py | 19 ++++++++++++++++++- tests/unit/v2/test_samples.py | 20 ++++++++++++++++++++ 6 files changed, 68 insertions(+), 17 deletions(-) diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index ce3c8ca..832fd57 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -11,6 +11,7 @@ import argparse import json from dataclasses import dataclass +from datetime import datetime, timezone from pathlib import Path from typing import Literal @@ -659,7 +660,9 @@ def _import_command( output.emit_advisory(f"Skipped row {row.row_number}: no accession") job = client.samples.import_samples(specs) document = _job_document(job) - document["skipped"] = [{"row_number": row.row_number} for row in skipped] + document["skipped"] = [ + {"row_number": row.row_number, "reason": "no accession"} for row in skipped + ] output.emit_result( f"Started import job {job.id} for {len(specs)} accession(s) " f"(status: {job.status}). Check progress with " @@ -712,9 +715,15 @@ def _job_summary(job: SampleImportJob) -> str: # _import_status_command) rather than repeated here, so a human # running this doesn't see the same sentence twice. return f"Job {job.id}: FAILED." + if job.started is not None: + return f"Job {job.id}: {job.status} (started {_format_timestamp(job.started)})." return f"Job {job.id}: {job.status}." +def _format_timestamp(timestamp: int) -> str: + return datetime.fromtimestamp(timestamp, tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") + + def _merge_metadata( pairs: list[str] | None, json_text: str | None, ) -> dict[str, str]: diff --git a/flowbio/v2/samples.py b/flowbio/v2/samples.py index ef49f4c..f354d7c 100644 --- a/flowbio/v2/samples.py +++ b/flowbio/v2/samples.py @@ -195,8 +195,10 @@ class SampleImportJob(BaseModel, frozen=True): id: SampleImportJobId = Field(description="Unique identifier for this import job.") status: SampleImportStatus = Field(description="The job's current lifecycle state.") - created: int = Field(description="Unix timestamp when the job was created.") - started: int | None = Field(default=None, description="Unix timestamp when the job started running, if it has.") + created: int | None = Field(default=None, description="Unix timestamp when the job was created.") + started: int | None = Field( + default=None, description="Unix timestamp when the job started running, if it has.", + ) finished: int | None = Field( default=None, description="Unix timestamp when the job finished (completed or failed), if it has.", ) diff --git a/source/cli.rst b/source/cli.rst index 0e68950..e41ae18 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -393,15 +393,17 @@ Every row is submitted **together as one server-side job**. This command does **not wait for it to finish** — it reports the job's id and initial status (almost always ``"RUNNING"``) and returns immediately. Check on it with ``samples import-status --job-id ID``; polling (if you want it) is up -to you, e.g. in a shell loop. +to you, e.g. in a shell loop. Building the same thing directly against the +library instead of the CLI? See :ref:`sample-imports`. **Output** — human: a confirmation line with the job id and a pointer to ``import-status``, plus one advisory per skipped row. ``--json``: the created job as a single document — ``id``, ``status``, ``created``/``started``/ -``finished`` (Unix timestamps; the latter two ``null`` until the job reaches -those stages), ``accessions``, ``sample_ids`` (empty until the job -completes), ``execution_id``, ``error``, and ``skipped`` (the row numbers of -any accession-less rows, empty if none). +``finished`` (Unix timestamps, ``null`` if not yet reached — or if the server +response omits one, which the client tolerates), ``accessions``, +``sample_ids`` (empty until the job completes), ``execution_id``, ``error``, +and ``skipped`` (``{"row_number": ..., "reason": "no accession"}`` for each +dropped row, empty if none). **Exit codes** — ``0`` the job was created (regardless of its eventual outcome — check that with ``import-status``); ``2`` the sheet isn't a @@ -446,12 +448,13 @@ transient failure (auth, network) breaks the loop instead of being read as printf '%s' "$out" | jq -r .status # COMPLETED / FAILED; empty if the command errored **Output** — human: a one-line summary including the sample ids on -``"COMPLETED"``, or — on ``"FAILED"`` — a plain ``FAILED.`` summary plus the -error as a separate advisory on stderr (so it isn't printed twice). ``--json``: -the job as a single document — ``id``, ``status``, ``created``/``started``/ -``finished`` (Unix timestamps, useful for judging how long a job has been -running when you've resumed polling one from elsewhere), ``accessions``, -``sample_ids``, ``execution_id``, ``error``. ``--json`` never prints prose to +``"COMPLETED"``, when it started (if known) on ``"RUNNING"``, or — on +``"FAILED"`` — a plain ``FAILED.`` summary plus the error as a separate +advisory on stderr (so it isn't printed twice). ``--json``: the job as a +single document — ``id``, ``status``, ``created``/``started``/``finished`` +(Unix timestamps, useful for judging how long a job has been running when +you've resumed polling one from elsewhere), ``accessions``, ``sample_ids``, +``execution_id``, ``error``. ``--json`` never prints prose to stderr (or anywhere but that one stdout document); the failure reason on a ``"FAILED"`` job is the document's ``error`` field, not a separate message. @@ -474,7 +477,7 @@ mapping above. Job 42: COMPLETED. Sample ids: 101, 102. $ flowbio samples import-status --job-id 42 --json - {"id": 42, "status": "COMPLETED", "accessions": ["ERR1160845", "ERR10677146"], "sample_ids": [101, 102], "execution_id": 7, "error": null} + {"id": 42, "status": "COMPLETED", "created": 1712345678, "started": 1712345680, "finished": 1712345900, "accessions": ["ERR1160845", "ERR10677146"], "sample_ids": [101, 102], "execution_id": 7, "error": null} ``api get`` ~~~~~~~~~~~ diff --git a/source/v2/samples.rst b/source/v2/samples.rst index 8098515..c055350 100644 --- a/source/v2/samples.rst +++ b/source/v2/samples.rst @@ -169,7 +169,7 @@ if the annotation has any warnings. .. _sample-imports: Importing samples from public repositories -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Use :meth:`~flowbio.v2.samples.SampleResource.import_samples` to create samples directly from public-repository run or experiment accessions diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index 3362b47..6f540c8 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -1191,7 +1191,7 @@ def test_blank_accession_row_is_reported_in_json_document( assert result.exit_code == 0 assert result.stderr == "" document = json.loads(result.stdout) - assert document["skipped"] == [{"row_number": 2}] + assert document["skipped"] == [{"row_number": 2, "reason": "no accession"}] @respx.mock def test_sheet_with_only_blank_accessions_is_usage_error( @@ -1248,6 +1248,23 @@ def test_reports_running_job(self, run_cli) -> None: assert "42" in result.stdout assert "RUNNING" in result.stdout + @respx.mock + def test_running_job_with_started_reports_when_it_started(self, run_cli) -> None: + respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json={ + "id": 42, "status": "RUNNING", "created": 1700000000, + "started": 1700000001, "finished": None, + "accessions": ["ERR1"], "sample_ids": [], "execution_id": None, "error": None, + }), + ) + + result = run_cli( + "--token", TOKEN, "samples", "import-status", "--job-id", "42", + ) + + assert result.exit_code == 0 + assert "started 2023-11-14" in result.stdout + @respx.mock def test_reports_completed_job_with_sample_ids(self, run_cli) -> None: respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( diff --git a/tests/unit/v2/test_samples.py b/tests/unit/v2/test_samples.py index 9b49029..7d8a99b 100644 --- a/tests/unit/v2/test_samples.py +++ b/tests/unit/v2/test_samples.py @@ -1245,3 +1245,23 @@ def test_raises_not_found_for_unknown_job(self) -> None: with pytest.raises(NotFoundError): client.samples.get_import(SampleImportJobId(999)) + + @respx.mock + def test_parses_job_missing_timestamps(self) -> None: + respx.get(f"{DEFAULT_BASE_URL}/v2/sample-imports/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json={ + "id": 42, + "status": "RUNNING", + "accessions": ["ERR1"], + "sample_ids": [], + "execution_id": None, + "error": None, + }), + ) + + client = Client() + result = client.samples.get_import(SampleImportJobId(42)) + + assert result.created is None + assert result.started is None + assert result.finished is None From 02189c9a658e17322cee9cb14aefbcaa1137cbe6 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 16:39:59 +0000 Subject: [PATCH 11/32] fix(samples): address round-10 findings - _format_timestamp now falls back to the raw number on ValueError/ OverflowError/OSError instead of letting a display-only computation crash a read-only status check. Added a test for an out-of-range (millisecond-precision) timestamp. - samples import's "skipped" JSON entries now use a reasons list and carry name, matching upload-batch's skipped-row shape exactly instead of a one-off {row_number, reason} pair a consumer of both commands would need a second code path for. - The stderr advisory and JSON entry for a skipped row now include its name when the sheet provided one, since an accession-less row often has no other identifying detail and "Skipped row 2" alone sends the user to the wrong place if they count from the header. - import-status's human summary now shows when a COMPLETED/FAILED job finished, not just when a RUNNING job started -- the previous round fixed half of the human/machine asymmetry the finding was about. - Fixed the heading underline in samples.rst that the previous round's attempt undershot by one character. - Docs: updated the skipped-entry description and the import/import-status examples to match. Rebuilt the docs locally to confirm they still render. Addresses Claude's round-10 review on flowbio#18. Refs FLOW-689. --- flowbio/cli/_samples.py | 32 +++++++++++++----- source/cli.rst | 25 +++++++------- source/v2/samples.rst | 2 +- tests/unit/cli/test_samples.py | 59 +++++++++++++++++++++++++++++++++- 4 files changed, 96 insertions(+), 22 deletions(-) diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index 832fd57..9aca30d 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -657,11 +657,12 @@ def _import_command( f"Check it has an 'accession' column and at least one filled-in row.", ) for row in skipped: - output.emit_advisory(f"Skipped row {row.row_number}: no accession") + output.emit_advisory(f"Skipped {_skipped_row_label(row)}: no accession") job = client.samples.import_samples(specs) document = _job_document(job) document["skipped"] = [ - {"row_number": row.row_number, "reason": "no accession"} for row in skipped + {"row_number": row.row_number, "name": row.name, "reasons": ["no accession"]} + for row in skipped ] output.emit_result( f"Started import job {job.id} for {len(specs)} accession(s) " @@ -672,6 +673,11 @@ def _import_command( return ExitCode.SUCCESS +def _skipped_row_label(row: AccessionSheetRow) -> str: + name_suffix = f" ({row.name})" if row.name else "" + return f"row {row.row_number}{name_suffix}" + + def _import_status_command( args: argparse.Namespace, client: Client, output: Output, ) -> ExitCode: @@ -709,19 +715,29 @@ def _job_document(job: SampleImportJob) -> dict[str, JsonValue]: def _job_summary(job: SampleImportJob) -> str: if job.status == "COMPLETED": ids = ", ".join(str(sample_id) for sample_id in job.sample_ids) or "none" - return f"Job {job.id}: COMPLETED. Sample ids: {ids}." + return f"Job {job.id}: COMPLETED{_timestamp_suffix('finished', job.finished)}. Sample ids: {ids}." if job.status == "FAILED": # The error, if any, is on stderr as an advisory (see # _import_status_command) rather than repeated here, so a human # running this doesn't see the same sentence twice. - return f"Job {job.id}: FAILED." - if job.started is not None: - return f"Job {job.id}: {job.status} (started {_format_timestamp(job.started)})." - return f"Job {job.id}: {job.status}." + return f"Job {job.id}: FAILED{_timestamp_suffix('finished', job.finished)}." + return f"Job {job.id}: {job.status}{_timestamp_suffix('started', job.started)}." + + +def _timestamp_suffix(label: str, timestamp: int | None) -> str: + if timestamp is None: + return "" + return f" ({label} {_format_timestamp(timestamp)})" def _format_timestamp(timestamp: int) -> str: - return datetime.fromtimestamp(timestamp, tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") + # The API is only known to send Unix-seconds timestamps, but this only + # renders a display string — falling back to the raw value on anything + # unexpected keeps a formatting surprise from failing a status check. + try: + return datetime.fromtimestamp(timestamp, tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") + except (ValueError, OverflowError, OSError): + return str(timestamp) def _merge_metadata( diff --git a/source/cli.rst b/source/cli.rst index e41ae18..5a01bb1 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -397,13 +397,14 @@ to you, e.g. in a shell loop. Building the same thing directly against the library instead of the CLI? See :ref:`sample-imports`. **Output** — human: a confirmation line with the job id and a pointer to -``import-status``, plus one advisory per skipped row. ``--json``: the created -job as a single document — ``id``, ``status``, ``created``/``started``/ -``finished`` (Unix timestamps, ``null`` if not yet reached — or if the server -response omits one, which the client tolerates), ``accessions``, -``sample_ids`` (empty until the job completes), ``execution_id``, ``error``, -and ``skipped`` (``{"row_number": ..., "reason": "no accession"}`` for each -dropped row, empty if none). +``import-status``, plus one advisory per skipped row (identified by name if +it has one). ``--json``: the created job as a single document — ``id``, +``status``, ``created``/``started``/``finished`` (Unix timestamps, ``null`` +if not yet reached — or if the server response omits one, which the client +tolerates), ``accessions``, ``sample_ids`` (empty until the job completes), +``execution_id``, ``error``, and ``skipped`` (``{"row_number": ..., "name": +..., "reasons": [...]}`` for each dropped row, matching ``upload-batch``'s +shape, empty if none). **Exit codes** — ``0`` the job was created (regardless of its eventual outcome — check that with ``import-status``); ``2`` the sheet isn't a @@ -447,10 +448,10 @@ transient failure (auth, network) breaks the loop instead of being read as done printf '%s' "$out" | jq -r .status # COMPLETED / FAILED; empty if the command errored -**Output** — human: a one-line summary including the sample ids on -``"COMPLETED"``, when it started (if known) on ``"RUNNING"``, or — on -``"FAILED"`` — a plain ``FAILED.`` summary plus the error as a separate -advisory on stderr (so it isn't printed twice). ``--json``: the job as a +**Output** — human: a one-line summary including the sample ids and when it +finished on ``"COMPLETED"``, when it started (if known) on ``"RUNNING"``, or +— on ``"FAILED"`` — when it finished plus the error as a separate advisory on +stderr (so it isn't printed twice). ``--json``: the job as a single document — ``id``, ``status``, ``created``/``started``/``finished`` (Unix timestamps, useful for judging how long a job has been running when you've resumed polling one from elsewhere), ``accessions``, ``sample_ids``, @@ -474,7 +475,7 @@ mapping above. .. code-block:: bash $ flowbio samples import-status --job-id 42 - Job 42: COMPLETED. Sample ids: 101, 102. + Job 42: COMPLETED (finished 2024-04-05 19:38:20 UTC). Sample ids: 101, 102. $ flowbio samples import-status --job-id 42 --json {"id": 42, "status": "COMPLETED", "created": 1712345678, "started": 1712345680, "finished": 1712345900, "accessions": ["ERR1160845", "ERR10677146"], "sample_ids": [101, 102], "execution_id": 7, "error": null} diff --git a/source/v2/samples.rst b/source/v2/samples.rst index c055350..5cfee62 100644 --- a/source/v2/samples.rst +++ b/source/v2/samples.rst @@ -169,7 +169,7 @@ if the annotation has any warnings. .. _sample-imports: Importing samples from public repositories -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Use :meth:`~flowbio.v2.samples.SampleResource.import_samples` to create samples directly from public-repository run or experiment accessions diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index 6f540c8..5fe07d8 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -1191,7 +1191,32 @@ def test_blank_accession_row_is_reported_in_json_document( assert result.exit_code == 0 assert result.stderr == "" document = json.loads(result.stdout) - assert document["skipped"] == [{"row_number": 2, "reason": "no accession"}] + assert document["skipped"] == [ + {"row_number": 2, "name": None, "reasons": ["no accession"]}, + ] + + @respx.mock + def test_blank_accession_row_with_name_is_identified_by_name( + self, run_cli, tmp_path: Path, + ) -> None: + respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "RUNNING", ["ERR1"], + )), + ) + sheet = _write_import_sheet( + tmp_path, + {"accession": "ERR1"}, + {"accession": "", "name": "liver_r2"}, + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", + "--sheet", str(sheet), "--sample-type", "rna_seq", + ) + + assert result.exit_code == 0 + assert "Skipped row 2 (liver_r2)" in result.stderr @respx.mock def test_sheet_with_only_blank_accessions_is_usage_error( @@ -1265,6 +1290,38 @@ def test_running_job_with_started_reports_when_it_started(self, run_cli) -> None assert result.exit_code == 0 assert "started 2023-11-14" in result.stdout + @respx.mock + def test_out_of_range_timestamp_falls_back_to_raw_value(self, run_cli) -> None: + respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json={ + "id": 42, "status": "RUNNING", "created": 1700000000, + "started": 1700000000000, "finished": None, + "accessions": ["ERR1"], "sample_ids": [], "execution_id": None, "error": None, + }), + ) + + result = run_cli( + "--token", TOKEN, "samples", "import-status", "--job-id", "42", + ) + + assert result.exit_code == 0 + assert "started 1700000000000" in result.stdout + + @respx.mock + def test_completed_job_reports_when_it_finished(self, run_cli) -> None: + respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json=_job_json( + 42, "COMPLETED", ["ERR1"], [101], + )), + ) + + result = run_cli( + "--token", TOKEN, "samples", "import-status", "--job-id", "42", + ) + + assert result.exit_code == 0 + assert "finished 2023-11-14" in result.stdout + @respx.mock def test_reports_completed_job_with_sample_ids(self, run_cli) -> None: respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( From 78115d4921ad43a0ec5bf2224871f5dc70e8efc2 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 17:19:06 +0000 Subject: [PATCH 12/32] refactor(samples): require accession, add per-row sample_type, simplify Per operator review comments on the PR: - AccessionSheetRow.accession is now a required str, not str | None. parse_accession_sheet rejects the whole sheet up front (CliUsageError, naming every offending row) if any row has no accession, or if the sheet has no rows at all, instead of silently skipping accession-less rows as the last several rounds had it doing. This removes the entire skip-tracking/"skipped" JSON-document machinery that existed only to support that behaviour: _skipped_row_label, _job_document's "skipped" key, and the per-row stderr advisories are all gone with it. - Added an optional sample_type column to the accession sheet: a row's own value overrides --sample-type for that row only, so a mixed-type sheet no longer has to be split into separate --sample-type invocations. AccessionSheetRow.to_spec(default_sample_type) builds the SampleImportSpec for a row, replacing the inline construction that used to live in _import_command. - Removed AccessionSheet.metadata_columns -- each row already carries its own metadata dict, so the sheet-level list was redundant and had no production caller. - _build_row's metadata-column comprehension now goes through the same cell() helper as accession/name/organism instead of duplicating its "strip, treat blank as absent" logic inline. - SampleImportJob.created/started/finished are now datetime | None instead of int | None -- pydantic parses the server's Unix-timestamp ints (and would parse ISO strings too) directly into proper datetime objects, so the CLI's display code no longer needs its own out-of-range fallback; formatting a valid datetime can't fail the way formatting an arbitrary int could. - _job_document is gone: both samples import and samples import-status now pass job.model_dump(mode="json") straight to Output.emit_result instead of a hand-maintained dict that had to be kept in sync with the model's fields by hand. Test suite and docs (source/cli.rst) updated throughout to match: the skip-related tests are replaced with tests asserting a blank accession (alone or mixed with valid rows) rejects the whole sheet, and a new TestAccessionSheetRowToSpec class covers the default/per-row sample_type behaviour. JSON-document assertions now expect ISO 8601 timestamp strings. Refs FLOW-689. --- flowbio/cli/_accession_sheet.py | 75 +++++++++++++------ flowbio/cli/_samples.py | 90 ++++++---------------- flowbio/v2/samples.py | 11 ++- source/cli.rst | 59 +++++++-------- tests/unit/cli/test_accession_sheet.py | 92 ++++++++++++++++++++--- tests/unit/cli/test_samples.py | 100 +++++++++++-------------- 6 files changed, 233 insertions(+), 194 deletions(-) diff --git a/flowbio/cli/_accession_sheet.py b/flowbio/cli/_accession_sheet.py index af16bfd..021a6d7 100644 --- a/flowbio/cli/_accession_sheet.py +++ b/flowbio/cli/_accession_sheet.py @@ -1,17 +1,18 @@ """CSV accession-sheet parsing for ``samples import``. -An accession sheet is a CSV with one row per accession to import: an +An accession sheet is a CSV with one row per accession to import: a required ``accession`` column (a public-repository run or experiment accession) plus -optional ``name``/``organism`` and per-accession metadata columns. This -mirrors ``_sheet.py``'s reads-based sample sheet, but the reserved columns -differ — there is nothing to upload (no ``reads1``/``reads2``) and the import -API has no project field, so ``RESERVED_COLUMNS`` is ``accession``, ``name``, -``organism`` instead. - -Rows are not validated here: the accession format, duplicates, sample type, -organism, and metadata rules are all checked server-side when the sheet is -submitted — duplicating that locally would just be a second, driftable copy -of the same rules. +optional ``name``/``organism``/``sample_type`` and per-accession metadata +columns. This mirrors ``_sheet.py``'s reads-based sample sheet, but the +reserved columns differ — there is nothing to upload (no ``reads1``/ +``reads2``) and the import API has no project field. + +Domain rules (accession format, duplicates, sample type, organism, metadata) +are all checked server-side when the sheet is submitted — duplicating that +locally would just be a second, driftable copy of the same rules. An +accession is different: it is the one column every row must have to mean +anything at all, so a missing one is rejected here rather than silently +skipped or shipped as an empty string the server would just reject anyway. """ from __future__ import annotations @@ -21,32 +22,47 @@ from flowbio.cli._exit_codes import CliUsageError from flowbio.cli._files import existing_file +from flowbio.v2.samples import SampleImportSpec, SampleTypeId -RESERVED_COLUMNS = ("accession", "name", "organism") +RESERVED_COLUMNS = ("accession", "name", "organism", "sample_type") @dataclass(frozen=True) class AccessionSheetRow: """One data row of an accession sheet. - ``accession`` may be ``None`` (empty cell, or no ``accession`` column at - all) — reported by the caller rather than rejected here, so a missing - accession is one visible problem instead of an opaque server rejection. + ``sample_type`` is only set when the sheet has its own ``sample_type`` + column for this row; :meth:`to_spec` falls back to the batch's + ``--sample-type`` when it's absent. """ row_number: int - accession: str | None + accession: str name: str | None organism: str | None + sample_type: SampleTypeId | None metadata: dict[str, str] + def to_spec(self, default_sample_type: SampleTypeId) -> SampleImportSpec: + """Build the :class:`~flowbio.v2.samples.SampleImportSpec` for this row. + + :param default_sample_type: The sample type to use when this row has + no ``sample_type`` of its own. + """ + return SampleImportSpec( + accession=self.accession, + sample_type=self.sample_type or default_sample_type, + name=self.name, + organism_id=self.organism, + metadata=self.metadata or None, + ) + @dataclass(frozen=True) class AccessionSheet: """A parsed accession sheet.""" path: Path - metadata_columns: list[str] rows: list[AccessionSheetRow] @@ -55,10 +71,11 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: :param path: The accession-sheet file. Must be a ``.csv`` — an ``.xlsx`` or ``.tsv`` is a usage error directing the user to export to CSV. - :returns: The parsed sheet with reserved/metadata columns separated and - empty cells dropped. Values are otherwise passed through unchanged — - including ``accession``, sent to the server exactly as entered. - :raises CliUsageError: If the file is not a readable ``.csv``. + :returns: The parsed sheet, with empty cells dropped. Values are otherwise + passed through unchanged — including ``accession``, sent to the + server exactly as entered. + :raises CliUsageError: If the file is not a readable ``.csv``, has no + rows, or has a row with no accession. """ if path.suffix.lower() != ".csv": raise CliUsageError( @@ -79,7 +96,15 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: _build_row(record, row_number, metadata_columns) for row_number, record in enumerate(reader, start=1) ] - return AccessionSheet(path=path, metadata_columns=metadata_columns, rows=rows) + if not rows: + raise CliUsageError(f"Accession sheet has no rows: {path}.") + missing = [row.row_number for row in rows if not row.accession] + if missing: + raise CliUsageError( + f"Accession sheet row(s) {', '.join(str(number) for number in missing)} " + f"have no accession: {path}.", + ) + return AccessionSheet(path=path, rows=rows) def _build_row( @@ -92,12 +117,14 @@ def cell(column: str) -> str | None: metadata = { column: value for column in metadata_columns - if (value := (record.get(column) or "").strip()) + if (value := cell(column)) is not None } + sample_type = cell("sample_type") return AccessionSheetRow( row_number=row_number, - accession=cell("accession"), + accession=cell("accession") or "", name=cell("name"), organism=cell("organism"), + sample_type=SampleTypeId(sample_type) if sample_type is not None else None, metadata=metadata, ) diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index 9aca30d..86c97ed 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -11,11 +11,11 @@ import argparse import json from dataclasses import dataclass -from datetime import datetime, timezone +from datetime import datetime from pathlib import Path from typing import Literal -from flowbio.cli._accession_sheet import AccessionSheetRow, parse_accession_sheet +from flowbio.cli._accession_sheet import parse_accession_sheet from flowbio.cli._exit_codes import CliUsageError, ExitCode from flowbio.cli._files import existing_file from flowbio.cli._output import Output, format_issue @@ -32,7 +32,6 @@ MetadataAttribute, SampleImportJob, SampleImportJobId, - SampleImportSpec, SampleTypeId, ) @@ -264,14 +263,20 @@ def _configure_import(import_parser: argparse.ArgumentParser) -> None: required=True, metavar="PATH", type=Path, - help="CSV accession sheet (accession, optional name/organism, plus metadata columns).", + help=( + "CSV accession sheet (required accession column, optional " + "name/organism/sample_type, plus metadata columns)." + ), ) import_parser.add_argument( "--sample-type", required=True, metavar="TYPE", type=SampleTypeId, - help="Sample type applied to every accession (sent as-is; validated server-side).", + help=( + "Default sample type (sent as-is; validated server-side), used for any " + "row without its own sample_type column." + ), ) @@ -623,8 +628,8 @@ def _import_command( ) -> ExitCode: """Kick off a batch import job from an accession sheet and report its id. - Every row is submitted as-is: the accession format, duplicates, sample - type, organism, and metadata rules are all validated server-side, so a + Every row is submitted as-is: the accession format, sample type, + organism, and metadata rules are all validated server-side, so a malformed sheet surfaces as a normal :class:`FlowApiError` rather than a local pre-flight rejection. This command does not wait for the job to finish — poll it yourself with ``samples import-status``. @@ -633,51 +638,21 @@ def _import_command( :param client: The authenticated Flow client. :param output: The result/error renderer. :returns: :attr:`ExitCode.SUCCESS` once the job has been kicked off. - :raises CliUsageError: If the sheet is not a readable ``.csv``, or if it - has no row with an accession — there is nothing an API call could - tell us about either that we can't see already. + :raises CliUsageError: If the sheet is not a readable ``.csv``, has no + rows, or has a row with no accession. """ sheet = parse_accession_sheet(args.sheet) - specs: list[SampleImportSpec] = [] - skipped: list[AccessionSheetRow] = [] - for row in sheet.rows: - if row.accession is None: - skipped.append(row) - continue - specs.append(SampleImportSpec( - accession=row.accession, - sample_type=args.sample_type, - name=row.name, - organism_id=row.organism, - metadata=row.metadata or None, - )) - if not specs: - raise CliUsageError( - f"Accession sheet has no row with an accession: {args.sheet}. " - f"Check it has an 'accession' column and at least one filled-in row.", - ) - for row in skipped: - output.emit_advisory(f"Skipped {_skipped_row_label(row)}: no accession") + specs = [row.to_spec(args.sample_type) for row in sheet.rows] job = client.samples.import_samples(specs) - document = _job_document(job) - document["skipped"] = [ - {"row_number": row.row_number, "name": row.name, "reasons": ["no accession"]} - for row in skipped - ] output.emit_result( f"Started import job {job.id} for {len(specs)} accession(s) " f"(status: {job.status}). Check progress with " f"'flowbio samples import-status --job-id {job.id}'.", - document, + job.model_dump(mode="json"), ) return ExitCode.SUCCESS -def _skipped_row_label(row: AccessionSheetRow) -> str: - name_suffix = f" ({row.name})" if row.name else "" - return f"row {row.row_number}{name_suffix}" - - def _import_status_command( args: argparse.Namespace, client: Client, output: Output, ) -> ExitCode: @@ -694,24 +669,10 @@ def _import_status_command( job = client.samples.get_import(args.job_id) if job.status == "FAILED": output.emit_advisory(f"Job {job.id} failed: {job.error or 'no error message returned'}") - output.emit_result(_job_summary(job), _job_document(job)) + output.emit_result(_job_summary(job), job.model_dump(mode="json")) return ExitCode.RUNTIME if job.status == "FAILED" else ExitCode.SUCCESS -def _job_document(job: SampleImportJob) -> dict[str, JsonValue]: - return { - "id": job.id, - "status": job.status, - "created": job.created, - "started": job.started, - "finished": job.finished, - "accessions": job.accessions, - "sample_ids": job.sample_ids, - "execution_id": job.execution_id, - "error": job.error, - } - - def _job_summary(job: SampleImportJob) -> str: if job.status == "COMPLETED": ids = ", ".join(str(sample_id) for sample_id in job.sample_ids) or "none" @@ -721,23 +682,14 @@ def _job_summary(job: SampleImportJob) -> str: # _import_status_command) rather than repeated here, so a human # running this doesn't see the same sentence twice. return f"Job {job.id}: FAILED{_timestamp_suffix('finished', job.finished)}." - return f"Job {job.id}: {job.status}{_timestamp_suffix('started', job.started)}." + label = "started" if job.started else "created" + return f"Job {job.id}: {job.status}{_timestamp_suffix(label, job.started or job.created)}." -def _timestamp_suffix(label: str, timestamp: int | None) -> str: +def _timestamp_suffix(label: str, timestamp: datetime | None) -> str: if timestamp is None: return "" - return f" ({label} {_format_timestamp(timestamp)})" - - -def _format_timestamp(timestamp: int) -> str: - # The API is only known to send Unix-seconds timestamps, but this only - # renders a display string — falling back to the raw value on anything - # unexpected keeps a formatting surprise from failing a status check. - try: - return datetime.fromtimestamp(timestamp, tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") - except (ValueError, OverflowError, OSError): - return str(timestamp) + return f" ({label} {timestamp.strftime('%Y-%m-%d %H:%M:%S UTC')})" def _merge_metadata( diff --git a/flowbio/v2/samples.py b/flowbio/v2/samples.py index f354d7c..27fb79e 100644 --- a/flowbio/v2/samples.py +++ b/flowbio/v2/samples.py @@ -28,6 +28,7 @@ from collections.abc import Sequence from dataclasses import dataclass +from datetime import datetime from pathlib import Path from typing import TYPE_CHECKING, Literal, NewType @@ -195,12 +196,10 @@ class SampleImportJob(BaseModel, frozen=True): id: SampleImportJobId = Field(description="Unique identifier for this import job.") status: SampleImportStatus = Field(description="The job's current lifecycle state.") - created: int | None = Field(default=None, description="Unix timestamp when the job was created.") - started: int | None = Field( - default=None, description="Unix timestamp when the job started running, if it has.", - ) - finished: int | None = Field( - default=None, description="Unix timestamp when the job finished (completed or failed), if it has.", + created: datetime | None = Field(default=None, description="When the job was created.") + started: datetime | None = Field(default=None, description="When the job started running, if it has.") + finished: datetime | None = Field( + default=None, description="When the job finished (completed or failed), if it has.", ) accessions: list[str] = Field(description="The accessions submitted with this job, in submission order.") sample_ids: list[int] = Field( diff --git a/source/cli.rst b/source/cli.rst index 5a01bb1..4b5172f 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -379,15 +379,17 @@ to every row — no files to upload yourself. flowbio samples import --sheet PATH --sample-type TYPE Run ``flowbio samples import --help`` for the full option list. The sheet is -a CSV with an ``accession`` column plus optional ``name``/``organism`` and -metadata columns (there is no ``batch-template`` equivalent for it, since it -has no reads files or project field). ``name`` defaults to the accession when -omitted. The sample type, accession format, duplicates, and metadata rules -are all sent as-is and validated **server-side** — this command only checks -that the sheet is a readable ``.csv`` with at least one row that has an -accession; rows without one are dropped (reported by row number on stderr) -rather than submitted as an empty string. Anything else invalid surfaces as -a normal API error, not a local rejection. +a CSV with a required ``accession`` column, plus optional ``name``/ +``organism``/``sample_type`` and metadata columns (there is no +``batch-template`` equivalent for it, since it has no reads files or project +field). ``name`` defaults to the accession when omitted. A row's own +``sample_type`` column, if present, overrides ``--sample-type`` for that row +only — useful for a mixed-type sheet. The sample type, accession format, and +metadata rules are all sent as-is and validated **server-side**; this command +only checks that the sheet is a readable ``.csv`` and that every row has an +accession — that column is the one thing every row must have to mean +anything, so a blank cell rejects the whole sheet up front rather than +shipping an empty string the server would just reject anyway. Every row is submitted **together as one server-side job**. This command does **not wait for it to finish** — it reports the job's id and initial @@ -397,19 +399,15 @@ to you, e.g. in a shell loop. Building the same thing directly against the library instead of the CLI? See :ref:`sample-imports`. **Output** — human: a confirmation line with the job id and a pointer to -``import-status``, plus one advisory per skipped row (identified by name if -it has one). ``--json``: the created job as a single document — ``id``, -``status``, ``created``/``started``/``finished`` (Unix timestamps, ``null`` -if not yet reached — or if the server response omits one, which the client -tolerates), ``accessions``, ``sample_ids`` (empty until the job completes), -``execution_id``, ``error``, and ``skipped`` (``{"row_number": ..., "name": -..., "reasons": [...]}`` for each dropped row, matching ``upload-batch``'s -shape, empty if none). +``import-status``. ``--json``: the created job as a single document — +``id``, ``status``, ``created``/``started``/``finished`` (ISO 8601 +timestamps, ``null`` if not yet reached), ``accessions``, ``sample_ids`` +(empty until the job completes), ``execution_id``, ``error``. **Exit codes** — ``0`` the job was created (regardless of its eventual outcome — check that with ``import-status``); ``2`` the sheet isn't a -readable ``.csv``, or has no row with an accession; ``1`` the API rejected -the batch (e.g. unknown sample type, missing required metadata, an +readable ``.csv``, has no rows, or has a row with no accession; ``1`` the API +rejected the batch (e.g. unknown sample type, missing required metadata, an unsupported accession format — these come back as an HTTP ``422``; ``5`` in the unlikely case it answers ``400`` instead); ``3`` authentication failure; otherwise the standard mapping above. @@ -422,7 +420,7 @@ otherwise the standard mapping above. Started import job 42 for 2 accession(s) (status: RUNNING). Check progress with 'flowbio samples import-status --job-id 42'. $ flowbio samples import --sheet ./accessions.csv --sample-type RNA-Seq --json - {"id": 42, "status": "RUNNING", "created": 1712345678, "started": null, "finished": null, "accessions": ["ERR1160845", "ERR10677146"], "sample_ids": [], "execution_id": 7, "error": null, "skipped": []} + {"id": 42, "status": "RUNNING", "created": "2024-04-05T19:34:38Z", "started": null, "finished": null, "accessions": ["ERR1160845", "ERR10677146"], "sample_ids": [], "execution_id": 7, "error": null} ``samples import-status`` ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -449,15 +447,16 @@ transient failure (auth, network) breaks the loop instead of being read as printf '%s' "$out" | jq -r .status # COMPLETED / FAILED; empty if the command errored **Output** — human: a one-line summary including the sample ids and when it -finished on ``"COMPLETED"``, when it started (if known) on ``"RUNNING"``, or -— on ``"FAILED"`` — when it finished plus the error as a separate advisory on -stderr (so it isn't printed twice). ``--json``: the job as a -single document — ``id``, ``status``, ``created``/``started``/``finished`` -(Unix timestamps, useful for judging how long a job has been running when -you've resumed polling one from elsewhere), ``accessions``, ``sample_ids``, -``execution_id``, ``error``. ``--json`` never prints prose to -stderr (or anywhere but that one stdout document); the failure reason on a -``"FAILED"`` job is the document's ``error`` field, not a separate message. +finished on ``"COMPLETED"``, when it started — or, if it hasn't yet, when it +was created — on ``"RUNNING"``, or — on ``"FAILED"`` — when it finished plus +the error as a separate advisory on stderr (so it isn't printed twice). +``--json``: the job as a single document — ``id``, ``status``, +``created``/``started``/``finished`` (ISO 8601 timestamps, useful for judging +how long a job has been running when you've resumed polling one from +elsewhere), ``accessions``, ``sample_ids``, ``execution_id``, ``error``. +``--json`` never prints prose to stderr (or anywhere but that one stdout +document); the failure reason on a ``"FAILED"`` job is the document's +``error`` field, not a separate message. **Exit codes** — ``0`` the job was fetched and is ``"RUNNING"`` or ``"COMPLETED"``; ``1`` either the job was fetched but is ``"FAILED"``, or the @@ -478,7 +477,7 @@ mapping above. Job 42: COMPLETED (finished 2024-04-05 19:38:20 UTC). Sample ids: 101, 102. $ flowbio samples import-status --job-id 42 --json - {"id": 42, "status": "COMPLETED", "created": 1712345678, "started": 1712345680, "finished": 1712345900, "accessions": ["ERR1160845", "ERR10677146"], "sample_ids": [101, 102], "execution_id": 7, "error": null} + {"id": 42, "status": "COMPLETED", "created": "2024-04-05T19:34:38Z", "started": "2024-04-05T19:34:40Z", "finished": "2024-04-05T19:38:20Z", "accessions": ["ERR1160845", "ERR10677146"], "sample_ids": [101, 102], "execution_id": 7, "error": null} ``api get`` ~~~~~~~~~~~ diff --git a/tests/unit/cli/test_accession_sheet.py b/tests/unit/cli/test_accession_sheet.py index 2ccd8ec..193b1be 100644 --- a/tests/unit/cli/test_accession_sheet.py +++ b/tests/unit/cli/test_accession_sheet.py @@ -5,8 +5,9 @@ from flowbio.cli._accession_sheet import parse_accession_sheet from flowbio.cli._exit_codes import CliUsageError +from flowbio.v2.samples import SampleImportSpec, SampleTypeId -HEADERS = ["accession", "name", "organism", "cell_type", "source", "source__annotation"] +HEADERS = ["accession", "name", "organism", "sample_type", "cell_type", "source", "source__annotation"] def _write_sheet( @@ -22,13 +23,6 @@ def _write_sheet( class TestParseAccessionSheet: - def test_separates_reserved_and_metadata_columns(self, tmp_path: Path) -> None: - sheet = parse_accession_sheet( - _write_sheet(tmp_path, {"accession": "ERR1160845"}), - ) - - assert sheet.metadata_columns == ["cell_type", "source", "source__annotation"] - def test_accession_is_passed_through_unchanged(self, tmp_path: Path) -> None: sheet = parse_accession_sheet( _write_sheet(tmp_path, {"accession": "err1160845"}), @@ -61,6 +55,20 @@ def test_name_and_organism_are_parsed_when_present(self, tmp_path: Path) -> None assert sheet.rows[0].name == "liver_r1" assert sheet.rows[0].organism == "Hs" + def test_sample_type_column_is_optional(self, tmp_path: Path) -> None: + sheet = parse_accession_sheet( + _write_sheet(tmp_path, {"accession": "ERR1160845"}), + ) + + assert sheet.rows[0].sample_type is None + + def test_sample_type_column_is_parsed_when_present(self, tmp_path: Path) -> None: + sheet = parse_accession_sheet(_write_sheet( + tmp_path, {"accession": "ERR1160845", "sample_type": "chip_seq"}, + )) + + assert sheet.rows[0].sample_type == "chip_seq" + def test_utf8_bom_is_stripped_from_first_header(self, tmp_path: Path) -> None: path = tmp_path / "sheet.csv" with path.open("w", newline="", encoding="utf-8-sig") as handle: @@ -70,7 +78,6 @@ def test_utf8_bom_is_stripped_from_first_header(self, tmp_path: Path) -> None: sheet = parse_accession_sheet(path) - assert sheet.metadata_columns == ["cell_type", "source", "source__annotation"] assert sheet.rows[0].accession == "ERR1160845" def test_row_numbers_are_one_based(self, tmp_path: Path) -> None: @@ -99,3 +106,70 @@ def test_tsv_sheet_rejected(self, tmp_path: Path) -> None: def test_missing_file_is_usage_error(self, tmp_path: Path) -> None: with pytest.raises(CliUsageError): parse_accession_sheet(tmp_path / "absent.csv") + + def test_header_only_sheet_is_usage_error(self, tmp_path: Path) -> None: + with pytest.raises(CliUsageError, match="no rows"): + parse_accession_sheet(_write_sheet(tmp_path)) + + def test_row_with_blank_accession_is_usage_error(self, tmp_path: Path) -> None: + with pytest.raises(CliUsageError, match="row"): + parse_accession_sheet(_write_sheet( + tmp_path, + {"accession": "ERR1160845"}, + {"accession": ""}, + )) + + def test_row_with_no_accession_column_is_usage_error(self, tmp_path: Path) -> None: + path = tmp_path / "sheet.csv" + with path.open("w", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=["run", "name"]) + writer.writeheader() + writer.writerow({"run": "ERR1160845", "name": "liver_r1"}) + + with pytest.raises(CliUsageError): + parse_accession_sheet(path) + + def test_usage_error_names_every_row_missing_an_accession(self, tmp_path: Path) -> None: + with pytest.raises(CliUsageError, match="1.*3"): + parse_accession_sheet(_write_sheet( + tmp_path, + {"accession": ""}, + {"accession": "ERR1"}, + {"accession": ""}, + )) + + +class TestAccessionSheetRowToSpec: + + def _row(self, tmp_path: Path, **overrides: str): + record = {"accession": "ERR1160845"} + record.update(overrides) + sheet = parse_accession_sheet(_write_sheet(tmp_path, record)) + return sheet.rows[0] + + def test_uses_default_sample_type_when_row_has_none(self, tmp_path: Path) -> None: + row = self._row(tmp_path) + + spec = row.to_spec(SampleTypeId("rna_seq")) + + assert spec == SampleImportSpec(accession="ERR1160845", sample_type="rna_seq") + + def test_row_sample_type_overrides_the_default(self, tmp_path: Path) -> None: + row = self._row(tmp_path, sample_type="chip_seq") + + spec = row.to_spec(SampleTypeId("rna_seq")) + + assert spec.sample_type == "chip_seq" + + def test_carries_name_organism_and_metadata(self, tmp_path: Path) -> None: + row = self._row(tmp_path, name="liver_r1", organism="Hs", cell_type="Neuron") + + spec = row.to_spec(SampleTypeId("rna_seq")) + + assert spec == SampleImportSpec( + accession="ERR1160845", + sample_type="rna_seq", + name="liver_r1", + organism_id="Hs", + metadata={"cell_type": "Neuron"}, + ) diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index 5fe07d8..4b26f71 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -926,7 +926,7 @@ def test_non_csv_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None: assert "CSV" in result.stderr -IMPORT_HEADERS = ["accession", "name", "organism", "cell_type", "source", "source__annotation"] +IMPORT_HEADERS = ["accession", "name", "organism", "sample_type", "cell_type", "source", "source__annotation"] def _write_import_sheet(directory: Path, *records: dict[str, str]) -> Path: @@ -1008,14 +1008,13 @@ def test_json_document_reports_job_fields( assert document == { "id": 42, "status": "RUNNING", - "created": 1700000000, + "created": "2023-11-14T22:13:20Z", "started": None, "finished": None, "accessions": ["ERR1"], "sample_ids": [], "execution_id": 7, "error": None, - "skipped": [], } @respx.mock @@ -1145,16 +1144,18 @@ def test_header_only_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None assert str(sheet) in result.stderr @respx.mock - def test_blank_accession_row_is_skipped(self, run_cli, tmp_path: Path) -> None: + def test_row_sample_type_overrides_the_default( + self, run_cli, tmp_path: Path, + ) -> None: route = respx.post(SAMPLE_IMPORTS_URL).mock( return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "RUNNING", ["ERR1"], + 1, "RUNNING", ["ERR1", "ERR2"], )), ) sheet = _write_import_sheet( tmp_path, - {"accession": "ERR1"}, - {"accession": ""}, + {"accession": "ERR1", "sample_type": "chip_seq"}, + {"accession": "ERR2"}, ) result = run_cli( @@ -1164,67 +1165,38 @@ def test_blank_accession_row_is_skipped(self, run_cli, tmp_path: Path) -> None: assert result.exit_code == 0 payload = json.loads(route.calls[0].request.content) - assert len(payload["imports"]) == 1 - assert payload["imports"][0]["accession"] == "ERR1" - assert "Skipped row 2" in result.stderr + sample_types = [entry["sample_type"] for entry in payload["imports"]] + assert sample_types == ["chip_seq", "rna_seq"] @respx.mock - def test_blank_accession_row_is_reported_in_json_document( + def test_sheet_with_only_blank_accessions_is_usage_error( self, run_cli, tmp_path: Path, ) -> None: - respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "RUNNING", ["ERR1"], - )), - ) - sheet = _write_import_sheet( - tmp_path, - {"accession": "ERR1"}, - {"accession": ""}, - ) + route = respx.post(SAMPLE_IMPORTS_URL) + sheet = _write_import_sheet(tmp_path, {"accession": ""}, {"accession": ""}) result = run_cli( "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", "--json", + "--sheet", str(sheet), "--sample-type", "rna_seq", ) - assert result.exit_code == 0 - assert result.stderr == "" - document = json.loads(result.stdout) - assert document["skipped"] == [ - {"row_number": 2, "name": None, "reasons": ["no accession"]}, - ] + assert result.exit_code == 2 + assert route.call_count == 0 + assert str(sheet) in result.stderr @respx.mock - def test_blank_accession_row_with_name_is_identified_by_name( + def test_mixed_valid_and_blank_accession_rows_is_usage_error( self, run_cli, tmp_path: Path, ) -> None: - respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "RUNNING", ["ERR1"], - )), - ) + # A blank accession is rejected outright rather than silently + # skipped, even when the rest of the sheet is otherwise fine. + route = respx.post(SAMPLE_IMPORTS_URL) sheet = _write_import_sheet( tmp_path, {"accession": "ERR1"}, - {"accession": "", "name": "liver_r2"}, - ) - - result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", + {"accession": ""}, ) - assert result.exit_code == 0 - assert "Skipped row 2 (liver_r2)" in result.stderr - - @respx.mock - def test_sheet_with_only_blank_accessions_is_usage_error( - self, run_cli, tmp_path: Path, - ) -> None: - route = respx.post(SAMPLE_IMPORTS_URL) - sheet = _write_import_sheet(tmp_path, {"accession": ""}, {"accession": ""}) - result = run_cli( "--token", TOKEN, "samples", "import", "--sheet", str(sheet), "--sample-type", "rna_seq", @@ -1232,7 +1204,6 @@ def test_sheet_with_only_blank_accessions_is_usage_error( assert result.exit_code == 2 assert route.call_count == 0 - assert str(sheet) in result.stderr @respx.mock def test_sheet_with_no_accession_column_is_usage_error( @@ -1291,7 +1262,7 @@ def test_running_job_with_started_reports_when_it_started(self, run_cli) -> None assert "started 2023-11-14" in result.stdout @respx.mock - def test_out_of_range_timestamp_falls_back_to_raw_value(self, run_cli) -> None: + def test_millisecond_scale_timestamp_is_parsed_correctly(self, run_cli) -> None: respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( return_value=httpx.Response(HTTPStatus.OK, json={ "id": 42, "status": "RUNNING", "created": 1700000000, @@ -1305,7 +1276,24 @@ def test_out_of_range_timestamp_falls_back_to_raw_value(self, run_cli) -> None: ) assert result.exit_code == 0 - assert "started 1700000000000" in result.stdout + assert "started 2023-11-14" in result.stdout + + @respx.mock + def test_running_job_falls_back_to_created_when_not_started(self, run_cli) -> None: + respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json={ + "id": 42, "status": "RUNNING", "created": 1700000000, + "started": None, "finished": None, + "accessions": ["ERR1"], "sample_ids": [], "execution_id": None, "error": None, + }), + ) + + result = run_cli( + "--token", TOKEN, "samples", "import-status", "--job-id", "42", + ) + + assert result.exit_code == 0 + assert "created 2023-11-14" in result.stdout @respx.mock def test_completed_job_reports_when_it_finished(self, run_cli) -> None: @@ -1414,9 +1402,9 @@ def test_json_document_matches_job_shape(self, run_cli) -> None: assert document == { "id": 42, "status": "COMPLETED", - "created": 1700000000, - "started": 1700000001, - "finished": 1700000002, + "created": "2023-11-14T22:13:20Z", + "started": "2023-11-14T22:13:21Z", + "finished": "2023-11-14T22:13:22Z", "accessions": ["ERR1"], "sample_ids": [101], "execution_id": 7, From e1fde614e4091ccdc27b4d2e109d7c75858e198a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 17:30:52 +0000 Subject: [PATCH 13/32] fix(samples): address round-12 Claude review findings - _timestamp_suffix now converts aware datetimes to UTC and treats naive ones as UTC explicitly, instead of labelling wall-clock fields "UTC" unconditionally (a real bug once created/started/finished became datetime instead of epoch int). - AccessionSheetRow now enforces non-empty accession by construction (__post_init__), and parse_accession_sheet no longer needs an "or ''" sentinel to satisfy typing before its own precondition check. - Missing-accession error message says "data row(s)" and gets its grammar right for the singular case; cli.rst documents the 1-based data-row numbering convention. - SampleImportJob.execution_id/error now default to None, matching the three timestamp fields, so a response that omits them doesn't raise an uncatchable ValidationError. - Removed a test that only pinned pydantic's own millisecond-timestamp auto-detection rather than any behaviour of this client. - cli.rst's samples import --json example shows execution_id as null (a freshly-kicked-off job has no pipeline execution yet), and notes sample_type is a reserved column name. Linear: FLOW-689 --- flowbio/cli/_accession_sheet.py | 51 +++++++++++++++++++++------------ flowbio/cli/_samples.py | 15 ++++++++-- flowbio/v2/samples.py | 8 ++++-- source/cli.rst | 18 +++++++----- tests/unit/cli/test_samples.py | 23 +++++++++++++-- tests/unit/v2/test_samples.py | 17 +++++++++++ 6 files changed, 99 insertions(+), 33 deletions(-) diff --git a/flowbio/cli/_accession_sheet.py b/flowbio/cli/_accession_sheet.py index 021a6d7..f29ec44 100644 --- a/flowbio/cli/_accession_sheet.py +++ b/flowbio/cli/_accession_sheet.py @@ -43,6 +43,10 @@ class AccessionSheetRow: sample_type: SampleTypeId | None metadata: dict[str, str] + def __post_init__(self) -> None: + if not self.accession: + raise ValueError("accession must not be empty") + def to_spec(self, default_sample_type: SampleTypeId) -> SampleImportSpec: """Build the :class:`~flowbio.v2.samples.SampleImportSpec` for this row. @@ -92,39 +96,50 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: metadata_columns = [ header for header in headers if header not in RESERVED_COLUMNS ] - rows = [ - _build_row(record, row_number, metadata_columns) - for row_number, record in enumerate(reader, start=1) - ] - if not rows: + records = list(enumerate(reader, start=1)) + if not records: raise CliUsageError(f"Accession sheet has no rows: {path}.") - missing = [row.row_number for row in rows if not row.accession] + missing = [ + row_number for row_number, record in records if not _cell(record, "accession") + ] if missing: + numbers = ", ".join(str(number) for number in missing) + # Data row 1 is the first row after the header, matching _sheet.py's + # convention (and upload-batch's documented "1-based row number"). + verb = "has" if len(missing) == 1 else "have" raise CliUsageError( - f"Accession sheet row(s) {', '.join(str(number) for number in missing)} " - f"have no accession: {path}.", + f"Accession sheet data row(s) {numbers} {verb} no accession: {path}.", ) + # The walrus filter is a no-op here: parse_accession_sheet already raised + # above if any record lacked an accession. Filtering on it (rather than + # asserting) is what gives the accession its narrowed str type below. + rows = [ + _build_row(record, row_number, metadata_columns, accession) + for row_number, record in records + if (accession := _cell(record, "accession")) is not None + ] return AccessionSheet(path=path, rows=rows) +def _cell(record: dict[str, str], column: str) -> str | None: + value = (record.get(column) or "").strip() + return value or None + + def _build_row( - record: dict[str, str], row_number: int, metadata_columns: list[str], + record: dict[str, str], row_number: int, metadata_columns: list[str], accession: str, ) -> AccessionSheetRow: - def cell(column: str) -> str | None: - value = (record.get(column) or "").strip() - return value or None - metadata = { column: value for column in metadata_columns - if (value := cell(column)) is not None + if (value := _cell(record, column)) is not None } - sample_type = cell("sample_type") + sample_type = _cell(record, "sample_type") return AccessionSheetRow( row_number=row_number, - accession=cell("accession") or "", - name=cell("name"), - organism=cell("organism"), + accession=accession, + name=_cell(record, "name"), + organism=_cell(record, "organism"), sample_type=SampleTypeId(sample_type) if sample_type is not None else None, metadata=metadata, ) diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index 86c97ed..ebe72c0 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -11,7 +11,7 @@ import argparse import json from dataclasses import dataclass -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path from typing import Literal @@ -676,7 +676,8 @@ def _import_status_command( def _job_summary(job: SampleImportJob) -> str: if job.status == "COMPLETED": ids = ", ".join(str(sample_id) for sample_id in job.sample_ids) or "none" - return f"Job {job.id}: COMPLETED{_timestamp_suffix('finished', job.finished)}. Sample ids: {ids}." + suffix = _timestamp_suffix("finished", job.finished) + return f"Job {job.id}: COMPLETED{suffix}. Sample ids: {ids}." if job.status == "FAILED": # The error, if any, is on stderr as an advisory (see # _import_status_command) rather than repeated here, so a human @@ -689,7 +690,15 @@ def _job_summary(job: SampleImportJob) -> str: def _timestamp_suffix(label: str, timestamp: datetime | None) -> str: if timestamp is None: return "" - return f" ({label} {timestamp.strftime('%Y-%m-%d %H:%M:%S UTC')})" + # A naive value (no tzinfo) is treated as already UTC rather than + # converted with astimezone(), which would assume the *local* system + # timezone instead. + utc_timestamp = ( + timestamp.astimezone(timezone.utc) + if timestamp.tzinfo is not None + else timestamp.replace(tzinfo=timezone.utc) + ) + return f" ({label} {utc_timestamp.strftime('%Y-%m-%d %H:%M:%S UTC')})" def _merge_metadata( diff --git a/flowbio/v2/samples.py b/flowbio/v2/samples.py index 27fb79e..be217c9 100644 --- a/flowbio/v2/samples.py +++ b/flowbio/v2/samples.py @@ -205,8 +205,12 @@ class SampleImportJob(BaseModel, frozen=True): sample_ids: list[int] = Field( description="The created samples' ids, corresponding to ``accessions`` once the job has completed.", ) - execution_id: int | None = Field(description="The pipeline execution backing this job, if one was created.") - error: str | None = Field(description='The failure reason, set only when status is "FAILED".') + execution_id: int | None = Field( + default=None, description="The pipeline execution backing this job, if one was created.", + ) + error: str | None = Field( + default=None, description='The failure reason, set only when status is "FAILED".', + ) class SampleResource: diff --git a/source/cli.rst b/source/cli.rst index 4b5172f..97d8bb0 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -382,14 +382,18 @@ Run ``flowbio samples import --help`` for the full option list. The sheet is a CSV with a required ``accession`` column, plus optional ``name``/ ``organism``/``sample_type`` and metadata columns (there is no ``batch-template`` equivalent for it, since it has no reads files or project -field). ``name`` defaults to the accession when omitted. A row's own -``sample_type`` column, if present, overrides ``--sample-type`` for that row -only — useful for a mixed-type sheet. The sample type, accession format, and -metadata rules are all sent as-is and validated **server-side**; this command -only checks that the sheet is a readable ``.csv`` and that every row has an +field; note ``sample_type`` is reserved for the per-row override, so a +metadata attribute of that exact name can't be sent through the sheet). +``name`` defaults to the accession when omitted. A row's own ``sample_type`` +column, if present, overrides ``--sample-type`` for that row only — useful +for a mixed-type sheet. The sample type, accession format, and metadata +rules are all sent as-is and validated **server-side**; this command only +checks that the sheet is a readable ``.csv`` and that every row has an accession — that column is the one thing every row must have to mean anything, so a blank cell rejects the whole sheet up front rather than -shipping an empty string the server would just reject anyway. +shipping an empty string the server would just reject anyway. Rows are +counted from ``1`` for the first data row, after the header (the same +convention as ``upload-batch``'s ``row_number``). Every row is submitted **together as one server-side job**. This command does **not wait for it to finish** — it reports the job's id and initial @@ -420,7 +424,7 @@ otherwise the standard mapping above. Started import job 42 for 2 accession(s) (status: RUNNING). Check progress with 'flowbio samples import-status --job-id 42'. $ flowbio samples import --sheet ./accessions.csv --sample-type RNA-Seq --json - {"id": 42, "status": "RUNNING", "created": "2024-04-05T19:34:38Z", "started": null, "finished": null, "accessions": ["ERR1160845", "ERR10677146"], "sample_ids": [], "execution_id": 7, "error": null} + {"id": 42, "status": "RUNNING", "created": "2024-04-05T19:34:38Z", "started": null, "finished": null, "accessions": ["ERR1160845", "ERR10677146"], "sample_ids": [], "execution_id": null, "error": null} ``samples import-status`` ~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index 4b26f71..5a7cfa6 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -1262,11 +1262,11 @@ def test_running_job_with_started_reports_when_it_started(self, run_cli) -> None assert "started 2023-11-14" in result.stdout @respx.mock - def test_millisecond_scale_timestamp_is_parsed_correctly(self, run_cli) -> None: + def test_started_with_non_utc_offset_is_reported_in_utc(self, run_cli) -> None: respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( return_value=httpx.Response(HTTPStatus.OK, json={ "id": 42, "status": "RUNNING", "created": 1700000000, - "started": 1700000000000, "finished": None, + "started": "2024-04-05T19:34:38+02:00", "finished": None, "accessions": ["ERR1"], "sample_ids": [], "execution_id": None, "error": None, }), ) @@ -1276,7 +1276,24 @@ def test_millisecond_scale_timestamp_is_parsed_correctly(self, run_cli) -> None: ) assert result.exit_code == 0 - assert "started 2023-11-14" in result.stdout + assert "started 2024-04-05 17:34:38 UTC" in result.stdout + + @respx.mock + def test_started_with_no_offset_is_treated_as_utc(self, run_cli) -> None: + respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json={ + "id": 42, "status": "RUNNING", "created": 1700000000, + "started": "2024-04-05T19:34:38", "finished": None, + "accessions": ["ERR1"], "sample_ids": [], "execution_id": None, "error": None, + }), + ) + + result = run_cli( + "--token", TOKEN, "samples", "import-status", "--job-id", "42", + ) + + assert result.exit_code == 0 + assert "started 2024-04-05 19:34:38 UTC" in result.stdout @respx.mock def test_running_job_falls_back_to_created_when_not_started(self, run_cli) -> None: diff --git a/tests/unit/v2/test_samples.py b/tests/unit/v2/test_samples.py index 7d8a99b..3f30458 100644 --- a/tests/unit/v2/test_samples.py +++ b/tests/unit/v2/test_samples.py @@ -1265,3 +1265,20 @@ def test_parses_job_missing_timestamps(self) -> None: assert result.created is None assert result.started is None assert result.finished is None + + @respx.mock + def test_parses_job_missing_execution_id_and_error(self) -> None: + respx.get(f"{DEFAULT_BASE_URL}/v2/sample-imports/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json={ + "id": 42, + "status": "RUNNING", + "accessions": ["ERR1"], + "sample_ids": [], + }), + ) + + client = Client() + result = client.samples.get_import(SampleImportJobId(42)) + + assert result.execution_id is None + assert result.error is None From 3c34f87d054ad2b93844301819234d6668668b6d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 17:41:55 +0000 Subject: [PATCH 14/32] fix(samples): address round-13 Claude review findings - SampleImportJob.accessions/sample_ids now default_factory=list, joining the timestamps/execution_id/error group as optional, since a POST kickoff response for a job that has created nothing yet plausibly omits them. Added a test parsing a payload of just {id, status} that covers all seven defaulted fields at once. - parse_accession_sheet is now a single pass collecting missing-accession row numbers and building rows together, replacing the two-pass walrus-filter version and the comment explaining why its filter could never exclude anything (a CLAUDE.md-flagged implementation-flow comment). - --sample-type is now optional: it's only required when some row has no sample_type column of its own. A sheet where every row specifies its own type no longer needs a --sample-type value that's silently discarded. AccessionSheetRow.to_spec documents and raises ValueError for the (CLI-unreachable) case of neither being available. - Added a test constructing AccessionSheetRow(accession="") directly, pinning the __post_init__ invariant independently of the parser. - Added a test for a blank sample_type cell (column present, empty) falling back to the default, distinct from the column being absent. - The library polling example in samples.rst now distinguishes FAILED from a timed-out RUNNING job instead of printing job.error (None) for both. - Fixed parse_accession_sheet's :returns: docstring, which claimed empty cells are dropped for accession too. Linear: FLOW-689 --- flowbio/cli/_accession_sheet.py | 37 ++++++++++++---------- flowbio/cli/_samples.py | 20 +++++++++--- flowbio/v2/samples.py | 5 ++- source/cli.rst | 31 ++++++++++--------- source/v2/samples.rst | 4 ++- tests/unit/cli/test_accession_sheet.py | 27 +++++++++++++++- tests/unit/cli/test_samples.py | 43 ++++++++++++++++++++++++++ tests/unit/v2/test_samples.py | 23 ++------------ 8 files changed, 133 insertions(+), 57 deletions(-) diff --git a/flowbio/cli/_accession_sheet.py b/flowbio/cli/_accession_sheet.py index f29ec44..135e6e6 100644 --- a/flowbio/cli/_accession_sheet.py +++ b/flowbio/cli/_accession_sheet.py @@ -47,15 +47,22 @@ def __post_init__(self) -> None: if not self.accession: raise ValueError("accession must not be empty") - def to_spec(self, default_sample_type: SampleTypeId) -> SampleImportSpec: + def to_spec(self, default_sample_type: SampleTypeId | None) -> SampleImportSpec: """Build the :class:`~flowbio.v2.samples.SampleImportSpec` for this row. :param default_sample_type: The sample type to use when this row has no ``sample_type`` of its own. + :raises ValueError: If this row has no ``sample_type`` of its own and + ``default_sample_type`` is ``None``. """ + sample_type = self.sample_type or default_sample_type + if sample_type is None: + raise ValueError( + f"row {self.row_number} has no sample_type and no default was given", + ) return SampleImportSpec( accession=self.accession, - sample_type=self.sample_type or default_sample_type, + sample_type=sample_type, name=self.name, organism_id=self.organism, metadata=self.metadata or None, @@ -75,9 +82,10 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: :param path: The accession-sheet file. Must be a ``.csv`` — an ``.xlsx`` or ``.tsv`` is a usage error directing the user to export to CSV. - :returns: The parsed sheet, with empty cells dropped. Values are otherwise - passed through unchanged — including ``accession``, sent to the - server exactly as entered. + :returns: The parsed sheet, with empty cells dropped (other than + ``accession``, which rejects the sheet instead — see below). Values + are otherwise passed through unchanged, including ``accession``, sent + to the server exactly as entered. :raises CliUsageError: If the file is not a readable ``.csv``, has no rows, or has a row with no accession. """ @@ -99,9 +107,14 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: records = list(enumerate(reader, start=1)) if not records: raise CliUsageError(f"Accession sheet has no rows: {path}.") - missing = [ - row_number for row_number, record in records if not _cell(record, "accession") - ] + rows: list[AccessionSheetRow] = [] + missing: list[int] = [] + for row_number, record in records: + accession = _cell(record, "accession") + if accession is None: + missing.append(row_number) + else: + rows.append(_build_row(record, row_number, metadata_columns, accession)) if missing: numbers = ", ".join(str(number) for number in missing) # Data row 1 is the first row after the header, matching _sheet.py's @@ -110,14 +123,6 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: raise CliUsageError( f"Accession sheet data row(s) {numbers} {verb} no accession: {path}.", ) - # The walrus filter is a no-op here: parse_accession_sheet already raised - # above if any record lacked an accession. Filtering on it (rather than - # asserting) is what gives the accession its narrowed str type below. - rows = [ - _build_row(record, row_number, metadata_columns, accession) - for row_number, record in records - if (accession := _cell(record, "accession")) is not None - ] return AccessionSheet(path=path, rows=rows) diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index ebe72c0..27af796 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -32,6 +32,7 @@ MetadataAttribute, SampleImportJob, SampleImportJobId, + SampleImportSpec, SampleTypeId, ) @@ -270,12 +271,11 @@ def _configure_import(import_parser: argparse.ArgumentParser) -> None: ) import_parser.add_argument( "--sample-type", - required=True, metavar="TYPE", type=SampleTypeId, help=( "Default sample type (sent as-is; validated server-side), used for any " - "row without its own sample_type column." + "row without its own sample_type column. Required unless every row has one." ), ) @@ -639,10 +639,22 @@ def _import_command( :param output: The result/error renderer. :returns: :attr:`ExitCode.SUCCESS` once the job has been kicked off. :raises CliUsageError: If the sheet is not a readable ``.csv``, has no - rows, or has a row with no accession. + rows, has a row with no accession, or has a row with no + ``sample_type`` of its own and no ``--sample-type`` given. """ sheet = parse_accession_sheet(args.sheet) - specs = [row.to_spec(args.sample_type) for row in sheet.rows] + missing_type = [ + row.row_number for row in sheet.rows + if row.sample_type is None and args.sample_type is None + ] + if missing_type: + numbers = ", ".join(str(number) for number in missing_type) + verb = "has" if len(missing_type) == 1 else "have" + raise CliUsageError( + f"Accession sheet data row(s) {numbers} {verb} no sample_type and " + f"--sample-type was not given: {args.sheet}.", + ) + specs: list[SampleImportSpec] = [row.to_spec(args.sample_type) for row in sheet.rows] job = client.samples.import_samples(specs) output.emit_result( f"Started import job {job.id} for {len(specs)} accession(s) " diff --git a/flowbio/v2/samples.py b/flowbio/v2/samples.py index be217c9..77a0044 100644 --- a/flowbio/v2/samples.py +++ b/flowbio/v2/samples.py @@ -201,8 +201,11 @@ class SampleImportJob(BaseModel, frozen=True): finished: datetime | None = Field( default=None, description="When the job finished (completed or failed), if it has.", ) - accessions: list[str] = Field(description="The accessions submitted with this job, in submission order.") + accessions: list[str] = Field( + default_factory=list, description="The accessions submitted with this job, in submission order.", + ) sample_ids: list[int] = Field( + default_factory=list, description="The created samples' ids, corresponding to ``accessions`` once the job has completed.", ) execution_id: int | None = Field( diff --git a/source/cli.rst b/source/cli.rst index 97d8bb0..4f276e6 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -376,7 +376,7 @@ to every row — no files to upload yourself. :: - flowbio samples import --sheet PATH --sample-type TYPE + flowbio samples import --sheet PATH [--sample-type TYPE] Run ``flowbio samples import --help`` for the full option list. The sheet is a CSV with a required ``accession`` column, plus optional ``name``/ @@ -386,14 +386,16 @@ field; note ``sample_type`` is reserved for the per-row override, so a metadata attribute of that exact name can't be sent through the sheet). ``name`` defaults to the accession when omitted. A row's own ``sample_type`` column, if present, overrides ``--sample-type`` for that row only — useful -for a mixed-type sheet. The sample type, accession format, and metadata -rules are all sent as-is and validated **server-side**; this command only -checks that the sheet is a readable ``.csv`` and that every row has an -accession — that column is the one thing every row must have to mean -anything, so a blank cell rejects the whole sheet up front rather than -shipping an empty string the server would just reject anyway. Rows are -counted from ``1`` for the first data row, after the header (the same -convention as ``upload-batch``'s ``row_number``). +for a mixed-type sheet; ``--sample-type`` itself is only required if some +row has no ``sample_type`` column of its own. The sample type, accession +format, and metadata rules are all sent as-is and validated **server-side**; +this command only checks that the sheet is a readable ``.csv``, that every +row has an accession, and that every row can resolve a sample type (its own +column, or ``--sample-type``) — a blank ``accession`` cell or a row with +neither rejects the whole sheet up front rather than shipping something the +server would just reject anyway. Rows are counted from ``1`` for the first +data row, after the header (the same convention as ``upload-batch``'s +``row_number``). Every row is submitted **together as one server-side job**. This command does **not wait for it to finish** — it reports the job's id and initial @@ -410,11 +412,12 @@ timestamps, ``null`` if not yet reached), ``accessions``, ``sample_ids`` **Exit codes** — ``0`` the job was created (regardless of its eventual outcome — check that with ``import-status``); ``2`` the sheet isn't a -readable ``.csv``, has no rows, or has a row with no accession; ``1`` the API -rejected the batch (e.g. unknown sample type, missing required metadata, an -unsupported accession format — these come back as an HTTP ``422``; ``5`` in -the unlikely case it answers ``400`` instead); ``3`` authentication failure; -otherwise the standard mapping above. +readable ``.csv``, has no rows, has a row with no accession, or has a row +with no resolvable sample type; ``1`` the API rejected the batch (e.g. +unknown sample type, missing required metadata, an unsupported accession +format — these come back as an HTTP ``422``; ``5`` in the unlikely case it +answers ``400`` instead); ``3`` authentication failure; otherwise the +standard mapping above. **Example** diff --git a/source/v2/samples.rst b/source/v2/samples.rst index 5cfee62..3ebeafd 100644 --- a/source/v2/samples.rst +++ b/source/v2/samples.rst @@ -200,8 +200,10 @@ leaves ``"RUNNING"`` — bound the wait so a stuck job doesn't loop forever:: if job.status == "COMPLETED": print(f"Imported samples: {job.sample_ids}") + elif job.status == "FAILED": + print(f"Import failed: {job.error}") else: - print(f"Import failed or still running: {job.error}") + print(f"Import did not finish within the deadline (status: {job.status})") ``job.accessions`` and ``job.sample_ids`` correspond positionally once the job has completed. diff --git a/tests/unit/cli/test_accession_sheet.py b/tests/unit/cli/test_accession_sheet.py index 193b1be..aa8f9fc 100644 --- a/tests/unit/cli/test_accession_sheet.py +++ b/tests/unit/cli/test_accession_sheet.py @@ -3,7 +3,7 @@ import pytest -from flowbio.cli._accession_sheet import parse_accession_sheet +from flowbio.cli._accession_sheet import AccessionSheetRow, parse_accession_sheet from flowbio.cli._exit_codes import CliUsageError from flowbio.v2.samples import SampleImportSpec, SampleTypeId @@ -139,6 +139,18 @@ def test_usage_error_names_every_row_missing_an_accession(self, tmp_path: Path) )) +def test_row_rejects_empty_accession_by_construction() -> None: + with pytest.raises(ValueError, match="accession"): + AccessionSheetRow( + row_number=1, + accession="", + name=None, + organism=None, + sample_type=None, + metadata={}, + ) + + class TestAccessionSheetRowToSpec: def _row(self, tmp_path: Path, **overrides: str): @@ -161,6 +173,19 @@ def test_row_sample_type_overrides_the_default(self, tmp_path: Path) -> None: assert spec.sample_type == "chip_seq" + def test_blank_sample_type_cell_falls_back_to_default(self, tmp_path: Path) -> None: + row = self._row(tmp_path, sample_type="") + + spec = row.to_spec(SampleTypeId("rna_seq")) + + assert spec.sample_type == "rna_seq" + + def test_raises_when_no_sample_type_or_default_available(self, tmp_path: Path) -> None: + row = self._row(tmp_path) + + with pytest.raises(ValueError, match="sample_type"): + row.to_spec(None) + def test_carries_name_organism_and_metadata(self, tmp_path: Path) -> None: row = self._row(tmp_path, name="liver_r1", organism="Hs", cell_type="Neuron") diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index 5a7cfa6..3d646d7 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -1168,6 +1168,49 @@ def test_row_sample_type_overrides_the_default( sample_types = [entry["sample_type"] for entry in payload["imports"]] assert sample_types == ["chip_seq", "rna_seq"] + @respx.mock + def test_sample_type_flag_is_optional_when_every_row_has_its_own( + self, run_cli, tmp_path: Path, + ) -> None: + route = respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "RUNNING", ["ERR1", "ERR2"], + )), + ) + sheet = _write_import_sheet( + tmp_path, + {"accession": "ERR1", "sample_type": "chip_seq"}, + {"accession": "ERR2", "sample_type": "atac_seq"}, + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), + ) + + assert result.exit_code == 0 + payload = json.loads(route.calls[0].request.content) + sample_types = [entry["sample_type"] for entry in payload["imports"]] + assert sample_types == ["chip_seq", "atac_seq"] + + @respx.mock + def test_row_missing_sample_type_without_default_is_usage_error( + self, run_cli, tmp_path: Path, + ) -> None: + route = respx.post(SAMPLE_IMPORTS_URL) + sheet = _write_import_sheet( + tmp_path, + {"accession": "ERR1", "sample_type": "chip_seq"}, + {"accession": "ERR2"}, + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), + ) + + assert result.exit_code == 2 + assert route.call_count == 0 + assert "2" in result.stderr + @respx.mock def test_sheet_with_only_blank_accessions_is_usage_error( self, run_cli, tmp_path: Path, diff --git a/tests/unit/v2/test_samples.py b/tests/unit/v2/test_samples.py index 3f30458..65fb582 100644 --- a/tests/unit/v2/test_samples.py +++ b/tests/unit/v2/test_samples.py @@ -1247,15 +1247,11 @@ def test_raises_not_found_for_unknown_job(self) -> None: client.samples.get_import(SampleImportJobId(999)) @respx.mock - def test_parses_job_missing_timestamps(self) -> None: + def test_parses_job_with_only_id_and_status(self) -> None: respx.get(f"{DEFAULT_BASE_URL}/v2/sample-imports/42").mock( return_value=httpx.Response(HTTPStatus.OK, json={ "id": 42, "status": "RUNNING", - "accessions": ["ERR1"], - "sample_ids": [], - "execution_id": None, - "error": None, }), ) @@ -1265,20 +1261,7 @@ def test_parses_job_missing_timestamps(self) -> None: assert result.created is None assert result.started is None assert result.finished is None - - @respx.mock - def test_parses_job_missing_execution_id_and_error(self) -> None: - respx.get(f"{DEFAULT_BASE_URL}/v2/sample-imports/42").mock( - return_value=httpx.Response(HTTPStatus.OK, json={ - "id": 42, - "status": "RUNNING", - "accessions": ["ERR1"], - "sample_ids": [], - }), - ) - - client = Client() - result = client.samples.get_import(SampleImportJobId(42)) - + assert result.accessions == [] + assert result.sample_ids == [] assert result.execution_id is None assert result.error is None From 84465685c95de78688ea7864798ea7a63dce03a9 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 17:50:18 +0000 Subject: [PATCH 15/32] fix(samples): address round-14 Claude review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - SampleImportJob's created/started/finished now go through an after-validator that assumes UTC for a naive value, so the rule holds for --json and library callers, not just the CLI's human renderer. _timestamp_suffix simplifies to astimezone(timezone.utc) since every timestamp reaching it is now guaranteed aware. - Fixed cli.rst's samples import summary sentence, which still said "applying one sample type to every row" after the per-row override landed two rounds ago. - Hoisted the args.sample_type is None check out of the missing_type comprehension in _import_command, so it's stated once instead of re-evaluated (and short-circuited) per row. - Fixed two tests: the incidental-digit assertion in test_row_missing_sample_type_without_default_is_usage_error now checks the actual message phrase, and test_missing_sample_type_is_usage_error gained the @respx.mock/ call-count assertion its siblings have. - Documented in cli.rst that sheet-level usage errors for 'samples import' are only detected after authentication, since whether --sample-type is required depends on the sheet's contents. Left as documented trade-offs (per the review's own framing): the SampleImportSpec-frozen-dataclass-with-a-dict-field hashability wrinkle, and the RNA-Seq/rna_seq casing convention mismatch between docs and tests — both cosmetic/informational rather than defects. Linear: FLOW-689 --- flowbio/cli/_samples.py | 32 ++++++++++++-------------------- flowbio/v2/samples.py | 15 +++++++++++++-- source/cli.rst | 9 ++++++--- tests/unit/cli/test_samples.py | 23 ++++++++++++++++++++++- tests/unit/v2/test_samples.py | 16 ++++++++++++++++ 5 files changed, 69 insertions(+), 26 deletions(-) diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index 27af796..f64224b 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -643,17 +643,15 @@ def _import_command( ``sample_type`` of its own and no ``--sample-type`` given. """ sheet = parse_accession_sheet(args.sheet) - missing_type = [ - row.row_number for row in sheet.rows - if row.sample_type is None and args.sample_type is None - ] - if missing_type: - numbers = ", ".join(str(number) for number in missing_type) - verb = "has" if len(missing_type) == 1 else "have" - raise CliUsageError( - f"Accession sheet data row(s) {numbers} {verb} no sample_type and " - f"--sample-type was not given: {args.sheet}.", - ) + if args.sample_type is None: + missing_type = [row.row_number for row in sheet.rows if row.sample_type is None] + if missing_type: + numbers = ", ".join(str(number) for number in missing_type) + verb = "has" if len(missing_type) == 1 else "have" + raise CliUsageError( + f"Accession sheet data row(s) {numbers} {verb} no sample_type and " + f"--sample-type was not given: {args.sheet}.", + ) specs: list[SampleImportSpec] = [row.to_spec(args.sample_type) for row in sheet.rows] job = client.samples.import_samples(specs) output.emit_result( @@ -702,15 +700,9 @@ def _job_summary(job: SampleImportJob) -> str: def _timestamp_suffix(label: str, timestamp: datetime | None) -> str: if timestamp is None: return "" - # A naive value (no tzinfo) is treated as already UTC rather than - # converted with astimezone(), which would assume the *local* system - # timezone instead. - utc_timestamp = ( - timestamp.astimezone(timezone.utc) - if timestamp.tzinfo is not None - else timestamp.replace(tzinfo=timezone.utc) - ) - return f" ({label} {utc_timestamp.strftime('%Y-%m-%d %H:%M:%S UTC')})" + # SampleImportJob normalises a naive value to UTC, so every timestamp + # reaching here is already aware. + return f" ({label} {timestamp.astimezone(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')})" def _merge_metadata( diff --git a/flowbio/v2/samples.py b/flowbio/v2/samples.py index 77a0044..4f4dcfc 100644 --- a/flowbio/v2/samples.py +++ b/flowbio/v2/samples.py @@ -28,11 +28,11 @@ from collections.abc import Sequence from dataclasses import dataclass -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path from typing import TYPE_CHECKING, Literal, NewType -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, field_validator from flowbio.v2._pagination import PageIterator from flowbio.v2.exceptions import ( @@ -215,6 +215,17 @@ class SampleImportJob(BaseModel, frozen=True): default=None, description='The failure reason, set only when status is "FAILED".', ) + @field_validator("created", "started", "finished", mode="after") + @classmethod + def _assume_utc_if_naive(cls, value: datetime | None) -> datetime | None: + # A naive value (no tzinfo) is treated as already UTC, matching what + # the server always means by these timestamps, rather than left + # ambiguous for every consumer (CLI, --json, library) to decide on + # its own. + if value is not None and value.tzinfo is None: + return value.replace(tzinfo=timezone.utc) + return value + class SampleResource: """Provides access to sample-related API endpoints. diff --git a/source/cli.rst b/source/cli.rst index 4f276e6..1a3db40 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -371,8 +371,8 @@ authentication failure; otherwise the standard mapping above. ~~~~~~~~~~~~~~~~~~ Kick off a batch import of samples from public-repository accessions (SRR/ -ERR/DRR run or SRX/ERX/DRX experiment accessions), applying one sample type -to every row — no files to upload yourself. +ERR/DRR run or SRX/ERX/DRX experiment accessions), applying a default sample +type that each row can override — no files to upload yourself. :: @@ -417,7 +417,10 @@ with no resolvable sample type; ``1`` the API rejected the batch (e.g. unknown sample type, missing required metadata, an unsupported accession format — these come back as an HTTP ``422``; ``5`` in the unlikely case it answers ``400`` instead); ``3`` authentication failure; otherwise the -standard mapping above. +standard mapping above. Because whether ``--sample-type`` is required +depends on the sheet's contents, these sheet-level usage errors (``2``) are +only detected after authentication succeeds, unlike a malformed flag value +(which fails before it). **Example** diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index 3d646d7..451accf 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -1120,7 +1120,9 @@ def test_missing_sheet_is_usage_error(self, run_cli) -> None: assert result.exit_code == 2 + @respx.mock def test_missing_sample_type_is_usage_error(self, run_cli, tmp_path: Path) -> None: + route = respx.post(SAMPLE_IMPORTS_URL) sheet = _write_import_sheet(tmp_path, {"accession": "ERR1"}) result = run_cli( @@ -1128,6 +1130,7 @@ def test_missing_sample_type_is_usage_error(self, run_cli, tmp_path: Path) -> No ) assert result.exit_code == 2 + assert route.call_count == 0 @respx.mock def test_header_only_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None: @@ -1209,7 +1212,7 @@ def test_row_missing_sample_type_without_default_is_usage_error( assert result.exit_code == 2 assert route.call_count == 0 - assert "2" in result.stderr + assert "data row(s) 2" in result.stderr @respx.mock def test_sheet_with_only_blank_accessions_is_usage_error( @@ -1338,6 +1341,24 @@ def test_started_with_no_offset_is_treated_as_utc(self, run_cli) -> None: assert result.exit_code == 0 assert "started 2024-04-05 19:34:38 UTC" in result.stdout + @respx.mock + def test_naive_started_is_reported_as_utc_in_json_too(self, run_cli) -> None: + respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json={ + "id": 42, "status": "RUNNING", "created": 1700000000, + "started": "2024-04-05T19:34:38", "finished": None, + "accessions": ["ERR1"], "sample_ids": [], "execution_id": None, "error": None, + }), + ) + + result = run_cli( + "--token", TOKEN, "samples", "import-status", "--job-id", "42", "--json", + ) + + assert result.exit_code == 0 + document = json.loads(result.stdout) + assert document["started"] == "2024-04-05T19:34:38Z" + @respx.mock def test_running_job_falls_back_to_created_when_not_started(self, run_cli) -> None: respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( diff --git a/tests/unit/v2/test_samples.py b/tests/unit/v2/test_samples.py index 65fb582..59cade8 100644 --- a/tests/unit/v2/test_samples.py +++ b/tests/unit/v2/test_samples.py @@ -1,4 +1,5 @@ import json +from datetime import datetime, timezone from http import HTTPStatus from pathlib import Path from unittest.mock import ANY, patch @@ -1265,3 +1266,18 @@ def test_parses_job_with_only_id_and_status(self) -> None: assert result.sample_ids == [] assert result.execution_id is None assert result.error is None + + @respx.mock + def test_naive_timestamp_is_treated_as_utc(self) -> None: + respx.get(f"{DEFAULT_BASE_URL}/v2/sample-imports/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json={ + "id": 42, + "status": "RUNNING", + "started": "2024-04-05T19:34:38", + }), + ) + + client = Client() + result = client.samples.get_import(SampleImportJobId(42)) + + assert result.started == datetime(2024, 4, 5, 19, 34, 38, tzinfo=timezone.utc) From 5e1c82b4e7c553b857398f236988e3d0325419c1 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 18:09:33 +0000 Subject: [PATCH 16/32] feat(samples): make sample_type an accession-sheet-only concept MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per operator direction: eliminated the possibility of supplying a sample type anywhere but the accession sheet's own column for 'samples import'. - Removed the --sample-type flag from 'samples import' entirely. - AccessionSheetRow.sample_type is now a required SampleTypeId (was optional with a CLI-supplied default). parse_accession_sheet rejects a sheet with any row missing accession or sample_type, naming every offending row for each column independently, the same way it already did for accession alone. - AccessionSheetRow.to_spec() takes no arguments now that there is nothing to resolve a default against. - This fully resolves round-15 finding 1 (an empty --sample-type value slipping past the required check) by removing the flag it depended on, rather than patching the identity/truthiness mismatch. Also fixed two more round-15 findings while in here: - The naive-timestamp UTC validator on SampleImportJob now also converts an already-aware non-UTC offset to UTC (astimezone), not just naive values — so --json and library callers get a uniformly UTC-normalised value, not merely a tz-aware one. Added a --json test for a +02:00 server timestamp. - Tightened two sheet-parsing tests whose match= patterns were satisfiable by digits/words in pytest's tmp_path rather than the actual error message. Left round-15 finding 2 (the missing-column message enumerating every row) as-is per operator direction — it errors, which is sufficient. Linear: FLOW-689 --- flowbio/cli/_accession_sheet.py | 98 +++++++------ flowbio/cli/_samples.py | 27 +--- flowbio/v2/samples.py | 8 +- source/cli.rst | 56 ++++---- tests/unit/cli/test_accession_sheet.py | 111 +++++++-------- tests/unit/cli/test_samples.py | 184 +++++++++++-------------- 6 files changed, 219 insertions(+), 265 deletions(-) diff --git a/flowbio/cli/_accession_sheet.py b/flowbio/cli/_accession_sheet.py index 135e6e6..0ed3e6a 100644 --- a/flowbio/cli/_accession_sheet.py +++ b/flowbio/cli/_accession_sheet.py @@ -1,18 +1,19 @@ """CSV accession-sheet parsing for ``samples import``. -An accession sheet is a CSV with one row per accession to import: a required -``accession`` column (a public-repository run or experiment accession) plus -optional ``name``/``organism``/``sample_type`` and per-accession metadata -columns. This mirrors ``_sheet.py``'s reads-based sample sheet, but the -reserved columns differ — there is nothing to upload (no ``reads1``/ -``reads2``) and the import API has no project field. +An accession sheet is a CSV with one row per accession to import: required +``accession`` and ``sample_type`` columns, plus optional ``name``/ +``organism`` and per-accession metadata columns. This mirrors ``_sheet.py``'s +reads-based sample sheet, but the reserved columns differ — there is nothing +to upload (no ``reads1``/``reads2``) and the import API has no project field. Domain rules (accession format, duplicates, sample type, organism, metadata) are all checked server-side when the sheet is submitted — duplicating that -locally would just be a second, driftable copy of the same rules. An -accession is different: it is the one column every row must have to mean -anything at all, so a missing one is rejected here rather than silently -skipped or shipped as an empty string the server would just reject anyway. +locally would just be a second, driftable copy of the same rules. +``accession`` and ``sample_type`` are different: they are the two columns +every row must have to mean anything at all, so a missing one is rejected +here rather than silently skipped or shipped as an empty string the server +would just reject anyway. There is deliberately no other way to supply a +sample type for ``samples import`` — the sheet is the single source of it. """ from __future__ import annotations @@ -29,40 +30,26 @@ @dataclass(frozen=True) class AccessionSheetRow: - """One data row of an accession sheet. - - ``sample_type`` is only set when the sheet has its own ``sample_type`` - column for this row; :meth:`to_spec` falls back to the batch's - ``--sample-type`` when it's absent. - """ + """One data row of an accession sheet.""" row_number: int accession: str name: str | None organism: str | None - sample_type: SampleTypeId | None + sample_type: SampleTypeId metadata: dict[str, str] def __post_init__(self) -> None: if not self.accession: raise ValueError("accession must not be empty") + if not self.sample_type: + raise ValueError("sample_type must not be empty") - def to_spec(self, default_sample_type: SampleTypeId | None) -> SampleImportSpec: - """Build the :class:`~flowbio.v2.samples.SampleImportSpec` for this row. - - :param default_sample_type: The sample type to use when this row has - no ``sample_type`` of its own. - :raises ValueError: If this row has no ``sample_type`` of its own and - ``default_sample_type`` is ``None``. - """ - sample_type = self.sample_type or default_sample_type - if sample_type is None: - raise ValueError( - f"row {self.row_number} has no sample_type and no default was given", - ) + def to_spec(self) -> SampleImportSpec: + """Build the :class:`~flowbio.v2.samples.SampleImportSpec` for this row.""" return SampleImportSpec( accession=self.accession, - sample_type=sample_type, + sample_type=self.sample_type, name=self.name, organism_id=self.organism, metadata=self.metadata or None, @@ -83,11 +70,11 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: :param path: The accession-sheet file. Must be a ``.csv`` — an ``.xlsx`` or ``.tsv`` is a usage error directing the user to export to CSV. :returns: The parsed sheet, with empty cells dropped (other than - ``accession``, which rejects the sheet instead — see below). Values - are otherwise passed through unchanged, including ``accession``, sent - to the server exactly as entered. + ``accession``/``sample_type``, which reject the sheet instead — see + below). Values are otherwise passed through unchanged, including + ``accession``, sent to the server exactly as entered. :raises CliUsageError: If the file is not a readable ``.csv``, has no - rows, or has a row with no accession. + rows, or has a row with no accession or no sample_type. """ if path.suffix.lower() != ".csv": raise CliUsageError( @@ -108,43 +95,54 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: if not records: raise CliUsageError(f"Accession sheet has no rows: {path}.") rows: list[AccessionSheetRow] = [] - missing: list[int] = [] + missing_accession: list[int] = [] + missing_sample_type: list[int] = [] for row_number, record in records: accession = _cell(record, "accession") + sample_type = _cell(record, "sample_type") if accession is None: - missing.append(row_number) - else: - rows.append(_build_row(record, row_number, metadata_columns, accession)) - if missing: - numbers = ", ".join(str(number) for number in missing) - # Data row 1 is the first row after the header, matching _sheet.py's - # convention (and upload-batch's documented "1-based row number"). - verb = "has" if len(missing) == 1 else "have" - raise CliUsageError( - f"Accession sheet data row(s) {numbers} {verb} no accession: {path}.", - ) + missing_accession.append(row_number) + if sample_type is None: + missing_sample_type.append(row_number) + if accession is not None and sample_type is not None: + rows.append(_build_row(record, row_number, metadata_columns, accession, sample_type)) + if missing_accession: + raise _missing_column_error("accession", missing_accession, path) + if missing_sample_type: + raise _missing_column_error("sample_type", missing_sample_type, path) return AccessionSheet(path=path, rows=rows) +def _missing_column_error(column: str, missing: list[int], path: Path) -> CliUsageError: + numbers = ", ".join(str(number) for number in missing) + # Data row 1 is the first row after the header, matching _sheet.py's + # convention (and upload-batch's documented "1-based row number"). + verb = "has" if len(missing) == 1 else "have" + return CliUsageError(f"Accession sheet data row(s) {numbers} {verb} no {column}: {path}.") + + def _cell(record: dict[str, str], column: str) -> str | None: value = (record.get(column) or "").strip() return value or None def _build_row( - record: dict[str, str], row_number: int, metadata_columns: list[str], accession: str, + record: dict[str, str], + row_number: int, + metadata_columns: list[str], + accession: str, + sample_type: str, ) -> AccessionSheetRow: metadata = { column: value for column in metadata_columns if (value := _cell(record, column)) is not None } - sample_type = _cell(record, "sample_type") return AccessionSheetRow( row_number=row_number, accession=accession, name=_cell(record, "name"), organism=_cell(record, "organism"), - sample_type=SampleTypeId(sample_type) if sample_type is not None else None, + sample_type=SampleTypeId(sample_type), metadata=metadata, ) diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index f64224b..5904509 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -265,17 +265,8 @@ def _configure_import(import_parser: argparse.ArgumentParser) -> None: metavar="PATH", type=Path, help=( - "CSV accession sheet (required accession column, optional " - "name/organism/sample_type, plus metadata columns)." - ), - ) - import_parser.add_argument( - "--sample-type", - metavar="TYPE", - type=SampleTypeId, - help=( - "Default sample type (sent as-is; validated server-side), used for any " - "row without its own sample_type column. Required unless every row has one." + "CSV accession sheet (required accession/sample_type columns, " + "optional name/organism, plus metadata columns)." ), ) @@ -639,20 +630,10 @@ def _import_command( :param output: The result/error renderer. :returns: :attr:`ExitCode.SUCCESS` once the job has been kicked off. :raises CliUsageError: If the sheet is not a readable ``.csv``, has no - rows, has a row with no accession, or has a row with no - ``sample_type`` of its own and no ``--sample-type`` given. + rows, or has a row with no accession or no sample_type. """ sheet = parse_accession_sheet(args.sheet) - if args.sample_type is None: - missing_type = [row.row_number for row in sheet.rows if row.sample_type is None] - if missing_type: - numbers = ", ".join(str(number) for number in missing_type) - verb = "has" if len(missing_type) == 1 else "have" - raise CliUsageError( - f"Accession sheet data row(s) {numbers} {verb} no sample_type and " - f"--sample-type was not given: {args.sheet}.", - ) - specs: list[SampleImportSpec] = [row.to_spec(args.sample_type) for row in sheet.rows] + specs: list[SampleImportSpec] = [row.to_spec() for row in sheet.rows] job = client.samples.import_samples(specs) output.emit_result( f"Started import job {job.id} for {len(specs)} accession(s) " diff --git a/flowbio/v2/samples.py b/flowbio/v2/samples.py index 4f4dcfc..26b48ea 100644 --- a/flowbio/v2/samples.py +++ b/flowbio/v2/samples.py @@ -217,14 +217,16 @@ class SampleImportJob(BaseModel, frozen=True): @field_validator("created", "started", "finished", mode="after") @classmethod - def _assume_utc_if_naive(cls, value: datetime | None) -> datetime | None: + def _normalize_to_utc(cls, value: datetime | None) -> datetime | None: + if value is None: + return None # A naive value (no tzinfo) is treated as already UTC, matching what # the server always means by these timestamps, rather than left # ambiguous for every consumer (CLI, --json, library) to decide on # its own. - if value is not None and value.tzinfo is None: + if value.tzinfo is None: return value.replace(tzinfo=timezone.utc) - return value + return value.astimezone(timezone.utc) class SampleResource: diff --git a/source/cli.rst b/source/cli.rst index 1a3db40..b7bf4b5 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -371,31 +371,28 @@ authentication failure; otherwise the standard mapping above. ~~~~~~~~~~~~~~~~~~ Kick off a batch import of samples from public-repository accessions (SRR/ -ERR/DRR run or SRX/ERX/DRX experiment accessions), applying a default sample -type that each row can override — no files to upload yourself. +ERR/DRR run or SRX/ERX/DRX experiment accessions) — no files to upload +yourself. :: - flowbio samples import --sheet PATH [--sample-type TYPE] + flowbio samples import --sheet PATH Run ``flowbio samples import --help`` for the full option list. The sheet is -a CSV with a required ``accession`` column, plus optional ``name``/ -``organism``/``sample_type`` and metadata columns (there is no -``batch-template`` equivalent for it, since it has no reads files or project -field; note ``sample_type`` is reserved for the per-row override, so a -metadata attribute of that exact name can't be sent through the sheet). -``name`` defaults to the accession when omitted. A row's own ``sample_type`` -column, if present, overrides ``--sample-type`` for that row only — useful -for a mixed-type sheet; ``--sample-type`` itself is only required if some -row has no ``sample_type`` column of its own. The sample type, accession -format, and metadata rules are all sent as-is and validated **server-side**; -this command only checks that the sheet is a readable ``.csv``, that every -row has an accession, and that every row can resolve a sample type (its own -column, or ``--sample-type``) — a blank ``accession`` cell or a row with -neither rejects the whole sheet up front rather than shipping something the -server would just reject anyway. Rows are counted from ``1`` for the first -data row, after the header (the same convention as ``upload-batch``'s -``row_number``). +a CSV with required ``accession``/``sample_type`` columns, plus optional +``name``/``organism`` and metadata columns (there is no ``batch-template`` +equivalent for it, since it has no reads files or project field). ``name`` +defaults to the accession when omitted. There is deliberately no +``--sample-type`` flag: the sheet's own column is the only way to supply a +sample type, so a mixed-type sheet needs no special handling and a +single-type sheet just repeats the same value down the column. The sample +type, accession format, and metadata rules are all sent as-is and validated +**server-side**; this command only checks that the sheet is a readable +``.csv`` and that every row has an accession and a sample type — a blank +cell in either column rejects the whole sheet up front rather than shipping +something the server would just reject anyway. Rows are counted from ``1`` +for the first data row, after the header (the same convention as +``upload-batch``'s ``row_number``). Every row is submitted **together as one server-side job**. This command does **not wait for it to finish** — it reports the job's id and initial @@ -412,24 +409,21 @@ timestamps, ``null`` if not yet reached), ``accessions``, ``sample_ids`` **Exit codes** — ``0`` the job was created (regardless of its eventual outcome — check that with ``import-status``); ``2`` the sheet isn't a -readable ``.csv``, has no rows, has a row with no accession, or has a row -with no resolvable sample type; ``1`` the API rejected the batch (e.g. -unknown sample type, missing required metadata, an unsupported accession -format — these come back as an HTTP ``422``; ``5`` in the unlikely case it -answers ``400`` instead); ``3`` authentication failure; otherwise the -standard mapping above. Because whether ``--sample-type`` is required -depends on the sheet's contents, these sheet-level usage errors (``2``) are -only detected after authentication succeeds, unlike a malformed flag value -(which fails before it). +readable ``.csv``, has no rows, or has a row with no accession or no +sample type; ``1`` the API rejected the batch (e.g. unknown sample type, +missing required metadata, an unsupported accession format — these come +back as an HTTP ``422``; ``5`` in the unlikely case it answers ``400`` +instead); ``3`` authentication failure; otherwise the standard mapping +above. **Example** .. code-block:: bash - $ flowbio samples import --sheet ./accessions.csv --sample-type RNA-Seq + $ flowbio samples import --sheet ./accessions.csv Started import job 42 for 2 accession(s) (status: RUNNING). Check progress with 'flowbio samples import-status --job-id 42'. - $ flowbio samples import --sheet ./accessions.csv --sample-type RNA-Seq --json + $ flowbio samples import --sheet ./accessions.csv --json {"id": 42, "status": "RUNNING", "created": "2024-04-05T19:34:38Z", "started": null, "finished": null, "accessions": ["ERR1160845", "ERR10677146"], "sample_ids": [], "execution_id": null, "error": null} ``samples import-status`` diff --git a/tests/unit/cli/test_accession_sheet.py b/tests/unit/cli/test_accession_sheet.py index aa8f9fc..0403c25 100644 --- a/tests/unit/cli/test_accession_sheet.py +++ b/tests/unit/cli/test_accession_sheet.py @@ -5,7 +5,7 @@ from flowbio.cli._accession_sheet import AccessionSheetRow, parse_accession_sheet from flowbio.cli._exit_codes import CliUsageError -from flowbio.v2.samples import SampleImportSpec, SampleTypeId +from flowbio.v2.samples import SampleImportSpec HEADERS = ["accession", "name", "organism", "sample_type", "cell_type", "source", "source__annotation"] @@ -21,11 +21,17 @@ def _write_sheet( return path +def _record(**overrides: str) -> dict[str, str]: + record = {"accession": "ERR1160845", "sample_type": "rna_seq"} + record.update(overrides) + return record + + class TestParseAccessionSheet: def test_accession_is_passed_through_unchanged(self, tmp_path: Path) -> None: sheet = parse_accession_sheet( - _write_sheet(tmp_path, {"accession": "err1160845"}), + _write_sheet(tmp_path, _record(accession="err1160845")), ) assert sheet.rows[0].accession == "err1160845" @@ -33,14 +39,14 @@ def test_accession_is_passed_through_unchanged(self, tmp_path: Path) -> None: def test_empty_cells_omitted_from_metadata(self, tmp_path: Path) -> None: sheet = parse_accession_sheet(_write_sheet( tmp_path, - {"accession": "ERR1160845", "cell_type": "", "source": "blood"}, + _record(cell_type="", source="blood"), )) assert sheet.rows[0].metadata == {"source": "blood"} def test_name_and_organism_are_optional(self, tmp_path: Path) -> None: sheet = parse_accession_sheet( - _write_sheet(tmp_path, {"accession": "ERR1160845"}), + _write_sheet(tmp_path, _record()), ) assert sheet.rows[0].name is None @@ -49,22 +55,15 @@ def test_name_and_organism_are_optional(self, tmp_path: Path) -> None: def test_name_and_organism_are_parsed_when_present(self, tmp_path: Path) -> None: sheet = parse_accession_sheet(_write_sheet( tmp_path, - {"accession": "ERR1160845", "name": "liver_r1", "organism": "Hs"}, + _record(name="liver_r1", organism="Hs"), )) assert sheet.rows[0].name == "liver_r1" assert sheet.rows[0].organism == "Hs" - def test_sample_type_column_is_optional(self, tmp_path: Path) -> None: - sheet = parse_accession_sheet( - _write_sheet(tmp_path, {"accession": "ERR1160845"}), - ) - - assert sheet.rows[0].sample_type is None - - def test_sample_type_column_is_parsed_when_present(self, tmp_path: Path) -> None: + def test_sample_type_is_parsed(self, tmp_path: Path) -> None: sheet = parse_accession_sheet(_write_sheet( - tmp_path, {"accession": "ERR1160845", "sample_type": "chip_seq"}, + tmp_path, _record(sample_type="chip_seq"), )) assert sheet.rows[0].sample_type == "chip_seq" @@ -74,7 +73,7 @@ def test_utf8_bom_is_stripped_from_first_header(self, tmp_path: Path) -> None: with path.open("w", newline="", encoding="utf-8-sig") as handle: writer = csv.DictWriter(handle, fieldnames=HEADERS) writer.writeheader() - writer.writerow({"accession": "ERR1160845"}) + writer.writerow(_record()) sheet = parse_accession_sheet(path) @@ -83,8 +82,8 @@ def test_utf8_bom_is_stripped_from_first_header(self, tmp_path: Path) -> None: def test_row_numbers_are_one_based(self, tmp_path: Path) -> None: sheet = parse_accession_sheet(_write_sheet( tmp_path, - {"accession": "ERR1160845"}, - {"accession": "ERR10677146"}, + _record(accession="ERR1160845"), + _record(accession="ERR10677146"), )) assert [row.row_number for row in sheet.rows] == [1, 2] @@ -112,32 +111,46 @@ def test_header_only_sheet_is_usage_error(self, tmp_path: Path) -> None: parse_accession_sheet(_write_sheet(tmp_path)) def test_row_with_blank_accession_is_usage_error(self, tmp_path: Path) -> None: - with pytest.raises(CliUsageError, match="row"): + with pytest.raises(CliUsageError, match=r"data row\(s\) 2 has no accession"): parse_accession_sheet(_write_sheet( tmp_path, - {"accession": "ERR1160845"}, - {"accession": ""}, + _record(accession="ERR1160845"), + _record(accession=""), )) def test_row_with_no_accession_column_is_usage_error(self, tmp_path: Path) -> None: path = tmp_path / "sheet.csv" with path.open("w", newline="") as handle: - writer = csv.DictWriter(handle, fieldnames=["run", "name"]) + writer = csv.DictWriter(handle, fieldnames=["run", "name", "sample_type"]) writer.writeheader() - writer.writerow({"run": "ERR1160845", "name": "liver_r1"}) + writer.writerow({"run": "ERR1160845", "name": "liver_r1", "sample_type": "rna_seq"}) with pytest.raises(CliUsageError): parse_accession_sheet(path) def test_usage_error_names_every_row_missing_an_accession(self, tmp_path: Path) -> None: - with pytest.raises(CliUsageError, match="1.*3"): + with pytest.raises(CliUsageError, match=r"data row\(s\) 1, 3 have no accession"): parse_accession_sheet(_write_sheet( tmp_path, - {"accession": ""}, - {"accession": "ERR1"}, - {"accession": ""}, + _record(accession=""), + _record(accession="ERR1"), + _record(accession=""), )) + def test_row_with_blank_sample_type_is_usage_error(self, tmp_path: Path) -> None: + with pytest.raises(CliUsageError, match=r"data row\(s\) 1 has no sample_type"): + parse_accession_sheet(_write_sheet(tmp_path, _record(sample_type=""))) + + def test_row_with_no_sample_type_column_is_usage_error(self, tmp_path: Path) -> None: + path = tmp_path / "sheet.csv" + with path.open("w", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=["accession", "name"]) + writer.writeheader() + writer.writerow({"accession": "ERR1160845", "name": "liver_r1"}) + + with pytest.raises(CliUsageError): + parse_accession_sheet(path) + def test_row_rejects_empty_accession_by_construction() -> None: with pytest.raises(ValueError, match="accession"): @@ -146,7 +159,19 @@ def test_row_rejects_empty_accession_by_construction() -> None: accession="", name=None, organism=None, - sample_type=None, + sample_type="rna_seq", + metadata={}, + ) + + +def test_row_rejects_empty_sample_type_by_construction() -> None: + with pytest.raises(ValueError, match="sample_type"): + AccessionSheetRow( + row_number=1, + accession="ERR1160845", + name=None, + organism=None, + sample_type="", metadata={}, ) @@ -154,42 +179,20 @@ def test_row_rejects_empty_accession_by_construction() -> None: class TestAccessionSheetRowToSpec: def _row(self, tmp_path: Path, **overrides: str): - record = {"accession": "ERR1160845"} - record.update(overrides) - sheet = parse_accession_sheet(_write_sheet(tmp_path, record)) + sheet = parse_accession_sheet(_write_sheet(tmp_path, _record(**overrides))) return sheet.rows[0] - def test_uses_default_sample_type_when_row_has_none(self, tmp_path: Path) -> None: - row = self._row(tmp_path) - - spec = row.to_spec(SampleTypeId("rna_seq")) - - assert spec == SampleImportSpec(accession="ERR1160845", sample_type="rna_seq") - - def test_row_sample_type_overrides_the_default(self, tmp_path: Path) -> None: + def test_uses_the_row_sample_type(self, tmp_path: Path) -> None: row = self._row(tmp_path, sample_type="chip_seq") - spec = row.to_spec(SampleTypeId("rna_seq")) - - assert spec.sample_type == "chip_seq" - - def test_blank_sample_type_cell_falls_back_to_default(self, tmp_path: Path) -> None: - row = self._row(tmp_path, sample_type="") - - spec = row.to_spec(SampleTypeId("rna_seq")) - - assert spec.sample_type == "rna_seq" - - def test_raises_when_no_sample_type_or_default_available(self, tmp_path: Path) -> None: - row = self._row(tmp_path) + spec = row.to_spec() - with pytest.raises(ValueError, match="sample_type"): - row.to_spec(None) + assert spec == SampleImportSpec(accession="ERR1160845", sample_type="chip_seq") def test_carries_name_organism_and_metadata(self, tmp_path: Path) -> None: row = self._row(tmp_path, name="liver_r1", organism="Hs", cell_type="Neuron") - spec = row.to_spec(SampleTypeId("rna_seq")) + spec = row.to_spec() assert spec == SampleImportSpec( accession="ERR1160845", diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index 451accf..3d2df5f 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -938,6 +938,12 @@ def _write_import_sheet(directory: Path, *records: dict[str, str]) -> Path: return path +def _import_record(**overrides: str) -> dict[str, str]: + record = {"accession": "ERR1", "sample_type": "rna_seq"} + record.update(overrides) + return record + + def _job_json( job_id: int, status: str, @@ -972,13 +978,12 @@ def test_kicks_off_job_and_reports_id_without_waiting( ) sheet = _write_import_sheet( tmp_path, - {"accession": "ERR1", "cell_type": "Neuron"}, - {"accession": "ERR2", "cell_type": "Fibroblast"}, + _import_record(accession="ERR1", cell_type="Neuron"), + _import_record(accession="ERR2", cell_type="Fibroblast"), ) result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), ) assert result.exit_code == 0 @@ -995,11 +1000,10 @@ def test_json_document_reports_job_fields( 42, "RUNNING", ["ERR1"], )), ) - sheet = _write_import_sheet(tmp_path, {"accession": "ERR1", "cell_type": "Neuron"}) + sheet = _write_import_sheet(tmp_path, _import_record(cell_type="Neuron")) result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", "--json", + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), "--json", ) assert result.exit_code == 0 @@ -1031,14 +1035,13 @@ def test_sends_every_row_without_local_validation( ) sheet = _write_import_sheet( tmp_path, - {"accession": "not-an-accession"}, - {"accession": "ERR1"}, - {"accession": "ERR1"}, + _import_record(accession="not-an-accession", sample_type="bogus"), + _import_record(accession="ERR1"), + _import_record(accession="ERR1"), ) result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), ) assert result.exit_code == 0 @@ -1056,14 +1059,12 @@ def test_sends_name_organism_and_metadata_in_payload( 1, "RUNNING", ["ERR1"], )), ) - sheet = _write_import_sheet(tmp_path, { - "accession": "ERR1", "name": "liver_r1", "organism": "Hs", - "cell_type": "Neuron", - }) + sheet = _write_import_sheet(tmp_path, _import_record( + name="liver_r1", organism="Hs", cell_type="Neuron", + )) run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), ) payload = json.loads(route.calls[0].request.content) @@ -1077,6 +1078,30 @@ def test_sends_name_organism_and_metadata_in_payload( }], } + @respx.mock + def test_each_row_uses_its_own_sample_type( + self, run_cli, tmp_path: Path, + ) -> None: + route = respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "RUNNING", ["ERR1", "ERR2"], + )), + ) + sheet = _write_import_sheet( + tmp_path, + _import_record(accession="ERR1", sample_type="chip_seq"), + _import_record(accession="ERR2", sample_type="atac_seq"), + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), + ) + + assert result.exit_code == 0 + payload = json.loads(route.calls[0].request.content) + sample_types = [entry["sample_type"] for entry in payload["imports"]] + assert sample_types == ["chip_seq", "atac_seq"] + @respx.mock def test_api_rejection_propagates_as_error( self, run_cli, tmp_path: Path, @@ -1087,11 +1112,10 @@ def test_api_rejection_propagates_as_error( json={"error": "sample type 'bogus' does not exist"}, ), ) - sheet = _write_import_sheet(tmp_path, {"accession": "ERR1"}) + sheet = _write_import_sheet(tmp_path, _import_record(sample_type="bogus")) result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "bogus", + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), ) assert result.exit_code == 1 @@ -1105,8 +1129,7 @@ def test_non_csv_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None: xlsx.write_bytes(b"PK\x03\x04") result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(xlsx), "--sample-type", "rna_seq", + "--token", TOKEN, "samples", "import", "--sheet", str(xlsx), ) assert result.exit_code == 2 @@ -1114,23 +1137,9 @@ def test_non_csv_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None: assert "CSV" in result.stderr def test_missing_sheet_is_usage_error(self, run_cli) -> None: - result = run_cli( - "--token", TOKEN, "samples", "import", "--sample-type", "rna_seq", - ) - - assert result.exit_code == 2 - - @respx.mock - def test_missing_sample_type_is_usage_error(self, run_cli, tmp_path: Path) -> None: - route = respx.post(SAMPLE_IMPORTS_URL) - sheet = _write_import_sheet(tmp_path, {"accession": "ERR1"}) - - result = run_cli( - "--token", TOKEN, "samples", "import", "--sheet", str(sheet), - ) + result = run_cli("--token", TOKEN, "samples", "import") assert result.exit_code == 2 - assert route.call_count == 0 @respx.mock def test_header_only_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None: @@ -1138,8 +1147,7 @@ def test_header_only_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None sheet = _write_import_sheet(tmp_path) result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), ) assert result.exit_code == 2 @@ -1147,63 +1155,14 @@ def test_header_only_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None assert str(sheet) in result.stderr @respx.mock - def test_row_sample_type_overrides_the_default( - self, run_cli, tmp_path: Path, - ) -> None: - route = respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "RUNNING", ["ERR1", "ERR2"], - )), - ) - sheet = _write_import_sheet( - tmp_path, - {"accession": "ERR1", "sample_type": "chip_seq"}, - {"accession": "ERR2"}, - ) - - result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", - ) - - assert result.exit_code == 0 - payload = json.loads(route.calls[0].request.content) - sample_types = [entry["sample_type"] for entry in payload["imports"]] - assert sample_types == ["chip_seq", "rna_seq"] - - @respx.mock - def test_sample_type_flag_is_optional_when_every_row_has_its_own( - self, run_cli, tmp_path: Path, - ) -> None: - route = respx.post(SAMPLE_IMPORTS_URL).mock( - return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( - 1, "RUNNING", ["ERR1", "ERR2"], - )), - ) - sheet = _write_import_sheet( - tmp_path, - {"accession": "ERR1", "sample_type": "chip_seq"}, - {"accession": "ERR2", "sample_type": "atac_seq"}, - ) - - result = run_cli( - "--token", TOKEN, "samples", "import", "--sheet", str(sheet), - ) - - assert result.exit_code == 0 - payload = json.loads(route.calls[0].request.content) - sample_types = [entry["sample_type"] for entry in payload["imports"]] - assert sample_types == ["chip_seq", "atac_seq"] - - @respx.mock - def test_row_missing_sample_type_without_default_is_usage_error( + def test_row_missing_sample_type_is_usage_error( self, run_cli, tmp_path: Path, ) -> None: route = respx.post(SAMPLE_IMPORTS_URL) sheet = _write_import_sheet( tmp_path, - {"accession": "ERR1", "sample_type": "chip_seq"}, - {"accession": "ERR2"}, + _import_record(accession="ERR1"), + _import_record(accession="ERR2", sample_type=""), ) result = run_cli( @@ -1212,18 +1171,19 @@ def test_row_missing_sample_type_without_default_is_usage_error( assert result.exit_code == 2 assert route.call_count == 0 - assert "data row(s) 2" in result.stderr + assert "data row(s) 2 has no sample_type" in result.stderr @respx.mock def test_sheet_with_only_blank_accessions_is_usage_error( self, run_cli, tmp_path: Path, ) -> None: route = respx.post(SAMPLE_IMPORTS_URL) - sheet = _write_import_sheet(tmp_path, {"accession": ""}, {"accession": ""}) + sheet = _write_import_sheet( + tmp_path, _import_record(accession=""), _import_record(accession=""), + ) result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), ) assert result.exit_code == 2 @@ -1239,13 +1199,12 @@ def test_mixed_valid_and_blank_accession_rows_is_usage_error( route = respx.post(SAMPLE_IMPORTS_URL) sheet = _write_import_sheet( tmp_path, - {"accession": "ERR1"}, - {"accession": ""}, + _import_record(accession="ERR1"), + _import_record(accession=""), ) result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), ) assert result.exit_code == 2 @@ -1258,13 +1217,12 @@ def test_sheet_with_no_accession_column_is_usage_error( route = respx.post(SAMPLE_IMPORTS_URL) sheet = tmp_path / "accessions.csv" with sheet.open("w", newline="") as handle: - writer = csv.DictWriter(handle, fieldnames=["run", "name"]) + writer = csv.DictWriter(handle, fieldnames=["run", "name", "sample_type"]) writer.writeheader() - writer.writerow({"run": "ERR1160845", "name": "liver_r1"}) + writer.writerow({"run": "ERR1160845", "name": "liver_r1", "sample_type": "rna_seq"}) result = run_cli( - "--token", TOKEN, "samples", "import", - "--sheet", str(sheet), "--sample-type", "rna_seq", + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), ) assert result.exit_code == 2 @@ -1359,6 +1317,24 @@ def test_naive_started_is_reported_as_utc_in_json_too(self, run_cli) -> None: document = json.loads(result.stdout) assert document["started"] == "2024-04-05T19:34:38Z" + @respx.mock + def test_non_utc_offset_started_is_reported_as_utc_in_json_too(self, run_cli) -> None: + respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( + return_value=httpx.Response(HTTPStatus.OK, json={ + "id": 42, "status": "RUNNING", "created": 1700000000, + "started": "2024-04-05T19:34:38+02:00", "finished": None, + "accessions": ["ERR1"], "sample_ids": [], "execution_id": None, "error": None, + }), + ) + + result = run_cli( + "--token", TOKEN, "samples", "import-status", "--job-id", "42", "--json", + ) + + assert result.exit_code == 0 + document = json.loads(result.stdout) + assert document["started"] == "2024-04-05T17:34:38Z" + @respx.mock def test_running_job_falls_back_to_created_when_not_started(self, run_cli) -> None: respx.get(f"{SAMPLE_IMPORTS_URL}/42").mock( From 60aff5b632f03bbdc8aad5e71d8fb78ff01f5703 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 18:18:25 +0000 Subject: [PATCH 17/32] fix(samples): address round-16 Claude review findings - parse_accession_sheet now raises a single CliUsageError naming both missing accessions and missing sample types when a sheet is broken in both columns, instead of only ever reporting the accession problem and requiring a second run to learn about the sample_type one. This restores the 'collect every problem up front' property this slice has kept through every redesign. - Renamed the error-building helper (_missing_column_error -> _missing_value_clause) to describe what it actually reports (blank cells across a set of rows, not a missing column), and it now builds one clause per column rather than a whole exception. - Tightened four CLI-level sheet-rejection tests that could previously pass for any locally-rejected sheet (they asserted only that the sheet path appeared in stderr, which every parse_accession_sheet error satisfies) to assert the distinguishing phrase instead. Added a test for the both-columns-broken case at both the CLI and parser level. - Added a copy-pasteable accession-sheet example to cli.rst, now that there's no --sample-type flag or template verb to get the header from and both required columns reject a blank cell. - Dropped the now-unnecessary list[SampleImportSpec] annotation (and the import it required) on _import_command's list comprehension. Left as informational/deferred, per the review's own framing: the two ValueError invariants in AccessionSheetRow.__post_init__, the frozen- dataclass-with-dict hashability wrinkle, and import_samples([])'s round-trip to the server for an empty list (consistent with 'let the API validate'). Linear: FLOW-689 --- flowbio/cli/_accession_sheet.py | 18 +++++++++++------ flowbio/cli/_samples.py | 3 +-- source/cli.rst | 8 ++++++++ tests/unit/cli/test_accession_sheet.py | 13 +++++++++++++ tests/unit/cli/test_samples.py | 27 +++++++++++++++++++++++--- 5 files changed, 58 insertions(+), 11 deletions(-) diff --git a/flowbio/cli/_accession_sheet.py b/flowbio/cli/_accession_sheet.py index 0ed3e6a..4752eeb 100644 --- a/flowbio/cli/_accession_sheet.py +++ b/flowbio/cli/_accession_sheet.py @@ -106,19 +106,25 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: missing_sample_type.append(row_number) if accession is not None and sample_type is not None: rows.append(_build_row(record, row_number, metadata_columns, accession, sample_type)) - if missing_accession: - raise _missing_column_error("accession", missing_accession, path) - if missing_sample_type: - raise _missing_column_error("sample_type", missing_sample_type, path) + if missing_accession or missing_sample_type: + clauses = [ + clause for clause in ( + _missing_value_clause("accession", missing_accession), + _missing_value_clause("sample_type", missing_sample_type), + ) if clause is not None + ] + raise CliUsageError(f"Accession sheet {'; '.join(clauses)}: {path}.") return AccessionSheet(path=path, rows=rows) -def _missing_column_error(column: str, missing: list[int], path: Path) -> CliUsageError: +def _missing_value_clause(column: str, missing: list[int]) -> str | None: + if not missing: + return None numbers = ", ".join(str(number) for number in missing) # Data row 1 is the first row after the header, matching _sheet.py's # convention (and upload-batch's documented "1-based row number"). verb = "has" if len(missing) == 1 else "have" - return CliUsageError(f"Accession sheet data row(s) {numbers} {verb} no {column}: {path}.") + return f"data row(s) {numbers} {verb} no {column}" def _cell(record: dict[str, str], column: str) -> str | None: diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index 5904509..00a90b4 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -32,7 +32,6 @@ MetadataAttribute, SampleImportJob, SampleImportJobId, - SampleImportSpec, SampleTypeId, ) @@ -633,7 +632,7 @@ def _import_command( rows, or has a row with no accession or no sample_type. """ sheet = parse_accession_sheet(args.sheet) - specs: list[SampleImportSpec] = [row.to_spec() for row in sheet.rows] + specs = [row.to_spec() for row in sheet.rows] job = client.samples.import_samples(specs) output.emit_result( f"Started import job {job.id} for {len(specs)} accession(s) " diff --git a/source/cli.rst b/source/cli.rst index b7bf4b5..5ff4d6f 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -418,6 +418,14 @@ above. **Example** +``accessions.csv``: + +.. code-block:: text + + accession,sample_type,name,organism + ERR1160845,RNA-Seq,liver_r1,Hs + ERR10677146,RNA-Seq,, + .. code-block:: bash $ flowbio samples import --sheet ./accessions.csv diff --git a/tests/unit/cli/test_accession_sheet.py b/tests/unit/cli/test_accession_sheet.py index 0403c25..7cbcbfd 100644 --- a/tests/unit/cli/test_accession_sheet.py +++ b/tests/unit/cli/test_accession_sheet.py @@ -151,6 +151,19 @@ def test_row_with_no_sample_type_column_is_usage_error(self, tmp_path: Path) -> with pytest.raises(CliUsageError): parse_accession_sheet(path) + def test_sheet_broken_in_both_columns_reports_both_in_one_error( + self, tmp_path: Path, + ) -> None: + with pytest.raises(CliUsageError) as excinfo: + parse_accession_sheet(_write_sheet( + tmp_path, + _record(accession=""), + _record(accession="ERR2", sample_type=""), + )) + + assert "data row(s) 1 has no accession" in str(excinfo.value) + assert "data row(s) 2 has no sample_type" in str(excinfo.value) + def test_row_rejects_empty_accession_by_construction() -> None: with pytest.raises(ValueError, match="accession"): diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index 3d2df5f..77b57b9 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -1152,7 +1152,7 @@ def test_header_only_sheet_is_usage_error(self, run_cli, tmp_path: Path) -> None assert result.exit_code == 2 assert route.call_count == 0 - assert str(sheet) in result.stderr + assert "no rows" in result.stderr @respx.mock def test_row_missing_sample_type_is_usage_error( @@ -1188,7 +1188,7 @@ def test_sheet_with_only_blank_accessions_is_usage_error( assert result.exit_code == 2 assert route.call_count == 0 - assert str(sheet) in result.stderr + assert "data row(s) 1, 2 have no accession" in result.stderr @respx.mock def test_mixed_valid_and_blank_accession_rows_is_usage_error( @@ -1209,6 +1209,27 @@ def test_mixed_valid_and_blank_accession_rows_is_usage_error( assert result.exit_code == 2 assert route.call_count == 0 + assert "data row(s) 2 has no accession" in result.stderr + + @respx.mock + def test_sheet_broken_in_both_columns_reports_both_in_one_error( + self, run_cli, tmp_path: Path, + ) -> None: + route = respx.post(SAMPLE_IMPORTS_URL) + sheet = _write_import_sheet( + tmp_path, + _import_record(accession=""), + _import_record(accession="ERR2", sample_type=""), + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), + ) + + assert result.exit_code == 2 + assert route.call_count == 0 + assert "data row(s) 1 has no accession" in result.stderr + assert "data row(s) 2 has no sample_type" in result.stderr @respx.mock def test_sheet_with_no_accession_column_is_usage_error( @@ -1227,7 +1248,7 @@ def test_sheet_with_no_accession_column_is_usage_error( assert result.exit_code == 2 assert route.call_count == 0 - assert str(sheet) in result.stderr + assert "data row(s) 1 has no accession" in result.stderr class TestSamplesImportStatus: From 433528aea21bfbfd09574c396addec12a5fa02c0 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 18:29:33 +0000 Subject: [PATCH 18/32] fix(samples): address round-17 Claude review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - parse_accession_sheet now skips a row whose every cell is blank (the trailing comma-only line a spreadsheet export leaves below the data) instead of treating it as a row missing accession and sample_type — that's spreadsheet noise, not a row with data but no accession, and rejecting a whole sheet on it was a bad first-run experience. A sheet of nothing but blank rows still reports 'no rows', same as header-only. - Documented, based on reading flow-api's _assign_metadata/ SampleType.validate_metadata directly: a __annotation metadata column is not given upload-batch's special handling for samples import — it's forwarded as an ordinary key that matches no real MetadataAttribute, so it's silently ignored rather than attached as an annotation. Added a test pinning the client-side forwarding behaviour (the ignoring itself is server-side and can't be tested from here). - Fixed parse_accession_sheet's docstring and cli.rst, which promised values are sent 'exactly as entered'/'as-is' when cells are in fact whitespace-stripped. - parse_accession_sheet no longer materialises every record into a list before processing (streams the reader directly, matching _sheet.py's style), and states the accession/sample_type presence check once instead of re-testing an already-decided condition. - Moved the row-numbering convention comment to the enumerate() call it actually explains, and dropped a '— see below' cross-reference the following :raises: line already covered. Linear: FLOW-689 --- flowbio/cli/_accession_sheet.py | 48 +++++++++++++++----------- source/cli.rst | 21 +++++++---- tests/unit/cli/test_accession_sheet.py | 27 +++++++++++++++ 3 files changed, 69 insertions(+), 27 deletions(-) diff --git a/flowbio/cli/_accession_sheet.py b/flowbio/cli/_accession_sheet.py index 4752eeb..8825267 100644 --- a/flowbio/cli/_accession_sheet.py +++ b/flowbio/cli/_accession_sheet.py @@ -14,10 +14,14 @@ here rather than silently skipped or shipped as an empty string the server would just reject anyway. There is deliberately no other way to supply a sample type for ``samples import`` — the sheet is the single source of it. +A row with every cell blank (e.g. a trailing comma-only line some spreadsheet +exports append below the data) is skipped rather than treated as a row +missing values, since there is nothing there to be missing. """ from __future__ import annotations import csv +from collections.abc import Sequence from dataclasses import dataclass from pathlib import Path @@ -69,10 +73,9 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: :param path: The accession-sheet file. Must be a ``.csv`` — an ``.xlsx`` or ``.tsv`` is a usage error directing the user to export to CSV. - :returns: The parsed sheet, with empty cells dropped (other than - ``accession``/``sample_type``, which reject the sheet instead — see - below). Values are otherwise passed through unchanged, including - ``accession``, sent to the server exactly as entered. + :returns: The parsed sheet, with empty cells dropped and surrounding + whitespace trimmed. Values are otherwise passed through unchanged, + including ``accession``, sent to the server as-entered. :raises CliUsageError: If the file is not a readable ``.csv``, has no rows, or has a row with no accession or no sample_type. """ @@ -82,6 +85,9 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: f"Export your spreadsheet to CSV first.", ) existing_file(path) + rows: list[AccessionSheetRow] = [] + missing_accession: list[int] = [] + missing_sample_type: list[int] = [] # utf-8-sig transparently strips a leading BOM, which spreadsheet tools # (notably Excel's "CSV UTF-8" export) prepend — otherwise the first header # parses as "accession" and every row reports a missing accession. @@ -91,21 +97,21 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: metadata_columns = [ header for header in headers if header not in RESERVED_COLUMNS ] - records = list(enumerate(reader, start=1)) - if not records: + # Row 1 is the first row after the header, matching _sheet.py's + # convention (and upload-batch's documented "1-based row number"). + for row_number, record in enumerate(reader, start=1): + if _is_blank_row(record, headers): + continue + accession = _cell(record, "accession") + sample_type = _cell(record, "sample_type") + if accession is None: + missing_accession.append(row_number) + if sample_type is None: + missing_sample_type.append(row_number) + if accession is not None and sample_type is not None: + rows.append(_build_row(record, row_number, metadata_columns, accession, sample_type)) + if not rows and not missing_accession and not missing_sample_type: raise CliUsageError(f"Accession sheet has no rows: {path}.") - rows: list[AccessionSheetRow] = [] - missing_accession: list[int] = [] - missing_sample_type: list[int] = [] - for row_number, record in records: - accession = _cell(record, "accession") - sample_type = _cell(record, "sample_type") - if accession is None: - missing_accession.append(row_number) - if sample_type is None: - missing_sample_type.append(row_number) - if accession is not None and sample_type is not None: - rows.append(_build_row(record, row_number, metadata_columns, accession, sample_type)) if missing_accession or missing_sample_type: clauses = [ clause for clause in ( @@ -117,12 +123,14 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: return AccessionSheet(path=path, rows=rows) +def _is_blank_row(record: dict[str, str], headers: Sequence[str]) -> bool: + return not any((record.get(header) or "").strip() for header in headers) + + def _missing_value_clause(column: str, missing: list[int]) -> str | None: if not missing: return None numbers = ", ".join(str(number) for number in missing) - # Data row 1 is the first row after the header, matching _sheet.py's - # convention (and upload-batch's documented "1-based row number"). verb = "has" if len(missing) == 1 else "have" return f"data row(s) {numbers} {verb} no {column}" diff --git a/source/cli.rst b/source/cli.rst index 5ff4d6f..2c66631 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -386,13 +386,20 @@ defaults to the accession when omitted. There is deliberately no ``--sample-type`` flag: the sheet's own column is the only way to supply a sample type, so a mixed-type sheet needs no special handling and a single-type sheet just repeats the same value down the column. The sample -type, accession format, and metadata rules are all sent as-is and validated -**server-side**; this command only checks that the sheet is a readable -``.csv`` and that every row has an accession and a sample type — a blank -cell in either column rejects the whole sheet up front rather than shipping -something the server would just reject anyway. Rows are counted from ``1`` -for the first data row, after the header (the same convention as -``upload-batch``'s ``row_number``). +type, accession format, and metadata rules are all sent as-is (surrounding +whitespace trimmed) and validated **server-side**; this command only checks +that the sheet is a readable ``.csv`` and that every row has an accession +and a sample type — a blank cell in either column rejects the whole sheet +up front rather than shipping something the server would just reject +anyway. A row with every cell blank (e.g. a trailing comma-only line some +spreadsheet exports leave below the data) is skipped rather than treated as +a row missing values. Rows are counted from ``1`` for the first data row, +after the header (the same convention as ``upload-batch``'s ``row_number``). + +Unlike ``upload-batch``, a metadata column named ``__annotation`` +is **not** given any special handling here — it is forwarded as an ordinary +metadata key, which the server does not recognise, so it is silently +ignored rather than attached as an annotation or rejected. Every row is submitted **together as one server-side job**. This command does **not wait for it to finish** — it reports the job's id and initial diff --git a/tests/unit/cli/test_accession_sheet.py b/tests/unit/cli/test_accession_sheet.py index 7cbcbfd..54ee957 100644 --- a/tests/unit/cli/test_accession_sheet.py +++ b/tests/unit/cli/test_accession_sheet.py @@ -164,6 +164,21 @@ def test_sheet_broken_in_both_columns_reports_both_in_one_error( assert "data row(s) 1 has no accession" in str(excinfo.value) assert "data row(s) 2 has no sample_type" in str(excinfo.value) + def test_trailing_blank_row_is_skipped(self, tmp_path: Path) -> None: + # A comma-only line, e.g. one a spreadsheet export leaves below the + # data — not a row with data but no accession, so it isn't an error. + sheet = parse_accession_sheet(_write_sheet( + tmp_path, + _record(accession="ERR1"), + {}, + )) + + assert [row.accession for row in sheet.rows] == ["ERR1"] + + def test_sheet_of_only_blank_rows_is_usage_error(self, tmp_path: Path) -> None: + with pytest.raises(CliUsageError, match="no rows"): + parse_accession_sheet(_write_sheet(tmp_path, {}, {})) + def test_row_rejects_empty_accession_by_construction() -> None: with pytest.raises(ValueError, match="accession"): @@ -214,3 +229,15 @@ def test_carries_name_organism_and_metadata(self, tmp_path: Path) -> None: organism_id="Hs", metadata={"cell_type": "Neuron"}, ) + + def test_annotation_suffixed_column_is_forwarded_as_a_plain_metadata_key( + self, tmp_path: Path, + ) -> None: + # Unlike upload-batch, there's no special handling of `__annotation` + # here — it's just another metadata column, and the server does not + # recognise it as one (see cli.rst). + row = self._row(tmp_path, source="blood", source__annotation="left lobe") + + spec = row.to_spec() + + assert spec.metadata == {"source": "blood", "source__annotation": "left lobe"} From d7a4bc2e7cd9862d757a34a7cde000882a05a44a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 18:38:11 +0000 Subject: [PATCH 19/32] fix(samples): address round-18 Claude review findings - Accession-sheet header names are now trimmed too, not just cell values: reassigning DictReader.fieldnames to the stripped list means a hand-authored header like 'accession, sample_type, name' (a space after each comma) is recognised correctly instead of treating ' sample_type'/' name' as bogus metadata columns and rejecting every row for a missing sample_type it visibly has. - Removed the now-redundant astimezone(timezone.utc) call in _timestamp_suffix (SampleImportJob's validator already guarantees a UTC-normalised value, aware or not, so the call was a no-op) along with its comment, and trimmed _job_summary's FAILED-branch comment to drop its cross-reference to _import_status_command. - Fixed a cli.rst sentence that said sample-type/accession-format/ metadata *rules* are sent as-is, when it's values that are sent and rules that are validated server-side. - parse_accession_sheet's own :returns: now mentions the blank-row skip as an observable property of what it returns, and states the accession/sample_type presence check once (an early continue) rather than a third redundant combined check. - Added tests: a header with spaces after commas, a blank row in the middle of a sheet (proving later rows keep their file-relative row numbers), and a CLI-level test that a trailing blank row is skipped and the remaining rows are still submitted. Linear: FLOW-689 --- flowbio/cli/_accession_sheet.py | 18 ++++++++++++------ flowbio/cli/_samples.py | 11 ++++------- source/cli.rst | 7 ++++--- tests/unit/cli/test_accession_sheet.py | 22 ++++++++++++++++++++++ tests/unit/cli/test_samples.py | 21 +++++++++++++++++++++ 5 files changed, 63 insertions(+), 16 deletions(-) diff --git a/flowbio/cli/_accession_sheet.py b/flowbio/cli/_accession_sheet.py index 8825267..cace45a 100644 --- a/flowbio/cli/_accession_sheet.py +++ b/flowbio/cli/_accession_sheet.py @@ -73,9 +73,10 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: :param path: The accession-sheet file. Must be a ``.csv`` — an ``.xlsx`` or ``.tsv`` is a usage error directing the user to export to CSV. - :returns: The parsed sheet, with empty cells dropped and surrounding - whitespace trimmed. Values are otherwise passed through unchanged, - including ``accession``, sent to the server as-entered. + :returns: The parsed sheet, with wholly-blank rows skipped, empty cells + dropped, and surrounding whitespace trimmed (including in header + names). Values are otherwise passed through unchanged, including + ``accession``, sent to the server as-entered. :raises CliUsageError: If the file is not a readable ``.csv``, has no rows, or has a row with no accession or no sample_type. """ @@ -93,7 +94,11 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: # parses as "accession" and every row reports a missing accession. with path.open(newline="", encoding="utf-8-sig") as handle: reader = csv.DictReader(handle) - headers = reader.fieldnames or [] + # A hand-authored header (unlike upload-batch's template-generated + # one) routinely has a stray space after a comma; reassigning + # fieldnames makes every row dict keyed by the trimmed name too. + headers = [header.strip() for header in reader.fieldnames or []] + reader.fieldnames = headers metadata_columns = [ header for header in headers if header not in RESERVED_COLUMNS ] @@ -108,8 +113,9 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: missing_accession.append(row_number) if sample_type is None: missing_sample_type.append(row_number) - if accession is not None and sample_type is not None: - rows.append(_build_row(record, row_number, metadata_columns, accession, sample_type)) + if accession is None or sample_type is None: + continue + rows.append(_build_row(record, row_number, metadata_columns, accession, sample_type)) if not rows and not missing_accession and not missing_sample_type: raise CliUsageError(f"Accession sheet has no rows: {path}.") if missing_accession or missing_sample_type: diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index 00a90b4..9633221 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -11,7 +11,7 @@ import argparse import json from dataclasses import dataclass -from datetime import datetime, timezone +from datetime import datetime from pathlib import Path from typing import Literal @@ -669,9 +669,8 @@ def _job_summary(job: SampleImportJob) -> str: suffix = _timestamp_suffix("finished", job.finished) return f"Job {job.id}: COMPLETED{suffix}. Sample ids: {ids}." if job.status == "FAILED": - # The error, if any, is on stderr as an advisory (see - # _import_status_command) rather than repeated here, so a human - # running this doesn't see the same sentence twice. + # A human sees the error as a separate advisory on stderr, so it + # isn't repeated here. return f"Job {job.id}: FAILED{_timestamp_suffix('finished', job.finished)}." label = "started" if job.started else "created" return f"Job {job.id}: {job.status}{_timestamp_suffix(label, job.started or job.created)}." @@ -680,9 +679,7 @@ def _job_summary(job: SampleImportJob) -> str: def _timestamp_suffix(label: str, timestamp: datetime | None) -> str: if timestamp is None: return "" - # SampleImportJob normalises a naive value to UTC, so every timestamp - # reaching here is already aware. - return f" ({label} {timestamp.astimezone(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')})" + return f" ({label} {timestamp.strftime('%Y-%m-%d %H:%M:%S UTC')})" def _merge_metadata( diff --git a/source/cli.rst b/source/cli.rst index 2c66631..153d227 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -385,9 +385,10 @@ equivalent for it, since it has no reads files or project field). ``name`` defaults to the accession when omitted. There is deliberately no ``--sample-type`` flag: the sheet's own column is the only way to supply a sample type, so a mixed-type sheet needs no special handling and a -single-type sheet just repeats the same value down the column. The sample -type, accession format, and metadata rules are all sent as-is (surrounding -whitespace trimmed) and validated **server-side**; this command only checks +single-type sheet just repeats the same value down the column. Every value +is sent as-is (surrounding whitespace trimmed, including in header names); +the accession format, sample type, and metadata rules are validated +**server-side**. This command only checks that the sheet is a readable ``.csv`` and that every row has an accession and a sample type — a blank cell in either column rejects the whole sheet up front rather than shipping something the server would just reject diff --git a/tests/unit/cli/test_accession_sheet.py b/tests/unit/cli/test_accession_sheet.py index 54ee957..64fac0f 100644 --- a/tests/unit/cli/test_accession_sheet.py +++ b/tests/unit/cli/test_accession_sheet.py @@ -88,6 +88,17 @@ def test_row_numbers_are_one_based(self, tmp_path: Path) -> None: assert [row.row_number for row in sheet.rows] == [1, 2] + def test_header_with_spaces_after_commas_is_still_recognised(self, tmp_path: Path) -> None: + path = tmp_path / "sheet.csv" + path.write_text("accession, sample_type, name\nERR1160845, rna_seq, liver_r1\n") + + sheet = parse_accession_sheet(path) + + assert sheet.rows[0].accession == "ERR1160845" + assert sheet.rows[0].sample_type == "rna_seq" + assert sheet.rows[0].name == "liver_r1" + assert sheet.rows[0].metadata == {} + def test_non_csv_xlsx_rejected_with_export_message(self, tmp_path: Path) -> None: xlsx = tmp_path / "sheet.xlsx" xlsx.write_bytes(b"PK") @@ -179,6 +190,17 @@ def test_sheet_of_only_blank_rows_is_usage_error(self, tmp_path: Path) -> None: with pytest.raises(CliUsageError, match="no rows"): parse_accession_sheet(_write_sheet(tmp_path, {}, {})) + def test_blank_row_in_the_middle_does_not_shift_later_row_numbers( + self, tmp_path: Path, + ) -> None: + with pytest.raises(CliUsageError, match=r"data row\(s\) 3 has no accession"): + parse_accession_sheet(_write_sheet( + tmp_path, + _record(accession="ERR1"), + {}, + _record(accession=""), + )) + def test_row_rejects_empty_accession_by_construction() -> None: with pytest.raises(ValueError, match="accession"): diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index 77b57b9..3f04081 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -1102,6 +1102,27 @@ def test_each_row_uses_its_own_sample_type( sample_types = [entry["sample_type"] for entry in payload["imports"]] assert sample_types == ["chip_seq", "atac_seq"] + @respx.mock + def test_trailing_blank_row_is_skipped( + self, run_cli, tmp_path: Path, + ) -> None: + route = respx.post(SAMPLE_IMPORTS_URL).mock( + return_value=httpx.Response(HTTPStatus.CREATED, json=_job_json( + 1, "RUNNING", ["ERR1"], + )), + ) + sheet = _write_import_sheet( + tmp_path, _import_record(accession="ERR1"), {}, + ) + + result = run_cli( + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), + ) + + assert result.exit_code == 0 + payload = json.loads(route.calls[0].request.content) + assert [entry["accession"] for entry in payload["imports"]] == ["ERR1"] + @respx.mock def test_api_rejection_propagates_as_error( self, run_cli, tmp_path: Path, From a607b79c4873feabcd1bfbeb244f42d64f847ced Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Jul 2026 15:09:07 +0000 Subject: [PATCH 20/32] fix(samples): reject an accession sheet with an unnamed column Per operator direction: an empty header column (e.g. a trailing comma in the header row, which spreadsheet exports routinely leave) is now a hard CliUsageError naming every unnamed column position, rather than becoming a metadata attribute literally named "". Without this, a stray value under the last such column would be submitted as metadata: {"": "..."} and reject the whole atomic batch server-side, while a value under an earlier one would be silently dropped (csv's DictReader zips duplicate field names last-wins). This is the column-axis counterpart of the wholly-blank-row skip: a blank cell in a named column is still just absent data, but a column with no name at all can't mean anything, so it's rejected outright rather than guessed at. Linear: FLOW-689 --- flowbio/cli/_accession_sheet.py | 10 ++++++++-- source/cli.rst | 16 +++++++++------- tests/unit/cli/test_accession_sheet.py | 16 ++++++++++++++++ 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/flowbio/cli/_accession_sheet.py b/flowbio/cli/_accession_sheet.py index cace45a..7263f18 100644 --- a/flowbio/cli/_accession_sheet.py +++ b/flowbio/cli/_accession_sheet.py @@ -77,8 +77,9 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: dropped, and surrounding whitespace trimmed (including in header names). Values are otherwise passed through unchanged, including ``accession``, sent to the server as-entered. - :raises CliUsageError: If the file is not a readable ``.csv``, has no - rows, or has a row with no accession or no sample_type. + :raises CliUsageError: If the file is not a readable ``.csv``, has an + unnamed column, has no rows, or has a row with no accession or no + sample_type. """ if path.suffix.lower() != ".csv": raise CliUsageError( @@ -99,6 +100,11 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: # fieldnames makes every row dict keyed by the trimmed name too. headers = [header.strip() for header in reader.fieldnames or []] reader.fieldnames = headers + unnamed = [position for position, header in enumerate(headers, start=1) if not header] + if unnamed: + positions = ", ".join(str(position) for position in unnamed) + verb = "is" if len(unnamed) == 1 else "are" + raise CliUsageError(f"Accession sheet column(s) {positions} {verb} unnamed: {path}.") metadata_columns = [ header for header in headers if header not in RESERVED_COLUMNS ] diff --git a/source/cli.rst b/source/cli.rst index 153d227..aea3da4 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -389,7 +389,9 @@ single-type sheet just repeats the same value down the column. Every value is sent as-is (surrounding whitespace trimmed, including in header names); the accession format, sample type, and metadata rules are validated **server-side**. This command only checks -that the sheet is a readable ``.csv`` and that every row has an accession +that the sheet is a readable ``.csv``, that every column has a name (a +trailing comma in the header row is rejected rather than becoming a +metadata attribute with an empty name), and that every row has an accession and a sample type — a blank cell in either column rejects the whole sheet up front rather than shipping something the server would just reject anyway. A row with every cell blank (e.g. a trailing comma-only line some @@ -417,12 +419,12 @@ timestamps, ``null`` if not yet reached), ``accessions``, ``sample_ids`` **Exit codes** — ``0`` the job was created (regardless of its eventual outcome — check that with ``import-status``); ``2`` the sheet isn't a -readable ``.csv``, has no rows, or has a row with no accession or no -sample type; ``1`` the API rejected the batch (e.g. unknown sample type, -missing required metadata, an unsupported accession format — these come -back as an HTTP ``422``; ``5`` in the unlikely case it answers ``400`` -instead); ``3`` authentication failure; otherwise the standard mapping -above. +readable ``.csv``, has an unnamed column, has no rows, or has a row with no +accession or no sample type; ``1`` the API rejected the batch (e.g. unknown +sample type, missing required metadata, an unsupported accession format — +these come back as an HTTP ``422``; ``5`` in the unlikely case it answers +``400`` instead); ``3`` authentication failure; otherwise the standard +mapping above. **Example** diff --git a/tests/unit/cli/test_accession_sheet.py b/tests/unit/cli/test_accession_sheet.py index 64fac0f..d6210cc 100644 --- a/tests/unit/cli/test_accession_sheet.py +++ b/tests/unit/cli/test_accession_sheet.py @@ -99,6 +99,22 @@ def test_header_with_spaces_after_commas_is_still_recognised(self, tmp_path: Pat assert sheet.rows[0].name == "liver_r1" assert sheet.rows[0].metadata == {} + def test_trailing_empty_header_column_is_usage_error(self, tmp_path: Path) -> None: + # A trailing comma in the header row (e.g. from a spreadsheet export) + # must not become a metadata attribute with an empty name. + path = tmp_path / "sheet.csv" + path.write_text("accession,sample_type,name,\nERR1,rna_seq,liver_r1,note\n") + + with pytest.raises(CliUsageError, match=r"column\(s\) 4 is unnamed"): + parse_accession_sheet(path) + + def test_multiple_unnamed_columns_are_all_named_in_the_error(self, tmp_path: Path) -> None: + path = tmp_path / "sheet.csv" + path.write_text("accession,sample_type,,\nERR1,rna_seq,,\n") + + with pytest.raises(CliUsageError, match=r"column\(s\) 3, 4 are unnamed"): + parse_accession_sheet(path) + def test_non_csv_xlsx_rejected_with_export_message(self, tmp_path: Path) -> None: xlsx = tmp_path / "sheet.xlsx" xlsx.write_bytes(b"PK") From a2e1f1a8c75f8ad5b3c714628d7c934206d940b4 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Jul 2026 15:16:49 +0000 Subject: [PATCH 21/32] fix(samples): reject duplicate accession-sheet columns too Extends the operator-directed unnamed-column rejection to duplicate column names, which have the same ambiguity (the parser can't know which column the user meant) and the same silent-loss failure mode csv.DictReader's last-wins zip produces for an unnamed column: a duplicated metadata column discards the earlier column's value, and a duplicated accession column whose second instance is blank rejects the sheet even though the first instance has one. Both checks now live in one _check_headers helper, each with a remedy clause since a user can't see a trailing header comma or a copy-pasted column name in their spreadsheet the way they'd see it in the raw CSV. Also, per Claude round-20 review and the operator's own inline comments on this file: - Removed two comments the operator flagged as unnecessary, keeping the Excel/BOM one. - _is_blank_row now calls _cell instead of duplicating its strip/ blank-as-absent logic inline. - _import_command's :raises: docstring, the module docstring, and cli.rst (description + exit codes) now all mention the unnamed/ duplicated column case, matching parse_accession_sheet's own. - Added a CLI-level test for the unnamed-column rejection, matching the coverage every other locally-rejected sheet already had. Linear: FLOW-689 --- flowbio/cli/_accession_sheet.py | 41 +++++++++++++++++--------- flowbio/cli/_samples.py | 5 ++-- source/cli.rst | 32 ++++++++++---------- tests/unit/cli/test_accession_sheet.py | 16 ++++++++++ tests/unit/cli/test_samples.py | 16 ++++++++++ 5 files changed, 79 insertions(+), 31 deletions(-) diff --git a/flowbio/cli/_accession_sheet.py b/flowbio/cli/_accession_sheet.py index 7263f18..8580974 100644 --- a/flowbio/cli/_accession_sheet.py +++ b/flowbio/cli/_accession_sheet.py @@ -16,7 +16,10 @@ sample type for ``samples import`` — the sheet is the single source of it. A row with every cell blank (e.g. a trailing comma-only line some spreadsheet exports append below the data) is skipped rather than treated as a row -missing values, since there is nothing there to be missing. +missing values, since there is nothing there to be missing. An unnamed or +duplicated header column is rejected outright rather than guessed at — a +column that could mean more than one thing, or nothing at all, isn't +something the parser can resolve on the user's behalf. """ from __future__ import annotations @@ -78,8 +81,8 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: names). Values are otherwise passed through unchanged, including ``accession``, sent to the server as-entered. :raises CliUsageError: If the file is not a readable ``.csv``, has an - unnamed column, has no rows, or has a row with no accession or no - sample_type. + unnamed or duplicated column, has no rows, or has a row with no + accession or no sample_type. """ if path.suffix.lower() != ".csv": raise CliUsageError( @@ -95,21 +98,12 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: # parses as "accession" and every row reports a missing accession. with path.open(newline="", encoding="utf-8-sig") as handle: reader = csv.DictReader(handle) - # A hand-authored header (unlike upload-batch's template-generated - # one) routinely has a stray space after a comma; reassigning - # fieldnames makes every row dict keyed by the trimmed name too. headers = [header.strip() for header in reader.fieldnames or []] reader.fieldnames = headers - unnamed = [position for position, header in enumerate(headers, start=1) if not header] - if unnamed: - positions = ", ".join(str(position) for position in unnamed) - verb = "is" if len(unnamed) == 1 else "are" - raise CliUsageError(f"Accession sheet column(s) {positions} {verb} unnamed: {path}.") + _check_headers(headers, path) metadata_columns = [ header for header in headers if header not in RESERVED_COLUMNS ] - # Row 1 is the first row after the header, matching _sheet.py's - # convention (and upload-batch's documented "1-based row number"). for row_number, record in enumerate(reader, start=1): if _is_blank_row(record, headers): continue @@ -135,8 +129,27 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: return AccessionSheet(path=path, rows=rows) +def _check_headers(headers: Sequence[str], path: Path) -> None: + unnamed = [position for position, header in enumerate(headers, start=1) if not header] + if unnamed: + positions = ", ".join(str(position) for position in unnamed) + verb = "is" if len(unnamed) == 1 else "are" + raise CliUsageError( + f"Accession sheet column(s) {positions} {verb} unnamed: {path}. " + f"Remove the trailing comma(s) from the header row, or give the column a name.", + ) + duplicates = sorted({header for header in headers if headers.count(header) > 1}) + if duplicates: + names = ", ".join(f"'{name}'" for name in duplicates) + verb = "is" if len(duplicates) == 1 else "are" + raise CliUsageError( + f"Accession sheet column name(s) {names} {verb} duplicated: {path}. " + f"Rename the repeated column(s) so each column is unique.", + ) + + def _is_blank_row(record: dict[str, str], headers: Sequence[str]) -> bool: - return not any((record.get(header) or "").strip() for header in headers) + return not any(_cell(record, header) for header in headers) def _missing_value_clause(column: str, missing: list[int]) -> str | None: diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index 9633221..4e77b06 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -628,8 +628,9 @@ def _import_command( :param client: The authenticated Flow client. :param output: The result/error renderer. :returns: :attr:`ExitCode.SUCCESS` once the job has been kicked off. - :raises CliUsageError: If the sheet is not a readable ``.csv``, has no - rows, or has a row with no accession or no sample_type. + :raises CliUsageError: If the sheet is not a readable ``.csv``, has an + unnamed or duplicated column, has no rows, or has a row with no + accession or no sample_type. """ sheet = parse_accession_sheet(args.sheet) specs = [row.to_spec() for row in sheet.rows] diff --git a/source/cli.rst b/source/cli.rst index aea3da4..9dfc840 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -389,15 +389,17 @@ single-type sheet just repeats the same value down the column. Every value is sent as-is (surrounding whitespace trimmed, including in header names); the accession format, sample type, and metadata rules are validated **server-side**. This command only checks -that the sheet is a readable ``.csv``, that every column has a name (a -trailing comma in the header row is rejected rather than becoming a -metadata attribute with an empty name), and that every row has an accession -and a sample type — a blank cell in either column rejects the whole sheet -up front rather than shipping something the server would just reject -anyway. A row with every cell blank (e.g. a trailing comma-only line some -spreadsheet exports leave below the data) is skipped rather than treated as -a row missing values. Rows are counted from ``1`` for the first data row, -after the header (the same convention as ``upload-batch``'s ``row_number``). +that the sheet is a readable ``.csv``, that every column has a unique name +(a trailing comma in the header row is rejected rather than becoming a +metadata attribute with an empty name, and a repeated column name is +rejected rather than one silently overwriting another), and that every row +has an accession and a sample type — a blank cell in either column rejects +the whole sheet up front rather than shipping something the server would +just reject anyway. A row with every cell blank (e.g. a trailing comma-only +line some spreadsheet exports leave below the data) is skipped rather than +treated as a row missing values. Rows are counted from ``1`` for the first +data row, after the header (the same convention as ``upload-batch``'s +``row_number``). Unlike ``upload-batch``, a metadata column named ``__annotation`` is **not** given any special handling here — it is forwarded as an ordinary @@ -419,12 +421,12 @@ timestamps, ``null`` if not yet reached), ``accessions``, ``sample_ids`` **Exit codes** — ``0`` the job was created (regardless of its eventual outcome — check that with ``import-status``); ``2`` the sheet isn't a -readable ``.csv``, has an unnamed column, has no rows, or has a row with no -accession or no sample type; ``1`` the API rejected the batch (e.g. unknown -sample type, missing required metadata, an unsupported accession format — -these come back as an HTTP ``422``; ``5`` in the unlikely case it answers -``400`` instead); ``3`` authentication failure; otherwise the standard -mapping above. +readable ``.csv``, has an unnamed or duplicated column, has no rows, or has +a row with no accession or no sample type; ``1`` the API rejected the batch +(e.g. unknown sample type, missing required metadata, an unsupported +accession format — these come back as an HTTP ``422``; ``5`` in the +unlikely case it answers ``400`` instead); ``3`` authentication failure; +otherwise the standard mapping above. **Example** diff --git a/tests/unit/cli/test_accession_sheet.py b/tests/unit/cli/test_accession_sheet.py index d6210cc..030968c 100644 --- a/tests/unit/cli/test_accession_sheet.py +++ b/tests/unit/cli/test_accession_sheet.py @@ -115,6 +115,22 @@ def test_multiple_unnamed_columns_are_all_named_in_the_error(self, tmp_path: Pat with pytest.raises(CliUsageError, match=r"column\(s\) 3, 4 are unnamed"): parse_accession_sheet(path) + def test_duplicated_metadata_column_is_usage_error(self, tmp_path: Path) -> None: + # Otherwise the earlier column's value is silently discarded, since + # csv.DictReader zips duplicate field names last-wins. + path = tmp_path / "sheet.csv" + path.write_text("accession,sample_type,treatment,treatment\nERR1,rna_seq,drugA,drugB\n") + + with pytest.raises(CliUsageError, match=r"column name\(s\) 'treatment' is duplicated"): + parse_accession_sheet(path) + + def test_duplicated_accession_column_is_usage_error(self, tmp_path: Path) -> None: + path = tmp_path / "sheet.csv" + path.write_text("accession,sample_type,accession\nERR1,rna_seq,ERR2\n") + + with pytest.raises(CliUsageError, match=r"column name\(s\) 'accession' is duplicated"): + parse_accession_sheet(path) + def test_non_csv_xlsx_rejected_with_export_message(self, tmp_path: Path) -> None: xlsx = tmp_path / "sheet.xlsx" xlsx.write_bytes(b"PK") diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index 3f04081..b963a01 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -1271,6 +1271,22 @@ def test_sheet_with_no_accession_column_is_usage_error( assert route.call_count == 0 assert "data row(s) 1 has no accession" in result.stderr + @respx.mock + def test_unnamed_header_column_is_usage_error( + self, run_cli, tmp_path: Path, + ) -> None: + route = respx.post(SAMPLE_IMPORTS_URL) + sheet = tmp_path / "accessions.csv" + sheet.write_text("accession,sample_type,name,\nERR1,rna_seq,liver_r1,note\n") + + result = run_cli( + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), + ) + + assert result.exit_code == 2 + assert route.call_count == 0 + assert "unnamed" in result.stderr + class TestSamplesImportStatus: From d6da4a75ae3947c0e276008c5121a3d037919495 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Jul 2026 15:25:35 +0000 Subject: [PATCH 22/32] fix(samples): reject a data row with more cells than the header MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per round-20 review: csv.DictReader stores a row's overflow cells under a None key (restkey), which parse_accession_sheet never read. Two consequences, now closed: - A row with one extra trailing value silently dropped that value (e.g. a metadata cell nobody read). - A shifted-right row (one stray leading comma) whose named cells were therefore all blank was treated as a wholly-blank row and skipped entirely by _is_blank_row — even when the overflow column held a real accession. An entire row could vanish with no error and no output at all. Both are the row-axis version of the column-axis problem the unnamed/ duplicate-column checks already close, so a row with more cells than the header is now rejected the same way, naming every offending row. Also: - Collected the unnamed-column and duplicate-column checks into one CliUsageError instead of two sequential ones, restoring the collect-every-problem-up-front property this slice has kept since round 16 (a sheet broken both ways no longer takes two fix-and-rerun cycles). - Consolidated the accession/sample_type/extra-cells 'missing X' clause-building into one _row_count_clause helper. - Restructured cli.rst's samples import section (the four local checks as a bullet list) and fixed a docs sentence that still described header names as values, per round 19/20 feedback. - Added tests for the new row-level rejection at both the parser and CLI level, a combined unnamed+duplicate-column test, a wholly empty file test, and tightened the CLI-level unnamed-column assertion to match its siblings. Linear: FLOW-689 --- flowbio/cli/_accession_sheet.py | 96 ++++++++++++++++---------- flowbio/cli/_samples.py | 5 +- source/cli.rst | 43 +++++++----- tests/unit/cli/test_accession_sheet.py | 37 ++++++++++ tests/unit/cli/test_samples.py | 34 ++++++++- 5 files changed, 158 insertions(+), 57 deletions(-) diff --git a/flowbio/cli/_accession_sheet.py b/flowbio/cli/_accession_sheet.py index 8580974..4b70d5f 100644 --- a/flowbio/cli/_accession_sheet.py +++ b/flowbio/cli/_accession_sheet.py @@ -8,18 +8,12 @@ Domain rules (accession format, duplicates, sample type, organism, metadata) are all checked server-side when the sheet is submitted — duplicating that -locally would just be a second, driftable copy of the same rules. -``accession`` and ``sample_type`` are different: they are the two columns -every row must have to mean anything at all, so a missing one is rejected -here rather than silently skipped or shipped as an empty string the server -would just reject anyway. There is deliberately no other way to supply a -sample type for ``samples import`` — the sheet is the single source of it. -A row with every cell blank (e.g. a trailing comma-only line some spreadsheet -exports append below the data) is skipped rather than treated as a row -missing values, since there is nothing there to be missing. An unnamed or -duplicated header column is rejected outright rather than guessed at — a -column that could mean more than one thing, or nothing at all, isn't -something the parser can resolve on the user's behalf. +locally would just be a second, driftable copy of the same rules. What *is* +checked locally is structural: every row must have an accession and a +sample type to mean anything at all, every header column must have a +unique, non-empty name, and every row must have exactly as many cells as +the header — any of these is a case the parser can't resolve on the user's +behalf, so it's rejected rather than guessed at. See :func:`parse_accession_sheet`. """ from __future__ import annotations @@ -81,8 +75,9 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: names). Values are otherwise passed through unchanged, including ``accession``, sent to the server as-entered. :raises CliUsageError: If the file is not a readable ``.csv``, has an - unnamed or duplicated column, has no rows, or has a row with no - accession or no sample_type. + unnamed or duplicated column, has no rows, has a row with more + cells than the header, or has a row with no accession or no + sample_type. """ if path.suffix.lower() != ".csv": raise CliUsageError( @@ -91,6 +86,7 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: ) existing_file(path) rows: list[AccessionSheetRow] = [] + extra_cells: list[int] = [] missing_accession: list[int] = [] missing_sample_type: list[int] = [] # utf-8-sig transparently strips a leading BOM, which spreadsheet tools @@ -105,6 +101,14 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: header for header in headers if header not in RESERVED_COLUMNS ] for row_number, record in enumerate(reader, start=1): + # csv.DictReader stores a row with more cells than the header + # under the None key; that overflow can't be attributed to any + # column, so it's rejected rather than silently dropped (or, if + # every named cell happens to be blank, the whole row silently + # skipped as though it carried nothing). + if record.get(None): + extra_cells.append(row_number) + continue if _is_blank_row(record, headers): continue accession = _cell(record, "accession") @@ -116,13 +120,14 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: if accession is None or sample_type is None: continue rows.append(_build_row(record, row_number, metadata_columns, accession, sample_type)) - if not rows and not missing_accession and not missing_sample_type: + if not rows and not extra_cells and not missing_accession and not missing_sample_type: raise CliUsageError(f"Accession sheet has no rows: {path}.") - if missing_accession or missing_sample_type: + if extra_cells or missing_accession or missing_sample_type: clauses = [ clause for clause in ( - _missing_value_clause("accession", missing_accession), - _missing_value_clause("sample_type", missing_sample_type), + _row_count_clause("more cells than the header", extra_cells), + _row_count_clause("no accession", missing_accession), + _row_count_clause("no sample_type", missing_sample_type), ) if clause is not None ] raise CliUsageError(f"Accession sheet {'; '.join(clauses)}: {path}.") @@ -131,33 +136,48 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: def _check_headers(headers: Sequence[str], path: Path) -> None: unnamed = [position for position, header in enumerate(headers, start=1) if not header] - if unnamed: - positions = ", ".join(str(position) for position in unnamed) - verb = "is" if len(unnamed) == 1 else "are" - raise CliUsageError( - f"Accession sheet column(s) {positions} {verb} unnamed: {path}. " - f"Remove the trailing comma(s) from the header row, or give the column a name.", - ) - duplicates = sorted({header for header in headers if headers.count(header) > 1}) - if duplicates: - names = ", ".join(f"'{name}'" for name in duplicates) - verb = "is" if len(duplicates) == 1 else "are" - raise CliUsageError( - f"Accession sheet column name(s) {names} {verb} duplicated: {path}. " - f"Rename the repeated column(s) so each column is unique.", - ) + duplicates = sorted({header for header in headers if header and headers.count(header) > 1}) + if not unnamed and not duplicates: + return + clauses = [ + clause for clause in ( + _unnamed_columns_clause(unnamed), + _duplicate_columns_clause(duplicates), + ) if clause is not None + ] + raise CliUsageError( + f"Accession sheet {'; '.join(clauses)}: {path}. Remove the trailing comma(s) " + f"from the header row, give unnamed columns a name, and rename any repeated " + f"column so each column is unique.", + ) + + +def _unnamed_columns_clause(unnamed: list[int]) -> str | None: + if not unnamed: + return None + positions = ", ".join(str(position) for position in unnamed) + verb = "is" if len(unnamed) == 1 else "are" + return f"column(s) {positions} {verb} unnamed" + + +def _duplicate_columns_clause(duplicates: list[str]) -> str | None: + if not duplicates: + return None + names = ", ".join(f"'{name}'" for name in duplicates) + verb = "is" if len(duplicates) == 1 else "are" + return f"column name(s) {names} {verb} duplicated" def _is_blank_row(record: dict[str, str], headers: Sequence[str]) -> bool: return not any(_cell(record, header) for header in headers) -def _missing_value_clause(column: str, missing: list[int]) -> str | None: - if not missing: +def _row_count_clause(reason: str, rows: list[int]) -> str | None: + if not rows: return None - numbers = ", ".join(str(number) for number in missing) - verb = "has" if len(missing) == 1 else "have" - return f"data row(s) {numbers} {verb} no {column}" + numbers = ", ".join(str(number) for number in rows) + verb = "has" if len(rows) == 1 else "have" + return f"data row(s) {numbers} {verb} {reason}" def _cell(record: dict[str, str], column: str) -> str | None: diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index 4e77b06..258f0d8 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -629,8 +629,9 @@ def _import_command( :param output: The result/error renderer. :returns: :attr:`ExitCode.SUCCESS` once the job has been kicked off. :raises CliUsageError: If the sheet is not a readable ``.csv``, has an - unnamed or duplicated column, has no rows, or has a row with no - accession or no sample_type. + unnamed or duplicated column, has no rows, has a row with more + cells than the header, or has a row with no accession or no + sample_type. """ sheet = parse_accession_sheet(args.sheet) specs = [row.to_spec() for row in sheet.rows] diff --git a/source/cli.rst b/source/cli.rst index 9dfc840..1494236 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -385,20 +385,30 @@ equivalent for it, since it has no reads files or project field). ``name`` defaults to the accession when omitted. There is deliberately no ``--sample-type`` flag: the sheet's own column is the only way to supply a sample type, so a mixed-type sheet needs no special handling and a -single-type sheet just repeats the same value down the column. Every value -is sent as-is (surrounding whitespace trimmed, including in header names); -the accession format, sample type, and metadata rules are validated -**server-side**. This command only checks -that the sheet is a readable ``.csv``, that every column has a unique name -(a trailing comma in the header row is rejected rather than becoming a -metadata attribute with an empty name, and a repeated column name is -rejected rather than one silently overwriting another), and that every row -has an accession and a sample type — a blank cell in either column rejects -the whole sheet up front rather than shipping something the server would -just reject anyway. A row with every cell blank (e.g. a trailing comma-only -line some spreadsheet exports leave below the data) is skipped rather than -treated as a row missing values. Rows are counted from ``1`` for the first -data row, after the header (the same convention as ``upload-batch``'s +single-type sheet just repeats the same value down the column. + +Every value is sent as-is, with surrounding whitespace trimmed; header +names are trimmed the same way. The accession format, sample type, and +metadata rules are all validated **server-side**. This command only +checks what's structural — anything it can't resolve on your behalf, it +rejects up front rather than guessing: + +- The sheet must be a readable ``.csv``, with at least one data row. +- Every header column must have a name, and no two columns may share one — + a trailing comma in the header row (or a copy-pasted column) is rejected + rather than becoming an empty-named metadata attribute, or one column + silently overwriting another. +- Every data row must have exactly as many cells as the header — a row + with extra cells (e.g. one stray comma) is rejected rather than dropping + the overflow, or silently skipping the row if that leaves every named + cell blank. +- Every row must have an accession and a sample type; a blank cell in + either column rejects the whole sheet. + +A row with every cell blank (e.g. a trailing comma-only line some +spreadsheet exports leave below the data) is skipped rather than treated +as a row missing values. Rows are counted from ``1`` for the first data +row, after the header (the same convention as ``upload-batch``'s ``row_number``). Unlike ``upload-batch``, a metadata column named ``__annotation`` @@ -421,8 +431,9 @@ timestamps, ``null`` if not yet reached), ``accessions``, ``sample_ids`` **Exit codes** — ``0`` the job was created (regardless of its eventual outcome — check that with ``import-status``); ``2`` the sheet isn't a -readable ``.csv``, has an unnamed or duplicated column, has no rows, or has -a row with no accession or no sample type; ``1`` the API rejected the batch +readable ``.csv``, has an unnamed or duplicated column, has no rows, has a +row with more cells than the header, or has a row with no accession or no +sample type; ``1`` the API rejected the batch (e.g. unknown sample type, missing required metadata, an unsupported accession format — these come back as an HTTP ``422``; ``5`` in the unlikely case it answers ``400`` instead); ``3`` authentication failure; diff --git a/tests/unit/cli/test_accession_sheet.py b/tests/unit/cli/test_accession_sheet.py index 030968c..25c56f6 100644 --- a/tests/unit/cli/test_accession_sheet.py +++ b/tests/unit/cli/test_accession_sheet.py @@ -131,6 +131,36 @@ def test_duplicated_accession_column_is_usage_error(self, tmp_path: Path) -> Non with pytest.raises(CliUsageError, match=r"column name\(s\) 'accession' is duplicated"): parse_accession_sheet(path) + def test_unnamed_and_duplicated_columns_reported_in_one_error(self, tmp_path: Path) -> None: + path = tmp_path / "sheet.csv" + path.write_text( + "accession,sample_type,treatment,treatment,\nERR1,rna_seq,drugA,drugB,note\n", + ) + + with pytest.raises(CliUsageError) as excinfo: + parse_accession_sheet(path) + + assert "column(s) 5 is unnamed" in str(excinfo.value) + assert "column name(s) 'treatment' is duplicated" in str(excinfo.value) + + def test_row_with_more_cells_than_the_header_is_usage_error(self, tmp_path: Path) -> None: + path = tmp_path / "sheet.csv" + path.write_text("accession,sample_type,name\nERR1,rna_seq,liver_r1,note\n") + + with pytest.raises(CliUsageError, match=r"data row\(s\) 1 has more cells than the header"): + parse_accession_sheet(path) + + def test_shifted_right_row_with_a_real_value_is_usage_error_not_skipped( + self, tmp_path: Path, + ) -> None: + # Every *named* cell is blank, but the accession landed in the + # overflow column — this must not be treated as a wholly-blank row. + path = tmp_path / "sheet.csv" + path.write_text("accession,sample_type,name\n,,,ERR1\n") + + with pytest.raises(CliUsageError, match=r"data row\(s\) 1 has more cells than the header"): + parse_accession_sheet(path) + def test_non_csv_xlsx_rejected_with_export_message(self, tmp_path: Path) -> None: xlsx = tmp_path / "sheet.xlsx" xlsx.write_bytes(b"PK") @@ -153,6 +183,13 @@ def test_header_only_sheet_is_usage_error(self, tmp_path: Path) -> None: with pytest.raises(CliUsageError, match="no rows"): parse_accession_sheet(_write_sheet(tmp_path)) + def test_wholly_empty_file_is_usage_error(self, tmp_path: Path) -> None: + path = tmp_path / "sheet.csv" + path.write_text("") + + with pytest.raises(CliUsageError, match="no rows"): + parse_accession_sheet(path) + def test_row_with_blank_accession_is_usage_error(self, tmp_path: Path) -> None: with pytest.raises(CliUsageError, match=r"data row\(s\) 2 has no accession"): parse_accession_sheet(_write_sheet( diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index b963a01..0563a32 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -1285,7 +1285,39 @@ def test_unnamed_header_column_is_usage_error( assert result.exit_code == 2 assert route.call_count == 0 - assert "unnamed" in result.stderr + assert "column(s) 4 is unnamed" in result.stderr + + @respx.mock + def test_duplicated_header_column_is_usage_error( + self, run_cli, tmp_path: Path, + ) -> None: + route = respx.post(SAMPLE_IMPORTS_URL) + sheet = tmp_path / "accessions.csv" + sheet.write_text("accession,sample_type,treatment,treatment\nERR1,rna_seq,drugA,drugB\n") + + result = run_cli( + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), + ) + + assert result.exit_code == 2 + assert route.call_count == 0 + assert "column name(s) 'treatment' is duplicated" in result.stderr + + @respx.mock + def test_row_with_more_cells_than_the_header_is_usage_error( + self, run_cli, tmp_path: Path, + ) -> None: + route = respx.post(SAMPLE_IMPORTS_URL) + sheet = tmp_path / "accessions.csv" + sheet.write_text("accession,sample_type,name\nERR1,rna_seq,liver_r1,note\n") + + result = run_cli( + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), + ) + + assert result.exit_code == 2 + assert route.call_count == 0 + assert "data row(s) 1 has more cells than the header" in result.stderr class TestSamplesImportStatus: From 3b2382a3abccbb3a94056190ff088d0cea33bde7 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Jul 2026 15:29:17 +0000 Subject: [PATCH 23/32] refactor(samples): simplify _import_spec_fields with dataclasses.asdict Per operator review comment: replaces the hand-built field-by-field dict with dataclasses.asdict() plus a rename (organism_id -> organism) and a single falsy-value filter, rather than four lines each checking a field for None/emptiness individually. Behaviour is unchanged (all 382 tests pass without modification) since every field is either required-and-non-empty or optional-and-None-when-absent. Linear: FLOW-689 --- flowbio/v2/samples.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/flowbio/v2/samples.py b/flowbio/v2/samples.py index 26b48ea..e0a4289 100644 --- a/flowbio/v2/samples.py +++ b/flowbio/v2/samples.py @@ -27,7 +27,7 @@ from __future__ import annotations from collections.abc import Sequence -from dataclasses import dataclass +from dataclasses import asdict, dataclass from datetime import datetime, timezone from pathlib import Path from typing import TYPE_CHECKING, Literal, NewType @@ -527,14 +527,9 @@ def get_import(self, job_id: SampleImportJobId) -> SampleImportJob: @staticmethod def _import_spec_fields(spec: SampleImportSpec) -> dict: - fields: dict = {"accession": spec.accession, "sample_type": spec.sample_type} - if spec.name is not None: - fields["name"] = spec.name - if spec.organism_id is not None: - fields["organism"] = spec.organism_id - if spec.metadata: - fields["metadata"] = spec.metadata - return fields + fields = asdict(spec) + fields["organism"] = fields.pop("organism_id") + return {key: value for key, value in fields.items() if value} def _create_metadata_attribute(self, item: dict) -> MetadataAttribute: item["required_for_sample_types"] = [ From 0bbee7291a2e48ad10814fef4b07dcec5953616d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Jul 2026 15:35:27 +0000 Subject: [PATCH 24/32] chore: bump version to 0.10.0 Release 'flowbio samples import'/'samples import-status' (public- repository accession imports) and client.samples.import_samples/ get_import. Backwards-compatible feature addition since 0.9.0. Linear: FLOW-689 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0316e01..28962a7 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="flowbio", - version="0.9.0", + version="0.10.0", description="A client for the Flow API.", long_description=long_description, long_description_content_type="text/markdown", From e40eb36403bfaf61e38373e25d8d489e7f84a3a6 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Jul 2026 15:43:30 +0000 Subject: [PATCH 25/32] fix(samples): don't reject a wholly-blank row over one extra comma MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per round-20 review (both automated runs agreed on this one): the new more-cells-than-header check tested csv.DictReader's overflow list for truthiness rather than content, so a wholly-blank trailing row one cell wider than the header (',,,' under a 3-column header) was rejected as 'more cells than the header' instead of being skipped as spreadsheet noise — undoing round 17's blank-row accommodation for that one shape. Now only rejected when the overflow actually carries a value, which keeps the shifted-right case (a real value landing in the overflow column) correctly caught. Also: - _import_spec_fields (the asdict refactor) now keeps the two required fields (accession, sample_type) unconditional and only filters the three optional ones, rather than filtering everything on truthiness — so a spec with an empty-string optional field still posts the two required keys. - The unnamed/duplicated-column error's remedy sentence now only names the fix(es) relevant to whichever clause(s) actually fired, instead of always listing all three regardless. - Resolved the disagreement between the two round-20 review runs on whether a short row (fewer cells than the header) should be rejected: left it accepted (csv.DictReader already treats missing trailing cells as blank, matching how an explicit empty cell is handled, and rejecting it risks false positives on legitimate hand-authored sheets that omit trailing optional columns) and corrected the docs/docstring, which had claimed rows need 'exactly' as many cells as the header when only 'more' was ever enforced. - Shortened cli.rst's exit-codes paragraph to point at the bullet list above it instead of re-enumerating every structural check a fourth time. - Added tests for the fixed blank-overflow case, the accepted-short- row case, and the asdict required-vs-optional fix. Linear: FLOW-689 --- flowbio/cli/_accession_sheet.py | 44 +++++++++++++------------- flowbio/v2/samples.py | 5 +-- source/cli.rst | 16 +++++----- tests/unit/cli/test_accession_sheet.py | 24 ++++++++++++++ tests/unit/v2/test_samples.py | 19 +++++++++++ 5 files changed, 76 insertions(+), 32 deletions(-) diff --git a/flowbio/cli/_accession_sheet.py b/flowbio/cli/_accession_sheet.py index 4b70d5f..e6da96f 100644 --- a/flowbio/cli/_accession_sheet.py +++ b/flowbio/cli/_accession_sheet.py @@ -11,9 +11,11 @@ locally would just be a second, driftable copy of the same rules. What *is* checked locally is structural: every row must have an accession and a sample type to mean anything at all, every header column must have a -unique, non-empty name, and every row must have exactly as many cells as -the header — any of these is a case the parser can't resolve on the user's -behalf, so it's rejected rather than guessed at. See :func:`parse_accession_sheet`. +unique, non-empty name, and no row may have more cells than the header — +any of these is a case the parser can't resolve on the user's behalf, so +it's rejected rather than guessed at. A row with *fewer* cells than the +header (e.g. trailing optional columns omitted) has its missing cells +treated as blank, same as an empty cell. See :func:`parse_accession_sheet`. """ from __future__ import annotations @@ -103,10 +105,10 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: for row_number, record in enumerate(reader, start=1): # csv.DictReader stores a row with more cells than the header # under the None key; that overflow can't be attributed to any - # column, so it's rejected rather than silently dropped (or, if - # every named cell happens to be blank, the whole row silently - # skipped as though it carried nothing). - if record.get(None): + # column, so a row whose overflow actually carries a value is + # rejected rather than silently dropped. + overflow_has_value = any((cell or "").strip() for cell in record.get(None) or []) + if overflow_has_value: extra_cells.append(row_number) continue if _is_blank_row(record, headers): @@ -139,30 +141,28 @@ def _check_headers(headers: Sequence[str], path: Path) -> None: duplicates = sorted({header for header in headers if header and headers.count(header) > 1}) if not unnamed and not duplicates: return - clauses = [ - clause for clause in ( - _unnamed_columns_clause(unnamed), - _duplicate_columns_clause(duplicates), - ) if clause is not None - ] + clauses: list[str] = [] + remedies: list[str] = [] + if unnamed: + clauses.append(_unnamed_columns_clause(unnamed)) + remedies.append("remove the trailing comma(s) from the header row, or give the column(s) a name") + if duplicates: + clauses.append(_duplicate_columns_clause(duplicates)) + remedies.append("rename the repeated column(s) so each column is unique") + remedy = "; and ".join(remedies) raise CliUsageError( - f"Accession sheet {'; '.join(clauses)}: {path}. Remove the trailing comma(s) " - f"from the header row, give unnamed columns a name, and rename any repeated " - f"column so each column is unique.", + f"Accession sheet {'; '.join(clauses)}: {path}. " + f"{remedy[0].upper()}{remedy[1:]}.", ) -def _unnamed_columns_clause(unnamed: list[int]) -> str | None: - if not unnamed: - return None +def _unnamed_columns_clause(unnamed: list[int]) -> str: positions = ", ".join(str(position) for position in unnamed) verb = "is" if len(unnamed) == 1 else "are" return f"column(s) {positions} {verb} unnamed" -def _duplicate_columns_clause(duplicates: list[str]) -> str | None: - if not duplicates: - return None +def _duplicate_columns_clause(duplicates: list[str]) -> str: names = ", ".join(f"'{name}'" for name in duplicates) verb = "is" if len(duplicates) == 1 else "are" return f"column name(s) {names} {verb} duplicated" diff --git a/flowbio/v2/samples.py b/flowbio/v2/samples.py index e0a4289..1cad297 100644 --- a/flowbio/v2/samples.py +++ b/flowbio/v2/samples.py @@ -528,8 +528,9 @@ def get_import(self, job_id: SampleImportJobId) -> SampleImportJob: @staticmethod def _import_spec_fields(spec: SampleImportSpec) -> dict: fields = asdict(spec) - fields["organism"] = fields.pop("organism_id") - return {key: value for key, value in fields.items() if value} + organism = fields.pop("organism_id") + optional = {"name": fields.pop("name"), "organism": organism, "metadata": fields.pop("metadata")} + return {**fields, **{key: value for key, value in optional.items() if value}} def _create_metadata_attribute(self, item: dict) -> MetadataAttribute: item["required_for_sample_types"] = [ diff --git a/source/cli.rst b/source/cli.rst index 1494236..15e0999 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -398,10 +398,11 @@ rejects up front rather than guessing: a trailing comma in the header row (or a copy-pasted column) is rejected rather than becoming an empty-named metadata attribute, or one column silently overwriting another. -- Every data row must have exactly as many cells as the header — a row - with extra cells (e.g. one stray comma) is rejected rather than dropping - the overflow, or silently skipping the row if that leaves every named - cell blank. +- No data row may have more cells than the header — an extra cell (e.g. one + stray comma) is rejected rather than dropping the overflow, or silently + skipping the row if that leaves every named cell blank. A row with + *fewer* cells than the header has its missing trailing cells treated as + blank, the same as an empty cell. - Every row must have an accession and a sample type; a blank cell in either column rejects the whole sheet. @@ -430,10 +431,9 @@ timestamps, ``null`` if not yet reached), ``accessions``, ``sample_ids`` (empty until the job completes), ``execution_id``, ``error``. **Exit codes** — ``0`` the job was created (regardless of its eventual -outcome — check that with ``import-status``); ``2`` the sheet isn't a -readable ``.csv``, has an unnamed or duplicated column, has no rows, has a -row with more cells than the header, or has a row with no accession or no -sample type; ``1`` the API rejected the batch +outcome — check that with ``import-status``); ``2`` the sheet is +structurally invalid (see the checks above), including not being a +readable ``.csv`` or having no rows; ``1`` the API rejected the batch (e.g. unknown sample type, missing required metadata, an unsupported accession format — these come back as an HTTP ``422``; ``5`` in the unlikely case it answers ``400`` instead); ``3`` authentication failure; diff --git a/tests/unit/cli/test_accession_sheet.py b/tests/unit/cli/test_accession_sheet.py index 25c56f6..3957e30 100644 --- a/tests/unit/cli/test_accession_sheet.py +++ b/tests/unit/cli/test_accession_sheet.py @@ -52,6 +52,17 @@ def test_name_and_organism_are_optional(self, tmp_path: Path) -> None: assert sheet.rows[0].name is None assert sheet.rows[0].organism is None + def test_row_shorter_than_the_header_has_missing_trailing_cells_treated_as_blank( + self, tmp_path: Path, + ) -> None: + path = tmp_path / "sheet.csv" + path.write_text("accession,sample_type,name,organism\nERR1,rna_seq\n") + + sheet = parse_accession_sheet(path) + + assert sheet.rows[0].name is None + assert sheet.rows[0].organism is None + def test_name_and_organism_are_parsed_when_present(self, tmp_path: Path) -> None: sheet = parse_accession_sheet(_write_sheet( tmp_path, @@ -161,6 +172,19 @@ def test_shifted_right_row_with_a_real_value_is_usage_error_not_skipped( with pytest.raises(CliUsageError, match=r"data row\(s\) 1 has more cells than the header"): parse_accession_sheet(path) + def test_wholly_blank_row_with_extra_trailing_comma_is_still_skipped( + self, tmp_path: Path, + ) -> None: + # A comma-only line one cell wider than the header (the overflow + # carries no value) is spreadsheet noise, not a row with data in + # the wrong place — must not be confused with the shifted-right case. + path = tmp_path / "sheet.csv" + path.write_text("accession,sample_type,name\nERR1,rna_seq,liver_r1\n,,,\n") + + sheet = parse_accession_sheet(path) + + assert [row.accession for row in sheet.rows] == ["ERR1"] + def test_non_csv_xlsx_rejected_with_export_message(self, tmp_path: Path) -> None: xlsx = tmp_path / "sheet.xlsx" xlsx.write_bytes(b"PK") diff --git a/tests/unit/v2/test_samples.py b/tests/unit/v2/test_samples.py index 59cade8..9444c6c 100644 --- a/tests/unit/v2/test_samples.py +++ b/tests/unit/v2/test_samples.py @@ -1143,6 +1143,25 @@ def test_sends_optional_fields_when_present(self) -> None: }], } + @respx.mock + def test_empty_string_optional_fields_are_omitted_not_the_required_ones(self) -> None: + route = respx.post(f"{DEFAULT_BASE_URL}/v2/sample-imports").mock( + return_value=httpx.Response(HTTPStatus.CREATED, json={ + "id": 1, "status": "RUNNING", "created": 1700000000, "accessions": ["ERR1"], + "sample_ids": [], "execution_id": None, "error": None, + }), + ) + + client = Client() + client.samples.import_samples([ + SampleImportSpec(accession="ERR1", sample_type="rna_seq", name="", organism_id=""), + ]) + + payload = json.loads(route.calls[0].request.content) + assert payload == { + "imports": [{"accession": "ERR1", "sample_type": "rna_seq"}], + } + @respx.mock def test_sends_multiple_imports_in_one_request(self) -> None: route = respx.post(f"{DEFAULT_BASE_URL}/v2/sample-imports").mock( From 34ec69ba93eca5d50bd54cfb51a7a5fa7e212403 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Jul 2026 15:53:52 +0000 Subject: [PATCH 26/32] fix(cli): reject short accession-sheet rows explicitly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A row with fewer cells than the header was previously accepted, with the missing trailing cells treated as blank — per explicit operator direction, short rows are now always rejected as a usage error, same as an over-wide row whose overflow carries a value. Folds in the remaining findings from the last automated review on this structural-validation behaviour: - rename the wrong-cell-count error's internal helper (_row_count_clause -> _row_reason_clause) since it doesn't count rows, it labels them with a reason - add a remedy clause to the wrong-cell-count error message pointing at the likely causes (unquoted comma, extra/missing cell) - correct the module and function docstrings, and the CLI docs, to precisely describe the finalised behaviour: a row with fewer cells is always rejected; a row with more cells is only rejected if the overflow isn't blank - add a test asserting a populated row with a harmless blank trailing overflow cell is still accepted, so that behaviour is now covered explicitly rather than just implied Linear: FLOW-689 --- flowbio/cli/_accession_sheet.py | 51 ++++++++++++++++---------- flowbio/cli/_samples.py | 5 ++- source/cli.rst | 12 +++--- tests/unit/cli/test_accession_sheet.py | 33 +++++++++++++---- tests/unit/cli/test_samples.py | 18 ++++++++- 5 files changed, 84 insertions(+), 35 deletions(-) diff --git a/flowbio/cli/_accession_sheet.py b/flowbio/cli/_accession_sheet.py index e6da96f..2c84621 100644 --- a/flowbio/cli/_accession_sheet.py +++ b/flowbio/cli/_accession_sheet.py @@ -11,11 +11,10 @@ locally would just be a second, driftable copy of the same rules. What *is* checked locally is structural: every row must have an accession and a sample type to mean anything at all, every header column must have a -unique, non-empty name, and no row may have more cells than the header — -any of these is a case the parser can't resolve on the user's behalf, so -it's rejected rather than guessed at. A row with *fewer* cells than the -header (e.g. trailing optional columns omitted) has its missing cells -treated as blank, same as an empty cell. See :func:`parse_accession_sheet`. +unique, non-empty name, and every row must have no fewer cells than the +header, and no more unless the extra ones are blank — any of these is a +case the parser can't resolve on the user's behalf, so it's rejected +rather than guessed at. See :func:`parse_accession_sheet`. """ from __future__ import annotations @@ -77,8 +76,9 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: names). Values are otherwise passed through unchanged, including ``accession``, sent to the server as-entered. :raises CliUsageError: If the file is not a readable ``.csv``, has an - unnamed or duplicated column, has no rows, has a row with more - cells than the header, or has a row with no accession or no + unnamed or duplicated column, has no rows, has a row with fewer + cells than the header or with more cells than the header where the + overflow isn't blank, or has a row with no accession or no sample_type. """ if path.suffix.lower() != ".csv": @@ -88,7 +88,7 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: ) existing_file(path) rows: list[AccessionSheetRow] = [] - extra_cells: list[int] = [] + wrong_cell_count: list[int] = [] missing_accession: list[int] = [] missing_sample_type: list[int] = [] # utf-8-sig transparently strips a leading BOM, which spreadsheet tools @@ -104,12 +104,17 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: ] for row_number, record in enumerate(reader, start=1): # csv.DictReader stores a row with more cells than the header - # under the None key; that overflow can't be attributed to any - # column, so a row whose overflow actually carries a value is - # rejected rather than silently dropped. + # under the None key, and fills a row with fewer from missing + # keys with None of its own — neither can be attributed to a + # column, so a row whose cell count doesn't match the header is + # rejected rather than guessed at. A short row is rejected + # outright (there's no benign "trailing noise" story for one — + # unlike a too-wide row, which a spreadsheet export can produce + # with an entirely blank overflow). + is_short = any(record.get(header) is None for header in headers) overflow_has_value = any((cell or "").strip() for cell in record.get(None) or []) - if overflow_has_value: - extra_cells.append(row_number) + if is_short or overflow_has_value: + wrong_cell_count.append(row_number) continue if _is_blank_row(record, headers): continue @@ -122,17 +127,23 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: if accession is None or sample_type is None: continue rows.append(_build_row(record, row_number, metadata_columns, accession, sample_type)) - if not rows and not extra_cells and not missing_accession and not missing_sample_type: + if not rows and not wrong_cell_count and not missing_accession and not missing_sample_type: raise CliUsageError(f"Accession sheet has no rows: {path}.") - if extra_cells or missing_accession or missing_sample_type: + if wrong_cell_count or missing_accession or missing_sample_type: clauses = [ clause for clause in ( - _row_count_clause("more cells than the header", extra_cells), - _row_count_clause("no accession", missing_accession), - _row_count_clause("no sample_type", missing_sample_type), + _row_reason_clause("a different number of cells than the header", wrong_cell_count), + _row_reason_clause("no accession", missing_accession), + _row_reason_clause("no sample_type", missing_sample_type), ) if clause is not None ] - raise CliUsageError(f"Accession sheet {'; '.join(clauses)}: {path}.") + message = f"Accession sheet {'; '.join(clauses)}: {path}." + if wrong_cell_count: + message += ( + " If a value legitimately contains a comma, quote it; otherwise remove " + "the extra cell(s), or fill in the missing one(s)." + ) + raise CliUsageError(message) return AccessionSheet(path=path, rows=rows) @@ -172,7 +183,7 @@ def _is_blank_row(record: dict[str, str], headers: Sequence[str]) -> bool: return not any(_cell(record, header) for header in headers) -def _row_count_clause(reason: str, rows: list[int]) -> str | None: +def _row_reason_clause(reason: str, rows: list[int]) -> str | None: if not rows: return None numbers = ", ".join(str(number) for number in rows) diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index 258f0d8..dfcac01 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -629,8 +629,9 @@ def _import_command( :param output: The result/error renderer. :returns: :attr:`ExitCode.SUCCESS` once the job has been kicked off. :raises CliUsageError: If the sheet is not a readable ``.csv``, has an - unnamed or duplicated column, has no rows, has a row with more - cells than the header, or has a row with no accession or no + unnamed or duplicated column, has no rows, has a row with fewer + cells than the header or with more cells than the header where the + overflow isn't blank, or has a row with no accession or no sample_type. """ sheet = parse_accession_sheet(args.sheet) diff --git a/source/cli.rst b/source/cli.rst index 15e0999..7c7f0fc 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -398,11 +398,13 @@ rejects up front rather than guessing: a trailing comma in the header row (or a copy-pasted column) is rejected rather than becoming an empty-named metadata attribute, or one column silently overwriting another. -- No data row may have more cells than the header — an extra cell (e.g. one - stray comma) is rejected rather than dropping the overflow, or silently - skipping the row if that leaves every named cell blank. A row with - *fewer* cells than the header has its missing trailing cells treated as - blank, the same as an empty cell. +- A data row must have at least as many cells as the header — a *short* + row (e.g. trailing optional columns omitted entirely, rather than left + as empty cells) is always rejected, not treated as having blank values. + A row with *more* cells than the header is rejected only if the overflow + carries a value (e.g. an unquoted comma inside a value shifting a real + value into the extra cell); a purely blank overflow (e.g. one stray + trailing comma) is dropped silently. - Every row must have an accession and a sample type; a blank cell in either column rejects the whole sheet. diff --git a/tests/unit/cli/test_accession_sheet.py b/tests/unit/cli/test_accession_sheet.py index 3957e30..9ab72e5 100644 --- a/tests/unit/cli/test_accession_sheet.py +++ b/tests/unit/cli/test_accession_sheet.py @@ -52,16 +52,16 @@ def test_name_and_organism_are_optional(self, tmp_path: Path) -> None: assert sheet.rows[0].name is None assert sheet.rows[0].organism is None - def test_row_shorter_than_the_header_has_missing_trailing_cells_treated_as_blank( + def test_row_shorter_than_the_header_is_usage_error( self, tmp_path: Path, ) -> None: path = tmp_path / "sheet.csv" path.write_text("accession,sample_type,name,organism\nERR1,rna_seq\n") - sheet = parse_accession_sheet(path) - - assert sheet.rows[0].name is None - assert sheet.rows[0].organism is None + with pytest.raises( + CliUsageError, match=r"data row\(s\) 1 has a different number of cells than the header", + ): + parse_accession_sheet(path) def test_name_and_organism_are_parsed_when_present(self, tmp_path: Path) -> None: sheet = parse_accession_sheet(_write_sheet( @@ -158,7 +158,9 @@ def test_row_with_more_cells_than_the_header_is_usage_error(self, tmp_path: Path path = tmp_path / "sheet.csv" path.write_text("accession,sample_type,name\nERR1,rna_seq,liver_r1,note\n") - with pytest.raises(CliUsageError, match=r"data row\(s\) 1 has more cells than the header"): + with pytest.raises( + CliUsageError, match=r"data row\(s\) 1 has a different number of cells than the header", + ): parse_accession_sheet(path) def test_shifted_right_row_with_a_real_value_is_usage_error_not_skipped( @@ -169,7 +171,9 @@ def test_shifted_right_row_with_a_real_value_is_usage_error_not_skipped( path = tmp_path / "sheet.csv" path.write_text("accession,sample_type,name\n,,,ERR1\n") - with pytest.raises(CliUsageError, match=r"data row\(s\) 1 has more cells than the header"): + with pytest.raises( + CliUsageError, match=r"data row\(s\) 1 has a different number of cells than the header", + ): parse_accession_sheet(path) def test_wholly_blank_row_with_extra_trailing_comma_is_still_skipped( @@ -185,6 +189,21 @@ def test_wholly_blank_row_with_extra_trailing_comma_is_still_skipped( assert [row.accession for row in sheet.rows] == ["ERR1"] + def test_row_with_a_blank_overflow_cell_is_still_accepted( + self, tmp_path: Path, + ) -> None: + # One cell wider than the header, but the overflow is blank and + # every named cell has its normal value — this is harmless trailing + # noise, not the shifted-right case above, where the row's *data* + # spilled into the overflow instead. + path = tmp_path / "sheet.csv" + path.write_text("accession,sample_type,name\nERR1,rna_seq,liver_r1,\n") + + sheet = parse_accession_sheet(path) + + assert sheet.rows[0].accession == "ERR1" + assert sheet.rows[0].name == "liver_r1" + def test_non_csv_xlsx_rejected_with_export_message(self, tmp_path: Path) -> None: xlsx = tmp_path / "sheet.xlsx" xlsx.write_bytes(b"PK") diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index 0563a32..7fa13fa 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -1317,7 +1317,23 @@ def test_row_with_more_cells_than_the_header_is_usage_error( assert result.exit_code == 2 assert route.call_count == 0 - assert "data row(s) 1 has more cells than the header" in result.stderr + assert "data row(s) 1 has a different number of cells than the header" in result.stderr + + @respx.mock + def test_row_with_fewer_cells_than_the_header_is_usage_error( + self, run_cli, tmp_path: Path, + ) -> None: + route = respx.post(SAMPLE_IMPORTS_URL) + sheet = tmp_path / "accessions.csv" + sheet.write_text("accession,sample_type,name,organism\nERR1,rna_seq\n") + + result = run_cli( + "--token", TOKEN, "samples", "import", "--sheet", str(sheet), + ) + + assert result.exit_code == 2 + assert route.call_count == 0 + assert "data row(s) 1 has a different number of cells than the header" in result.stderr class TestSamplesImportStatus: From 40caadba4e64ea0ab2e2f8dae0104c6a1dba0a58 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Jul 2026 16:08:29 +0000 Subject: [PATCH 27/32] fix(cli): skip a short wholly-blank row instead of rejecting it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-20 automated review: the row-width check ran before the blank-row check, so a trailing comma-only line below the data (e.g. spreadsheet noise) was silently skipped at the header's exact width or wider, but became a hard usage error if it happened to have fewer commas than the header — contradicting the docs, which promise any wholly-blank line is skipped regardless of shape. Reordered so the blank check (now aware of the overflow too) runs first; a short row is still rejected outright if it has any data, and a shifted-right row with a real value in its overflow is still rejected, same as before. Also split the combined 'wrong cell count' error into distinct short-row and over-wide-row buckets, each with its own clause and its own remedy — previously a short row's error carried remove-the-extra-cell advice that could never apply to it, and there was no way to tell which kind of row was which in a mixed sheet. Plus: a one-line docstring on _import_spec_fields recording that it sends every dataclass field verbatim except the three it renames/filters (so a newly-added optional field isn't silently sent as null without a second thought); 'is dropped silently' -> 'is ignored' in cli.rst (was ambiguous about whether the row or the cell was being dropped); trimmed an overly-mechanical comment the operator flagged as unnecessary. Linear: FLOW-689 --- flowbio/cli/_accession_sheet.py | 49 +++++++++++++++----------- flowbio/v2/samples.py | 7 ++++ source/cli.rst | 25 ++++++------- tests/unit/cli/test_accession_sheet.py | 38 +++++++++++++++++--- tests/unit/cli/test_samples.py | 4 +-- 5 files changed, 83 insertions(+), 40 deletions(-) diff --git a/flowbio/cli/_accession_sheet.py b/flowbio/cli/_accession_sheet.py index 2c84621..1c37765 100644 --- a/flowbio/cli/_accession_sheet.py +++ b/flowbio/cli/_accession_sheet.py @@ -88,7 +88,8 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: ) existing_file(path) rows: list[AccessionSheetRow] = [] - wrong_cell_count: list[int] = [] + short_rows: list[int] = [] + overflow_rows: list[int] = [] missing_accession: list[int] = [] missing_sample_type: list[int] = [] # utf-8-sig transparently strips a leading BOM, which spreadsheet tools @@ -103,20 +104,17 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: header for header in headers if header not in RESERVED_COLUMNS ] for row_number, record in enumerate(reader, start=1): - # csv.DictReader stores a row with more cells than the header - # under the None key, and fills a row with fewer from missing - # keys with None of its own — neither can be attributed to a - # column, so a row whose cell count doesn't match the header is - # rejected rather than guessed at. A short row is rejected - # outright (there's no benign "trailing noise" story for one — - # unlike a too-wide row, which a spreadsheet export can produce - # with an entirely blank overflow). - is_short = any(record.get(header) is None for header in headers) overflow_has_value = any((cell or "").strip() for cell in record.get(None) or []) - if is_short or overflow_has_value: - wrong_cell_count.append(row_number) + if not overflow_has_value and _is_blank_row(record, headers): continue - if _is_blank_row(record, headers): + # A short row's missing cells could just as easily be trailing + # optional columns as data landing in the wrong place, so — + # unlike a too-wide row's blank overflow — it's rejected outright. + if any(record.get(header) is None for header in headers): + short_rows.append(row_number) + continue + if overflow_has_value: + overflow_rows.append(row_number) continue accession = _cell(record, "accession") sample_type = _cell(record, "sample_type") @@ -127,22 +125,31 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: if accession is None or sample_type is None: continue rows.append(_build_row(record, row_number, metadata_columns, accession, sample_type)) - if not rows and not wrong_cell_count and not missing_accession and not missing_sample_type: + sheet_is_empty = ( + not rows and not short_rows and not overflow_rows + and not missing_accession and not missing_sample_type + ) + if sheet_is_empty: raise CliUsageError(f"Accession sheet has no rows: {path}.") - if wrong_cell_count or missing_accession or missing_sample_type: + if short_rows or overflow_rows or missing_accession or missing_sample_type: clauses = [ clause for clause in ( - _row_reason_clause("a different number of cells than the header", wrong_cell_count), + _row_reason_clause("fewer cells than the header", short_rows), + _row_reason_clause("more cells than the header", overflow_rows), _row_reason_clause("no accession", missing_accession), _row_reason_clause("no sample_type", missing_sample_type), ) if clause is not None ] message = f"Accession sheet {'; '.join(clauses)}: {path}." - if wrong_cell_count: - message += ( - " If a value legitimately contains a comma, quote it; otherwise remove " - "the extra cell(s), or fill in the missing one(s)." + remedies: list[str] = [] + if short_rows: + remedies.append("fill in the missing cell(s), or remove the row") + if overflow_rows: + remedies.append( + "if a value legitimately contains a comma, quote it; otherwise " + "remove the extra cell(s)", ) + message += "".join(f" {remedy[0].upper()}{remedy[1:]}." for remedy in remedies) raise CliUsageError(message) return AccessionSheet(path=path, rows=rows) @@ -179,7 +186,7 @@ def _duplicate_columns_clause(duplicates: list[str]) -> str: return f"column name(s) {names} {verb} duplicated" -def _is_blank_row(record: dict[str, str], headers: Sequence[str]) -> bool: +def _is_blank_row(record: dict[str, str], headers: list[str]) -> bool: return not any(_cell(record, header) for header in headers) diff --git a/flowbio/v2/samples.py b/flowbio/v2/samples.py index 1cad297..75e31b6 100644 --- a/flowbio/v2/samples.py +++ b/flowbio/v2/samples.py @@ -527,6 +527,13 @@ def get_import(self, job_id: SampleImportJobId) -> SampleImportJob: @staticmethod def _import_spec_fields(spec: SampleImportSpec) -> dict: + """Build the wire payload for one accession. + + Every field is sent under its dataclass name as-is; only ``name``, + ``organism`` (renamed from ``organism_id``), and ``metadata`` are + omitted when empty. A field added to :class:`SampleImportSpec` is + sent even when unset unless it's also added to ``optional`` here. + """ fields = asdict(spec) organism = fields.pop("organism_id") optional = {"name": fields.pop("name"), "organism": organism, "metadata": fields.pop("metadata")} diff --git a/source/cli.rst b/source/cli.rst index 7c7f0fc..eb1f1f2 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -398,21 +398,22 @@ rejects up front rather than guessing: a trailing comma in the header row (or a copy-pasted column) is rejected rather than becoming an empty-named metadata attribute, or one column silently overwriting another. -- A data row must have at least as many cells as the header — a *short* - row (e.g. trailing optional columns omitted entirely, rather than left - as empty cells) is always rejected, not treated as having blank values. - A row with *more* cells than the header is rejected only if the overflow - carries a value (e.g. an unquoted comma inside a value shifting a real - value into the extra cell); a purely blank overflow (e.g. one stray - trailing comma) is dropped silently. +- A data row with data in it must have as many cells as the header — a + *short* row (e.g. trailing optional columns omitted entirely, rather + than left as empty cells) is rejected, not treated as having blank + values. A row with *more* cells than the header is rejected only if the + overflow carries a value (e.g. an unquoted comma inside a value shifting + a real value into the extra cell); a purely blank overflow (e.g. one + stray trailing comma) is ignored. - Every row must have an accession and a sample type; a blank cell in either column rejects the whole sheet. -A row with every cell blank (e.g. a trailing comma-only line some -spreadsheet exports leave below the data) is skipped rather than treated -as a row missing values. Rows are counted from ``1`` for the first data -row, after the header (the same convention as ``upload-batch``'s -``row_number``). +A row with every cell blank — whatever its width, including one with fewer +cells than the header (e.g. a trailing comma-only line some spreadsheet +exports leave below the data, however many commas it happens to have) — is +skipped rather than treated as a row missing values. Rows are counted from +``1`` for the first data row, after the header (the same convention as +``upload-batch``'s ``row_number``). Unlike ``upload-batch``, a metadata column named ``__annotation`` is **not** given any special handling here — it is forwarded as an ordinary diff --git a/tests/unit/cli/test_accession_sheet.py b/tests/unit/cli/test_accession_sheet.py index 9ab72e5..aea0ccb 100644 --- a/tests/unit/cli/test_accession_sheet.py +++ b/tests/unit/cli/test_accession_sheet.py @@ -59,11 +59,9 @@ def test_row_shorter_than_the_header_is_usage_error( path.write_text("accession,sample_type,name,organism\nERR1,rna_seq\n") with pytest.raises( - CliUsageError, match=r"data row\(s\) 1 has a different number of cells than the header", + CliUsageError, match=r"data row\(s\) 1 has fewer cells than the header", ): parse_accession_sheet(path) - - def test_name_and_organism_are_parsed_when_present(self, tmp_path: Path) -> None: sheet = parse_accession_sheet(_write_sheet( tmp_path, _record(name="liver_r1", organism="Hs"), @@ -159,7 +157,7 @@ def test_row_with_more_cells_than_the_header_is_usage_error(self, tmp_path: Path path.write_text("accession,sample_type,name\nERR1,rna_seq,liver_r1,note\n") with pytest.raises( - CliUsageError, match=r"data row\(s\) 1 has a different number of cells than the header", + CliUsageError, match=r"data row\(s\) 1 has more cells than the header", ): parse_accession_sheet(path) @@ -172,7 +170,7 @@ def test_shifted_right_row_with_a_real_value_is_usage_error_not_skipped( path.write_text("accession,sample_type,name\n,,,ERR1\n") with pytest.raises( - CliUsageError, match=r"data row\(s\) 1 has a different number of cells than the header", + CliUsageError, match=r"data row\(s\) 1 has more cells than the header", ): parse_accession_sheet(path) @@ -204,6 +202,36 @@ def test_row_with_a_blank_overflow_cell_is_still_accepted( assert sheet.rows[0].accession == "ERR1" assert sheet.rows[0].name == "liver_r1" + def test_wholly_blank_row_shorter_than_the_header_is_still_skipped( + self, tmp_path: Path, + ) -> None: + # A blank trailing line can have fewer commas than the header width + # too (some spreadsheet exports trim trailing empty cells) — this + # must be treated the same as the exact-width blank-row case above, + # not as a short row with data that can't be attributed to a column. + path = tmp_path / "sheet.csv" + path.write_text("accession,sample_type,name\nERR1,rna_seq,liver_r1\n,\n") + + sheet = parse_accession_sheet(path) + + assert [row.accession for row in sheet.rows] == ["ERR1"] + + def test_sheet_with_both_a_short_and_an_over_wide_row_reports_both( + self, tmp_path: Path, + ) -> None: + path = tmp_path / "sheet.csv" + path.write_text( + "accession,sample_type,name\nERR1,rna_seq\nERR2,rna_seq,liver_r1,note\n", + ) + + with pytest.raises(CliUsageError) as excinfo: + parse_accession_sheet(path) + + assert "data row(s) 1 has fewer cells than the header" in str(excinfo.value) + assert "data row(s) 2 has more cells than the header" in str(excinfo.value) + assert "Fill in the missing cell(s), or remove the row" in str(excinfo.value) + assert "quote it; otherwise remove the extra cell(s)" in str(excinfo.value) + def test_non_csv_xlsx_rejected_with_export_message(self, tmp_path: Path) -> None: xlsx = tmp_path / "sheet.xlsx" xlsx.write_bytes(b"PK") diff --git a/tests/unit/cli/test_samples.py b/tests/unit/cli/test_samples.py index 7fa13fa..bdd8fb5 100644 --- a/tests/unit/cli/test_samples.py +++ b/tests/unit/cli/test_samples.py @@ -1317,7 +1317,7 @@ def test_row_with_more_cells_than_the_header_is_usage_error( assert result.exit_code == 2 assert route.call_count == 0 - assert "data row(s) 1 has a different number of cells than the header" in result.stderr + assert "data row(s) 1 has more cells than the header" in result.stderr @respx.mock def test_row_with_fewer_cells_than_the_header_is_usage_error( @@ -1333,7 +1333,7 @@ def test_row_with_fewer_cells_than_the_header_is_usage_error( assert result.exit_code == 2 assert route.call_count == 0 - assert "data row(s) 1 has a different number of cells than the header" in result.stderr + assert "data row(s) 1 has fewer cells than the header" in result.stderr class TestSamplesImportStatus: From b990dc393201112b80f5aa17414730d0aecea7ee Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Jul 2026 16:23:08 +0000 Subject: [PATCH 28/32] fix(cli): reject a sheet whose header row is blank Round-20 review, finding 1: a blank first line parses via csv.DictReader as a present-but-empty fieldnames list, distinct from an empty file (fieldnames=None). Left unguarded, every data row's cells landed under the overflow restkey against a zero-column header, so the sheet was rejected as 'more cells than the header' with a quote-your-commas remedy that pointed away from the actual problem. Now raised explicitly as 'has no header row' before that can happen; the true-empty-file case is unaffected and still reaches the existing 'has no rows' check. Also, from the same round: - accurate record typing: record's cells really are 'str | None' (None for a short row) with an overflow 'list[str]' under the restkey None, which the dict[str, str] annotations on _cell/_is_blank_row/_build_row denied even though the short/overflow checks built on top of them now depend on exactly that shape. Introduced a ParsedRow alias and an _overflow_cells helper so the types match what the code actually reads. - parameterised _import_spec_fields's return type, now that its neighbour _build_sample_fields is the only other bare dict return in the class. - reworded the short-row remedy: the likeliest cause is a hand-authored row that omitted trailing optional columns entirely (as cli.rst already says), not one missing a value it was supposed to have. - collapsed _import_command's duplicated :raises: list into a cross- reference to parse_accession_sheet, the copy most likely to drift. - _check_headers's remedy join ('; and') read oddly with two clauses; now two plain sentences like the row-level remedies. - _check_headers takes list[str] now, matching every other boundary in the module. Linear: FLOW-689 --- flowbio/cli/_accession_sheet.py | 51 ++++++++++++++++++-------- flowbio/cli/_samples.py | 7 +--- flowbio/v2/samples.py | 2 +- tests/unit/cli/test_accession_sheet.py | 12 +++++- 4 files changed, 49 insertions(+), 23 deletions(-) diff --git a/flowbio/cli/_accession_sheet.py b/flowbio/cli/_accession_sheet.py index 1c37765..6d91e74 100644 --- a/flowbio/cli/_accession_sheet.py +++ b/flowbio/cli/_accession_sheet.py @@ -19,7 +19,7 @@ from __future__ import annotations import csv -from collections.abc import Sequence +from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path @@ -29,6 +29,11 @@ RESERVED_COLUMNS = ("accession", "name", "organism", "sample_type") +# csv.DictReader's row shape: a header's cell, or None if the row was too +# short to reach it; a list[str] of any cells beyond the header, under the +# key None, for a row that was too long. +ParsedRow = Mapping[str | None, str | list[str] | None] + @dataclass(frozen=True) class AccessionSheetRow: @@ -75,11 +80,11 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: dropped, and surrounding whitespace trimmed (including in header names). Values are otherwise passed through unchanged, including ``accession``, sent to the server as-entered. - :raises CliUsageError: If the file is not a readable ``.csv``, has an - unnamed or duplicated column, has no rows, has a row with fewer - cells than the header or with more cells than the header where the - overflow isn't blank, or has a row with no accession or no - sample_type. + :raises CliUsageError: If the file is not a readable ``.csv``, has no + header row, has an unnamed or duplicated column, has no rows, has + a row with fewer cells than the header or with more cells than the + header where the overflow isn't blank, or has a row with no + accession or no sample_type. """ if path.suffix.lower() != ".csv": raise CliUsageError( @@ -97,6 +102,12 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: # parses as "accession" and every row reports a missing accession. with path.open(newline="", encoding="utf-8-sig") as handle: reader = csv.DictReader(handle) + # A present-but-empty fieldnames list means the first line was + # blank, not absent (an empty file gives fieldnames=None instead, + # caught by the "no rows" check below); left unchecked, every data + # row would overflow a zero-column header instead. + if reader.fieldnames == []: + raise CliUsageError(f"Accession sheet has no header row: {path}.") headers = [header.strip() for header in reader.fieldnames or []] reader.fieldnames = headers _check_headers(headers, path) @@ -104,7 +115,7 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: header for header in headers if header not in RESERVED_COLUMNS ] for row_number, record in enumerate(reader, start=1): - overflow_has_value = any((cell or "").strip() for cell in record.get(None) or []) + overflow_has_value = any(cell.strip() for cell in _overflow_cells(record)) if not overflow_has_value and _is_blank_row(record, headers): continue # A short row's missing cells could just as easily be trailing @@ -143,7 +154,10 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: message = f"Accession sheet {'; '.join(clauses)}: {path}." remedies: list[str] = [] if short_rows: - remedies.append("fill in the missing cell(s), or remove the row") + remedies.append( + "add the missing trailing comma(s), leaving the cell(s) blank if that " + "column doesn't apply to this row, or fill in a value", + ) if overflow_rows: remedies.append( "if a value legitimately contains a comma, quote it; otherwise " @@ -154,7 +168,7 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: return AccessionSheet(path=path, rows=rows) -def _check_headers(headers: Sequence[str], path: Path) -> None: +def _check_headers(headers: list[str], path: Path) -> None: unnamed = [position for position, header in enumerate(headers, start=1) if not header] duplicates = sorted({header for header in headers if header and headers.count(header) > 1}) if not unnamed and not duplicates: @@ -167,10 +181,9 @@ def _check_headers(headers: Sequence[str], path: Path) -> None: if duplicates: clauses.append(_duplicate_columns_clause(duplicates)) remedies.append("rename the repeated column(s) so each column is unique") - remedy = "; and ".join(remedies) + remedy = ". ".join(f"{r[0].upper()}{r[1:]}" for r in remedies) raise CliUsageError( - f"Accession sheet {'; '.join(clauses)}: {path}. " - f"{remedy[0].upper()}{remedy[1:]}.", + f"Accession sheet {'; '.join(clauses)}: {path}. {remedy}.", ) @@ -186,10 +199,15 @@ def _duplicate_columns_clause(duplicates: list[str]) -> str: return f"column name(s) {names} {verb} duplicated" -def _is_blank_row(record: dict[str, str], headers: list[str]) -> bool: +def _is_blank_row(record: ParsedRow, headers: list[str]) -> bool: return not any(_cell(record, header) for header in headers) +def _overflow_cells(record: ParsedRow) -> list[str]: + overflow = record.get(None) + return overflow if isinstance(overflow, list) else [] + + def _row_reason_clause(reason: str, rows: list[int]) -> str | None: if not rows: return None @@ -198,13 +216,14 @@ def _row_reason_clause(reason: str, rows: list[int]) -> str | None: return f"data row(s) {numbers} {verb} {reason}" -def _cell(record: dict[str, str], column: str) -> str | None: - value = (record.get(column) or "").strip() +def _cell(record: ParsedRow, column: str) -> str | None: + raw = record.get(column) + value = (raw if isinstance(raw, str) else "").strip() return value or None def _build_row( - record: dict[str, str], + record: ParsedRow, row_number: int, metadata_columns: list[str], accession: str, diff --git a/flowbio/cli/_samples.py b/flowbio/cli/_samples.py index dfcac01..e681d32 100644 --- a/flowbio/cli/_samples.py +++ b/flowbio/cli/_samples.py @@ -628,11 +628,8 @@ def _import_command( :param client: The authenticated Flow client. :param output: The result/error renderer. :returns: :attr:`ExitCode.SUCCESS` once the job has been kicked off. - :raises CliUsageError: If the sheet is not a readable ``.csv``, has an - unnamed or duplicated column, has no rows, has a row with fewer - cells than the header or with more cells than the header where the - overflow isn't blank, or has a row with no accession or no - sample_type. + :raises CliUsageError: If the accession sheet is structurally invalid + (see :func:`~flowbio.cli._accession_sheet.parse_accession_sheet`). """ sheet = parse_accession_sheet(args.sheet) specs = [row.to_spec() for row in sheet.rows] diff --git a/flowbio/v2/samples.py b/flowbio/v2/samples.py index 75e31b6..885bcc6 100644 --- a/flowbio/v2/samples.py +++ b/flowbio/v2/samples.py @@ -526,7 +526,7 @@ def get_import(self, job_id: SampleImportJobId) -> SampleImportJob: return SampleImportJob(**self._transport.get(f"/v2/sample-imports/{job_id}")) @staticmethod - def _import_spec_fields(spec: SampleImportSpec) -> dict: + def _import_spec_fields(spec: SampleImportSpec) -> dict[str, str | dict[str, str]]: """Build the wire payload for one accession. Every field is sent under its dataclass name as-is; only ``name``, diff --git a/tests/unit/cli/test_accession_sheet.py b/tests/unit/cli/test_accession_sheet.py index aea0ccb..cfffc2a 100644 --- a/tests/unit/cli/test_accession_sheet.py +++ b/tests/unit/cli/test_accession_sheet.py @@ -229,7 +229,7 @@ def test_sheet_with_both_a_short_and_an_over_wide_row_reports_both( assert "data row(s) 1 has fewer cells than the header" in str(excinfo.value) assert "data row(s) 2 has more cells than the header" in str(excinfo.value) - assert "Fill in the missing cell(s), or remove the row" in str(excinfo.value) + assert "Add the missing trailing comma(s)" in str(excinfo.value) assert "quote it; otherwise remove the extra cell(s)" in str(excinfo.value) def test_non_csv_xlsx_rejected_with_export_message(self, tmp_path: Path) -> None: @@ -261,6 +261,16 @@ def test_wholly_empty_file_is_usage_error(self, tmp_path: Path) -> None: with pytest.raises(CliUsageError, match="no rows"): parse_accession_sheet(path) + def test_blank_first_line_is_usage_error(self, tmp_path: Path) -> None: + # A blank line where the header belongs must not be read as a + # present, zero-column header — that would make every data row's + # cells overflow it instead of reporting the real problem. + path = tmp_path / "sheet.csv" + path.write_text("\nERR1,rna_seq\n") + + with pytest.raises(CliUsageError, match="no header row"): + parse_accession_sheet(path) + def test_row_with_blank_accession_is_usage_error(self, tmp_path: Path) -> None: with pytest.raises(CliUsageError, match=r"data row\(s\) 2 has no accession"): parse_accession_sheet(_write_sheet( From 598dd09a4c6dbec1d40d58f24a4eae5c24aef412 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Jul 2026 16:33:14 +0000 Subject: [PATCH 29/32] docs(cli): note that a wholly-empty line doesn't consume a row number MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-20 review, finding 1: csv.DictReader discards a line with no commas at all before yielding it, so it doesn't get a row number, while a comma-only blank line does (and is then skipped by _is_blank_row) — the two look identical in a spreadsheet but count differently, and row numbers are now the entire payload of four errors. Renumbering from the underlying csv.reader would be more correct but is a bigger, riskier change for what the previous round already treated as an acceptable precision tradeoff (the missing-column message enumerating every row is similarly waived); documenting the gap is the proportionate fix here. Added a test pinning it. Also, from the same round: - _import_spec_fields: named the optional/omit-when-empty field set directly instead of enforcing it via which keys got popped out of a dict literal, so it visibly matches its own docstring. - trimmed the two comments the review flagged as narrating flow rather than stating the one non-obvious fact, and moved ParsedRow's doc into a docstring literal to match its siblings in v2/samples.py. - added a test confirming a quoted comma-containing value is parsed as one cell, which is what the over-wide-row error's remedy recommends doing. Left as-is, explained in the PR: consolidating the two message-builders and three clause helpers (finding 2) and shortening the module docstring now that :raises: repeats it (finding 6) — both flagged as 'nothing wrong today', and the last round of this cap window is not the place to start a broader refactor. Linear: FLOW-689 --- flowbio/cli/_accession_sheet.py | 12 +++++------- flowbio/v2/samples.py | 6 +++--- source/cli.rst | 3 ++- tests/unit/cli/test_accession_sheet.py | 24 ++++++++++++++++++++++++ 4 files changed, 34 insertions(+), 11 deletions(-) diff --git a/flowbio/cli/_accession_sheet.py b/flowbio/cli/_accession_sheet.py index 6d91e74..7076416 100644 --- a/flowbio/cli/_accession_sheet.py +++ b/flowbio/cli/_accession_sheet.py @@ -29,10 +29,10 @@ RESERVED_COLUMNS = ("accession", "name", "organism", "sample_type") -# csv.DictReader's row shape: a header's cell, or None if the row was too -# short to reach it; a list[str] of any cells beyond the header, under the -# key None, for a row that was too long. ParsedRow = Mapping[str | None, str | list[str] | None] +"""One row as ``csv.DictReader`` yields it: a header's cell, or ``None`` if +the row was too short to reach it; the extra cells of a too-long row, as a +``list[str]``, under the key ``None``.""" @dataclass(frozen=True) @@ -102,10 +102,8 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: # parses as "accession" and every row reports a missing accession. with path.open(newline="", encoding="utf-8-sig") as handle: reader = csv.DictReader(handle) - # A present-but-empty fieldnames list means the first line was - # blank, not absent (an empty file gives fieldnames=None instead, - # caught by the "no rows" check below); left unchecked, every data - # row would overflow a zero-column header instead. + # A blank first line parses as fieldnames == [] — distinct from an + # absent one, which is fieldnames is None. if reader.fieldnames == []: raise CliUsageError(f"Accession sheet has no header row: {path}.") headers = [header.strip() for header in reader.fieldnames or []] diff --git a/flowbio/v2/samples.py b/flowbio/v2/samples.py index 885bcc6..e18923e 100644 --- a/flowbio/v2/samples.py +++ b/flowbio/v2/samples.py @@ -535,9 +535,9 @@ def _import_spec_fields(spec: SampleImportSpec) -> dict[str, str | dict[str, str sent even when unset unless it's also added to ``optional`` here. """ fields = asdict(spec) - organism = fields.pop("organism_id") - optional = {"name": fields.pop("name"), "organism": organism, "metadata": fields.pop("metadata")} - return {**fields, **{key: value for key, value in optional.items() if value}} + fields["organism"] = fields.pop("organism_id") + optional = ("name", "organism", "metadata") + return {key: value for key, value in fields.items() if key not in optional or value} def _create_metadata_attribute(self, item: dict) -> MetadataAttribute: item["required_for_sample_types"] = [ diff --git a/source/cli.rst b/source/cli.rst index eb1f1f2..7d7911b 100644 --- a/source/cli.rst +++ b/source/cli.rst @@ -413,7 +413,8 @@ cells than the header (e.g. a trailing comma-only line some spreadsheet exports leave below the data, however many commas it happens to have) — is skipped rather than treated as a row missing values. Rows are counted from ``1`` for the first data row, after the header (the same convention as -``upload-batch``'s ``row_number``). +``upload-batch``'s ``row_number``) — except a line with no commas at all +(as opposed to one with only blank cells), which doesn't consume a number. Unlike ``upload-batch``, a metadata column named ``__annotation`` is **not** given any special handling here — it is forwarded as an ordinary diff --git a/tests/unit/cli/test_accession_sheet.py b/tests/unit/cli/test_accession_sheet.py index cfffc2a..31e0e2f 100644 --- a/tests/unit/cli/test_accession_sheet.py +++ b/tests/unit/cli/test_accession_sheet.py @@ -202,6 +202,18 @@ def test_row_with_a_blank_overflow_cell_is_still_accepted( assert sheet.rows[0].accession == "ERR1" assert sheet.rows[0].name == "liver_r1" + def test_quoted_value_containing_a_comma_is_not_treated_as_overflow( + self, tmp_path: Path, + ) -> None: + # The over-wide-row error's remedy tells the user to quote a value + # that legitimately contains a comma — confirm that actually works. + path = tmp_path / "sheet.csv" + path.write_text('accession,sample_type,name\nERR1,rna_seq,"liver, left lobe"\n') + + sheet = parse_accession_sheet(path) + + assert sheet.rows[0].name == "liver, left lobe" + def test_wholly_blank_row_shorter_than_the_header_is_still_skipped( self, tmp_path: Path, ) -> None: @@ -351,6 +363,18 @@ def test_blank_row_in_the_middle_does_not_shift_later_row_numbers( _record(accession=""), )) + def test_a_line_with_no_commas_at_all_does_not_consume_a_row_number( + self, tmp_path: Path, + ) -> None: + # Unlike a comma-only blank row (above), a line with nothing on it + # at all isn't yielded as a row by csv.DictReader in the first + # place, so it doesn't take a row number — documented in cli.rst. + path = tmp_path / "sheet.csv" + path.write_text("accession,sample_type\nERR1,rna_seq\n\n,rna_seq\n") + + with pytest.raises(CliUsageError, match=r"data row\(s\) 2 has no accession"): + parse_accession_sheet(path) + def test_row_rejects_empty_accession_by_construction() -> None: with pytest.raises(ValueError, match="accession"): From 0bd4f627fe05f4aeaf8ae6fc15be473ac7484b51 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Jul 2026 11:41:40 +0000 Subject: [PATCH 30/32] test(cli): split test_row_shorter_than_the_header_is_usage_error in two Round-20 finding 1: an earlier edit inserted the short-row rejection check into the middle of what used to be test_name_and_organism_are_parsed instead of adding a sibling test, fusing an unrelated name/organism assertion onto the end of a differently-named test. Both halves already passed, so nothing was broken, but a name/organism regression would have been reported under a test about row width. Split back into two. Linear: FLOW-689 --- tests/unit/cli/test_accession_sheet.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/cli/test_accession_sheet.py b/tests/unit/cli/test_accession_sheet.py index 31e0e2f..51ac7a4 100644 --- a/tests/unit/cli/test_accession_sheet.py +++ b/tests/unit/cli/test_accession_sheet.py @@ -62,6 +62,8 @@ def test_row_shorter_than_the_header_is_usage_error( CliUsageError, match=r"data row\(s\) 1 has fewer cells than the header", ): parse_accession_sheet(path) + + def test_name_and_organism_are_parsed(self, tmp_path: Path) -> None: sheet = parse_accession_sheet(_write_sheet( tmp_path, _record(name="liver_r1", organism="Hs"), From 7e9bfdf762fbda7e99d70d2a383e150156b0f1df Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Jul 2026 11:45:56 +0000 Subject: [PATCH 31/32] refactor(cli): split parse_accession_sheet into three functions Per operator request: the function had grown to ~90 lines across four concerns (suffix/existence checks, header validation, the per-row classification loop, and building the combined structural error). Extracted the header-validation block into _read_header (returns the header and metadata columns, raising on a missing/blank header row or a header problem) and the post-loop 'sheet has no rows, or N rows failed a check' logic into _check_rows, leaving parse_accession_sheet as the suffix check plus the row loop. No behavioural change: every raised message, accepted/rejected row shape, and existing test is unchanged (392 passed, same as before this commit). Linear: FLOW-689 --- flowbio/cli/_accession_sheet.py | 91 +++++++++++++++++++-------------- 1 file changed, 52 insertions(+), 39 deletions(-) diff --git a/flowbio/cli/_accession_sheet.py b/flowbio/cli/_accession_sheet.py index 7076416..8f17173 100644 --- a/flowbio/cli/_accession_sheet.py +++ b/flowbio/cli/_accession_sheet.py @@ -102,16 +102,7 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: # parses as "accession" and every row reports a missing accession. with path.open(newline="", encoding="utf-8-sig") as handle: reader = csv.DictReader(handle) - # A blank first line parses as fieldnames == [] — distinct from an - # absent one, which is fieldnames is None. - if reader.fieldnames == []: - raise CliUsageError(f"Accession sheet has no header row: {path}.") - headers = [header.strip() for header in reader.fieldnames or []] - reader.fieldnames = headers - _check_headers(headers, path) - metadata_columns = [ - header for header in headers if header not in RESERVED_COLUMNS - ] + headers, metadata_columns = _read_header(reader, path) for row_number, record in enumerate(reader, start=1): overflow_has_value = any(cell.strip() for cell in _overflow_cells(record)) if not overflow_has_value and _is_blank_row(record, headers): @@ -134,38 +125,60 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: if accession is None or sample_type is None: continue rows.append(_build_row(record, row_number, metadata_columns, accession, sample_type)) - sheet_is_empty = ( - not rows and not short_rows and not overflow_rows - and not missing_accession and not missing_sample_type - ) - if sheet_is_empty: - raise CliUsageError(f"Accession sheet has no rows: {path}.") - if short_rows or overflow_rows or missing_accession or missing_sample_type: - clauses = [ - clause for clause in ( - _row_reason_clause("fewer cells than the header", short_rows), - _row_reason_clause("more cells than the header", overflow_rows), - _row_reason_clause("no accession", missing_accession), - _row_reason_clause("no sample_type", missing_sample_type), - ) if clause is not None - ] - message = f"Accession sheet {'; '.join(clauses)}: {path}." - remedies: list[str] = [] - if short_rows: - remedies.append( - "add the missing trailing comma(s), leaving the cell(s) blank if that " - "column doesn't apply to this row, or fill in a value", - ) - if overflow_rows: - remedies.append( - "if a value legitimately contains a comma, quote it; otherwise " - "remove the extra cell(s)", - ) - message += "".join(f" {remedy[0].upper()}{remedy[1:]}." for remedy in remedies) - raise CliUsageError(message) + _check_rows(path, rows, short_rows, overflow_rows, missing_accession, missing_sample_type) return AccessionSheet(path=path, rows=rows) +def _read_header(reader: csv.DictReader[str], path: Path) -> tuple[list[str], list[str]]: + """Validate the sheet's header row and return its columns and metadata columns.""" + # A blank first line parses as fieldnames == [] — distinct from an + # absent one, which is fieldnames is None. + if reader.fieldnames == []: + raise CliUsageError(f"Accession sheet has no header row: {path}.") + headers = [header.strip() for header in reader.fieldnames or []] + reader.fieldnames = headers + _check_headers(headers, path) + metadata_columns = [header for header in headers if header not in RESERVED_COLUMNS] + return headers, metadata_columns + + +def _check_rows( + path: Path, + rows: list[AccessionSheetRow], + short_rows: list[int], + overflow_rows: list[int], + missing_accession: list[int], + missing_sample_type: list[int], +) -> None: + """Raise if the sheet had no rows at all, or any row failed a structural check.""" + if not (rows or short_rows or overflow_rows or missing_accession or missing_sample_type): + raise CliUsageError(f"Accession sheet has no rows: {path}.") + if not (short_rows or overflow_rows or missing_accession or missing_sample_type): + return + clauses = [ + clause for clause in ( + _row_reason_clause("fewer cells than the header", short_rows), + _row_reason_clause("more cells than the header", overflow_rows), + _row_reason_clause("no accession", missing_accession), + _row_reason_clause("no sample_type", missing_sample_type), + ) if clause is not None + ] + message = f"Accession sheet {'; '.join(clauses)}: {path}." + remedies: list[str] = [] + if short_rows: + remedies.append( + "add the missing trailing comma(s), leaving the cell(s) blank if that " + "column doesn't apply to this row, or fill in a value", + ) + if overflow_rows: + remedies.append( + "if a value legitimately contains a comma, quote it; otherwise " + "remove the extra cell(s)", + ) + message += "".join(f" {remedy[0].upper()}{remedy[1:]}." for remedy in remedies) + raise CliUsageError(message) + + def _check_headers(headers: list[str], path: Path) -> None: unnamed = [position for position, header in enumerate(headers, start=1) if not header] duplicates = sorted({header for header in headers if header and headers.count(header) > 1}) From 63944a8b71a0217d3eb5d7b2f070f68d175030c9 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Jul 2026 12:04:53 +0000 Subject: [PATCH 32/32] refactor(cli): drop the redundant path parameter from error helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per operator review: _read_header, _check_rows, and (necessarily, since _read_header calls it) _check_headers only ever used path to echo it back in their own error message — which is already the file the user passed via --sheet, so printing it back adds nothing. Dropped the parameter from all three and the ': ' from their messages. parse_accession_sheet itself still takes and uses path (for the .csv suffix check, existing_file, and opening the file) and still reports it in its own '.csv file' message, which wasn't part of this request. No test asserted on the path appearing in any of these messages, so nothing needed updating; 392 passed. Linear: FLOW-689 --- flowbio/cli/_accession_sheet.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/flowbio/cli/_accession_sheet.py b/flowbio/cli/_accession_sheet.py index 8f17173..866f484 100644 --- a/flowbio/cli/_accession_sheet.py +++ b/flowbio/cli/_accession_sheet.py @@ -102,7 +102,7 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: # parses as "accession" and every row reports a missing accession. with path.open(newline="", encoding="utf-8-sig") as handle: reader = csv.DictReader(handle) - headers, metadata_columns = _read_header(reader, path) + headers, metadata_columns = _read_header(reader) for row_number, record in enumerate(reader, start=1): overflow_has_value = any(cell.strip() for cell in _overflow_cells(record)) if not overflow_has_value and _is_blank_row(record, headers): @@ -125,25 +125,24 @@ def parse_accession_sheet(path: Path) -> AccessionSheet: if accession is None or sample_type is None: continue rows.append(_build_row(record, row_number, metadata_columns, accession, sample_type)) - _check_rows(path, rows, short_rows, overflow_rows, missing_accession, missing_sample_type) + _check_rows(rows, short_rows, overflow_rows, missing_accession, missing_sample_type) return AccessionSheet(path=path, rows=rows) -def _read_header(reader: csv.DictReader[str], path: Path) -> tuple[list[str], list[str]]: +def _read_header(reader: csv.DictReader[str]) -> tuple[list[str], list[str]]: """Validate the sheet's header row and return its columns and metadata columns.""" # A blank first line parses as fieldnames == [] — distinct from an # absent one, which is fieldnames is None. if reader.fieldnames == []: - raise CliUsageError(f"Accession sheet has no header row: {path}.") + raise CliUsageError("Accession sheet has no header row.") headers = [header.strip() for header in reader.fieldnames or []] reader.fieldnames = headers - _check_headers(headers, path) + _check_headers(headers) metadata_columns = [header for header in headers if header not in RESERVED_COLUMNS] return headers, metadata_columns def _check_rows( - path: Path, rows: list[AccessionSheetRow], short_rows: list[int], overflow_rows: list[int], @@ -152,7 +151,7 @@ def _check_rows( ) -> None: """Raise if the sheet had no rows at all, or any row failed a structural check.""" if not (rows or short_rows or overflow_rows or missing_accession or missing_sample_type): - raise CliUsageError(f"Accession sheet has no rows: {path}.") + raise CliUsageError("Accession sheet has no rows.") if not (short_rows or overflow_rows or missing_accession or missing_sample_type): return clauses = [ @@ -163,7 +162,7 @@ def _check_rows( _row_reason_clause("no sample_type", missing_sample_type), ) if clause is not None ] - message = f"Accession sheet {'; '.join(clauses)}: {path}." + message = f"Accession sheet {'; '.join(clauses)}." remedies: list[str] = [] if short_rows: remedies.append( @@ -179,7 +178,7 @@ def _check_rows( raise CliUsageError(message) -def _check_headers(headers: list[str], path: Path) -> None: +def _check_headers(headers: list[str]) -> None: unnamed = [position for position, header in enumerate(headers, start=1) if not header] duplicates = sorted({header for header in headers if header and headers.count(header) > 1}) if not unnamed and not duplicates: @@ -193,9 +192,7 @@ def _check_headers(headers: list[str], path: Path) -> None: clauses.append(_duplicate_columns_clause(duplicates)) remedies.append("rename the repeated column(s) so each column is unique") remedy = ". ".join(f"{r[0].upper()}{r[1:]}" for r in remedies) - raise CliUsageError( - f"Accession sheet {'; '.join(clauses)}: {path}. {remedy}.", - ) + raise CliUsageError(f"Accession sheet {'; '.join(clauses)}. {remedy}.") def _unnamed_columns_clause(unnamed: list[int]) -> str: