diff --git a/docs/usage.md b/docs/usage.md index e3885f1..a2be905 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -239,7 +239,74 @@ pridepy download-px-raw-files \ | --- | --- | --- | | `-a, --accession` | ProteomeXchange accession (e.g. `PXD039236`). `--px` is a deprecated alias | required | | `-o, --output-folder` | Destination directory | required | +| `-p, --protocol` | Transfer protocol: `ftp`, `aspera`, `globus`, `s3` (FTP-first with fallback) | `ftp` | +| `-w, --parallel-files` | Download 1–32 files concurrently (across-file concurrency) | `1` | +| `-t, --threads` | Parallel HTTP Range threads per file (1–32) for fast per-file downloads | `1` | | `--skip-if-downloaded-already` | Skip files already present locally | off | +| `--preserve-structure` | Recreate the dataset's subdirectory layout under the output folder | off | +| `--iprox-user` | Your registered iProX username (required with `--protocol aspera`; env fallback: `IPROX_USER`) | — | +| `--aspera-key` | Path to an Aspera private key for iProX (optional with `--protocol aspera`; env fallback: `IPROX_ASPERA_KEY`) | — | + +iProX Aspera defaults to password authentication (your iProX account +password), supplied via the `IPROX_ASPERA_PASSWORD` env var or an +interactive hidden prompt. Pass `--aspera-key` instead if you have a +registered Aspera private key. Exactly one credential is required — +`pridepy` fails fast rather than letting `ascp` block on an interactive +prompt, so batch/sbatch jobs must set `IPROX_ASPERA_PASSWORD` (or +`--aspera-key`) up front. + +### Fast downloads: parallel files and per-file segments + +Combine `-w` (files in parallel) and `-t` (Range segments per file) for fast bulk downloads. +The total concurrent connections is approximately `parallel_files × threads`. + +**Parallel across files (recommended for most users, no account required):** + +```bash +# Download up to 8 files concurrently from ProteomeXchange +pridepy download-px-raw-files \ + -a PXD077178 \ + -o ./PXD077178 \ + -w 8 +``` + +**Combine parallel files with per-file segments:** + +```bash +# Download 8 files in parallel, each split into 4 Range segments +pridepy download-px-raw-files \ + -a PXD077178 \ + -o ./out \ + -w 8 \ + -t 4 +``` + +### Fast downloads with iProX Aspera (account required) + +iProX offers Aspera for very large bulk transfers. Aspera is faster than HTTP +on high-bandwidth connections but requires an iProX account. Combine +`--protocol aspera` with `--iprox-user` and a credential — either the +`IPROX_ASPERA_PASSWORD` env var (password auth, the default) or +`--aspera-key` (a registered Aspera private key): + +```bash +# password auth (env var — safe for sbatch; no prompt/hang) +IPROX_ASPERA_PASSWORD=... pridepy download-px-raw-files -a PXD077178 -o ./out --protocol aspera --iprox-user + +# or with a registered key +pridepy download-px-raw-files -a PXD077178 -o ./out --protocol aspera --iprox-user --aspera-key /path/to/key +``` + +If run interactively without `IPROX_ASPERA_PASSWORD` or `--aspera-key`, +`pridepy` prompts for the password (hidden input). In non-interactive/batch +contexts (e.g. `sbatch`) you must set `IPROX_ASPERA_PASSWORD` or pass +`--aspera-key` up front — otherwise the command errors immediately instead +of hanging on a prompt. + +Aspera transfers a single batched session per dataset (`ascp --file-list`) +and always preserves the iProX `IPX.../IPX.../` source directory tree under +the output folder — `--preserve-structure` / flattening does not apply to +the Aspera path. ### Go directly to the hosting repository (native MassIVE / JPOST / iProX accessions) @@ -268,7 +335,7 @@ How each repository is enumerated: - **MassIVE** walks the FTPS tree at `massive-ftp.ucsd.edu` (the server requires TLS). MassIVE distributes datasets across versioned root directories (`/v01`–`/vNN`); `pridepy` discovers the correct root automatically. If FTP/FTPS is blocked by the network, `pridepy` falls back to HTTPS: it lists the dataset from the GNPS2 file index (`datasetcache.gnps2.org`) and downloads each file from the ProteoSAFe endpoint at `massive.ucsd.edu` (byte-identical to the FTPS copy). - **JPOST** lists files through the JSON PROXI endpoint at `https://repository.jpostdb.org/proxi/datasets/` and downloads from `ftp.jpostdb.org` over plain FTP. The PROXI listing avoids the source-IP connection limit JPOST enforces on FTP. -- **iProX** fetches the dataset's ProteomeXchange XML from `http://download.iprox.org//PX_.xml`, then downloads each referenced file from the same host over anonymous HTTP (with `Range` support for resume). iProX also exposes Aspera (`faspe://`) with username/password for very large bulk transfers; `pridepy` uses the public HTTP endpoint so no iProX credentials are required. +- **iProX** fetches the dataset's ProteomeXchange XML from `http://download.iprox.org//PX_.xml`, then downloads each referenced file from the same host over anonymous HTTP (with `Range` support for resume). iProX also exposes Aspera with username/password for very large bulk transfers; `pridepy` uses the public HTTP endpoint so no iProX credentials are required. `download-all-public-raw-files` retrieves the files stored under the dataset's `raw/` collection. These direct downloads support resume (REST for FTP, diff --git a/pridepy/download/client.py b/pridepy/download/client.py index 5dd10f7..144ad30 100644 --- a/pridepy/download/client.py +++ b/pridepy/download/client.py @@ -318,8 +318,23 @@ def download_px_raw_files( output_folder: str, skip_if_downloaded_already: bool = True, flatten: bool = True, + parallel_files: int = 1, + download_threads: int = 1, + protocol: str = "ftp", + iprox_user: Optional[str] = None, + aspera_key: Optional[str] = None, + aspera_password: Optional[str] = None, ) -> None: """Delegate to :meth:`ProteomeXchangeProvider.download_from_accession_or_url`.""" return ProteomeXchangeProvider().download_from_accession_or_url( - px_id_or_url, output_folder, skip_if_downloaded_already, flatten=flatten + px_id_or_url, + output_folder, + skip_if_downloaded_already, + flatten=flatten, + parallel_files=parallel_files, + download_threads=download_threads, + protocol=protocol, + iprox_user=iprox_user, + aspera_key=aspera_key, + aspera_password=aspera_password, ) diff --git a/pridepy/download/iprox.py b/pridepy/download/iprox.py index db5e7fc..322c8e5 100644 --- a/pridepy/download/iprox.py +++ b/pridepy/download/iprox.py @@ -14,6 +14,8 @@ import logging import os import re +import subprocess +import tempfile import defusedxml.ElementTree as ET from typing import ClassVar, Dict, List, Optional from urllib.parse import urlparse @@ -34,6 +36,8 @@ class IproxProvider(Provider): PX_XML_URL_TEMPLATE: ClassVar[str] = ( "http://download.iprox.org/{accession}/PX_{accession}.xml" ) + ASPERA_HOST: ClassVar[str] = "download.iprox.org" + ASPERA_PORT: ClassVar[str] = "33001" # iProX PX XML uses the same PSI-MS cvParam "name" values as JPOST PROXI, # so we reuse JpostProvider's category map. PX_CATEGORY_MAP: ClassVar[Dict[str, str]] = JpostProvider.PROXI_CATEGORY_MAP @@ -45,6 +49,84 @@ def matches(accession: str) -> bool: return False return bool(re.fullmatch(r"IPX\d{7,10}", accession.upper())) + @staticmethod + def _ascp_binary() -> str: + # Reuse PRIDE's bundled ascp binary resolution. + from pridepy.download.pride import PrideProvider + return PrideProvider.get_ascp_binary() + + @classmethod + def aspera_download( + cls, + urls: List[str], + output_folder: str, + user: Optional[str], + key_path: Optional[str] = None, + password: Optional[str] = None, + maximum_bandwidth: str = "500M", + ) -> None: + """Download iProX-hosted URLs in one batched ``ascp --file-list`` session. + + Auth resolution is fail-fast and never lets ``ascp`` fall back to an + interactive ``Password:`` prompt (which would hang a batch/sbatch + job): pass ``key_path`` for key-based auth (``-i ``), or + ``password`` for password auth (via the ``ASPERA_SCP_PASS`` + env var — iProX's own account password, not a key). Exactly one of + the two must be supplied by the caller. ``ascp --mode recv + --file-list`` always recreates the remote ``/IPX.../IPX.../`` source + tree under ``output_folder`` — this transfer path does not support + flattening. + """ + if not user: + raise ValueError( + "iProX Aspera requires --iprox-user (your registered iProX " + "username)." + ) + env = dict(os.environ) + if key_path: + if not os.path.isfile(key_path): + raise ValueError( + f"iProX Aspera key not found: {key_path}. Pass " + "--aspera-key to your registered Aspera private " + "key, or use the default HTTP transport." + ) + elif password: + env["ASPERA_SCP_PASS"] = password + else: + raise ValueError( + "iProX Aspera needs a credential: set IPROX_ASPERA_PASSWORD " + "(your iProX account password) or pass --aspera-key . " + "HTTP is the default alternative." + ) + ascp = cls._ascp_binary() + remote_paths = [urlparse(u).path for u in urls] + os.makedirs(output_folder, exist_ok=True) + fd, list_path = tempfile.mkstemp(prefix="iprox_aspera_", suffix=".txt", text=True) + try: + with os.fdopen(fd, "w") as fh: + for path in remote_paths: + fh.write(path + "\n") + argv = [ + ascp, "-T", "-l", maximum_bandwidth, "-P", cls.ASPERA_PORT, + "-k", "1", + ] + (["-i", key_path] if key_path else []) + [ + "--mode", "recv", + "--host", cls.ASPERA_HOST, "--file-list", list_path, + "--user", user, output_folder, + ] + logging.info( + "iProX Aspera: transferring %d file(s) via ascp --file-list", + len(remote_paths), + ) + try: + subprocess.run(argv, check=True, env=env, stdin=subprocess.DEVNULL) + except subprocess.CalledProcessError as e: + raise RuntimeError( + f"iProX Aspera transfer failed (exit {e.returncode})" + ) from e + finally: + os.remove(list_path) + @staticmethod def _get_public_root(accession: str) -> str: return f"/{accession.upper()}" diff --git a/pridepy/download/pride.py b/pridepy/download/pride.py index 80f9348..d75f478 100644 --- a/pridepy/download/pride.py +++ b/pridepy/download/pride.py @@ -53,6 +53,14 @@ class PrideProvider(Provider): ARCHIVE_FTP_URL_PREFIX: ClassVar[str] = "ftp://ftp.pride.ebi.ac.uk/" ARCHIVE_HTTPS_URL_PREFIX: ClassVar[str] = "https://ftp.pride.ebi.ac.uk/" S3_URL: ClassVar[str] = "https://hh.fire.sdo.ebi.ac.uk" + # EBI-internal FIRE S3 endpoint. Only reachable from within EBI + # infrastructure (compute/login nodes), where it is markedly faster and + # more reliable than the public FTP/HTTPS/Globus paths — which is the + # whole point of the ``fire`` protocol. Overridable via the + # ``PRIDEPY_FIRE_ENDPOINT`` env var for sites with a different host. + FIRE_S3_URL: ClassVar[str] = os.environ.get( + "PRIDEPY_FIRE_ENDPOINT", "https://hl.fire.sdo.ebi.ac.uk" + ) S3_BUCKET: ClassVar[str] = "pride-public" PROTOCOL_ORDER: ClassVar[List[str]] = ["aspera", "s3", "ftp", "globus"] @@ -138,7 +146,16 @@ def get_submitted_file_path_prefix(self, accession): def _protocol_sequence(protocol: str) -> List[str]: """ Build the ordered list of protocols to try for a requested download mode. + + ``fire`` (the EBI-internal FIRE S3 endpoint) is only ever tried when it + is explicitly requested — it is never folded into another protocol's + fallback chain, because it is unreachable outside EBI and would just + waste retries there. When requested, it is tried first and then falls + back to the public protocols so a run started outside EBI still + completes. """ + if protocol == "fire": + return ["fire"] + PrideProvider.PROTOCOL_ORDER if protocol not in PrideProvider.PROTOCOL_ORDER: return [] return [protocol] + [p for p in PrideProvider.PROTOCOL_ORDER if p != protocol] @@ -218,7 +235,9 @@ def _get_download_url(file_record: Dict, protocol: str) -> str: PrideProvider.ARCHIVE_HTTPS_URL_PREFIX, 1, ) - if protocol == "s3": + if protocol in ("s3", "fire"): + # Both S3 modes derive the object key from the FTP path; they + # differ only in the FIRE endpoint host (external vs EBI-internal). return ftp_url raise ValueError(f"Unsupported protocol: {protocol}") @@ -511,17 +530,26 @@ def download_files_from_globus( @staticmethod def download_files_from_s3( - file_list_json: List[Dict], output_folder: str, skip_if_downloaded_already + file_list_json: List[Dict], + output_folder: str, + skip_if_downloaded_already, + endpoint_url: Optional[str] = None, ): """ - Download files using S3 transfer URL with a progress bar and retry logic. + Download files from a FIRE S3 endpoint with a progress bar and retry logic. + :param file_list_json: file list in JSON format :param output_folder: folder to download the files :param skip_if_downloaded_already: Boolean value to skip the download if the file has already been downloaded. + :param endpoint_url: FIRE S3 endpoint. Defaults to the public + :attr:`S3_URL` (``hh.fire``); pass :attr:`FIRE_S3_URL` (``hl.fire``) + for the EBI-internal ``fire`` protocol. """ if not os.path.isdir(output_folder): os.makedirs(output_folder, exist_ok=True) + endpoint_url = endpoint_url or PrideProvider.S3_URL + # Retry and timeout config retry_config = Config( retries={"max_attempts": 5, "mode": "standard"}, @@ -533,19 +561,21 @@ def download_files_from_s3( s3_resource = boto3.resource( "s3", config=retry_config, - endpoint_url=PrideProvider.S3_URL, + endpoint_url=endpoint_url, ) bucket = s3_resource.Bucket(PrideProvider.S3_BUCKET) + logging.info( + "Downloading %d file(s) from FIRE S3 endpoint %s (bucket %s)", + len(file_list_json), endpoint_url, PrideProvider.S3_BUCKET, + ) failed: List[str] = [] for file in file_list_json: try: - # Determine S3 or FTP path - download_url = ( - file["publicFileLocations"][0]["value"] - if file["publicFileLocations"][0]["name"] == "FTP Protocol" - else file["publicFileLocations"][1]["value"] - ) + # Resolve the canonical FTP URL, then map it to the S3 object + # key: pride-public mirrors the archive layout, so the key is + # the archive-relative path (YYYY/MM/ACCESSION/filename). + download_url = PrideProvider._get_download_url(file, "ftp") ftp_base_url = "ftp://ftp.pride.ebi.ac.uk/pride/data/archive/" s3_path = download_url.replace(ftp_base_url, "") @@ -731,11 +761,14 @@ def _batch_download_by_protocol( download_threads=download_threads, ) return - if protocol == "s3": + if protocol in ("s3", "fire"): PrideProvider.download_files_from_s3( file_list, output_folder, skip_if_downloaded_already=skip_if_downloaded_already, + endpoint_url=( + PrideProvider.FIRE_S3_URL if protocol == "fire" else PrideProvider.S3_URL + ), ) return raise ValueError(f"Unsupported protocol: {protocol}") @@ -925,9 +958,9 @@ def _download_files_batch( :param aspera_maximum_bandwidth: parameter in Aspera sets the maximum bandwidth for the transfer. :param skip_if_downloaded_already: Boolean value to skip the download if the file has already been downloaded. """ - protocols_supported = ["ftp", "aspera", "globus", "s3"] + protocols_supported = ["ftp", "aspera", "globus", "s3", "fire"] if protocol not in protocols_supported: - logging.error("Protocol should be one of ftp, aspera, globus, s3") + logging.error("Protocol should be one of ftp, aspera, globus, s3, fire") return os.makedirs(output_folder, exist_ok=True) @@ -943,8 +976,13 @@ def _download_files_batch( protocol_sequence = PrideProvider._protocol_sequence(protocol) primary_protocol = protocol_sequence[0] - # Retry with the primary protocol first, then fall back to others - fallback_sequence = protocol_sequence + # Retry with the primary protocol first, then fall back to others. + # ``fire`` is excluded from the per-file fallback: a FIRE failure is an + # endpoint-level condition (the EBI-internal host is unreachable), so it + # fails identically for every file. Re-attempting it per file in Phase 2 + # would just burn retries on connections that cannot succeed — send those + # files straight to the public fallback protocols instead. + fallback_sequence = [p for p in protocol_sequence if p != "fire"] # Phase 1: batch download with the requested protocol. Reuses a single # FTP/S3 connection for all files (the previous behaviour) instead of diff --git a/pridepy/download/proteomexchange.py b/pridepy/download/proteomexchange.py index 32ea72e..9b82703 100644 --- a/pridepy/download/proteomexchange.py +++ b/pridepy/download/proteomexchange.py @@ -26,7 +26,7 @@ import posixpath import re import defusedxml.ElementTree as ET -from typing import ClassVar, Dict, List +from typing import ClassVar, Dict, List, Optional from urllib.parse import urlparse from pridepy.download.base import Provider @@ -170,23 +170,71 @@ def download_from_accession_or_url( output_folder: str, skip_if_downloaded_already: bool = True, flatten: bool = True, + parallel_files: int = 1, + download_threads: int = 1, + protocol: str = "ftp", + iprox_user: Optional[str] = None, + aspera_key: Optional[str] = None, + aspera_password: Optional[str] = None, ) -> None: """End-to-end: resolve XML, list files, partition by scheme, download. Convenience for the ``download-px-raw-files`` CLI command — combines :meth:`list_files` and :meth:`download_files` with the original ``download_px_raw_files`` defaults (skip-if-downloaded-already - defaults to ``True``, no parallel workers). + defaults to ``True``). ``parallel_files`` controls across-file + concurrency and ``download_threads`` controls per-file HTTP Range + segments; ``protocol`` flows into :meth:`download_files` (ftp/http(s) + are handled directly today). + + When ``protocol == "aspera"``, iProX-hosted files are routed through + :meth:`IproxProvider.aspera_download` instead of the HTTP/FTP path + (opt-in, requires ``iprox_user`` plus either ``aspera_key`` or + ``aspera_password``; the transfer always preserves the source + directory tree). """ records = self.list_files(px_id_or_url) if not records: logging.info("No Associated raw file URIs found in PX XML") return + + if protocol.lower() == "aspera": + from pridepy.download.iprox import IproxProvider + iprox_urls = [] + for r in records: + loc = self.get_download_url(r, protocol) + host = (urlparse(loc).hostname or "").lower() + if host == "download.iprox.org": + iprox_urls.append(loc) + if not iprox_urls: + raise ValueError( + "Aspera requested but no iProX-hosted files found in this dataset." + ) + if len(iprox_urls) < len(records): + logging.warning( + "%d of %d file(s) are NOT hosted on iProX and were NOT " + "downloaded: --protocol aspera only handles iProX-hosted " + "files. Use the default HTTP transport (omit --protocol, " + "or pass --protocol ftp) to download the full set.", + len(records) - len(iprox_urls), + len(records), + ) + IproxProvider.aspera_download( + urls=iprox_urls, + output_folder=output_folder, + user=iprox_user, + key_path=aspera_key, + password=aspera_password, + ) + return + self.download_files( accession=px_id_or_url, records=records, output_folder=output_folder, skip_if_downloaded_already=skip_if_downloaded_already, - protocol="ftp", + protocol=protocol, flatten=flatten, + parallel_files=parallel_files, + download_threads=download_threads, ) diff --git a/pridepy/download/transport.py b/pridepy/download/transport.py index feae6ab..5f6882e 100644 --- a/pridepy/download/transport.py +++ b/pridepy/download/transport.py @@ -19,6 +19,11 @@ from pridepy.util.api_handling import Util +# Combined cap on parallel_files (-w) x download_threads (-t): each factor is +# independently clamped to 32, but nested they can reach 1024 concurrent HTTP +# connections. Clamp the product to keep peak connections reasonable. +MAX_TOTAL_HTTP_CONNECTIONS = 64 + def _safe_join(output_folder: str, relative_path: str) -> str: """Join ``output_folder`` with a dataset-relative path. @@ -845,6 +850,19 @@ def _rel(idx: int) -> Optional[str]: failed: List[str] = [] workers = max(1, min(parallel_files, len(http_urls))) + if workers * download_threads > MAX_TOTAL_HTTP_CONNECTIONS: + clamped_threads = max(1, MAX_TOTAL_HTTP_CONNECTIONS // workers) + logging.warning( + "parallel_files (%d) x download_threads (%d) = %d exceeds the " + "combined connection cap of %d; reducing download_threads to %d " + "to keep peak HTTP connections bounded.", + workers, + download_threads, + workers * download_threads, + MAX_TOTAL_HTTP_CONNECTIONS, + clamped_threads, + ) + download_threads = clamped_threads if workers > 1: logging.info( f"Downloading {len(http_urls)} HTTP(S) file(s) with {workers} parallel workers" diff --git a/pridepy/pridepy.py b/pridepy/pridepy.py index 17723cf..91d3d67 100644 --- a/pridepy/pridepy.py +++ b/pridepy/pridepy.py @@ -1,12 +1,18 @@ #!/usr/bin/env python3 import asyncio import logging +import os +import sys +from typing import Optional + import click from pridepy.download.client import Client as Files from pridepy.pdc import download_pdc_files as run_pdc_download from pridepy.project.project import Project -PROTOCOL_CHOICES = click.Choice(["ftp", "aspera", "globus", "s3"], case_sensitive=False) +PROTOCOL_CHOICES = click.Choice( + ["ftp", "aspera", "globus", "s3", "fire"], case_sensitive=False +) @click.group() @@ -25,7 +31,10 @@ def main(): "--protocol", default="ftp", type=PROTOCOL_CHOICES, - help="Protocol to use for download: ftp, aspera, globus, s3. Default is ftp with fallback enabled.", + help="Protocol to use for download: ftp, aspera, globus, s3, fire. " + "'fire' uses the EBI-internal FIRE S3 endpoint (only reachable inside EBI " + "infrastructure; falls back to the public protocols elsewhere). " + "Default is ftp with fallback enabled.", ) @click.option( "-o", @@ -60,6 +69,15 @@ def main(): type=click.IntRange(1, 32), help="Number of threads for each file download. Default is 1.", ) +@click.option( + "-w", + "--parallel-files", + "parallel_files", + default=1, + type=click.IntRange(1, 32), + help="Number of files to download in parallel (across-file concurrency). " + "Combine with -t/--threads (per-file segments). Default is 1.", +) @click.option( "--preserve-structure", is_flag=True, @@ -75,6 +93,7 @@ def download_all_public_raw_files( aspera_maximum_bandwidth: str = "50M", checksum_check: bool = False, download_threads: int = 1, + parallel_files: int = 1, preserve_structure: bool = False, ): """ @@ -88,6 +107,7 @@ def download_all_public_raw_files( aspera_maximum_bandwidth (str): Maximum bandwidth for Aspera protocol. Default is 100M. checksum_check (bool): Flag to download checksum file for the project. Default is False. download_threads (int): Number of threads for each file download. Default is 1. + parallel_files (int): Number of files to download in parallel. Default is 1. """ raw_files = Files() @@ -105,6 +125,7 @@ def download_all_public_raw_files( aspera_maximum_bandwidth=aspera_maximum_bandwidth, checksum_check=checksum_check, download_threads=download_threads, + parallel_files=parallel_files, flatten=not preserve_structure, ) @@ -119,7 +140,10 @@ def download_all_public_raw_files( "--protocol", default="ftp", type=PROTOCOL_CHOICES, - help="Protocol to use for download: ftp, aspera, globus, s3. Default is ftp with fallback enabled.", + help="Protocol to use for download: ftp, aspera, globus, s3, fire. " + "'fire' uses the EBI-internal FIRE S3 endpoint (only reachable inside EBI " + "infrastructure; falls back to the public protocols elsewhere). " + "Default is ftp with fallback enabled.", ) @click.option( "-o", @@ -161,6 +185,15 @@ def download_all_public_raw_files( type=click.IntRange(1, 32), help="Number of threads for each file download. Default is 1.", ) +@click.option( + "-w", + "--parallel-files", + "parallel_files", + default=1, + type=click.IntRange(1, 32), + help="Number of files to download in parallel (across-file concurrency). " + "Combine with -t/--threads (per-file segments). Default is 1.", +) @click.option( "--preserve-structure", is_flag=True, @@ -177,6 +210,7 @@ def download_all_public_category_files( checksum_check: bool = False, category: str = "RAW", download_threads: int = 1, + parallel_files: int = 1, preserve_structure: bool = False, ): """ @@ -191,6 +225,7 @@ def download_all_public_category_files( checksum_check (bool): If True, downloads the checksum file for the project. category (str): Comma-separated categories of files to download (e.g. RAW or RAW,SEARCH). download_threads (int): Number of threads for each file download. Default is 1. + parallel_files (int): Number of files to download in parallel. Default is 1. """ valid_categories = {"RAW", "PEAK", "SEARCH", "RESULT", "SPECTRUM_LIBRARY", "OTHER", "FASTA"} @@ -218,6 +253,7 @@ def download_all_public_category_files( checksum_check=checksum_check, categories=categories, download_threads=download_threads, + parallel_files=parallel_files, flatten=not preserve_structure, ) @@ -232,7 +268,10 @@ def download_all_public_category_files( "--protocol", default="ftp", type=PROTOCOL_CHOICES, - help="Protocol to use for download: ftp, aspera, globus, s3. Default is ftp with fallback enabled.", + help="Protocol to use for download: ftp, aspera, globus, s3, fire. " + "'fire' uses the EBI-internal FIRE S3 endpoint (only reachable inside EBI " + "infrastructure; falls back to the public protocols elsewhere). " + "Default is ftp with fallback enabled.", ) @click.option("-f", "--file-name", required=True, help="fileName to be downloaded") @click.option( @@ -332,6 +371,30 @@ def download_file_by_name( default=False, help="Skip the download if the file has already been downloaded.", ) +@click.option( + "-p", + "--protocol", + default="ftp", + type=PROTOCOL_CHOICES, + help="Protocol to use for download: ftp, aspera, globus, s3. Default is ftp with fallback enabled.", +) +@click.option( + "-t", + "--threads", + "download_threads", + default=1, + type=click.IntRange(1, 32), + help="Number of threads for each file download. Default is 1.", +) +@click.option( + "-w", + "--parallel-files", + "parallel_files", + default=1, + type=click.IntRange(1, 32), + help="Number of files to download in parallel (across-file concurrency). " + "Combine with -t/--threads (per-file segments). Default is 1.", +) @click.option( "--preserve-structure", is_flag=True, @@ -339,20 +402,54 @@ def download_file_by_name( help="Recreate the dataset's subdirectory layout under the output folder. " "By default files are downloaded flat into the output folder.", ) +@click.option( + "--iprox-user", + "iprox_user", + envvar="IPROX_USER", + default=None, + type=str, + help="Your registered iProX username. Required with --protocol aspera.", +) +@click.option( + "--aspera-key", + "aspera_key", + envvar="IPROX_ASPERA_KEY", + default=None, + type=str, + help="Optional Aspera private key for iProX (only with --protocol " + "aspera). If omitted, password auth is used via the " + "IPROX_ASPERA_PASSWORD env var or a secure prompt.", +) def download_px_raw_files( accession: str, output_folder: str, skip_if_downloaded_already: bool, + protocol: str = "ftp", + download_threads: int = 1, + parallel_files: int = 1, preserve_structure: bool = False, + iprox_user: Optional[str] = None, + aspera_key: Optional[str] = None, ): """CLI wrapper to download raw files via ProteomeXchange XML.""" files = Files() logging.info(f"PX accession/URL: {accession}") + + aspera_password = os.environ.get("IPROX_ASPERA_PASSWORD") + if protocol.lower() == "aspera" and not aspera_key and not aspera_password and sys.stdin.isatty(): + aspera_password = click.prompt("iProX Aspera password", hide_input=True) + files.download_px_raw_files( accession, output_folder, skip_if_downloaded_already, flatten=not preserve_structure, + protocol=protocol, + download_threads=download_threads, + parallel_files=parallel_files, + iprox_user=iprox_user, + aspera_key=aspera_key, + aspera_password=aspera_password, ) @@ -553,7 +650,10 @@ def _read_url_arguments(url_list_path, urls_csv=None): "--protocol", default="ftp", type=PROTOCOL_CHOICES, - help="Protocol to use for download: ftp, aspera, globus, s3. Default is ftp with fallback enabled.", + help="Protocol to use for download: ftp, aspera, globus, s3, fire. " + "'fire' uses the EBI-internal FIRE S3 endpoint (only reachable inside EBI " + "infrastructure; falls back to the public protocols elsewhere). " + "Default is ftp with fallback enabled.", ) @click.option( "-F", @@ -602,6 +702,15 @@ def _read_url_arguments(url_list_path, urls_csv=None): type=click.IntRange(1, 32), help="Number of threads for each file download. Default is 1.", ) +@click.option( + "-w", + "--parallel-files", + "parallel_files", + default=1, + type=click.IntRange(1, 32), + help="Number of files to download in parallel (across-file concurrency). " + "Combine with -t/--threads (per-file segments). Default is 1.", +) @click.option( "--preserve-structure", is_flag=True, @@ -619,6 +728,7 @@ def download_files_by_list( aspera_maximum_bandwidth, checksum_check, download_threads, + parallel_files: int = 1, preserve_structure: bool = False, ): """Download a named subset of files from a PRIDE project.""" @@ -635,6 +745,7 @@ def download_files_by_list( aspera_maximum_bandwidth=aspera_maximum_bandwidth, checksum_check=checksum_check, download_threads=download_threads, + parallel_files=parallel_files, flatten=not preserve_structure, ) @@ -693,6 +804,15 @@ def download_files_by_list( type=click.IntRange(1, 32), help="Number of threads for each file download. Default is 1.", ) +@click.option( + "-w", + "--parallel-files", + "parallel_files", + default=1, + type=click.IntRange(1, 32), + help="Number of files to download in parallel (across-file concurrency). " + "Combine with -t/--threads (per-file segments). Default is 1.", +) def download_files_by_url( url_list_path, urls_csv, @@ -701,6 +821,7 @@ def download_files_by_url( protocol, checksum_check, download_threads, + parallel_files: int = 1, ): """Download files from raw URLs (http/https/ftp), dispatched by scheme.""" urls = _read_url_arguments(url_list_path, urls_csv) @@ -711,6 +832,7 @@ def download_files_by_url( skip_if_downloaded_already=skip_if_downloaded_already, protocol=protocol, download_threads=download_threads, + parallel_files=parallel_files, checksum_check=checksum_check, ) diff --git a/pridepy/tests/test_cli_flatten.py b/pridepy/tests/test_cli_flatten.py index 92c00df..01f887c 100644 --- a/pridepy/tests/test_cli_flatten.py +++ b/pridepy/tests/test_cli_flatten.py @@ -23,7 +23,23 @@ def test_download_all_public_raw_files_flattens_by_default(self): kwargs = files_cls.return_value.download_all_raw_files.call_args.kwargs assert kwargs["flatten"] is True assert kwargs["download_threads"] == 1 - assert "parallel_files" not in kwargs + assert kwargs["parallel_files"] == 1 + + def test_download_all_public_raw_files_parallel_files(self): + with patch("pridepy.pridepy.Files") as files_cls: + self._invoke( + [ + "download-all-public-raw-files", + "-a", + "MSV000012345", + "-o", + "/tmp/x", + "-w", + "8", + ] + ) + kwargs = files_cls.return_value.download_all_raw_files.call_args.kwargs + assert kwargs["parallel_files"] == 8 def test_download_all_public_raw_files_threads(self): with patch("pridepy.pridepy.Files") as files_cls: @@ -40,7 +56,7 @@ def test_download_all_public_raw_files_threads(self): ) kwargs = files_cls.return_value.download_all_raw_files.call_args.kwargs assert kwargs["download_threads"] == 4 - assert "parallel_files" not in kwargs + assert kwargs["parallel_files"] == 1 def test_download_all_public_raw_files_preserve_structure(self): with patch("pridepy.pridepy.Files") as files_cls: @@ -74,7 +90,25 @@ def test_download_all_public_category_files_preserve_structure(self): kwargs = files_cls.return_value.download_all_category_files.call_args.kwargs assert kwargs["flatten"] is False assert kwargs["download_threads"] == 1 - assert "parallel_files" not in kwargs + assert kwargs["parallel_files"] == 1 + + def test_download_all_public_category_files_parallel_files(self): + with patch("pridepy.pridepy.Files") as files_cls: + self._invoke( + [ + "download-all-public-category-files", + "-a", + "MSV000012345", + "-o", + "/tmp/x", + "-c", + "RAW", + "-w", + "8", + ] + ) + kwargs = files_cls.return_value.download_all_category_files.call_args.kwargs + assert kwargs["parallel_files"] == 8 def test_download_files_by_list_preserve_structure(self): with patch("pridepy.pridepy.Files") as files_cls: @@ -93,7 +127,25 @@ def test_download_files_by_list_preserve_structure(self): kwargs = files_cls.return_value.download_files_by_list.call_args.kwargs assert kwargs["flatten"] is False assert kwargs["download_threads"] == 1 - assert "parallel_files" not in kwargs + assert kwargs["parallel_files"] == 1 + + def test_download_files_by_list_parallel_files(self): + with patch("pridepy.pridepy.Files") as files_cls: + self._invoke( + [ + "download-files-by-list", + "-a", + "MSV000012345", + "-o", + "/tmp/x", + "-f", + "a.raw", + "-w", + "8", + ] + ) + kwargs = files_cls.return_value.download_files_by_list.call_args.kwargs + assert kwargs["parallel_files"] == 8 def test_download_files_by_url_threads(self): with patch("pridepy.pridepy.Files") as files_cls: @@ -110,7 +162,23 @@ def test_download_files_by_url_threads(self): ) kwargs = files_cls.download_files_by_url.call_args.kwargs assert kwargs["download_threads"] == 4 - assert "parallel_files" not in kwargs + assert kwargs["parallel_files"] == 1 + + def test_download_files_by_url_parallel_files(self): + with patch("pridepy.pridepy.Files") as files_cls: + self._invoke( + [ + "download-files-by-url", + "-u", + "https://example.org/a.raw", + "-o", + "/tmp/x", + "-w", + "8", + ] + ) + kwargs = files_cls.download_files_by_url.call_args.kwargs + assert kwargs["parallel_files"] == 8 def test_download_px_raw_files_preserve_structure(self): with patch("pridepy.pridepy.Files") as files_cls: @@ -126,3 +194,25 @@ def test_download_px_raw_files_preserve_structure(self): ) kwargs = files_cls.return_value.download_px_raw_files.call_args.kwargs assert kwargs["flatten"] is False + + def test_download_px_raw_files_protocol_threads_parallel(self): + with patch("pridepy.pridepy.Files") as files_cls: + self._invoke( + [ + "download-px-raw-files", + "-a", + "PXD000001", + "-o", + "/tmp/x", + "-w", + "8", + "-t", + "4", + "-p", + "ftp", + ] + ) + kwargs = files_cls.return_value.download_px_raw_files.call_args.kwargs + assert kwargs["parallel_files"] == 8 + assert kwargs["download_threads"] == 4 + assert kwargs["protocol"] == "ftp" diff --git a/pridepy/tests/test_download_resilience.py b/pridepy/tests/test_download_resilience.py index 532f624..c2b8cc7 100644 --- a/pridepy/tests/test_download_resilience.py +++ b/pridepy/tests/test_download_resilience.py @@ -247,6 +247,39 @@ def test_download_http_urls_raises_when_a_file_fails(self): max_retries=1, ) + def test_download_http_urls_clamps_combined_connection_cap(self): + """parallel_files x download_threads must be clamped to + transport.MAX_TOTAL_HTTP_CONNECTIONS, with a warning explaining why, + so e.g. -w 32 -t 32 doesn't open 1024 connections.""" + seen_threads = [] + + def fake_http_download_one(url, output_folder, skip_if_downloaded_already, + max_retries=3, position=0, relative_path=None, + download_threads=1): + seen_threads.append(download_threads) + + with tempfile.TemporaryDirectory() as tmp_dir: + with patch.object( + transport, "_http_download_one", side_effect=fake_http_download_one + ): + with self.assertLogs(level="WARNING") as log_ctx: + transport.download_http_urls( + http_urls=[ + "https://example.org/a.raw", + "https://example.org/b.raw", + "https://example.org/c.raw", + ], + output_folder=tmp_dir, + skip_if_downloaded_already=False, + parallel_files=32, + download_threads=32, + ) + assert any("connection cap" in m.lower() for m in log_ctx.output) + workers = min(32, 3) + expected_threads = max(1, transport.MAX_TOTAL_HTTP_CONNECTIONS // workers) + assert workers * expected_threads <= transport.MAX_TOTAL_HTTP_CONNECTIONS + assert all(t == expected_threads for t in seen_threads) + def test_download_ftp_urls_raises_when_a_file_fails(self): """A failed FTP transfer must surface as an exception.""" with tempfile.TemporaryDirectory() as tmp_dir: diff --git a/pridepy/tests/test_iprox_aspera.py b/pridepy/tests/test_iprox_aspera.py new file mode 100644 index 0000000..9df029c --- /dev/null +++ b/pridepy/tests/test_iprox_aspera.py @@ -0,0 +1,274 @@ +"""iProX Aspera command construction + credential handling.""" +import os +import re +import subprocess +import tempfile +from unittest import TestCase +from unittest.mock import patch, MagicMock + +import pytest + +from pridepy.download.iprox import IproxProvider + + +def _make_key_file(tmp): + key_path = os.path.join(tmp, "aspera.key") + with open(key_path, "w") as f: + f.write("fake-private-key") + return key_path + + +class TestIproxAspera(TestCase): + def test_builds_ascp_key_based_file_list_argv(self): + urls = [ + "http://download.iprox.org/IPX0003474000/IPX0003474001/a.raw", + "http://download.iprox.org/IPX0002031000/IPX0002031001/b.raw", + ] + captured_list_contents = {} + + def fake_run(argv, **kwargs): + list_idx = argv.index("--file-list") + 1 + list_path = argv[list_idx] + with open(list_path) as f: + captured_list_contents["lines"] = f.read().splitlines() + return MagicMock(returncode=0) + + with tempfile.TemporaryDirectory() as tmp, \ + patch("pridepy.download.iprox.subprocess.run", side_effect=fake_run) as run, \ + patch.object(IproxProvider, "_ascp_binary", return_value="/bin/ascp"): + key_path = _make_key_file(tmp) + output_folder = os.path.join(tmp, "out") + IproxProvider.aspera_download( + urls=urls, + output_folder=output_folder, + user="daicx", + key_path=key_path, + maximum_bandwidth="500M", + ) + + args, kwargs = run.call_args + argv = args[0] + + assert argv[0] == "/bin/ascp" + assert "-T" in argv + assert argv[argv.index("-l") + 1] == "500M" + assert argv[argv.index("-P") + 1] == "33001" + assert argv[argv.index("-k") + 1] == "1" + assert argv[argv.index("-i") + 1] == key_path + assert argv[argv.index("--mode") + 1] == "recv" + assert argv[argv.index("--host") + 1] == "download.iprox.org" + assert argv[argv.index("--user") + 1] == "daicx" + assert argv[-1] == output_folder + + # key-based auth: no ASPERA_SCP_PASS set in the subprocess env + assert "ASPERA_SCP_PASS" not in kwargs.get("env", {}) + assert "ASPERA_SCP_PASS" not in argv + assert not any("secret" in str(a) for a in argv) + + # stdin is always closed so a rejected/missing credential errors + # instead of blocking on an interactive Password: prompt + assert kwargs.get("stdin") == subprocess.DEVNULL + + # file-list contains exactly the URL paths, one per line + assert captured_list_contents["lines"] == [ + "/IPX0003474000/IPX0003474001/a.raw", + "/IPX0002031000/IPX0002031001/b.raw", + ] + + def test_builds_ascp_password_based_file_list_argv(self): + urls = ["http://download.iprox.org/IPX0003474000/IPX0003474001/a.raw"] + + with tempfile.TemporaryDirectory() as tmp, \ + patch("pridepy.download.iprox.subprocess.run") as run, \ + patch.object(IproxProvider, "_ascp_binary", return_value="/bin/ascp"): + run.return_value = MagicMock(returncode=0) + output_folder = os.path.join(tmp, "out") + IproxProvider.aspera_download( + urls=urls, + output_folder=output_folder, + user="daicx", + password="secret", + maximum_bandwidth="500M", + ) + + args, kwargs = run.call_args + argv = args[0] + + # password auth: no -i flag at all + assert "-i" not in argv + assert argv[argv.index("--mode") + 1] == "recv" + assert argv[argv.index("--host") + 1] == "download.iprox.org" + assert argv[argv.index("--user") + 1] == "daicx" + assert argv[-1] == output_folder + + # credential goes through the env, never on argv + assert kwargs.get("env", {}).get("ASPERA_SCP_PASS") == "secret" + assert not any("secret" in str(a) for a in argv) + + # stdin closed so ascp can't fall back to an interactive prompt + assert kwargs.get("stdin") == subprocess.DEVNULL + + def test_temp_file_list_is_removed_after_run(self): + url = "http://download.iprox.org/IPX0003578000/IPX0003578001/a.raw" + captured = {} + + def fake_run(argv, **kwargs): + captured["list_path"] = argv[argv.index("--file-list") + 1] + return MagicMock(returncode=0) + + with tempfile.TemporaryDirectory() as tmp, \ + patch("pridepy.download.iprox.subprocess.run", side_effect=fake_run), \ + patch.object(IproxProvider, "_ascp_binary", return_value="/bin/ascp"): + key_path = _make_key_file(tmp) + IproxProvider.aspera_download( + urls=[url], + output_folder=os.path.join(tmp, "out"), + user="daicx", + key_path=key_path, + ) + assert not os.path.exists(captured["list_path"]) + + def test_missing_user_raises(self): + with tempfile.TemporaryDirectory() as tmp: + key_path = _make_key_file(tmp) + with pytest.raises(ValueError, match="--iprox-user"): + IproxProvider.aspera_download( + urls=["http://download.iprox.org/IPX1/a.raw"], + output_folder=os.path.join(tmp, "out"), + user=None, + key_path=key_path, + ) + + def test_missing_credential_raises(self): + """Neither key nor password supplied: fail fast, never call ascp.""" + with tempfile.TemporaryDirectory() as tmp, \ + patch("pridepy.download.iprox.subprocess.run") as run: + with pytest.raises(ValueError, match="IPROX_ASPERA_PASSWORD"): + IproxProvider.aspera_download( + urls=["http://download.iprox.org/IPX1/a.raw"], + output_folder=os.path.join(tmp, "out"), + user="daicx", + ) + run.assert_not_called() + + def test_nonexistent_key_path_raises(self): + with pytest.raises(ValueError, match="--aspera-key"): + IproxProvider.aspera_download( + urls=["http://download.iprox.org/IPX1/a.raw"], + output_folder="/tmp/x", + user="daicx", + key_path="/no/such/key/file", + ) + + def test_failed_transfer_raises_runtime_error(self): + url = "http://download.iprox.org/IPX0003578000/IPX0003578001/a.raw" + with tempfile.TemporaryDirectory() as tmp, \ + patch("pridepy.download.iprox.subprocess.run") as run, \ + patch.object(IproxProvider, "_ascp_binary", return_value="/bin/ascp"): + key_path = _make_key_file(tmp) + run.side_effect = subprocess.CalledProcessError(1, ["ascp"]) + with pytest.raises(RuntimeError, match=re.escape("exit 1")): + IproxProvider.aspera_download( + urls=[url], + output_folder=os.path.join(tmp, "out"), + user="daicx", + key_path=key_path, + ) + + +class TestPxAsperaRouting(TestCase): + def test_px_aspera_routes_to_iprox_with_key(self): + from pridepy.download.proteomexchange import ProteomeXchangeProvider + prov = ProteomeXchangeProvider() + rec = { + "publicFileLocations": [ + {"name": "FTP Protocol", + "value": "http://download.iprox.org/IPX1/IPX2/a.raw"} + ], + "relativePath": "IPX2/a.raw", + } + with tempfile.TemporaryDirectory() as tmp: + key_path = _make_key_file(tmp) + with patch.object(prov, "list_files", return_value=[rec]), \ + patch("pridepy.download.iprox.IproxProvider.aspera_download") as asp: + prov.download_from_accession_or_url( + "PXD000001", "/tmp/x", protocol="aspera", + iprox_user="daicx", aspera_key=key_path, + ) + asp.assert_called_once() + assert asp.call_args.kwargs["user"] == "daicx" + assert asp.call_args.kwargs["key_path"] == key_path + assert asp.call_args.kwargs["password"] is None + + def test_px_aspera_routes_to_iprox_with_password(self): + from pridepy.download.proteomexchange import ProteomeXchangeProvider + prov = ProteomeXchangeProvider() + rec = { + "publicFileLocations": [ + {"name": "FTP Protocol", + "value": "http://download.iprox.org/IPX1/IPX2/a.raw"} + ], + "relativePath": "IPX2/a.raw", + } + with patch.object(prov, "list_files", return_value=[rec]), \ + patch("pridepy.download.iprox.IproxProvider.aspera_download") as asp: + prov.download_from_accession_or_url( + "PXD000001", "/tmp/x", protocol="aspera", + iprox_user="daicx", aspera_password="secret", + ) + asp.assert_called_once() + assert asp.call_args.kwargs["user"] == "daicx" + assert asp.call_args.kwargs["key_path"] is None + assert asp.call_args.kwargs["password"] == "secret" + + def test_px_aspera_rejects_spoofed_host(self): + """A URL on a lookalike host (substring match, not exact) must NOT be + routed to iProX Aspera.""" + from pridepy.download.proteomexchange import ProteomeXchangeProvider + prov = ProteomeXchangeProvider() + rec = { + "publicFileLocations": [ + {"name": "FTP Protocol", + "value": "http://download.iprox.org.evil.example/IPX1/a.raw"} + ], + "relativePath": "a.raw", + } + with patch.object(prov, "list_files", return_value=[rec]), \ + patch("pridepy.download.iprox.IproxProvider.aspera_download") as asp: + with pytest.raises(ValueError, match="no iProX-hosted files"): + prov.download_from_accession_or_url( + "PXD000001", "/tmp/x", protocol="aspera", + iprox_user="daicx", aspera_key="/some/key", + ) + asp.assert_not_called() + + def test_px_aspera_warns_on_mixed_dataset(self): + """Non-iProX files in a mixed dataset are dropped from the aspera + transfer; a warning should be logged naming how many were skipped.""" + from pridepy.download.proteomexchange import ProteomeXchangeProvider + prov = ProteomeXchangeProvider() + records = [ + { + "publicFileLocations": [ + {"name": "FTP Protocol", + "value": "http://download.iprox.org/IPX1/a.raw"} + ], + "relativePath": "a.raw", + }, + { + "publicFileLocations": [ + {"name": "FTP Protocol", + "value": "ftp://massive-ftp.ucsd.edu/MSV1/b.raw"} + ], + "relativePath": "b.raw", + }, + ] + with patch.object(prov, "list_files", return_value=records), \ + patch("pridepy.download.iprox.IproxProvider.aspera_download") as asp, \ + self.assertLogs(level="WARNING") as log_ctx: + prov.download_from_accession_or_url( + "PXD000001", "/tmp/x", protocol="aspera", + iprox_user="daicx", aspera_key="/some/key", + ) + asp.assert_called_once() + assert any("not" in m.lower() and "iprox" in m.lower() for m in log_ctx.output) diff --git a/pridepy/tests/test_review_fixes.py b/pridepy/tests/test_review_fixes.py index 242941c..2458d40 100644 --- a/pridepy/tests/test_review_fixes.py +++ b/pridepy/tests/test_review_fixes.py @@ -142,6 +142,68 @@ def test_s3_batch_raises_when_a_file_fails(self): records, tmp_dir, skip_if_downloaded_already=False ) + def test_fire_protocol_uses_internal_endpoint_and_derives_key(self): + """`fire` must hit the EBI-internal FIRE endpoint and map the FTP path + to the correct pride-public S3 object key.""" + records = [_pride_record("a.raw", accession="PXD002137", date="2015/08")] + mock_obj = Mock() + mock_obj.content_length = 10 + mock_bucket = Mock() + mock_bucket.Object.return_value = mock_obj + mock_resource = Mock() + mock_resource.Bucket.return_value = mock_bucket + with tempfile.TemporaryDirectory() as tmp_dir: + with patch( + "pridepy.download.pride.boto3.resource", return_value=mock_resource + ) as mock_boto: + PrideProvider._batch_download_by_protocol( + records, + tmp_dir, + protocol="fire", + skip_if_downloaded_already=False, + aspera_maximum_bandwidth="100M", + ) + # boto3.resource was created against the internal hl.fire endpoint. + assert mock_boto.call_args.kwargs["endpoint_url"] == PrideProvider.FIRE_S3_URL + assert "hl.fire.sdo.ebi.ac.uk" in PrideProvider.FIRE_S3_URL + # The object key is the archive-relative path, no ftp:// prefix. + mock_bucket.Object.assert_called_once_with("2015/08/PXD002137/a.raw") + + def test_fire_protocol_sequence_requested_only(self): + """`fire` is tried first when requested, then the public protocols; + it is never folded into another protocol's fallback chain.""" + assert PrideProvider._protocol_sequence("fire") == [ + "fire", "aspera", "s3", "ftp", "globus", + ] + assert "fire" not in PrideProvider._protocol_sequence("ftp") + assert "fire" not in PrideProvider._protocol_sequence("s3") + + def test_fire_not_retried_in_phase2_fallback(self): + """After a FIRE (endpoint-level) failure, the per-file Phase-2 fallback + must go straight to the public protocols and never re-attempt fire.""" + record = _pride_record("a.raw", accession="PXD002137", date="2015/08") + captured = {} + + def _fake_fallback(*, file_record, output_folder, protocol_sequence, **kw): + captured["seq"] = protocol_sequence + return True # pretend a public protocol succeeded + + with tempfile.TemporaryDirectory() as tmp_dir: + with patch.object(PrideProvider, "_batch_download_by_protocol"), \ + patch("pridepy.download.pride._provider_util.validate_download", + return_value=(False, "missing")), \ + patch.object(PrideProvider, "_download_with_fallback", + side_effect=_fake_fallback): + PrideProvider._download_files_batch( + file_list_json=[record], + accession="PXD002137", + output_folder=tmp_dir, + skip_if_downloaded_already=False, + protocol="fire", + ) + assert captured["seq"] == ["aspera", "s3", "ftp", "globus"] + assert "fire" not in captured["seq"] + def test_download_files_forwards_protocol_to_get_download_url(self): seen = []