diff --git a/docs/sphinx-docs/source/user/menu_bar.rst b/docs/sphinx-docs/source/user/menu_bar.rst index 0ab0c88d84..a969e2b2f4 100644 --- a/docs/sphinx-docs/source/user/menu_bar.rst +++ b/docs/sphinx-docs/source/user/menu_bar.rst @@ -11,7 +11,8 @@ The File option allows you load data into *SasView* for analysis, or to save the Data can be loaded one file at a time, or by selecting multiple files, or by loading an entire folder of files (in which case *SasView* will attempt to make an intelligent guess as to what to load based on the file formats it recognises in the folder!). Data can also be loaded by dragging and dropping files directly -onto Data Explorer. +onto Data Explorer. Additionally, datasets can be downloaded directly from the SASBDB (Small Angle Scattering +Biological Data Bank) using **File > Load from SASBDB...** (see :ref:`SASBDB_Download` for details). A *SasView* session can also be saved and reloaded as an 'Analysis' (an individual model fit or invariant calculation, etc), or as a 'Project' (everything you have done since starting your *SasView* session). diff --git a/docs/sphinx-docs/source/user/tools.rst b/docs/sphinx-docs/source/user/tools.rst index 23c6fcc674..f30eb2c9d9 100644 --- a/docs/sphinx-docs/source/user/tools.rst +++ b/docs/sphinx-docs/source/user/tools.rst @@ -32,3 +32,5 @@ Tools & Utilities MuMag Tool + SASBDB Download + diff --git a/src/sas/qtgui/MainWindow/GuiManager.py b/src/sas/qtgui/MainWindow/GuiManager.py index e3d787116e..f0502df3e2 100644 --- a/src/sas/qtgui/MainWindow/GuiManager.py +++ b/src/sas/qtgui/MainWindow/GuiManager.py @@ -668,6 +668,7 @@ def addTriggers(self): # File self._workspace.actionLoadData.triggered.connect(self.actionLoadData) self._workspace.actionLoad_Data_Folder.triggered.connect(self.actionLoad_Data_Folder) + self._workspace.actionLoad_SASBDB.triggered.connect(self.actionLoad_SASBDB) self._workspace.actionOpen_Project.triggered.connect(self.actionOpen_Project) self._workspace.actionOpen_Analysis.triggered.connect(self.actionOpen_Analysis) self._workspace.actionSave.triggered.connect(self.actionSave_Project) @@ -765,6 +766,24 @@ def actionLoad_Data_Folder(self): """ self.filesWidget.loadFolder() + def actionLoad_SASBDB(self): + """ + Menu File/Load from SASBDB + + Opens a dialog to download and load a dataset from SASBDB. + """ + from sas.qtgui.Utilities.SASBDB.sasbdb_loader import load_downloaded_dataset + from sas.qtgui.Utilities.SASBDB.SASBDBDownloadDialog import SASBDBDownloadDialog + + dialog = SASBDBDownloadDialog(parent=self._workspace) + if dialog.exec(): + load_downloaded_dataset( + self.filesWidget, + self._workspace, + dialog.getDownloadedFilepath(), + dialog.getDatasetInfo(), + ) + def actionOpen_Project(self): """ Menu Open Project diff --git a/src/sas/qtgui/MainWindow/UI/MainWindowUI.ui b/src/sas/qtgui/MainWindow/UI/MainWindowUI.ui index 65a8656e34..9a32105729 100755 --- a/src/sas/qtgui/MainWindow/UI/MainWindowUI.ui +++ b/src/sas/qtgui/MainWindow/UI/MainWindowUI.ui @@ -33,6 +33,7 @@ + @@ -310,6 +311,14 @@ Load Data Folder + + + Load from SASBDB... + + + Download and load dataset from SASBDB + + Open Project diff --git a/src/sas/qtgui/Utilities/GuiUtils.py b/src/sas/qtgui/Utilities/GuiUtils.py index 67bace0f06..94e3d47dc6 100644 --- a/src/sas/qtgui/Utilities/GuiUtils.py +++ b/src/sas/qtgui/Utilities/GuiUtils.py @@ -39,6 +39,7 @@ from sas.qtgui.Plotting.Plottables import Chisq, Plottable, PlottableFit1D, PlottableTheory1D, Text, View from sas.qtgui.Plotting.PlotterData import Data1D, Data2D, DataRole from sas.qtgui.Utilities.BackgroundColor import BG_DEFAULT, BG_WARNING +from sas.qtgui.Utilities.SASBDB.sasbdb_display import append_sasbdb_data_summary from sas.sascalc.fit.AbstractFitEngine import FitData1D, FitData2D, FResult from sas.system import HELP_SYSTEM from sas.system.user import PATH_LIKE @@ -549,7 +550,8 @@ def retrieveData1d(data): #logger.error(msg) raise ValueError(msg) - text = data.__str__() + text = append_sasbdb_data_summary(data.__str__(), data) + text += 'Data Min Max:\n' text += 'X_min = %s: X_max = %s\n' % (xmin, max(data.x)) text += 'Y_min = %s: Y_max = %s\n' % (ymin, max(data.y)) @@ -593,7 +595,8 @@ def retrieveData2d(data): msg = "Incorrect type passed to retrieveData2d" raise AttributeError(msg) - text = data.__str__() + text = append_sasbdb_data_summary(data.__str__(), data) + text += 'Data Min Max:\n' text += 'I_min = %s\n' % min(data.data) text += 'I_max = %s\n\n' % max(data.data) diff --git a/src/sas/qtgui/Utilities/SASBDB/SASBDBDownloadDialog.py b/src/sas/qtgui/Utilities/SASBDB/SASBDBDownloadDialog.py new file mode 100644 index 0000000000..ac84c65631 --- /dev/null +++ b/src/sas/qtgui/Utilities/SASBDB/SASBDBDownloadDialog.py @@ -0,0 +1,90 @@ +""" +SASBDB Dataset Download Dialog. +""" + +import logging +import os +import tempfile + +from PySide6 import QtWidgets + +from .sasbdb_api import SASBDBDatasetInfo, downloadDataset, validateDatasetId +from .sasbdb_display import metadata_summary +from .UI.SASBDBDownloadDialogUI import Ui_SASBDBDownloadDialogUI + +logger = logging.getLogger(__name__) + + +class SASBDBDownloadDialog(QtWidgets.QDialog, Ui_SASBDBDownloadDialogUI): + """Dialog for downloading datasets from SASBDB.""" + + def __init__(self, parent=None): + super().__init__(parent) + self.setupUi(self) + self.downloaded_filepath: str | None = None + self.dataset_info: SASBDBDatasetInfo | None = None + + self.cmdDownload.clicked.connect(self.onDownload) + self.cmdCancel.clicked.connect(self.reject) + self.cmdHelp.clicked.connect(self.onHelp) + self.txtDatasetId.returnPressed.connect(self.onDownload) + self.txtDatasetId.setFocus() + + def onDownload(self): + dataset_id = self.txtDatasetId.text().strip() + if not dataset_id: + self._showError("Please enter a dataset identifier.") + return + + normalized_id, validation_error = validateDatasetId(dataset_id) + if validation_error: + self._showError(validation_error) + return + + self._set_busy(True) + self.lblStatus.setText("Downloading dataset...") + + try: + filepath, dataset_info = downloadDataset(normalized_id, tempfile.gettempdir()) + self.dataset_info = dataset_info + + if filepath and os.path.exists(filepath): + self.downloaded_filepath = filepath + summary = metadata_summary(dataset_info) if dataset_info else "" + status = f"Successfully downloaded dataset {normalized_id}" + if summary: + status += f"\n{summary}" + self.lblStatus.setText(status) + self.accept() + else: + self._showError( + f"Failed to download dataset {normalized_id}.\n" + "Please check the dataset identifier and try again." + ) + except Exception as e: + logger.error(f"Error downloading dataset {normalized_id}: {e}", exc_info=True) + self._showError(f"Error downloading dataset:\n{e}") + finally: + self._set_busy(False) + + def _set_busy(self, busy: bool): + self.cmdDownload.setEnabled(not busy) + self.cmdCancel.setEnabled(not busy) + self.progressBar.setVisible(busy) + if busy: + self.progressBar.setRange(0, 0) + QtWidgets.QApplication.processEvents() + + def _showError(self, message: str): + self.lblStatus.setText(f"{message}") + QtWidgets.QMessageBox.warning(self, "Download Error", message) + + def getDownloadedFilepath(self) -> str | None: + return self.downloaded_filepath + + def getDatasetInfo(self) -> SASBDBDatasetInfo | None: + return self.dataset_info + + def onHelp(self): + from sas.qtgui.Utilities import GuiUtils + GuiUtils.showHelp("user/qtgui/Utilities/SASBDB/sasbdb_download_help.html") diff --git a/src/sas/qtgui/Utilities/SASBDB/UI/SASBDBDownloadDialogUI.ui b/src/sas/qtgui/Utilities/SASBDB/UI/SASBDBDownloadDialogUI.ui new file mode 100644 index 0000000000..9a2ad0a09e --- /dev/null +++ b/src/sas/qtgui/Utilities/SASBDB/UI/SASBDBDownloadDialogUI.ui @@ -0,0 +1,123 @@ + + + SASBDBDownloadDialogUI + + + + 0 + 0 + 450 + 200 + + + + Load from SASBDB + + + + + + Enter SASBDB Dataset Identifier: + + + + + + + e.g., SASDN24 + + + + + + + + + + true + + + + + + + 0 + + + 0 + + + 0 + + + false + + + + + + + Qt::Vertical + + + + 20 + 40 + + + + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + Download and Load + + + true + + + + + + + Help + + + + + + + Cancel + + + + + + + + + + + cmdCancel + clicked() + SASBDBDownloadDialogUI + reject() + + + + diff --git a/src/sas/qtgui/Utilities/SASBDB/UI/__init__.py b/src/sas/qtgui/Utilities/SASBDB/UI/__init__.py new file mode 100644 index 0000000000..88400a3f7c --- /dev/null +++ b/src/sas/qtgui/Utilities/SASBDB/UI/__init__.py @@ -0,0 +1,4 @@ +""" +UI components for SASBDB utilities. +""" + diff --git a/src/sas/qtgui/Utilities/SASBDB/__init__.py b/src/sas/qtgui/Utilities/SASBDB/__init__.py new file mode 100644 index 0000000000..5ce0e3c888 --- /dev/null +++ b/src/sas/qtgui/Utilities/SASBDB/__init__.py @@ -0,0 +1,7 @@ +""" +SASBDB (Small Angle Scattering Biological Data Bank) utilities. + +This package provides functionality for interacting with SASBDB, +including downloading datasets and exporting data to SASBDB format. +""" + diff --git a/src/sas/qtgui/Utilities/SASBDB/media/sasbdb_download_help.rst b/src/sas/qtgui/Utilities/SASBDB/media/sasbdb_download_help.rst new file mode 100644 index 0000000000..054edcf1ab --- /dev/null +++ b/src/sas/qtgui/Utilities/SASBDB/media/sasbdb_download_help.rst @@ -0,0 +1,115 @@ +.. sasbdb_download_help.rst + +.. _SASBDB_Download: + +Loading Datasets from SASBDB +============================= + +Description +----------- + +The SASBDB (Small Angle Scattering Biological Data Bank) download feature allows you to directly download and load datasets from the SASBDB database into SasView. This feature provides easy access to published small-angle scattering data and automatically populates metadata from the SASBDB entry. + +Accessing the Feature +--------------------- + +To access the SASBDB download feature: + +1. Select **File > Load from SASBDB...** from the main menu +2. A dialog will appear prompting you to enter a SASBDB dataset identifier + +Dataset Identifier +------------------ + +Enter a valid SASBDB dataset identifier in the input field. The identifier can be: + +- A 7-character SASBDB code (e.g., ``SASDN24``) +- The identifier is case-insensitive and will be automatically normalized + +Examples of valid identifiers: +- ``SASDN24`` +- ``sasdn24`` (will be converted to uppercase) + +Download Process +---------------- + +When you click **Download and Load**: + +1. **Validation**: The dataset identifier is validated +2. **Metadata Retrieval**: The system fetches metadata from the SASBDB REST API +3. **Data Download**: The experimental data file is downloaded to a temporary location +4. **Data Loading**: The downloaded data is automatically loaded into SasView +5. **Metadata Population**: Metadata from SASBDB is automatically populated into the loaded dataset + +Metadata Population +-------------------- + +When a dataset is loaded from SASBDB, the following metadata is automatically extracted and populated: + +**Sample Information:** +- Sample ID (from molecule short name, if available) +- Sample details section populated with: + - Sequence (when available) + - Molecule/sample description + - UniProt code (when available) + - Oligomerization and number of molecules (when available) + - Source organism (when available) + - Temperature + - Concentration + - Buffer description and pH + +**Instrument Information:** +- Instrument/beamline name +- Detector information +- Location (city, country) +- Source type (X-ray synchrotron, neutron, etc.) + +**Experimental Parameters:** +- Wavelength +- Temperature +- Q-range (if available) + +**Structural Parameters:** +- Radius of gyration (Rg) with errors +- I(0) with errors +- Maximum dimension (Dmax) +- Molecular weight (MW) with method +- Porod volume + +**Publication Information:** +- Authors +- DOI +- PMID + +Viewing Metadata +---------------- + +After loading a SASBDB dataset, you can view the populated metadata by: + +1. Right-clicking on the loaded dataset in the Data Explorer +2. Selecting **Data Info** from the context menu +3. The metadata will be displayed in a clean, formatted section labeled "SASBDB Metadata" + +The metadata is displayed in a compact format with key information organized by category: +- Entry code and instrument information +- Sample ID and sample details (sequence, UniProt, oligomerization, concentration, buffer, etc.) +- Source wavelength +- Structural parameters (Rg, I(0), Dmax, MW, etc.) +- Publication information + + +Tips +---- + +- **Dataset Identifiers**: You can find SASBDB dataset identifiers in published papers or on the SASBDB website (https://www.sasbdb.org) +- **Metadata**: All metadata is automatically extracted from the SASBDB entry, so you don't need to manually enter it +- **Data Format**: The downloaded data is in a standard format compatible with SasView +- **Offline Use**: Once downloaded, the data file is stored locally and can be used offline + +Related Documentation +--------------------- + +- :ref:`SASBDB Export ` - Export your data to SASBDB format +- `SASBDB Website `_ - Browse and search the SASBDB database +- `SASBDB REST API Documentation `_ - Technical API reference + diff --git a/src/sas/qtgui/Utilities/SASBDB/sasbdb_api.py b/src/sas/qtgui/Utilities/SASBDB/sasbdb_api.py new file mode 100644 index 0000000000..0e912d7157 --- /dev/null +++ b/src/sas/qtgui/Utilities/SASBDB/sasbdb_api.py @@ -0,0 +1,184 @@ +""" +SASBDB REST API client module. +""" + +import logging +import os +import re +import tempfile + +import requests + +from .sasbdb_parse import SASBDBDatasetInfo, parseMetadata + +logger = logging.getLogger(__name__) + +SASBDB_API_BASE = "https://www.sasbdb.org/rest-api" +SASBDB_SITE_BASE = "https://www.sasbdb.org" +_SASBDB_ID_PATTERN = re.compile(r"^SAS[A-Z]{2}\d+$") +INVALID_DATASET_ID_MESSAGE = "Enter the full 7-character SASBDB code (e.g. SASDN24)." + +_URL_FIELDS = ( + "intensities_data", "intensitiesData", "data_file_url", "dataFileUrl", + "data_file", "dataFile", "scattering_data_url", "scatteringDataUrl", + "experimental_data_url", "experimentalDataUrl", "download_url", "downloadUrl", + "file_url", "fileUrl", "files", "data_files", "dataFiles", + "experimental_files", "scattering_files", "experimental_data", + "experimentalData", "scattering_data", "scatteringData", +) +_URL_NESTED = ("entry", "data", "files", "experimental_data", "scattering_data") +_URL_ITEM_KEYS = ("url", "path", "file", "file_url", "download_url") +_KNOWN_EXTENSIONS = {".dat", ".txt", ".csv", ".out", ".asc"} + + +def getDatasetMetadata(dataset_id: str) -> dict | None: + """Fetch dataset metadata from the SASBDB API.""" + normalized_id = _normalizeDatasetId(dataset_id) + if not normalized_id: + logger.error("Invalid dataset ID format: %s", dataset_id) + return None + + endpoint = f"{SASBDB_API_BASE}/entry/summary/{normalized_id}/" + try: + logger.info("Fetching dataset metadata from: %s", endpoint) + response = requests.get(endpoint, headers={"accept": "application/json"}, timeout=30) + response.raise_for_status() + logger.info("Successfully retrieved metadata for dataset %s", normalized_id) + return response.json() + except requests.exceptions.HTTPError as error: + if error.response is not None and error.response.status_code == 404: + logger.error("Dataset %s not found (404)", normalized_id) + else: + logger.error("HTTP error fetching dataset %s: %s", normalized_id, error) + except requests.exceptions.RequestException as error: + logger.error("Network error fetching dataset %s: %s", normalized_id, error) + except ValueError as error: + logger.error("Invalid JSON response for dataset %s: %s", normalized_id, error) + return None + + +def getDataFileUrl(metadata: dict) -> str | None: + """Extract a data file URL from dataset metadata.""" + if not isinstance(metadata, dict): + return None + + for field_name in _URL_FIELDS: + if field_name in metadata: + url = _resolve_url_value(metadata[field_name]) + if url: + return url + + for nested_key in _URL_NESTED: + nested = metadata.get(nested_key) + if isinstance(nested, dict): + url = getDataFileUrl(nested) + if url: + return url + + logger.warning("Could not find data file URL in metadata") + logger.debug("Metadata keys: %s", list(metadata.keys())) + return None + + +def downloadDataFile(url: str, filepath: str) -> bool: + """Download a data file from the given URL.""" + try: + logger.info("Downloading data file from: %s", url) + response = requests.get(url, timeout=60, stream=True) + response.raise_for_status() + os.makedirs(os.path.dirname(filepath), exist_ok=True) + with open(filepath, "wb") as handle: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + handle.write(chunk) + logger.info("Successfully downloaded data file to: %s", filepath) + return True + except requests.exceptions.RequestException as error: + logger.error("Error downloading data file from %s: %s", url, error) + except OSError as error: + logger.error("Error writing data file to %s: %s", filepath, error) + return False + + +def validateDatasetId(dataset_id: str) -> tuple[str | None, str | None]: + """Validate and normalize a SASBDB dataset identifier.""" + normalized = _normalizeDatasetId(dataset_id) + if normalized: + return normalized, None + return None, INVALID_DATASET_ID_MESSAGE + + +def downloadDataset( + dataset_id: str, output_dir: str | None = None, +) -> tuple[str | None, SASBDBDatasetInfo | None]: + """Fetch metadata, download the intensity file, and return path + parsed info.""" + metadata = getDatasetMetadata(dataset_id) + if not metadata: + return None, None + + dataset_info = parseMetadata(metadata) + data_url = dataset_info.intensities_data_url or getDataFileUrl(metadata) + if not data_url: + logger.error("Could not find data file URL in metadata for dataset %s", dataset_id) + return None, dataset_info + + dataset_info.intensities_data_url = data_url + output_dir = output_dir or tempfile.gettempdir() + normalized_id = _normalizeDatasetId(dataset_id) + filename = f"SASBDB_{normalized_id}{_guessFileExtension(data_url, metadata)}" + filepath = os.path.join(output_dir, filename) + + if downloadDataFile(data_url, filepath): + return filepath, dataset_info + return None, dataset_info + + +def _normalizeDatasetId(dataset_id: str) -> str | None: + if not dataset_id: + return None + normalized = dataset_id.strip().upper() + if len(normalized) != 7 or not _SASBDB_ID_PATTERN.match(normalized): + logger.warning("Invalid SASBDB dataset ID: %r", dataset_id) + return None + return normalized + + +def _resolve_url_value(value) -> str | None: + if isinstance(value, str): + return _absolute_url(value) + if isinstance(value, list) and value: + return _resolve_url_value(value[0]) + if isinstance(value, dict): + for key in _URL_ITEM_KEYS: + if key in value: + return _resolve_url_value(value[key]) + return None + + +def _absolute_url(url: str) -> str | None: + if url.startswith("/"): + return f"{SASBDB_SITE_BASE}{url}" + if url.startswith("http"): + return url + return None + + +def _guessFileExtension(url: str, metadata: dict) -> str: + if "." in url: + ext = "." + url.split(".")[-1].split("?")[0] + if ext.lower() in _KNOWN_EXTENSIONS: + return ext + + if isinstance(metadata, dict): + for field_name in ("file_type", "fileType", "format", "data_format"): + file_type = metadata.get(field_name) + if file_type is None: + continue + file_type = str(file_type).lower() + if "csv" in file_type: + return ".csv" + if "txt" in file_type or "text" in file_type: + return ".txt" + if "dat" in file_type or "data" in file_type: + return ".dat" + return ".dat" diff --git a/src/sas/qtgui/Utilities/SASBDB/sasbdb_display.py b/src/sas/qtgui/Utilities/SASBDB/sasbdb_display.py new file mode 100644 index 0000000000..385b2e53a5 --- /dev/null +++ b/src/sas/qtgui/Utilities/SASBDB/sasbdb_display.py @@ -0,0 +1,332 @@ +""" +Format SASBDB metadata for dialogs, logs, and data summary panels. +""" + +import ast +import re + +from .sasbdb_api import SASBDBDatasetInfo + + +def is_sasbdb_data(data) -> bool: + meta = getattr(data, "meta_data", None) or {} + return "SASBDB_code" in meta + + +def metadata_summary(info: SASBDBDatasetInfo) -> str: + """Format dataset metadata for dialog status and logging.""" + lines = [] + if info.title: + lines.append(f"Title: {info.title}") + if info.sample_name: + lines.append(f"Sample: {info.sample_name}") + if info.molecule_name: + lines.append(f"Molecule: {info.molecule_name}") + if info.concentration: + lines.append(f"Concentration: {info.concentration} {info.concentration_unit}") + if info.buffer_description: + lines.append(f"Buffer: {info.buffer_description}") + if info.instrument: + lines.append(f"Instrument: {info.instrument}") + if info.wavelength: + lines.append(f"Wavelength: {info.wavelength} {info.wavelength_unit}") + if info.temperature: + lines.append(f"Temperature: {info.temperature} {info.temperature_unit}") + if info.rg is not None: + lines.append(_format_rg_line(info.rg, info.rg_error, style="label")) + if info.i0 is not None: + lines.append(_format_i0_line(info.i0, info.i0_error, style="label")) + if info.dmax is not None: + lines.append(f"Dmax: {info.dmax} Å") + if info.molecular_weight is not None: + lines.append(f"MW: {info.molecular_weight:.1f} kDa") + if info.publication_doi: + lines.append(f"DOI: {info.publication_doi}") + return "\n".join(lines) + + +def append_sasbdb_data_summary(text: str, data) -> str: + """Clean dict repr lines and append SASBDB metadata for data summary panels.""" + if not is_sasbdb_data(data): + return text + return _clean_dict_lines(text) + format_data_panel(data) + + +def format_data_panel(data) -> str: + """Format SASBDB metadata from a loaded data object.""" + meta = getattr(data, "meta_data", None) or {} + if "SASBDB_code" not in meta: + return "" + + lines = [ + "\n" + "=" * 50, + "SASBDB Metadata", + "=" * 50, + ] + + header_parts = [] + if meta.get("SASBDB_code"): + header_parts.append(f"Code: {meta['SASBDB_code']}") + + instrument_str, location_str = _instrument_from_data(data, meta) + if instrument_str: + header_parts.append(f"Instrument: {instrument_str}") + if location_str: + header_parts.append(location_str) + + detector_info = _detector_from_meta(meta) + if detector_info: + header_parts.append(f"Detector: {detector_info}") + + if header_parts: + lines.append(" | ".join(header_parts)) + + sample_line = _sample_line_from_data(data) + if sample_line: + lines.append(sample_line) + + if hasattr(data, "source") and data.source: + source = data.source + if hasattr(source, "wavelength") and source.wavelength is not None: + wl_unit = getattr(source, "wavelength_unit", "Å") or "Å" + lines.append(f"Wavelength: {source.wavelength} {wl_unit}") + + structural = _structural_lines_from_meta(meta) + if structural: + lines.append("Structural: " + " | ".join(structural)) + + publication = _publication_lines_from_meta(meta) + if publication: + lines.append("Publication: " + " | ".join(publication)) + + lines.extend(["=" * 50, ""]) + return "\n".join(line + "\n" for line in lines) + + +def _format_rg_line(rg, rg_error=None, style="label") -> str: + if style == "panel": + text = f"Rg = {rg:.2f}" + if rg_error is not None: + text += f" ± {rg_error:.2f}" + return text + " Å" + rg_text = f"Rg: {rg:.2f}" + if rg_error: + rg_text += f" ± {rg_error:.2f}" + return f"{rg_text} Å" + + +def _format_i0_line(i0, i0_error=None, style="label") -> str: + if style == "panel": + text = f"I(0) = {i0:.4e}" + if i0_error is not None: + text += f" ± {i0_error:.4e}" + return text + text = f"I(0): {i0}" + if i0_error: + text += f" ± {i0_error}" + return text + + +def _instrument_parts_from_dict(parsed_dict: dict) -> list[str]: + parts = [] + if parsed_dict.get("name"): + parts.append(parsed_dict["name"]) + if parsed_dict.get("beamline_name"): + parts.append(f"Beamline: {parsed_dict['beamline_name']}") + if parsed_dict.get("type_of_source"): + parts.append(f"Source: {parsed_dict['type_of_source']}") + if parsed_dict.get("city"): + city = parsed_dict["city"] + if parsed_dict.get("country"): + parts.append(f"{city}, {parsed_dict['country']}") + else: + parts.append(city) + elif parsed_dict.get("country"): + parts.append(parsed_dict["country"]) + + detector_info = None + detector = parsed_dict.get("detector") + if isinstance(detector, dict): + detector_info = detector.get("name") or detector.get("type") + elif detector: + detector_info = str(detector) + if detector_info: + parts.append(f"Detector: {detector_info}") + return parts + + +def _instrument_from_data(data, meta: dict) -> tuple[str | None, str | None]: + instrument_str = None + location_str = None + + if hasattr(data, "instrument") and data.instrument: + if isinstance(data.instrument, str): + instrument_str = data.instrument + elif isinstance(data.instrument, dict): + instrument_str = data.instrument.get("name") or data.instrument.get("beamline_name") + if data.instrument.get("city") and data.instrument.get("country"): + location_str = f"{data.instrument['city']}, {data.instrument['country']}" + elif data.instrument.get("city"): + location_str = data.instrument["city"] + + if not instrument_str: + for key, val in meta.items(): + if isinstance(val, dict) and ( + "beamline_name" in val or "name" in val or "type_of_source" in val + ): + instrument_str = val.get("beamline_name") or val.get("name") + if val.get("city") and val.get("country"): + location_str = f"{val['city']}, {val['country']}" + elif val.get("city"): + location_str = val["city"] + break + if key in ("instrument", "source", "beamline") and isinstance(val, str): + instrument_str = val + break + + return instrument_str, location_str + + +def _detector_from_meta(meta: dict) -> str | None: + for key, val in meta.items(): + if isinstance(val, dict) and ("detector" in key.lower() or "type" in val): + detector_name = val.get("name") or val.get("type") + if detector_name: + return detector_name + if "detector" in key.lower() and isinstance(val, str): + return val + return None + + +def _sample_line_from_data(data) -> str | None: + if not hasattr(data, "sample") or not data.sample: + return None + + sample = data.sample + sample_parts = [] + if hasattr(sample, "name") and sample.name: + sample_parts.append(sample.name) + if hasattr(sample, "temperature") and sample.temperature is not None: + temp_unit = getattr(sample, "temperature_unit", "K") or "K" + sample_parts.append(f"T = {sample.temperature} {temp_unit}") + if hasattr(sample, "details") and sample.details: + for detail in sample.details: + if detail.startswith("Concentration:"): + sample_parts.append(detail.replace("Concentration: ", "")) + elif detail.startswith("Buffer:"): + sample_parts.append(detail.replace("Buffer: ", "")) + + if sample_parts: + return f"Sample: {' | '.join(sample_parts)}" + return None + + +def _structural_lines_from_meta(meta: dict) -> list[str]: + lines = [] + if meta.get("SASBDB_Rg") is not None: + lines.append(_format_rg_line(meta["SASBDB_Rg"], meta.get("SASBDB_Rg_error"), style="panel")) + if meta.get("SASBDB_I0") is not None: + lines.append(_format_i0_line(meta["SASBDB_I0"], meta.get("SASBDB_I0_error"), style="panel")) + if meta.get("SASBDB_Dmax") is not None: + lines.append(f"Dmax = {meta['SASBDB_Dmax']:.2f} Å") + if meta.get("SASBDB_MW") is not None: + mw = f"MW = {meta['SASBDB_MW']:.2f} kDa" + if meta.get("SASBDB_MW_method"): + mw += f" ({meta['SASBDB_MW_method']})" + lines.append(mw) + if meta.get("SASBDB_Porod_volume") is not None: + lines.append(f"Porod Volume = {meta['SASBDB_Porod_volume']:.0f} ų") + return lines + + +def _publication_lines_from_meta(meta: dict) -> list[str]: + lines = [] + if meta.get("SASBDB_authors"): + lines.append(f"Authors: {_truncate_authors(meta['SASBDB_authors'])}") + if meta.get("SASBDB_DOI"): + lines.append(f"DOI: {meta['SASBDB_DOI']}") + if meta.get("SASBDB_PMID"): + lines.append(f"PMID: {meta['SASBDB_PMID']}") + return lines + + +def _truncate_authors(authors: str, max_len: int = 50) -> str: + if len(authors) <= max_len: + return authors + + names = [name.strip() for name in authors.split(",") if name.strip()] + if not names: + return authors + + shown = names[0] + for name in names[1:]: + candidate = f"{shown}, {name}" + if len(candidate) + len(" et al.") > max_len: + return f"{shown} et al." + shown = candidate + + if len(shown) > max_len: + return f"{shown[: max_len - len(' et al.')].rstrip(', ')} et al." + return shown + + +def _format_dict_line(line: str) -> str: + stripped = line.strip() + if not ("{" in stripped and "'" in stripped and ":" in stripped): + return line + + label_match = re.match(r"^(\w+)\s*:\s*(.+)", stripped) + if label_match: + label = label_match.group(1) + dict_str = label_match.group(2) + elif stripped.startswith("{'") or stripped.startswith("{"): + label = None + dict_str = stripped + else: + return line + + try: + parsed_dict = ast.literal_eval(dict_str) + if not isinstance(parsed_dict, dict): + return line + + parts = _instrument_parts_from_dict(parsed_dict) + if parts: + formatted = " | ".join(parts) + return f"{label}: {formatted}" if label else formatted + + simple_parts = [ + f"{key}: {value}" + for key, value in parsed_dict.items() + if value is not None and not isinstance(value, dict) + ] + if simple_parts and len(simple_parts) <= 5: + formatted = " | ".join(simple_parts) + return f"{label}: {formatted}" if label else formatted + except (ValueError, SyntaxError): + pass + + return line + + +def _clean_dict_lines(text: str) -> str: + cleaned_lines = [] + for line in text.split("\n"): + stripped = line.strip() + if ( + "{" in stripped + and "'" in stripped + and ":" in stripped + and ( + stripped.startswith("{'") + or re.search(r":\s*\{", stripped) + or re.search(r"\{'[^']+':", stripped) + ) + ): + if stripped.count("{") > 0 and stripped.count("}") > 0: + cleaned_lines.append(_format_dict_line(line)) + else: + cleaned_lines.append(line) + else: + cleaned_lines.append(line) + return "\n".join(cleaned_lines) diff --git a/src/sas/qtgui/Utilities/SASBDB/sasbdb_loader.py b/src/sas/qtgui/Utilities/SASBDB/sasbdb_loader.py new file mode 100644 index 0000000000..e560f26bef --- /dev/null +++ b/src/sas/qtgui/Utilities/SASBDB/sasbdb_loader.py @@ -0,0 +1,158 @@ +""" +Load SASBDB datasets into SasView. +""" + +import logging +import os + +from PySide6.QtWidgets import QMessageBox + +from sasdata.dataloader.data_info import Sample, Source + +from .sasbdb_api import SASBDBDatasetInfo +from .sasbdb_display import metadata_summary + +logger = logging.getLogger(__name__) + +_META_FIELDS = ( + ("SASBDB_Rg", "rg", ("SASBDB_Rg_error", "rg_error")), + ("SASBDB_I0", "i0", ("SASBDB_I0_error", "i0_error")), + ("SASBDB_Dmax", "dmax", None), + ("SASBDB_MW", "molecular_weight", ("SASBDB_MW_method", "molecular_weight_method")), + ("SASBDB_Porod_volume", "porod_volume", None), + ("SASBDB_molecule", "molecule_name", None), + ("SASBDB_molecule_type", "molecule_type", None), + ("SASBDB_oligomeric_state", "oligomeric_state", None), + ("SASBDB_DOI", "publication_doi", None), + ("SASBDB_PMID", "publication_pmid", None), +) + + +def load_downloaded_dataset(files_widget, parent, filepath: str | None, + dataset_info: SASBDBDatasetInfo | None) -> None: + """Load a downloaded SASBDB file and apply metadata.""" + if not filepath or not os.path.exists(filepath): + return + + try: + loaded_data, load_message = files_widget.readData([filepath]) + if not loaded_data: + logger.error("Failed to load SASBDB dataset from %s: %s", filepath, load_message) + QMessageBox.warning( + parent, + "Load Error", + f"Failed to load downloaded dataset:\n{load_message}", + ) + return + + if dataset_info: + populate_metadata(loaded_data, dataset_info) + + entry_id = dataset_info.code or dataset_info.entry_id if dataset_info else None + logger.info( + "Successfully loaded SASBDB dataset %s from %s", + entry_id or "unknown", + filepath, + ) + if dataset_info: + for line in metadata_summary(dataset_info).splitlines(): + logger.info(" %s", line) + except Exception as e: + logger.error(f"Error loading downloaded SASBDB dataset: {e}", exc_info=True) + QMessageBox.warning(parent, "Load Error", f"Failed to load downloaded dataset:\n{e}") + + +def populate_metadata(loaded_data: dict, dataset_info: SASBDBDatasetInfo) -> None: + """Populate SASBDB metadata into loaded data objects.""" + for data_id, data in loaded_data.items(): + try: + _apply_metadata(data, dataset_info) + except Exception as e: + logger.warning(f"Error populating metadata for data {data_id}: {e}") + + +def _apply_metadata(data, info: SASBDBDatasetInfo) -> None: + if info.title and not data.title: + data.title = info.title + if info.instrument and not data.instrument: + data.instrument = info.instrument + if info.code: + if not data.run: + data.run = [] + if info.code not in data.run: + data.run.append(info.code) + + if data.sample is None: + data.sample = Sample() + if not getattr(data.sample, "details", None): + data.sample.details = [] + + sample_id = info.molecule_short_name or info.sample_name or info.code + if sample_id: + data.sample.ID = sample_id + + details = [] + if info.molecule_name: + detail = f"Molecule: {info.molecule_name}" + if info.molecule_type: + detail += f" ({info.molecule_type})" + details.append(detail) + elif info.sample_name: + details.append(f"Sample: {info.sample_name}") + if info.sample_description: + details.append(f"Description: {info.sample_description}") + if info.sequence: + details.append(f"Sequence: {info.sequence}") + if info.uniprot_code: + details.append(f"UniProt: {info.uniprot_code}") + oligomerization = info.oligomerization or info.oligomeric_state + if oligomerization: + details.append(f"Oligomerization: {oligomerization}") + if info.number_of_molecules: + details.append(f"Number of molecules: {info.number_of_molecules}") + if info.source_organism: + details.append(f"Source organism: {info.source_organism}") + if info.temperature is not None: + data.sample.temperature = info.temperature + if info.temperature_unit: + data.sample.temperature_unit = info.temperature_unit + temp = f"Temperature: {info.temperature}" + if info.temperature_unit: + temp += f" {info.temperature_unit}" + details.append(temp) + if info.concentration is not None: + conc = f"Concentration: {info.concentration}" + if info.concentration_unit: + conc += f" {info.concentration_unit}" + details.append(conc) + if info.buffer_description: + buffer = f"Buffer: {info.buffer_description}" + if info.ph is not None: + buffer += f" (pH {info.ph})" + details.append(buffer) + + for detail in details: + if detail not in data.sample.details: + data.sample.details.append(detail) + + if data.source is None: + data.source = Source() + if info.wavelength is not None: + data.source.wavelength = info.wavelength + data.source.wavelength_unit = info.wavelength_unit + + if not getattr(data, "meta_data", None): + data.meta_data = {} + data.meta_data["SASBDB_code"] = info.code or info.entry_id + for key, attr, extra in _META_FIELDS: + value = getattr(info, attr, None) + if value is None: + continue + data.meta_data[key] = value + if extra: + extra_key, extra_attr = extra + extra_value = getattr(info, extra_attr, None) + if extra_value is not None: + data.meta_data[extra_key] = extra_value + if info.authors: + data.meta_data["SASBDB_authors"] = ", ".join(info.authors) diff --git a/src/sas/qtgui/Utilities/SASBDB/sasbdb_parse.py b/src/sas/qtgui/Utilities/SASBDB/sasbdb_parse.py new file mode 100644 index 0000000000..1b393ab8e6 --- /dev/null +++ b/src/sas/qtgui/Utilities/SASBDB/sasbdb_parse.py @@ -0,0 +1,230 @@ +"""Parse SASBDB REST API metadata into structured objects.""" + +import logging +from dataclasses import dataclass, field + +logger = logging.getLogger(__name__) + +_NESTED_PATHS = ("sample", "molecule", "experiment", "experimental", "metadata", "info") +_SEQUENCE_KEYS = frozenset( + {"sequence", "fasta_sequence", "fasta", "primary_sequence", "sequence_string"} +) + +_STR_FIELDS = { + "entry_id": ("id", "entry_id", "sasbdb_id"), + "code": ("code", "accession_code", "entry_code"), + "title": ("title", "entry_title", "sample_title"), + "sample_name": ("sample_name", "sample", "name", "sample_name_full"), + "sample_description": ("sample_description", "description", "sample_description_full"), + "concentration_unit": ("concentration_unit", "conc_unit", "concentration_units"), + "molecule_name": ( + "molecule_name", "macromolecule_name", "protein_name", "molecule", + "macromolecule", "protein", "name", "long_name", + ), + "molecule_short_name": ("short_name", "molecule_short_name", "shortName", "short"), + "molecule_type": ( + "molecule_type", "macromolecule_type", "sample_type", "type", + "molecule_type_full", "molecular_type", + ), + "uniprot_code": ("uniprot_code", "uniprot", "uniprot_id", "uniprot_accession", "uniprot_ac"), + "source_organism": ("source_organism", "organism", "organism_name"), + "number_of_molecules": ( + "number_of_molecules", "num_molecules", "copy_number", "number_molecules", + ), + "oligomerization": ( + "oligomerization", "oligomeric_state", "oligomer_state", "complex_state", + ), + "molecular_weight_method": ("mw_method", "molecular_weight_method"), + "buffer_description": ("buffer", "buffer_description", "buffer_composition"), + "instrument": ("instrument", "beamline", "instrument_name"), + "detector": ("detector", "detector_name", "detector_type"), + "publication_title": ("publication_title", "pub_title", "paper_title"), + "publication_doi": ("doi", "publication_doi", "pub_doi"), + "publication_pmid": ("pmid", "publication_pmid", "pubmed_id"), + "intensities_data_url": ("intensities_data", "data_url", "intensities_url"), + "pddf_data_url": ("pddf_data", "pddf_url", "pr_data"), +} + +_FLOAT_FIELDS = { + "concentration": ("concentration", "sample_concentration", "conc", "sample_conc"), + "molecular_weight": ("molecular_weight", "mw", "exp_mw", "experimental_mw", "mw_kda"), + "ph": ("ph", "buffer_ph"), + "wavelength": ("wavelength", "xray_wavelength", "neutron_wavelength"), + "temperature": ("temperature", "sample_temperature", "temp"), + "rg": ("rg", "radius_of_gyration", "guinier_rg"), + "rg_error": ("rg_error", "rg_err", "guinier_rg_error"), + "i0": ("i0", "i_zero", "guinier_i0"), + "i0_error": ("i0_error", "i0_err", "guinier_i0_error"), + "dmax": ("dmax", "d_max", "maximum_dimension"), + "porod_volume": ("porod_volume", "volume", "porod_vol"), + "q_min": ("q_min", "qmin", "s_min", "smin"), + "q_max": ("q_max", "qmax", "s_max", "smax"), +} + + +@dataclass +class SASBDBDatasetInfo: + """Parsed metadata from a SASBDB dataset entry.""" + entry_id: str = "" + code: str = "" + title: str = "" + sample_name: str = "" + sample_description: str = "" + concentration: float | None = None + concentration_unit: str = "mg/mL" + molecule_name: str = "" + molecule_short_name: str = "" + molecule_type: str = "" + sequence: str = "" + uniprot_code: str = "" + source_organism: str = "" + number_of_molecules: str = "" + oligomerization: str = "" + molecular_weight: float | None = None + molecular_weight_method: str = "" + oligomeric_state: str = "" + buffer_description: str = "" + ph: float | None = None + instrument: str = "" + detector: str = "" + wavelength: float | None = None + wavelength_unit: str = "Å" + temperature: float | None = None + temperature_unit: str = "K" + rg: float | None = None + rg_error: float | None = None + i0: float | None = None + i0_error: float | None = None + dmax: float | None = None + porod_volume: float | None = None + q_min: float | None = None + q_max: float | None = None + publication_title: str = "" + publication_doi: str = "" + publication_pmid: str = "" + authors: list = field(default_factory=list) + intensities_data_url: str = "" + pddf_data_url: str = "" + raw_metadata: dict = field(default_factory=dict) + + +def parseMetadata(metadata: dict) -> SASBDBDatasetInfo: + """Parse SASBDB API response into a structured SASBDBDatasetInfo object.""" + info = SASBDBDatasetInfo() + if not isinstance(metadata, dict): + return info + + info.raw_metadata = metadata + logger.debug("SASBDB API response keys: %s", list(metadata.keys())) + + for attr, keys in _STR_FIELDS.items(): + setattr(info, attr, _get_str(metadata, *keys)) + for attr, keys in _FLOAT_FIELDS.items(): + setattr(info, attr, _get_float(metadata, *keys)) + + info.sequence = _get_sequence(metadata) + info.concentration_unit = info.concentration_unit or "mg/mL" + info.oligomeric_state = info.oligomerization or _get_str( + metadata, "oligomeric_state", "oligomer_state", "oligomerization" + ) + + authors = metadata.get("authors") or metadata.get("author_list") or [] + if isinstance(authors, list): + info.authors = [str(author) for author in authors] + elif isinstance(authors, str): + info.authors = [authors] + + return info + + +def _get_str(data: dict, *keys: str) -> str: + value = _find_shallow(data, keys, as_float=False) + return value if isinstance(value, str) else _find_deep(data, keys, as_float=False) or "" + + +def _get_float(data: dict, *keys: str) -> float | None: + value = _find_shallow(data, keys, as_float=True) + return value if isinstance(value, float) else _find_deep(data, keys, as_float=True) + + +def _find_shallow(data: dict, keys: tuple[str, ...], *, as_float: bool): + for key in keys: + if key in data and data[key] is not None: + converted = _convert_value(data[key], as_float=as_float) + if _is_present(converted, as_float): + return converted + + for path in _NESTED_PATHS: + nested = data.get(path) + if isinstance(nested, dict): + for key in keys: + if key in nested and nested[key] is not None: + converted = _convert_value(nested[key], as_float=as_float) + if _is_present(converted, as_float): + return converted + return None + + +def _find_deep(data, keys: tuple[str, ...], *, as_float: bool): + key_set = set(keys) + + def walk(obj): + if isinstance(obj, dict): + for key, value in obj.items(): + if key in key_set: + converted = _convert_value(value, as_float=as_float) + if _is_present(converted, as_float): + return converted + for value in obj.values(): + found = walk(value) + if found is not None: + return found + elif isinstance(obj, list): + for item in obj: + found = walk(item) + if found is not None: + return found + return None + + return walk(data) + + +def _convert_value(value, *, as_float: bool): + if value is None: + return None + if as_float: + try: + return float(value) + except (ValueError, TypeError): + return None + if isinstance(value, str): + return value.strip() + if isinstance(value, (int, float, bool)): + return str(value) + return "" + + +def _is_present(value, as_float: bool) -> bool: + if as_float: + return value is not None + return bool(value) + + +def _get_sequence(data: dict) -> str: + def walk(obj) -> str: + if isinstance(obj, dict): + for key in _SEQUENCE_KEYS: + if key in obj and isinstance(obj[key], str) and obj[key].strip(): + return obj[key].strip() + for value in obj.values(): + found = walk(value) + if found: + return found + elif isinstance(obj, list): + for item in obj: + found = walk(item) + if found: + return found + return "" + + return walk(data) if isinstance(data, dict) else ""