From a564e61e1591c30dc646a647d981262f9607b0bb Mon Sep 17 00:00:00 2001 From: Wojtek Potrzebowski Date: Wed, 14 Jan 2026 22:00:26 +0100 Subject: [PATCH 01/12] Fetching SASBDB entries by ID and loading them into SasView --- src/sas/qtgui/MainWindow/GuiManager.py | 192 +++++++ src/sas/qtgui/MainWindow/UI/MainWindowUI.ui | 13 + src/sas/qtgui/Utilities/GuiUtils.py | 252 +++++++++ .../Utilities/SASBDB/SASBDBDownloadDialog.py | 193 +++++++ .../SASBDB/UI/SASBDBDownloadDialogUI.ui | 116 ++++ src/sas/qtgui/Utilities/SASBDB/UI/__init__.py | 4 + src/sas/qtgui/Utilities/SASBDB/__init__.py | 7 + src/sas/qtgui/Utilities/SASBDB/sasbdb_api.py | 506 ++++++++++++++++++ 8 files changed, 1283 insertions(+) create mode 100644 src/sas/qtgui/Utilities/SASBDB/SASBDBDownloadDialog.py create mode 100644 src/sas/qtgui/Utilities/SASBDB/UI/SASBDBDownloadDialogUI.ui create mode 100644 src/sas/qtgui/Utilities/SASBDB/UI/__init__.py create mode 100644 src/sas/qtgui/Utilities/SASBDB/__init__.py create mode 100644 src/sas/qtgui/Utilities/SASBDB/sasbdb_api.py diff --git a/src/sas/qtgui/MainWindow/GuiManager.py b/src/sas/qtgui/MainWindow/GuiManager.py index 961ec9b358..a304d814b8 100644 --- a/src/sas/qtgui/MainWindow/GuiManager.py +++ b/src/sas/qtgui/MainWindow/GuiManager.py @@ -704,6 +704,7 @@ def addTriggers(self): # File self._workspace.actionLoadData.triggered.connect(self.actionLoadData) self._workspace.actionLoad_Data_Folder.triggered.connect(self.actionLoad_Data_Folder) + self._workspace.actionLoad_SASBDB.triggered.connect(self.actionLoad_SASBDB) self._workspace.actionOpen_Project.triggered.connect(self.actionOpen_Project) self._workspace.actionOpen_Analysis.triggered.connect(self.actionOpen_Analysis) self._workspace.actionSave.triggered.connect(self.actionSave_Project) @@ -800,6 +801,197 @@ def actionLoad_Data_Folder(self): Menu File/Load Data Folder """ self.filesWidget.loadFolder() + + def actionLoad_SASBDB(self): + """ + Menu File/Load from SASBDB + + Opens a dialog to download and load a dataset from SASBDB. + """ + from sas.qtgui.Utilities.SASBDB.SASBDBDownloadDialog import SASBDBDownloadDialog + + dialog = SASBDBDownloadDialog(parent=self._workspace) + if dialog.exec(): + # Get the downloaded file path and metadata + filepath = dialog.getDownloadedFilepath() + dataset_info = dialog.getDatasetInfo() + + if filepath and os.path.exists(filepath): + try: + # Load the downloaded file into SasView + # readData returns (output_dict, message) + loaded_data, load_message = self.filesWidget.readData([filepath]) + + # Populate additional metadata from SASBDB into loaded data + if dataset_info and loaded_data: + self._populateSASBDBMetadata(loaded_data, dataset_info) + + # Log metadata summary + if dataset_info: + entry_id = dataset_info.code or dataset_info.entry_id + logger.info(f"Successfully loaded SASBDB dataset {entry_id} from {filepath}") + if dataset_info.title: + logger.info(f" Title: {dataset_info.title}") + if dataset_info.rg is not None: + logger.info(f" Rg: {dataset_info.rg:.2f} Å") + if dataset_info.molecular_weight is not None: + logger.info(f" MW: {dataset_info.molecular_weight:.1f} kDa") + else: + logger.info(f"Successfully loaded SASBDB dataset from {filepath}") + + except Exception as e: + logger.error(f"Error loading downloaded SASBDB dataset: {e}", exc_info=True) + QMessageBox.warning( + self._workspace, + "Load Error", + f"Failed to load downloaded dataset:\n{str(e)}" + ) + + def _populateSASBDBMetadata(self, loaded_data: dict, dataset_info): + """ + Populate SASBDB metadata into loaded data objects. + + Updates the sample, source, and other metadata properties of loaded + data with information from SASBDB. + + :param loaded_data: Dictionary of loaded data objects (keyed by id) + :param dataset_info: SASBDBDatasetInfo object with parsed metadata + """ + from sasdata.dataloader.data_info import Sample, Source + + for data_id, data in loaded_data.items(): + try: + # Debug: log what we have in dataset_info + logger.debug(f"Populating metadata for {data_id}: molecule_name={dataset_info.molecule_name}, " + f"sample_name={dataset_info.sample_name}, temperature={dataset_info.temperature}, " + f"concentration={dataset_info.concentration}, buffer={dataset_info.buffer_description}") + + # Update title + if dataset_info.title and not data.title: + data.title = dataset_info.title + + # Update instrument from SASBDB instrument metadata + if dataset_info.instrument and not data.instrument: + data.instrument = dataset_info.instrument + + # Update run info with SASBDB code + if dataset_info.code: + if not data.run: + data.run = [] + if dataset_info.code not in data.run: + data.run.append(dataset_info.code) + + # Populate sample information from Molecule metadata + if data.sample is None: + data.sample = Sample() + + # Ensure sample.details is initialized + if not hasattr(data.sample, 'details') or data.sample.details is None: + data.sample.details = [] + + # Use molecule name as sample name (molecule IS the sample in SASBDB context) + # Always set sample name if we have molecule_name or sample_name + if dataset_info.molecule_name: + sample_name = dataset_info.molecule_name + # Add molecule type if available + if dataset_info.molecule_type: + sample_name += f" ({dataset_info.molecule_type})" + # Add oligomeric state if available + if dataset_info.oligomeric_state: + sample_name += f" - {dataset_info.oligomeric_state}" + data.sample.name = sample_name + logger.debug(f"Set sample.name to: {sample_name}") + elif dataset_info.sample_name: + # Fallback to sample_name if molecule_name not available + data.sample.name = dataset_info.sample_name + logger.debug(f"Set sample.name to: {dataset_info.sample_name}") + + # Set temperature + if dataset_info.temperature is not None: + data.sample.temperature = dataset_info.temperature + if dataset_info.temperature_unit: + data.sample.temperature_unit = dataset_info.temperature_unit + logger.debug(f"Set sample.temperature to: {dataset_info.temperature} {dataset_info.temperature_unit}") + + # Store concentration in sample details + if dataset_info.concentration is not None: + conc_str = f"Concentration: {dataset_info.concentration}" + if dataset_info.concentration_unit: + conc_str += f" {dataset_info.concentration_unit}" + data.sample.details.append(conc_str) + logger.debug(f"Added to sample.details: {conc_str}") + + # Add buffer info to sample details + if dataset_info.buffer_description: + buffer_str = f"Buffer: {dataset_info.buffer_description}" + if dataset_info.ph is not None: + buffer_str += f" (pH {dataset_info.ph})" + data.sample.details.append(buffer_str) + logger.debug(f"Added to sample.details: {buffer_str}") + + # Log sample info for debugging + logger.debug(f"Sample populated for {data_id}: name={getattr(data.sample, 'name', None)}, " + f"temperature={getattr(data.sample, 'temperature', None)}, " + f"details={getattr(data.sample, 'details', [])}") + + # Populate source information + if data.source is None: + data.source = Source() + + if dataset_info.wavelength is not None: + data.source.wavelength = dataset_info.wavelength + data.source.wavelength_unit = dataset_info.wavelength_unit + + # Store additional SASBDB metadata in meta_data dictionary + if not hasattr(data, 'meta_data') or data.meta_data is None: + data.meta_data = {} + + # SASBDB-specific metadata + data.meta_data['SASBDB_code'] = dataset_info.code or dataset_info.entry_id + + if dataset_info.rg is not None: + data.meta_data['SASBDB_Rg'] = dataset_info.rg + if dataset_info.rg_error is not None: + data.meta_data['SASBDB_Rg_error'] = dataset_info.rg_error + + if dataset_info.i0 is not None: + data.meta_data['SASBDB_I0'] = dataset_info.i0 + if dataset_info.i0_error is not None: + data.meta_data['SASBDB_I0_error'] = dataset_info.i0_error + + if dataset_info.dmax is not None: + data.meta_data['SASBDB_Dmax'] = dataset_info.dmax + + if dataset_info.molecular_weight is not None: + data.meta_data['SASBDB_MW'] = dataset_info.molecular_weight + if dataset_info.molecular_weight_method: + data.meta_data['SASBDB_MW_method'] = dataset_info.molecular_weight_method + + if dataset_info.porod_volume is not None: + data.meta_data['SASBDB_Porod_volume'] = dataset_info.porod_volume + + if dataset_info.molecule_name: + data.meta_data['SASBDB_molecule'] = dataset_info.molecule_name + + if dataset_info.molecule_type: + data.meta_data['SASBDB_molecule_type'] = dataset_info.molecule_type + + if dataset_info.oligomeric_state: + data.meta_data['SASBDB_oligomeric_state'] = dataset_info.oligomeric_state + + if dataset_info.publication_doi: + data.meta_data['SASBDB_DOI'] = dataset_info.publication_doi + + if dataset_info.publication_pmid: + data.meta_data['SASBDB_PMID'] = dataset_info.publication_pmid + + if dataset_info.authors: + data.meta_data['SASBDB_authors'] = ', '.join(dataset_info.authors) + + logger.debug(f"Populated SASBDB metadata for data: {data_id}") + + except Exception as e: + logger.warning(f"Error populating metadata for data {data_id}: {e}") def actionOpen_Project(self): """ diff --git a/src/sas/qtgui/MainWindow/UI/MainWindowUI.ui b/src/sas/qtgui/MainWindow/UI/MainWindowUI.ui index 65a8656e34..61aee0e236 100755 --- a/src/sas/qtgui/MainWindow/UI/MainWindowUI.ui +++ b/src/sas/qtgui/MainWindow/UI/MainWindowUI.ui @@ -33,6 +33,7 @@ + @@ -310,6 +311,18 @@ Load Data Folder + + + + :/res/file_send-128.png:/res/file_send-128.png + + + Load from SASBDB... + + + Download and load dataset from SASBDB + + Open Project diff --git a/src/sas/qtgui/Utilities/GuiUtils.py b/src/sas/qtgui/Utilities/GuiUtils.py index c7bebbbceb..1f94693761 100644 --- a/src/sas/qtgui/Utilities/GuiUtils.py +++ b/src/sas/qtgui/Utilities/GuiUtils.py @@ -533,6 +533,220 @@ def openLink(url): raise AttributeError(msg) +def _formatDictValue(value, max_depth=2) -> str: + """ + Format a dictionary value for display, extracting useful information. + + :param value: Dictionary or other value to format + :param max_depth: Maximum nesting depth to process + :return: Formatted string representation + """ + if not isinstance(value, dict): + return str(value) + + if max_depth <= 0: + return "{...}" + + # Extract common useful fields from dictionaries + parts = [] + + # Instrument/source dictionaries + if 'name' in value: + parts.append(value['name']) + if 'beamline_name' in value: + parts.append(f"Beamline: {value['beamline_name']}") + if 'type' in value: + parts.append(f"Type: {value['type']}") + if 'city' in value and 'country' in value: + parts.append(f"{value['city']}, {value['country']}") + elif 'city' in value: + parts.append(value['city']) + elif 'country' in value: + parts.append(value['country']) + + # Detector dictionaries + if 'detector' in str(value).lower() or 'type' in value: + if 'name' in value: + parts.append(f"Detector: {value['name']}") + if 'type' in value and 'name' not in value: + parts.append(f"Detector: {value['type']}") + if 'resolution' in value: + parts.append(f"Resolution: {value['resolution']}") + + if parts: + return " | ".join(parts) + + # Fallback: show key-value pairs for shallow dicts + if len(value) <= 3: + return ", ".join(f"{k}: {v}" for k, v in value.items() if not isinstance(v, dict)) + + return "{...}" + + +def _formatSASBDBMetadata(data) -> str: + """ + Format SASBDB metadata from data object for display. + + Displays instrument, sample, source info and meta_data dictionary + for datasets loaded from SASBDB in a clean, concise format. + + :param data: Data1D or Data2D object + :return: Formatted string with SASBDB metadata, or empty string if none + """ + text = "" + + # Check if meta_data exists and contains SASBDB info + meta = getattr(data, 'meta_data', None) or {} + + # Check if this is SASBDB data + if 'SASBDB_code' not in meta: + return text + + text += "\n" + "=" * 50 + "\n" + text += "SASBDB Metadata\n" + text += "=" * 50 + "\n" + + # Entry and instrument info (compact header) + header_parts = [] + if meta.get('SASBDB_code'): + header_parts.append(f"Code: {meta['SASBDB_code']}") + + # Extract instrument info from metadata (handle both string and dict formats) + instrument_str = None + location_str = None + + if hasattr(data, 'instrument') and data.instrument: + if isinstance(data.instrument, str): + instrument_str = data.instrument + elif isinstance(data.instrument, dict): + # Extract from dict + instrument_str = data.instrument.get('name') or data.instrument.get('beamline_name') + if data.instrument.get('city') and data.instrument.get('country'): + location_str = f"{data.instrument['city']}, {data.instrument['country']}" + elif data.instrument.get('city'): + location_str = data.instrument['city'] + + # Also check meta_data for instrument info (search all keys for dict values) + if not instrument_str: + for key, val in meta.items(): + if isinstance(val, dict): + # Check if this looks like an instrument dict + if 'beamline_name' in val or 'name' in val or 'type_of_source' in val: + instrument_str = val.get('beamline_name') or val.get('name') + if val.get('city') and val.get('country'): + location_str = f"{val['city']}, {val['country']}" + elif val.get('city'): + location_str = val['city'] + break + elif key in ['instrument', 'source', 'beamline'] and isinstance(val, str): + instrument_str = val + break + + if instrument_str: + header_parts.append(f"Instrument: {instrument_str}") + if location_str: + header_parts.append(location_str) + + # Extract detector info if available (search all keys for detector dicts) + detector_info = None + for key, val in meta.items(): + if isinstance(val, dict) and ('detector' in key.lower() or 'type' in val): + detector_name = val.get('name') or val.get('type') + if detector_name: + detector_info = detector_name + break + elif 'detector' in key.lower() and isinstance(val, str): + detector_info = val + break + + if detector_info: + header_parts.append(f"Detector: {detector_info}") + + if header_parts: + text += " | ".join(header_parts) + "\n" + + # Sample info (consolidated) + if hasattr(data, 'sample') and data.sample: + sample = data.sample + sample_parts = [] + + if hasattr(sample, 'name') and sample.name: + sample_parts.append(sample.name) + + # Add temperature if available + if hasattr(sample, 'temperature') and sample.temperature is not None: + temp_unit = getattr(sample, 'temperature_unit', 'K') or 'K' + sample_parts.append(f"T = {sample.temperature} {temp_unit}") + + # Add concentration and buffer from details (if present) + if hasattr(sample, 'details') and sample.details: + for detail in sample.details: + if detail.startswith("Concentration:"): + sample_parts.append(detail.replace("Concentration: ", "")) + elif detail.startswith("Buffer:"): + sample_parts.append(detail.replace("Buffer: ", "")) + + if sample_parts: + text += f"Sample: {' | '.join(sample_parts)}\n" + + # Source wavelength (if available) + if hasattr(data, 'source') and data.source: + source = data.source + if hasattr(source, 'wavelength') and source.wavelength is not None: + wl_unit = getattr(source, 'wavelength_unit', 'Å') or 'Å' + text += f"Wavelength: {source.wavelength} {wl_unit}\n" + + # Structural parameters (compact format) + structural_parts = [] + if meta.get('SASBDB_Rg') is not None: + rg_str = f"Rg = {meta['SASBDB_Rg']:.2f}" + if meta.get('SASBDB_Rg_error') is not None: + rg_str += f" ± {meta['SASBDB_Rg_error']:.2f}" + rg_str += " Å" + structural_parts.append(rg_str) + + if meta.get('SASBDB_I0') is not None: + i0_str = f"I(0) = {meta['SASBDB_I0']:.4e}" + if meta.get('SASBDB_I0_error') is not None: + i0_str += f" ± {meta['SASBDB_I0_error']:.4e}" + structural_parts.append(i0_str) + + if meta.get('SASBDB_Dmax') is not None: + structural_parts.append(f"Dmax = {meta['SASBDB_Dmax']:.2f} Å") + + if meta.get('SASBDB_MW') is not None: + mw_str = f"MW = {meta['SASBDB_MW']:.2f} kDa" + if meta.get('SASBDB_MW_method'): + mw_str += f" ({meta['SASBDB_MW_method']})" + structural_parts.append(mw_str) + + if meta.get('SASBDB_Porod_volume') is not None: + structural_parts.append(f"Porod Volume = {meta['SASBDB_Porod_volume']:.0f} ų") + + if structural_parts: + text += "Structural: " + " | ".join(structural_parts) + "\n" + + # Publication info (compact, only if available) + pub_parts = [] + if meta.get('SASBDB_authors'): + # Truncate authors if too long + authors = meta['SASBDB_authors'] + if len(authors) > 50: + authors = authors[:47] + "..." + pub_parts.append(f"Authors: {authors}") + if meta.get('SASBDB_DOI'): + pub_parts.append(f"DOI: {meta['SASBDB_DOI']}") + if meta.get('SASBDB_PMID'): + pub_parts.append(f"PMID: {meta['SASBDB_PMID']}") + + if pub_parts: + text += "Publication: " + " | ".join(pub_parts) + "\n" + + text += "=" * 50 + "\n\n" + + return text + + def retrieveData1d(data): """ Retrieve 1D data from file and construct its text @@ -551,6 +765,25 @@ def retrieveData1d(data): raise ValueError(msg) text = data.__str__() + + # Clean up raw dictionary representations from the output + import re + # Remove lines that are just raw dictionary representations + lines = text.split('\n') + cleaned_lines = [] + for line in lines: + # Skip lines that are just raw dictionary representations + stripped = line.strip() + if stripped.startswith("{'") and "'" in stripped and ":" in stripped: + # Check if it looks like a dict (has multiple key-value pairs) + if stripped.count("'") >= 4 and ":" in stripped: + continue # Skip this line + cleaned_lines.append(line) + text = '\n'.join(cleaned_lines) + + # Add SASBDB metadata if present + text += _formatSASBDBMetadata(data) + text += 'Data Min Max:\n' text += 'X_min = %s: X_max = %s\n' % (xmin, max(data.x)) text += 'Y_min = %s: Y_max = %s\n' % (ymin, max(data.y)) @@ -595,6 +828,25 @@ def retrieveData2d(data): raise AttributeError(msg) text = data.__str__() + + # Clean up raw dictionary representations from the output + import re + # Remove lines that are just raw dictionary representations + lines = text.split('\n') + cleaned_lines = [] + for line in lines: + # Skip lines that are just raw dictionary representations + stripped = line.strip() + if stripped.startswith("{'") and "'" in stripped and ":" in stripped: + # Check if it looks like a dict (has multiple key-value pairs) + if stripped.count("'") >= 4 and ":" in stripped: + continue # Skip this line + cleaned_lines.append(line) + text = '\n'.join(cleaned_lines) + + # Add SASBDB metadata if present + text += _formatSASBDBMetadata(data) + text += 'Data Min Max:\n' text += 'I_min = %s\n' % min(data.data) text += 'I_max = %s\n\n' % max(data.data) diff --git a/src/sas/qtgui/Utilities/SASBDB/SASBDBDownloadDialog.py b/src/sas/qtgui/Utilities/SASBDB/SASBDBDownloadDialog.py new file mode 100644 index 0000000000..4e680c9448 --- /dev/null +++ b/src/sas/qtgui/Utilities/SASBDB/SASBDBDownloadDialog.py @@ -0,0 +1,193 @@ +""" +SASBDB Dataset Download Dialog. + +Provides a dialog interface for downloading datasets from SASBDB +and loading them into SasView. +""" + +import logging +import os +import tempfile +from typing import Optional + +from PySide6 import QtWidgets + +from .sasbdb_api import downloadDataset, SASBDBDatasetInfo +from .UI.SASBDBDownloadDialogUI import Ui_SASBDBDownloadDialogUI + +logger = logging.getLogger(__name__) + + +class SASBDBDownloadDialog(QtWidgets.QDialog, Ui_SASBDBDownloadDialogUI): + """ + Dialog for downloading datasets from SASBDB. + + Allows users to enter a SASBDB dataset identifier, downloads + the dataset, and loads it into SasView. + """ + + def __init__(self, parent=None): + """ + Initialize the download dialog. + + :param parent: Parent widget + """ + super().__init__(parent) + self.setupUi(self) + + # Store downloaded file path and metadata + self.downloaded_filepath: Optional[str] = None + self.dataset_info: Optional[SASBDBDatasetInfo] = None + + # Connect signals + self.cmdDownload.clicked.connect(self.onDownload) + self.cmdCancel.clicked.connect(self.reject) + self.txtDatasetId.returnPressed.connect(self.onDownload) + + # Set focus on dataset ID input + self.txtDatasetId.setFocus() + + def onDownload(self): + """ + Handle download button click. + + Validates the dataset ID, downloads the dataset, + and loads it into SasView. + """ + dataset_id = self.txtDatasetId.text().strip() + + if not dataset_id: + self._showError("Please enter a dataset identifier.") + return + + # Disable download button during download + self.cmdDownload.setEnabled(False) + self.cmdCancel.setEnabled(False) + self.progressBar.setVisible(True) + self.progressBar.setRange(0, 0) # Indeterminate progress + self.lblStatus.setText("Downloading dataset...") + QtWidgets.QApplication.processEvents() + + try: + # Download the dataset + output_dir = tempfile.gettempdir() + filepath, dataset_info = downloadDataset(dataset_id, output_dir) + + # Store the metadata + self.dataset_info = dataset_info + + if filepath and os.path.exists(filepath): + self.downloaded_filepath = filepath + + # Build success message with metadata summary + success_msg = f"Successfully downloaded dataset {dataset_id}" + if dataset_info: + details = self._formatMetadataSummary(dataset_info) + if details: + success_msg += f"\n{details}" + + self.lblStatus.setText(success_msg) + self.progressBar.setVisible(False) + + # Log detailed metadata + if dataset_info: + self._logMetadata(dataset_info) + + self.accept() # Close dialog and return success + else: + self._showError(f"Failed to download dataset {dataset_id}.\n" + "Please check the dataset identifier and try again.") + self.progressBar.setVisible(False) + self.cmdDownload.setEnabled(True) + self.cmdCancel.setEnabled(True) + + except Exception as e: + logger.error(f"Error downloading dataset {dataset_id}: {e}", exc_info=True) + self._showError(f"Error downloading dataset:\n{str(e)}") + self.progressBar.setVisible(False) + self.cmdDownload.setEnabled(True) + self.cmdCancel.setEnabled(True) + + def _formatMetadataSummary(self, info: SASBDBDatasetInfo) -> str: + """ + Format a brief summary of the dataset metadata. + + :param info: Parsed dataset info + :return: Formatted summary string + """ + parts = [] + + if info.title: + parts.append(f"Title: {info.title}") + if info.sample_name: + parts.append(f"Sample: {info.sample_name}") + if info.rg is not None: + rg_str = f"Rg: {info.rg:.2f}" + if info.rg_error: + rg_str += f" ± {info.rg_error:.2f}" + rg_str += " Å" + parts.append(rg_str) + if info.molecular_weight is not None: + parts.append(f"MW: {info.molecular_weight:.1f} kDa") + + return "\n".join(parts) + + def _logMetadata(self, info: SASBDBDatasetInfo): + """ + Log detailed metadata information. + + :param info: Parsed dataset info + """ + logger.info(f"SASBDB Dataset: {info.code or info.entry_id}") + if info.title: + logger.info(f" Title: {info.title}") + if info.sample_name: + logger.info(f" Sample: {info.sample_name}") + if info.molecule_name: + logger.info(f" Molecule: {info.molecule_name}") + if info.concentration: + logger.info(f" Concentration: {info.concentration} {info.concentration_unit}") + if info.buffer_description: + logger.info(f" Buffer: {info.buffer_description}") + if info.instrument: + logger.info(f" Instrument: {info.instrument}") + if info.wavelength: + logger.info(f" Wavelength: {info.wavelength} {info.wavelength_unit}") + if info.temperature: + logger.info(f" Temperature: {info.temperature} {info.temperature_unit}") + if info.rg is not None: + logger.info(f" Rg: {info.rg} ± {info.rg_error or 0} Å") + if info.i0 is not None: + logger.info(f" I(0): {info.i0} ± {info.i0_error or 0}") + if info.dmax is not None: + logger.info(f" Dmax: {info.dmax} Å") + if info.molecular_weight is not None: + logger.info(f" MW: {info.molecular_weight} kDa") + if info.publication_doi: + logger.info(f" DOI: {info.publication_doi}") + + def _showError(self, message: str): + """ + Display an error message to the user. + + :param message: Error message to display + """ + self.lblStatus.setText(f"{message}") + QtWidgets.QMessageBox.warning(self, "Download Error", message) + + def getDownloadedFilepath(self) -> Optional[str]: + """ + Get the path to the downloaded file. + + :return: Path to downloaded file, or None if download failed + """ + return self.downloaded_filepath + + def getDatasetInfo(self) -> Optional[SASBDBDatasetInfo]: + """ + Get the parsed dataset metadata. + + :return: SASBDBDatasetInfo object, or None if metadata not available + """ + return self.dataset_info + diff --git a/src/sas/qtgui/Utilities/SASBDB/UI/SASBDBDownloadDialogUI.ui b/src/sas/qtgui/Utilities/SASBDB/UI/SASBDBDownloadDialogUI.ui new file mode 100644 index 0000000000..7efee88cd9 --- /dev/null +++ b/src/sas/qtgui/Utilities/SASBDB/UI/SASBDBDownloadDialogUI.ui @@ -0,0 +1,116 @@ + + + SASBDBDownloadDialogUI + + + + 0 + 0 + 450 + 200 + + + + Load from SASBDB + + + + + + Enter SASBDB Dataset Identifier: + + + + + + + e.g., 1234 or SASDB1234 + + + + + + + + + + true + + + + + + + 0 + + + 0 + + + 0 + + + false + + + + + + + Qt::Vertical + + + + 20 + 40 + + + + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + Download and Load + + + true + + + + + + + Cancel + + + + + + + + + + + cmdCancel + clicked() + SASBDBDownloadDialogUI + reject() + + + + diff --git a/src/sas/qtgui/Utilities/SASBDB/UI/__init__.py b/src/sas/qtgui/Utilities/SASBDB/UI/__init__.py new file mode 100644 index 0000000000..88400a3f7c --- /dev/null +++ b/src/sas/qtgui/Utilities/SASBDB/UI/__init__.py @@ -0,0 +1,4 @@ +""" +UI components for SASBDB utilities. +""" + diff --git a/src/sas/qtgui/Utilities/SASBDB/__init__.py b/src/sas/qtgui/Utilities/SASBDB/__init__.py new file mode 100644 index 0000000000..5ce0e3c888 --- /dev/null +++ b/src/sas/qtgui/Utilities/SASBDB/__init__.py @@ -0,0 +1,7 @@ +""" +SASBDB (Small Angle Scattering Biological Data Bank) utilities. + +This package provides functionality for interacting with SASBDB, +including downloading datasets and exporting data to SASBDB format. +""" + diff --git a/src/sas/qtgui/Utilities/SASBDB/sasbdb_api.py b/src/sas/qtgui/Utilities/SASBDB/sasbdb_api.py new file mode 100644 index 0000000000..9f67950f62 --- /dev/null +++ b/src/sas/qtgui/Utilities/SASBDB/sasbdb_api.py @@ -0,0 +1,506 @@ +""" +SASBDB REST API client module. + +Provides functions to interact with the SASBDB REST API for downloading +datasets and retrieving metadata. +""" + +import logging +import os +import tempfile +from dataclasses import dataclass, field +from typing import Optional + +import requests + +logger = logging.getLogger(__name__) + +# Base API URL - SASBDB REST API +SASBDB_API_BASE = "https://www.sasbdb.org/rest-api" + + +@dataclass +class SASBDBDatasetInfo: + """ + Parsed metadata from a SASBDB dataset entry. + + Contains key information extracted from the SASBDB REST API response. + """ + # Identifiers + entry_id: str = "" + code: str = "" + title: str = "" + + # Sample information + sample_name: str = "" + sample_description: str = "" + concentration: Optional[float] = None + concentration_unit: str = "mg/mL" + + # Molecule information + molecule_name: str = "" + molecule_type: str = "" + molecular_weight: Optional[float] = None # Experimental MW in kDa + molecular_weight_method: str = "" + oligomeric_state: str = "" + + # Buffer information + buffer_description: str = "" + ph: Optional[float] = None + + # Experimental parameters + instrument: str = "" + detector: str = "" + wavelength: Optional[float] = None # in Angstrom + wavelength_unit: str = "Å" + temperature: Optional[float] = None # in Kelvin or Celsius + temperature_unit: str = "K" + + # Analysis results + rg: Optional[float] = None # Radius of gyration in Angstrom + rg_error: Optional[float] = None + i0: Optional[float] = None # I(0) + i0_error: Optional[float] = None + dmax: Optional[float] = None # Maximum dimension in Angstrom + porod_volume: Optional[float] = None + + # Q-range + q_min: Optional[float] = None + q_max: Optional[float] = None + + # Publication + publication_title: str = "" + publication_doi: str = "" + publication_pmid: str = "" + authors: list = field(default_factory=list) + + # Data files + intensities_data_url: str = "" + pddf_data_url: str = "" + + # Raw metadata for additional fields + raw_metadata: dict = field(default_factory=dict) + + +def parseMetadata(metadata: dict) -> SASBDBDatasetInfo: + """ + Parse SASBDB API response into a structured SASBDBDatasetInfo object. + + :param metadata: Raw JSON dictionary from SASBDB API + :return: Parsed SASBDBDatasetInfo object + """ + info = SASBDBDatasetInfo() + + if not isinstance(metadata, dict): + return info + + # Store raw metadata for reference + info.raw_metadata = metadata + + # Log top-level keys for debugging + logger.debug(f"SASBDB API response keys: {list(metadata.keys())}") + + # Identifiers + info.entry_id = _get_str(metadata, 'id', 'entry_id', 'sasbdb_id') + info.code = _get_str(metadata, 'code', 'accession_code', 'entry_code') + info.title = _get_str(metadata, 'title', 'entry_title', 'sample_title') + + # Sample information - try more variations + info.sample_name = _get_str(metadata, 'sample_name', 'sample', 'name', 'sample_name_full') + info.sample_description = _get_str(metadata, 'sample_description', 'description', 'sample_description_full') + info.concentration = _get_float(metadata, 'concentration', 'sample_concentration', 'conc', 'sample_conc') + info.concentration_unit = _get_str(metadata, 'concentration_unit', 'conc_unit', 'concentration_units') or "mg/mL" + + # Molecule information - try more variations + info.molecule_name = _get_str(metadata, 'molecule_name', 'macromolecule_name', 'protein_name', + 'molecule', 'macromolecule', 'protein', 'name') + info.molecule_type = _get_str(metadata, 'molecule_type', 'macromolecule_type', 'sample_type', + 'type', 'molecule_type_full') + info.molecular_weight = _get_float(metadata, 'molecular_weight', 'mw', 'exp_mw', + 'experimental_mw', 'mw_kda') + info.molecular_weight_method = _get_str(metadata, 'mw_method', 'molecular_weight_method') + info.oligomeric_state = _get_str(metadata, 'oligomeric_state', 'oligomer_state', 'oligomerization') + + # Buffer information + info.buffer_description = _get_str(metadata, 'buffer', 'buffer_description', 'buffer_composition') + info.ph = _get_float(metadata, 'ph', 'buffer_ph') + + # Experimental parameters + info.instrument = _get_str(metadata, 'instrument', 'beamline', 'instrument_name') + info.detector = _get_str(metadata, 'detector', 'detector_name', 'detector_type') + info.wavelength = _get_float(metadata, 'wavelength', 'xray_wavelength', 'neutron_wavelength') + info.temperature = _get_float(metadata, 'temperature', 'sample_temperature', 'temp') + + # Analysis results - Guinier + info.rg = _get_float(metadata, 'rg', 'radius_of_gyration', 'guinier_rg') + info.rg_error = _get_float(metadata, 'rg_error', 'rg_err', 'guinier_rg_error') + info.i0 = _get_float(metadata, 'i0', 'i_zero', 'guinier_i0') + info.i0_error = _get_float(metadata, 'i0_error', 'i0_err', 'guinier_i0_error') + + # Analysis results - P(r) + info.dmax = _get_float(metadata, 'dmax', 'd_max', 'maximum_dimension') + info.porod_volume = _get_float(metadata, 'porod_volume', 'volume', 'porod_vol') + + # Q-range + info.q_min = _get_float(metadata, 'q_min', 'qmin', 's_min', 'smin') + info.q_max = _get_float(metadata, 'q_max', 'qmax', 's_max', 'smax') + + # Publication + info.publication_title = _get_str(metadata, 'publication_title', 'pub_title', 'paper_title') + info.publication_doi = _get_str(metadata, 'doi', 'publication_doi', 'pub_doi') + info.publication_pmid = _get_str(metadata, 'pmid', 'publication_pmid', 'pubmed_id') + + # Authors + authors = metadata.get('authors') or metadata.get('author_list') or [] + if isinstance(authors, list): + info.authors = [str(a) for a in authors] + elif isinstance(authors, str): + info.authors = [authors] + + # Data file URLs + info.intensities_data_url = _get_str(metadata, 'intensities_data', 'data_url', 'intensities_url') + info.pddf_data_url = _get_str(metadata, 'pddf_data', 'pddf_url', 'pr_data') + + return info + + +def _get_str(data: dict, *keys: str) -> str: + """ + Get string value from dictionary, trying multiple possible keys. + Also searches in nested dictionaries (sample, molecule, experiment, etc.). + + :param data: Dictionary to search + :param keys: Possible key names to try + :return: String value or empty string if not found + """ + # First try top-level keys + for key in keys: + if key in data and data[key] is not None: + return str(data[key]) + + # Then search in common nested structures + nested_paths = ['sample', 'molecule', 'experiment', 'experimental', 'metadata', 'info'] + for path in nested_paths: + if path in data and isinstance(data[path], dict): + for key in keys: + if key in data[path] and data[path][key] is not None: + return str(data[path][key]) + + return "" + + +def _get_float(data: dict, *keys: str) -> Optional[float]: + """ + Get float value from dictionary, trying multiple possible keys. + Also searches in nested dictionaries (sample, molecule, experiment, etc.). + + :param data: Dictionary to search + :param keys: Possible key names to try + :return: Float value or None if not found + """ + # First try top-level keys + for key in keys: + if key in data and data[key] is not None: + try: + return float(data[key]) + except (ValueError, TypeError): + continue + + # Then search in common nested structures + nested_paths = ['sample', 'molecule', 'experiment', 'experimental', 'metadata', 'info'] + for path in nested_paths: + if path in data and isinstance(data[path], dict): + for key in keys: + if key in data[path] and data[path][key] is not None: + try: + return float(data[path][key]) + except (ValueError, TypeError): + continue + + return None + + +def getDatasetMetadata(dataset_id: str) -> Optional[dict]: + """ + Fetch dataset metadata from SASBDB API. + + :param dataset_id: SASBDB dataset identifier (e.g., "SASDN24" - full 7-character ID) + :return: Dictionary containing dataset metadata, or None if error + """ + # Normalize dataset ID (uppercase, strip whitespace, ensure 7 characters) + normalized_id = _normalizeDatasetId(dataset_id) + if not normalized_id: + logger.error(f"Invalid dataset ID format: {dataset_id}") + return None + + # Use the correct REST API endpoint: /rest-api/entry/summary/{id}/ + endpoint = f"{SASBDB_API_BASE}/entry/summary/{normalized_id}/" + + try: + logger.info(f"Fetching dataset metadata from: {endpoint}") + headers = {"accept": "application/json"} + response = requests.get(endpoint, headers=headers, timeout=30) + response.raise_for_status() + + # Parse JSON response + metadata = response.json() + logger.info(f"Successfully retrieved metadata for dataset {normalized_id}") + return metadata + + except requests.exceptions.HTTPError as e: + if e.response.status_code == 404: + logger.error(f"Dataset {normalized_id} not found (404)") + else: + logger.error(f"HTTP error fetching dataset {normalized_id}: {e}") + return None + except requests.exceptions.RequestException as e: + logger.error(f"Network error fetching dataset {normalized_id}: {e}") + return None + except ValueError as e: + logger.error(f"Invalid JSON response for dataset {normalized_id}: {e}") + return None + + +def getDataFileUrl(metadata: dict) -> Optional[str]: + """ + Extract data file URL from dataset metadata. + + Looks for common field names in the metadata JSON that might contain + the data file URL or path. Also checks for SASBDB-specific fields. + + :param metadata: Dictionary containing dataset metadata + :return: URL string for the data file, or None if not found + """ + if not isinstance(metadata, dict): + return None + + # SASBDB-specific field names that might contain data file URLs + # Priority: intensities_data is the primary field for SASBDB + possible_fields = [ + # SASBDB primary data field + 'intensities_data', + 'intensitiesData', + # Direct URL fields + 'data_file_url', + 'dataFileUrl', + 'data_file', + 'dataFile', + 'scattering_data_url', + 'scatteringDataUrl', + 'experimental_data_url', + 'experimentalDataUrl', + 'download_url', + 'downloadUrl', + 'file_url', + 'fileUrl', + # File list fields + 'files', + 'data_files', + 'dataFiles', + 'experimental_files', + 'scattering_files', + # SASBDB API specific fields + 'experimental_data', + 'experimentalData', + 'scattering_data', + 'scatteringData', + ] + + # Check top-level fields + for field in possible_fields: + if field in metadata: + value = metadata[field] + if isinstance(value, str) and (value.startswith('http') or value.startswith('/')): + # If relative URL, make it absolute + if value.startswith('/'): + return f"https://www.sasbdb.org{value}" + return value + elif isinstance(value, list) and len(value) > 0: + # If it's a list, take the first item + first_item = value[0] + if isinstance(first_item, str): + if first_item.startswith('/'): + return f"https://www.sasbdb.org{first_item}" + elif first_item.startswith('http'): + return first_item + elif isinstance(first_item, dict): + # Check for 'url' or 'path' in the item + for url_field in ['url', 'path', 'file', 'file_url', 'download_url']: + if url_field in first_item: + url = first_item[url_field] + if isinstance(url, str): + if url.startswith('/'): + return f"https://www.sasbdb.org{url}" + elif url.startswith('http'): + return url + + # Check nested structures (common in REST APIs) + for nested_key in ['entry', 'data', 'files', 'experimental_data', 'scattering_data']: + if nested_key in metadata and isinstance(metadata[nested_key], dict): + result = getDataFileUrl(metadata[nested_key]) + if result: + return result + + # If we have an entry ID, try constructing a download URL + # Format might be: /rest-api/entry/{id}/download/ or similar + entry_id = metadata.get('entry_id') or metadata.get('id') or metadata.get('sasbdb_id') + if entry_id: + # Try common download endpoint patterns + download_endpoints = [ + f"{SASBDB_API_BASE}/entry/{entry_id}/download/", + f"{SASBDB_API_BASE}/entry/{entry_id}/data/", + f"{SASBDB_API_BASE}/entry/{entry_id}/file/", + ] + # Return first endpoint (we'll try them in downloadDataFile if needed) + return download_endpoints[0] + + logger.warning("Could not find data file URL in metadata") + logger.debug(f"Metadata keys: {list(metadata.keys())}") + return None + + +def downloadDataFile(url: str, filepath: str) -> bool: + """ + Download a data file from the given URL to the specified filepath. + + :param url: URL of the data file to download + :param filepath: Local filepath where the file should be saved + :return: True if download successful, False otherwise + """ + try: + logger.info(f"Downloading data file from: {url}") + response = requests.get(url, timeout=60, stream=True) + response.raise_for_status() + + # Create directory if it doesn't exist + os.makedirs(os.path.dirname(filepath), exist_ok=True) + + # Write file in chunks to handle large files + with open(filepath, 'wb') as f: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + f.write(chunk) + + logger.info(f"Successfully downloaded data file to: {filepath}") + return True + + except requests.exceptions.RequestException as e: + logger.error(f"Error downloading data file from {url}: {e}") + return False + except IOError as e: + logger.error(f"Error writing data file to {filepath}: {e}") + return False + + +def _normalizeDatasetId(dataset_id: str) -> Optional[str]: + """ + Normalize a SASBDB dataset identifier. + + SASBDB identifiers are 7 characters long, typically in format: + - "SASDN24" (prefix + number) + - "SASDB1234" (if 4-digit number) + - etc. + + The API requires the full 7-character identifier. + + :param dataset_id: Input dataset identifier + :return: Normalized identifier (uppercase, stripped), or None if invalid + """ + if not dataset_id: + return None + + # Remove whitespace and convert to uppercase + normalized = dataset_id.strip().upper() + + # SASBDB identifiers are typically 7 characters + # Accept identifiers that are 4-10 characters to be flexible + if len(normalized) < 4 or len(normalized) > 10: + logger.warning(f"Dataset ID length unusual: {len(normalized)} characters for '{normalized}'") + # Still try it, but warn + + # Return the normalized identifier as-is (API expects full identifier) + return normalized + + +def downloadDataset(dataset_id: str, output_dir: Optional[str] = None) -> tuple[Optional[str], Optional[SASBDBDatasetInfo]]: + """ + Download a complete dataset from SASBDB. + + This is a convenience function that: + 1. Fetches metadata + 2. Extracts data file URL + 3. Downloads the data file + 4. Returns the local filepath and parsed metadata + + :param dataset_id: SASBDB dataset identifier + :param output_dir: Directory to save the file (defaults to temp directory) + :return: Tuple of (path to downloaded file, parsed metadata info) or (None, None) if error + """ + # Get metadata + metadata = getDatasetMetadata(dataset_id) + if not metadata: + return None, None + + # Parse metadata into structured object + dataset_info = parseMetadata(metadata) + + # Get data file URL + data_url = getDataFileUrl(metadata) + if not data_url: + logger.error(f"Could not find data file URL in metadata for dataset {dataset_id}") + return None, dataset_info # Return metadata even if download fails + + # Store the data URL in the info object + if not dataset_info.intensities_data_url: + dataset_info.intensities_data_url = data_url + + # Determine output directory + if output_dir is None: + output_dir = tempfile.gettempdir() + + # Generate filename + normalized_id = _normalizeDatasetId(dataset_id) + # Try to determine file extension from URL or metadata + file_extension = _guessFileExtension(data_url, metadata) + filename = f"SASBDB_{normalized_id}{file_extension}" + filepath = os.path.join(output_dir, filename) + + # Download the file + if downloadDataFile(data_url, filepath): + return filepath, dataset_info + + return None, dataset_info + + +def _guessFileExtension(url: str, metadata: dict) -> str: + """ + Guess the file extension from URL or metadata. + + :param url: Data file URL + :param metadata: Dataset metadata + :return: File extension (e.g., ".dat", ".txt", ".csv") + """ + # Check URL for extension + if '.' in url: + parts = url.split('.') + if len(parts) > 1: + ext = '.' + parts[-1].split('?')[0] # Remove query parameters + if ext.lower() in ['.dat', '.txt', '.csv', '.out', '.asc']: + return ext + + # Check metadata for file type hints + if isinstance(metadata, dict): + file_type_fields = ['file_type', 'fileType', 'format', 'data_format'] + for field in file_type_fields: + if field in metadata: + file_type = str(metadata[field]).lower() + if 'csv' in file_type: + return '.csv' + elif 'txt' in file_type or 'text' in file_type: + return '.txt' + elif 'dat' in file_type or 'data' in file_type: + return '.dat' + + # Default to .dat for scattering data + return '.dat' + From 78ff44b1078c5f1d1087f55ec0bacca2c84e3e16 Mon Sep 17 00:00:00 2001 From: Wojtek Potrzebowski Date: Thu, 15 Jan 2026 13:47:45 +0100 Subject: [PATCH 02/12] Adding documentation --- docs/sphinx-docs/source/user/menu_bar.rst | 3 +- docs/sphinx-docs/source/user/tools.rst | 2 + src/sas/qtgui/Utilities/GuiUtils.py | 146 ++++++++++++++++-- .../Utilities/SASBDB/SASBDBDownloadDialog.py | 30 ++++ .../SASBDB/UI/SASBDBDownloadDialogUI.ui | 7 + .../SASBDB/media/sasbdb_download_help.rst | 135 ++++++++++++++++ 6 files changed, 306 insertions(+), 17 deletions(-) create mode 100644 src/sas/qtgui/Utilities/SASBDB/media/sasbdb_download_help.rst diff --git a/docs/sphinx-docs/source/user/menu_bar.rst b/docs/sphinx-docs/source/user/menu_bar.rst index 0ab0c88d84..a969e2b2f4 100644 --- a/docs/sphinx-docs/source/user/menu_bar.rst +++ b/docs/sphinx-docs/source/user/menu_bar.rst @@ -11,7 +11,8 @@ The File option allows you load data into *SasView* for analysis, or to save the Data can be loaded one file at a time, or by selecting multiple files, or by loading an entire folder of files (in which case *SasView* will attempt to make an intelligent guess as to what to load based on the file formats it recognises in the folder!). Data can also be loaded by dragging and dropping files directly -onto Data Explorer. +onto Data Explorer. Additionally, datasets can be downloaded directly from the SASBDB (Small Angle Scattering +Biological Data Bank) using **File > Load from SASBDB...** (see :ref:`SASBDB_Download` for details). A *SasView* session can also be saved and reloaded as an 'Analysis' (an individual model fit or invariant calculation, etc), or as a 'Project' (everything you have done since starting your *SasView* session). diff --git a/docs/sphinx-docs/source/user/tools.rst b/docs/sphinx-docs/source/user/tools.rst index 23c6fcc674..f30eb2c9d9 100644 --- a/docs/sphinx-docs/source/user/tools.rst +++ b/docs/sphinx-docs/source/user/tools.rst @@ -32,3 +32,5 @@ Tools & Utilities MuMag Tool + SASBDB Download + diff --git a/src/sas/qtgui/Utilities/GuiUtils.py b/src/sas/qtgui/Utilities/GuiUtils.py index 1f94693761..e00d32df9b 100644 --- a/src/sas/qtgui/Utilities/GuiUtils.py +++ b/src/sas/qtgui/Utilities/GuiUtils.py @@ -583,6 +583,100 @@ def _formatDictValue(value, max_depth=2) -> str: return "{...}" +def _formatDictLine(line: str) -> str: + """ + Extract and format dictionary information from a line. + + :param line: Line that may contain a dictionary representation + :return: Formatted line with dictionary info extracted, or original line if no dict found + """ + import ast + import re + + stripped = line.strip() + + # Check if line contains a dictionary + if not ("{" in stripped and "'" in stripped and ":" in stripped): + return line + + # Try to extract the label (e.g., "Instrument:", "Detector:") + label_match = re.match(r'^(\w+)\s*:\s*(.+)', stripped) + if label_match: + label = label_match.group(1) + dict_str = label_match.group(2) + elif stripped.startswith("{'") or stripped.startswith("{"): + label = None + dict_str = stripped + else: + return line + + # Try to parse the dictionary string + try: + # Use ast.literal_eval to safely parse the dictionary + parsed_dict = ast.literal_eval(dict_str) + if not isinstance(parsed_dict, dict): + return line + + # Format the dictionary + parts = [] + + # Extract instrument/source info + if 'name' in parsed_dict and parsed_dict['name']: + parts.append(parsed_dict['name']) + if 'beamline_name' in parsed_dict and parsed_dict['beamline_name']: + parts.append(f"Beamline: {parsed_dict['beamline_name']}") + if 'type_of_source' in parsed_dict and parsed_dict['type_of_source']: + parts.append(f"Source: {parsed_dict['type_of_source']}") + if 'city' in parsed_dict and parsed_dict['city']: + city = parsed_dict['city'] + if 'country' in parsed_dict and parsed_dict['country']: + parts.append(f"{city}, {parsed_dict['country']}") + else: + parts.append(city) + elif 'country' in parsed_dict and parsed_dict['country']: + parts.append(parsed_dict['country']) + + # Extract detector info (nested or direct) + detector_info = None + if 'detector' in parsed_dict and isinstance(parsed_dict['detector'], dict): + det = parsed_dict['detector'] + if 'name' in det and det['name']: + detector_info = det['name'] + elif 'type' in det and det['type']: + detector_info = det['type'] + elif 'detector' in parsed_dict: + detector_info = str(parsed_dict['detector']) + + if detector_info: + parts.append(f"Detector: {detector_info}") + + # If we extracted useful info, format it + if parts: + formatted = " | ".join(parts) + if label: + return f"{label}: {formatted}" + else: + return formatted + + # Fallback: show key-value pairs for simple dicts + simple_parts = [] + for k, v in parsed_dict.items(): + if v is not None and not isinstance(v, dict): + simple_parts.append(f"{k}: {v}") + if simple_parts and len(simple_parts) <= 5: + formatted = " | ".join(simple_parts) + if label: + return f"{label}: {formatted}" + else: + return formatted + + except (ValueError, SyntaxError): + # If parsing fails, return original line + pass + + return line + + def _formatSASBDBMetadata(data) -> str: """ Format SASBDB metadata from data object for display. @@ -766,19 +860,29 @@ def retrieveData1d(data): text = data.__str__() - # Clean up raw dictionary representations from the output + # Format dictionary representations instead of removing them import re - # Remove lines that are just raw dictionary representations lines = text.split('\n') cleaned_lines = [] for line in lines: - # Skip lines that are just raw dictionary representations stripped = line.strip() - if stripped.startswith("{'") and "'" in stripped and ":" in stripped: - # Check if it looks like a dict (has multiple key-value pairs) - if stripped.count("'") >= 4 and ":" in stripped: - continue # Skip this line - cleaned_lines.append(line) + # Check if line contains a dictionary representation + if ("{" in stripped and "'" in stripped and ":" in stripped and + (stripped.startswith("{'") or + re.search(r':\s*\{', stripped) or # Matches "Instrument: {" + re.search(r"\{'[^']+':", stripped))): # Matches "{'key':" + # Count opening and closing braces to detect dict structures + open_braces = stripped.count('{') + close_braces = stripped.count('}') + # If it looks like a dictionary, format it instead of skipping + if open_braces > 0 and close_braces > 0: + formatted_line = _formatDictLine(line) + if formatted_line != line: # Only add if it was successfully formatted + cleaned_lines.append(formatted_line) + else: + cleaned_lines.append(line) + else: + cleaned_lines.append(line) text = '\n'.join(cleaned_lines) # Add SASBDB metadata if present @@ -829,19 +933,29 @@ def retrieveData2d(data): text = data.__str__() - # Clean up raw dictionary representations from the output + # Format dictionary representations instead of removing them import re - # Remove lines that are just raw dictionary representations lines = text.split('\n') cleaned_lines = [] for line in lines: - # Skip lines that are just raw dictionary representations stripped = line.strip() - if stripped.startswith("{'") and "'" in stripped and ":" in stripped: - # Check if it looks like a dict (has multiple key-value pairs) - if stripped.count("'") >= 4 and ":" in stripped: - continue # Skip this line - cleaned_lines.append(line) + # Check if line contains a dictionary representation + if ("{" in stripped and "'" in stripped and ":" in stripped and + (stripped.startswith("{'") or + re.search(r':\s*\{', stripped) or # Matches "Instrument: {" + re.search(r"\{'[^']+':", stripped))): # Matches "{'key':" + # Count opening and closing braces to detect dict structures + open_braces = stripped.count('{') + close_braces = stripped.count('}') + # If it looks like a dictionary, format it instead of skipping + if open_braces > 0 and close_braces > 0: + formatted_line = _formatDictLine(line) + if formatted_line != line: # Only add if it was successfully formatted + cleaned_lines.append(formatted_line) + else: + cleaned_lines.append(line) + else: + cleaned_lines.append(line) text = '\n'.join(cleaned_lines) # Add SASBDB metadata if present diff --git a/src/sas/qtgui/Utilities/SASBDB/SASBDBDownloadDialog.py b/src/sas/qtgui/Utilities/SASBDB/SASBDBDownloadDialog.py index 4e680c9448..9b0ef622ad 100644 --- a/src/sas/qtgui/Utilities/SASBDB/SASBDBDownloadDialog.py +++ b/src/sas/qtgui/Utilities/SASBDB/SASBDBDownloadDialog.py @@ -42,6 +42,7 @@ def __init__(self, parent=None): # Connect signals self.cmdDownload.clicked.connect(self.onDownload) self.cmdCancel.clicked.connect(self.reject) + self.cmdHelp.clicked.connect(self.onHelp) self.txtDatasetId.returnPressed.connect(self.onDownload) # Set focus on dataset ID input @@ -190,4 +191,33 @@ def getDatasetInfo(self) -> Optional[SASBDBDatasetInfo]: :return: SASBDBDatasetInfo object, or None if metadata not available """ return self.dataset_info + + def onHelp(self): + """ + Show the SASBDB download help documentation. + """ + from sas.qtgui.Utilities import GuiUtils + help_location = "user/qtgui/Utilities/SASBDB/sasbdb_download_help.html" + try: + # Try to get guiManager from parent workspace + parent = self.parent() + if parent: + # Check if parent has guiManager attribute + if hasattr(parent, 'guiManager') and hasattr(parent.guiManager, 'showHelp'): + parent.guiManager.showHelp(help_location) + return + # Check if parent itself has showHelp + elif hasattr(parent, 'showHelp'): + parent.showHelp(help_location) + return + + # Fallback to GuiUtils + GuiUtils.showHelp(help_location) + except Exception as e: + logger.warning(f"Could not display help: {e}") + # Final fallback to GuiUtils + try: + GuiUtils.showHelp(help_location) + except Exception as e2: + logger.error(f"Failed to display help: {e2}") diff --git a/src/sas/qtgui/Utilities/SASBDB/UI/SASBDBDownloadDialogUI.ui b/src/sas/qtgui/Utilities/SASBDB/UI/SASBDBDownloadDialogUI.ui index 7efee88cd9..8dd7f45f81 100644 --- a/src/sas/qtgui/Utilities/SASBDB/UI/SASBDBDownloadDialogUI.ui +++ b/src/sas/qtgui/Utilities/SASBDB/UI/SASBDBDownloadDialogUI.ui @@ -92,6 +92,13 @@ + + + + Help + + + diff --git a/src/sas/qtgui/Utilities/SASBDB/media/sasbdb_download_help.rst b/src/sas/qtgui/Utilities/SASBDB/media/sasbdb_download_help.rst new file mode 100644 index 0000000000..b547655df2 --- /dev/null +++ b/src/sas/qtgui/Utilities/SASBDB/media/sasbdb_download_help.rst @@ -0,0 +1,135 @@ +.. sasbdb_download_help.rst + +.. _SASBDB_Download: + +Loading Datasets from SASBDB +============================= + +Description +----------- + +The SASBDB (Small Angle Scattering Biological Data Bank) download feature allows you to directly download and load datasets from the SASBDB database into SasView. This feature provides easy access to published small-angle scattering data and automatically populates metadata from the SASBDB entry. + +Accessing the Feature +--------------------- + +To access the SASBDB download feature: + +1. Select **File > Load from SASBDB...** from the main menu +2. A dialog will appear prompting you to enter a SASBDB dataset identifier + +Dataset Identifier +------------------ + +Enter a valid SASBDB dataset identifier in the input field. The identifier can be: + +- A 7-character SASBDB code (e.g., ``SASDN24``, ``SASDB1234``) +- The identifier is case-insensitive and will be automatically normalized + +Examples of valid identifiers: +- ``SASDN24`` +- ``sasdn24`` (will be converted to uppercase) + +Download Process +---------------- + +When you click **Download and Load**: + +1. **Validation**: The dataset identifier is validated +2. **Metadata Retrieval**: The system fetches metadata from the SASBDB REST API +3. **Data Download**: The experimental data file is downloaded to a temporary location +4. **Data Loading**: The downloaded data is automatically loaded into SasView +5. **Metadata Population**: Metadata from SASBDB is automatically populated into the loaded dataset + +Progress Indicator +------------------ + +During the download process, you will see: + +- A progress bar indicating the download is in progress +- Status messages showing the current operation +- A success message with a summary of the loaded dataset + +Metadata Population +-------------------- + +When a dataset is loaded from SASBDB, the following metadata is automatically extracted and populated: + +**Sample Information:** +- Sample name (from molecule name, type, and oligomeric state) +- Temperature +- Concentration +- Buffer description and pH + +**Instrument Information:** +- Instrument/beamline name +- Detector information +- Location (city, country) +- Source type (X-ray synchrotron, neutron, etc.) + +**Experimental Parameters:** +- Wavelength +- Temperature +- Q-range (if available) + +**Structural Parameters:** +- Radius of gyration (Rg) with errors +- I(0) with errors +- Maximum dimension (Dmax) +- Molecular weight (MW) with method +- Porod volume + +**Publication Information:** +- Authors +- DOI +- PMID + +Viewing Metadata +---------------- + +After loading a SASBDB dataset, you can view the populated metadata by: + +1. Right-clicking on the loaded dataset in the Data Explorer +2. Selecting **Data Info** from the context menu +3. The metadata will be displayed in a clean, formatted section labeled "SASBDB Metadata" + +The metadata is displayed in a compact format with key information organized by category: +- Entry code and instrument information +- Sample details (name, temperature, concentration, buffer) +- Source wavelength +- Structural parameters (Rg, I(0), Dmax, MW, etc.) +- Publication information + +Error Handling +-------------- + +If an error occurs during download: + +- An error message will be displayed explaining the issue +- Common errors include: + - Invalid dataset identifier format + - Dataset not found in SASBDB + - Network connection issues + - API service unavailable + +If you encounter errors: + +1. Verify the dataset identifier is correct +2. Check your internet connection +3. Try again after a few moments if the SASBDB service is temporarily unavailable + +Tips +---- + +- **Dataset Identifiers**: You can find SASBDB dataset identifiers in published papers or on the SASBDB website (https://www.sasbdb.org) +- **Metadata**: All metadata is automatically extracted from the SASBDB entry, so you don't need to manually enter it +- **Data Format**: The downloaded data is in a standard format compatible with SasView +- **Offline Use**: Once downloaded, the data file is stored locally and can be used offline + +Related Documentation +--------------------- + +- :ref:`SASBDB Export ` - Export your data to SASBDB format +- `SASBDB Website `_ - Browse and search the SASBDB database +- `SASBDB REST API Documentation `_ - Technical API reference + From 7219c8f6824de11d23ffd0a2a7879ea8d2cb2b66 Mon Sep 17 00:00:00 2001 From: Wojtek Potrzebowski Date: Wed, 4 Mar 2026 18:21:39 +0100 Subject: [PATCH 03/12] Updated loading metadata --- src/sas/qtgui/MainWindow/GuiManager.py | 75 ++++++-- .../SASBDB/media/sasbdb_download_help.rst | 16 +- src/sas/qtgui/Utilities/SASBDB/sasbdb_api.py | 166 +++++++++++++++++- 3 files changed, 230 insertions(+), 27 deletions(-) diff --git a/src/sas/qtgui/MainWindow/GuiManager.py b/src/sas/qtgui/MainWindow/GuiManager.py index a304d814b8..ce2c62fe41 100644 --- a/src/sas/qtgui/MainWindow/GuiManager.py +++ b/src/sas/qtgui/MainWindow/GuiManager.py @@ -889,22 +889,55 @@ def _populateSASBDBMetadata(self, loaded_data: dict, dataset_info): if not hasattr(data.sample, 'details') or data.sample.details is None: data.sample.details = [] - # Use molecule name as sample name (molecule IS the sample in SASBDB context) - # Always set sample name if we have molecule_name or sample_name + # Map molecule short name into Sample ID when available. + sample_id = "" + if dataset_info.molecule_short_name: + sample_id = dataset_info.molecule_short_name + elif dataset_info.sample_name: + # Fallback for entries without molecule short name. + sample_id = dataset_info.sample_name + elif dataset_info.code: + sample_id = dataset_info.code + if sample_id: + data.sample.ID = sample_id + logger.debug(f"Set sample.ID to: {sample_id}") + + # Build human-readable sample details from remaining metadata. + sample_details = [] if dataset_info.molecule_name: - sample_name = dataset_info.molecule_name - # Add molecule type if available + molecule_str = f"Molecule: {dataset_info.molecule_name}" if dataset_info.molecule_type: - sample_name += f" ({dataset_info.molecule_type})" - # Add oligomeric state if available - if dataset_info.oligomeric_state: - sample_name += f" - {dataset_info.oligomeric_state}" - data.sample.name = sample_name - logger.debug(f"Set sample.name to: {sample_name}") + molecule_str += f" ({dataset_info.molecule_type})" + sample_details.append(molecule_str) elif dataset_info.sample_name: - # Fallback to sample_name if molecule_name not available - data.sample.name = dataset_info.sample_name - logger.debug(f"Set sample.name to: {dataset_info.sample_name}") + sample_details.append(f"Sample: {dataset_info.sample_name}") + + if dataset_info.sample_description: + sample_details.append( + f"Description: {dataset_info.sample_description}" + ) + + if dataset_info.sequence: + sample_details.append(f"Sequence: {dataset_info.sequence}") + + if dataset_info.uniprot_code: + sample_details.append(f"UniProt: {dataset_info.uniprot_code}") + + oligomerization = ( + dataset_info.oligomerization or dataset_info.oligomeric_state + ) + if oligomerization: + sample_details.append(f"Oligomerization: {oligomerization}") + + if dataset_info.number_of_molecules: + sample_details.append( + f"Number of molecules: {dataset_info.number_of_molecules}" + ) + + if dataset_info.source_organism: + sample_details.append( + f"Source organism: {dataset_info.source_organism}" + ) # Set temperature if dataset_info.temperature is not None: @@ -912,22 +945,30 @@ def _populateSASBDBMetadata(self, loaded_data: dict, dataset_info): if dataset_info.temperature_unit: data.sample.temperature_unit = dataset_info.temperature_unit logger.debug(f"Set sample.temperature to: {dataset_info.temperature} {dataset_info.temperature_unit}") + temp_unit = dataset_info.temperature_unit or "" + temp_str = f"Temperature: {dataset_info.temperature}" + if temp_unit: + temp_str += f" {temp_unit}" + sample_details.append(temp_str) # Store concentration in sample details if dataset_info.concentration is not None: conc_str = f"Concentration: {dataset_info.concentration}" if dataset_info.concentration_unit: conc_str += f" {dataset_info.concentration_unit}" - data.sample.details.append(conc_str) - logger.debug(f"Added to sample.details: {conc_str}") + sample_details.append(conc_str) # Add buffer info to sample details if dataset_info.buffer_description: buffer_str = f"Buffer: {dataset_info.buffer_description}" if dataset_info.ph is not None: buffer_str += f" (pH {dataset_info.ph})" - data.sample.details.append(buffer_str) - logger.debug(f"Added to sample.details: {buffer_str}") + sample_details.append(buffer_str) + + for detail in sample_details: + if detail and detail not in data.sample.details: + data.sample.details.append(detail) + logger.debug(f"Added to sample.details: {detail}") # Log sample info for debugging logger.debug(f"Sample populated for {data_id}: name={getattr(data.sample, 'name', None)}, " diff --git a/src/sas/qtgui/Utilities/SASBDB/media/sasbdb_download_help.rst b/src/sas/qtgui/Utilities/SASBDB/media/sasbdb_download_help.rst index b547655df2..00d7a30d43 100644 --- a/src/sas/qtgui/Utilities/SASBDB/media/sasbdb_download_help.rst +++ b/src/sas/qtgui/Utilities/SASBDB/media/sasbdb_download_help.rst @@ -56,10 +56,16 @@ Metadata Population When a dataset is loaded from SASBDB, the following metadata is automatically extracted and populated: **Sample Information:** -- Sample name (from molecule name, type, and oligomeric state) -- Temperature -- Concentration -- Buffer description and pH +- Sample ID (from molecule short name, if available) +- Sample details section populated with: + - Sequence (when available) + - Molecule/sample description + - UniProt code (when available) + - Oligomerization and number of molecules (when available) + - Source organism (when available) + - Temperature + - Concentration + - Buffer description and pH **Instrument Information:** - Instrument/beamline name @@ -95,7 +101,7 @@ After loading a SASBDB dataset, you can view the populated metadata by: The metadata is displayed in a compact format with key information organized by category: - Entry code and instrument information -- Sample details (name, temperature, concentration, buffer) +- Sample ID and sample details (sequence, UniProt, oligomerization, concentration, buffer, etc.) - Source wavelength - Structural parameters (Rg, I(0), Dmax, MW, etc.) - Publication information diff --git a/src/sas/qtgui/Utilities/SASBDB/sasbdb_api.py b/src/sas/qtgui/Utilities/SASBDB/sasbdb_api.py index 9f67950f62..08c69a0329 100644 --- a/src/sas/qtgui/Utilities/SASBDB/sasbdb_api.py +++ b/src/sas/qtgui/Utilities/SASBDB/sasbdb_api.py @@ -39,7 +39,13 @@ class SASBDBDatasetInfo: # Molecule information molecule_name: str = "" + molecule_short_name: str = "" molecule_type: str = "" + sequence: str = "" + uniprot_code: str = "" + source_organism: str = "" + number_of_molecules: str = "" + oligomerization: str = "" molecular_weight: Optional[float] = None # Experimental MW in kDa molecular_weight_method: str = "" oligomeric_state: str = "" @@ -112,14 +118,61 @@ def parseMetadata(metadata: dict) -> SASBDBDatasetInfo: info.concentration_unit = _get_str(metadata, 'concentration_unit', 'conc_unit', 'concentration_units') or "mg/mL" # Molecule information - try more variations - info.molecule_name = _get_str(metadata, 'molecule_name', 'macromolecule_name', 'protein_name', - 'molecule', 'macromolecule', 'protein', 'name') - info.molecule_type = _get_str(metadata, 'molecule_type', 'macromolecule_type', 'sample_type', - 'type', 'molecule_type_full') + info.molecule_name = _get_str( + metadata, + 'molecule_name', + 'macromolecule_name', + 'protein_name', + 'molecule', + 'macromolecule', + 'protein', + 'name', + ) or _get_deep_str( + metadata, 'long_name', 'molecule_name', 'macromolecule_name', 'protein_name' + ) + info.molecule_short_name = _get_str( + metadata, 'short_name', 'molecule_short_name', 'shortName', 'short' + ) or _get_deep_str(metadata, 'short_name', 'molecule_short_name', 'shortName') + info.molecule_type = _get_str( + metadata, 'molecule_type', 'macromolecule_type', 'sample_type', + 'type', 'molecule_type_full' + ) or _get_deep_str( + metadata, 'molecule_type', 'macromolecule_type', 'molecular_type' + ) + info.sequence = _get_sequence(metadata) + info.uniprot_code = _get_str( + metadata, + 'uniprot_code', + 'uniprot', + 'uniprot_id', + 'uniprot_accession', + 'uniprot_ac', + ) or _get_deep_str( + metadata, 'uniprot_code', 'uniprot', 'uniprot_id', 'uniprot_accession' + ) + info.source_organism = _get_str( + metadata, 'source_organism', 'organism', 'organism_name' + ) or _get_deep_str( + metadata, 'source_organism', 'organism', 'organism_name' + ) + info.number_of_molecules = _get_str( + metadata, 'number_of_molecules', 'num_molecules', 'copy_number' + ) or _get_deep_str( + metadata, 'number_of_molecules', 'number_molecules', 'num_molecules', + 'copy_number' + ) + info.oligomerization = _get_str( + metadata, 'oligomerization', 'oligomeric_state', 'oligomer_state' + ) or _get_deep_str( + metadata, 'oligomerization', 'oligomeric_state', 'oligomer_state', + 'complex_state' + ) info.molecular_weight = _get_float(metadata, 'molecular_weight', 'mw', 'exp_mw', 'experimental_mw', 'mw_kda') info.molecular_weight_method = _get_str(metadata, 'mw_method', 'molecular_weight_method') - info.oligomeric_state = _get_str(metadata, 'oligomeric_state', 'oligomer_state', 'oligomerization') + info.oligomeric_state = info.oligomerization or _get_str( + metadata, 'oligomeric_state', 'oligomer_state', 'oligomerization' + ) # Buffer information info.buffer_description = _get_str(metadata, 'buffer', 'buffer_description', 'buffer_composition') @@ -189,6 +242,109 @@ def _get_str(data: dict, *keys: str) -> str: return "" +def _get_sequence(data: dict) -> str: + """ + Extract a protein/nucleotide sequence from nested SASBDB metadata. + + This helper searches recursively through dictionaries and lists since + sequence information may appear under molecule lists or nested blocks. + + :param data: Dictionary to search + :return: Sequence string or empty string if not found + """ + sequence_keys = { + 'sequence', + 'fasta_sequence', + 'fasta', + 'primary_sequence', + 'sequence_string', + } + + def _from_value(value) -> str: + """Normalize potential sequence values to a plain string.""" + if value is None: + return "" + if isinstance(value, str): + return value.strip() + if isinstance(value, list): + parts = [] + for item in value: + item_str = _from_value(item) + if item_str: + parts.append(item_str) + if parts: + return " ".join(parts) + return "" + if isinstance(value, dict): + # Common wrappers for sequence strings. + for nested_key in ("value", "text", "seq"): + nested = value.get(nested_key) + nested_str = _from_value(nested) + if nested_str: + return nested_str + return "" + + def _search(obj) -> str: + if isinstance(obj, dict): + for key in sequence_keys: + if key in obj: + sequence = _from_value(obj[key]) + if sequence: + return sequence + for value in obj.values(): + sequence = _search(value) + if sequence: + return sequence + elif isinstance(obj, list): + for item in obj: + sequence = _search(item) + if sequence: + return sequence + return "" + + return _search(data) + + +def _get_deep_str(data: dict, *keys: str) -> str: + """ + Recursively search nested dict/list structures for the first key match. + + :param data: Dictionary to search + :param keys: Candidate field names to locate + :return: String value or empty string if not found + """ + key_set = set(keys) + + def _normalize(value) -> str: + if value is None: + return "" + if isinstance(value, str): + return value.strip() + if isinstance(value, (int, float, bool)): + return str(value) + return "" + + def _search(obj) -> str: + if isinstance(obj, dict): + for key, value in obj.items(): + if key in key_set: + normalized = _normalize(value) + if normalized: + return normalized + for value in obj.values(): + found = _search(value) + if found: + return found + elif isinstance(obj, list): + for item in obj: + found = _search(item) + if found: + return found + return "" + + return _search(data) + + def _get_float(data: dict, *keys: str) -> Optional[float]: """ Get float value from dictionary, trying multiple possible keys. From 2b165f25fed907df9f68496346f7197ffdb58195 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci-lite[bot]" <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Date: Wed, 1 Apr 2026 09:15:49 +0000 Subject: [PATCH 04/12] [pre-commit.ci lite] apply automatic fixes for ruff linting errors --- src/sas/qtgui/MainWindow/GuiManager.py | 70 ++++----- src/sas/qtgui/Utilities/GuiUtils.py | 112 ++++++------- .../Utilities/SASBDB/SASBDBDownloadDialog.py | 59 ++++--- src/sas/qtgui/Utilities/SASBDB/sasbdb_api.py | 147 +++++++++--------- 4 files changed, 193 insertions(+), 195 deletions(-) diff --git a/src/sas/qtgui/MainWindow/GuiManager.py b/src/sas/qtgui/MainWindow/GuiManager.py index ce2c62fe41..19534f5000 100644 --- a/src/sas/qtgui/MainWindow/GuiManager.py +++ b/src/sas/qtgui/MainWindow/GuiManager.py @@ -801,7 +801,7 @@ def actionLoad_Data_Folder(self): Menu File/Load Data Folder """ self.filesWidget.loadFolder() - + def actionLoad_SASBDB(self): """ Menu File/Load from SASBDB @@ -809,23 +809,23 @@ def actionLoad_SASBDB(self): Opens a dialog to download and load a dataset from SASBDB. """ from sas.qtgui.Utilities.SASBDB.SASBDBDownloadDialog import SASBDBDownloadDialog - + dialog = SASBDBDownloadDialog(parent=self._workspace) if dialog.exec(): # Get the downloaded file path and metadata filepath = dialog.getDownloadedFilepath() dataset_info = dialog.getDatasetInfo() - + if filepath and os.path.exists(filepath): try: # Load the downloaded file into SasView # readData returns (output_dict, message) loaded_data, load_message = self.filesWidget.readData([filepath]) - + # Populate additional metadata from SASBDB into loaded data if dataset_info and loaded_data: self._populateSASBDBMetadata(loaded_data, dataset_info) - + # Log metadata summary if dataset_info: entry_id = dataset_info.code or dataset_info.entry_id @@ -838,7 +838,7 @@ def actionLoad_SASBDB(self): logger.info(f" MW: {dataset_info.molecular_weight:.1f} kDa") else: logger.info(f"Successfully loaded SASBDB dataset from {filepath}") - + except Exception as e: logger.error(f"Error loading downloaded SASBDB dataset: {e}", exc_info=True) QMessageBox.warning( @@ -846,7 +846,7 @@ def actionLoad_SASBDB(self): "Load Error", f"Failed to load downloaded dataset:\n{str(e)}" ) - + def _populateSASBDBMetadata(self, loaded_data: dict, dataset_info): """ Populate SASBDB metadata into loaded data objects. @@ -858,37 +858,37 @@ def _populateSASBDBMetadata(self, loaded_data: dict, dataset_info): :param dataset_info: SASBDBDatasetInfo object with parsed metadata """ from sasdata.dataloader.data_info import Sample, Source - + for data_id, data in loaded_data.items(): try: # Debug: log what we have in dataset_info logger.debug(f"Populating metadata for {data_id}: molecule_name={dataset_info.molecule_name}, " f"sample_name={dataset_info.sample_name}, temperature={dataset_info.temperature}, " f"concentration={dataset_info.concentration}, buffer={dataset_info.buffer_description}") - + # Update title if dataset_info.title and not data.title: data.title = dataset_info.title - + # Update instrument from SASBDB instrument metadata if dataset_info.instrument and not data.instrument: data.instrument = dataset_info.instrument - + # Update run info with SASBDB code if dataset_info.code: if not data.run: data.run = [] if dataset_info.code not in data.run: data.run.append(dataset_info.code) - + # Populate sample information from Molecule metadata if data.sample is None: data.sample = Sample() - + # Ensure sample.details is initialized if not hasattr(data.sample, 'details') or data.sample.details is None: data.sample.details = [] - + # Map molecule short name into Sample ID when available. sample_id = "" if dataset_info.molecule_short_name: @@ -938,7 +938,7 @@ def _populateSASBDBMetadata(self, loaded_data: dict, dataset_info): sample_details.append( f"Source organism: {dataset_info.source_organism}" ) - + # Set temperature if dataset_info.temperature is not None: data.sample.temperature = dataset_info.temperature @@ -950,14 +950,14 @@ def _populateSASBDBMetadata(self, loaded_data: dict, dataset_info): if temp_unit: temp_str += f" {temp_unit}" sample_details.append(temp_str) - + # Store concentration in sample details if dataset_info.concentration is not None: conc_str = f"Concentration: {dataset_info.concentration}" if dataset_info.concentration_unit: conc_str += f" {dataset_info.concentration_unit}" sample_details.append(conc_str) - + # Add buffer info to sample details if dataset_info.buffer_description: buffer_str = f"Buffer: {dataset_info.buffer_description}" @@ -969,68 +969,68 @@ def _populateSASBDBMetadata(self, loaded_data: dict, dataset_info): if detail and detail not in data.sample.details: data.sample.details.append(detail) logger.debug(f"Added to sample.details: {detail}") - + # Log sample info for debugging logger.debug(f"Sample populated for {data_id}: name={getattr(data.sample, 'name', None)}, " f"temperature={getattr(data.sample, 'temperature', None)}, " f"details={getattr(data.sample, 'details', [])}") - + # Populate source information if data.source is None: data.source = Source() - + if dataset_info.wavelength is not None: data.source.wavelength = dataset_info.wavelength data.source.wavelength_unit = dataset_info.wavelength_unit - + # Store additional SASBDB metadata in meta_data dictionary if not hasattr(data, 'meta_data') or data.meta_data is None: data.meta_data = {} - + # SASBDB-specific metadata data.meta_data['SASBDB_code'] = dataset_info.code or dataset_info.entry_id - + if dataset_info.rg is not None: data.meta_data['SASBDB_Rg'] = dataset_info.rg if dataset_info.rg_error is not None: data.meta_data['SASBDB_Rg_error'] = dataset_info.rg_error - + if dataset_info.i0 is not None: data.meta_data['SASBDB_I0'] = dataset_info.i0 if dataset_info.i0_error is not None: data.meta_data['SASBDB_I0_error'] = dataset_info.i0_error - + if dataset_info.dmax is not None: data.meta_data['SASBDB_Dmax'] = dataset_info.dmax - + if dataset_info.molecular_weight is not None: data.meta_data['SASBDB_MW'] = dataset_info.molecular_weight if dataset_info.molecular_weight_method: data.meta_data['SASBDB_MW_method'] = dataset_info.molecular_weight_method - + if dataset_info.porod_volume is not None: data.meta_data['SASBDB_Porod_volume'] = dataset_info.porod_volume - + if dataset_info.molecule_name: data.meta_data['SASBDB_molecule'] = dataset_info.molecule_name - + if dataset_info.molecule_type: data.meta_data['SASBDB_molecule_type'] = dataset_info.molecule_type - + if dataset_info.oligomeric_state: data.meta_data['SASBDB_oligomeric_state'] = dataset_info.oligomeric_state - + if dataset_info.publication_doi: data.meta_data['SASBDB_DOI'] = dataset_info.publication_doi - + if dataset_info.publication_pmid: data.meta_data['SASBDB_PMID'] = dataset_info.publication_pmid - + if dataset_info.authors: data.meta_data['SASBDB_authors'] = ', '.join(dataset_info.authors) - + logger.debug(f"Populated SASBDB metadata for data: {data_id}") - + except Exception as e: logger.warning(f"Error populating metadata for data {data_id}: {e}") diff --git a/src/sas/qtgui/Utilities/GuiUtils.py b/src/sas/qtgui/Utilities/GuiUtils.py index e00d32df9b..5dfbfd4c5a 100644 --- a/src/sas/qtgui/Utilities/GuiUtils.py +++ b/src/sas/qtgui/Utilities/GuiUtils.py @@ -543,13 +543,13 @@ def _formatDictValue(value, max_depth=2) -> str: """ if not isinstance(value, dict): return str(value) - + if max_depth <= 0: return "{...}" - + # Extract common useful fields from dictionaries parts = [] - + # Instrument/source dictionaries if 'name' in value: parts.append(value['name']) @@ -563,7 +563,7 @@ def _formatDictValue(value, max_depth=2) -> str: parts.append(value['city']) elif 'country' in value: parts.append(value['country']) - + # Detector dictionaries if 'detector' in str(value).lower() or 'type' in value: if 'name' in value: @@ -572,14 +572,14 @@ def _formatDictValue(value, max_depth=2) -> str: parts.append(f"Detector: {value['type']}") if 'resolution' in value: parts.append(f"Resolution: {value['resolution']}") - + if parts: return " | ".join(parts) - + # Fallback: show key-value pairs for shallow dicts if len(value) <= 3: return ", ".join(f"{k}: {v}" for k, v in value.items() if not isinstance(v, dict)) - + return "{...}" @@ -592,13 +592,13 @@ def _formatDictLine(line: str) -> str: """ import ast import re - + stripped = line.strip() - + # Check if line contains a dictionary if not ("{" in stripped and "'" in stripped and ":" in stripped): return line - + # Try to extract the label (e.g., "Instrument:", "Detector:") label_match = re.match(r'^(\w+)\s*:\s*(.+)', stripped) if label_match: @@ -609,17 +609,17 @@ def _formatDictLine(line: str) -> str: dict_str = stripped else: return line - + # Try to parse the dictionary string try: # Use ast.literal_eval to safely parse the dictionary parsed_dict = ast.literal_eval(dict_str) if not isinstance(parsed_dict, dict): return line - + # Format the dictionary parts = [] - + # Extract instrument/source info if 'name' in parsed_dict and parsed_dict['name']: parts.append(parsed_dict['name']) @@ -635,7 +635,7 @@ def _formatDictLine(line: str) -> str: parts.append(city) elif 'country' in parsed_dict and parsed_dict['country']: parts.append(parsed_dict['country']) - + # Extract detector info (nested or direct) detector_info = None if 'detector' in parsed_dict and isinstance(parsed_dict['detector'], dict): @@ -646,10 +646,10 @@ def _formatDictLine(line: str) -> str: detector_info = det['type'] elif 'detector' in parsed_dict: detector_info = str(parsed_dict['detector']) - + if detector_info: parts.append(f"Detector: {detector_info}") - + # If we extracted useful info, format it if parts: formatted = " | ".join(parts) @@ -657,7 +657,7 @@ def _formatDictLine(line: str) -> str: return f"{label}: {formatted}" else: return formatted - + # Fallback: show key-value pairs for simple dicts simple_parts = [] for k, v in parsed_dict.items(): @@ -669,11 +669,11 @@ def _formatDictLine(line: str) -> str: return f"{label}: {formatted}" else: return formatted - + except (ValueError, SyntaxError): # If parsing fails, return original line pass - + return line @@ -688,27 +688,27 @@ def _formatSASBDBMetadata(data) -> str: :return: Formatted string with SASBDB metadata, or empty string if none """ text = "" - + # Check if meta_data exists and contains SASBDB info meta = getattr(data, 'meta_data', None) or {} - + # Check if this is SASBDB data if 'SASBDB_code' not in meta: return text - + text += "\n" + "=" * 50 + "\n" text += "SASBDB Metadata\n" text += "=" * 50 + "\n" - + # Entry and instrument info (compact header) header_parts = [] if meta.get('SASBDB_code'): header_parts.append(f"Code: {meta['SASBDB_code']}") - + # Extract instrument info from metadata (handle both string and dict formats) instrument_str = None location_str = None - + if hasattr(data, 'instrument') and data.instrument: if isinstance(data.instrument, str): instrument_str = data.instrument @@ -719,7 +719,7 @@ def _formatSASBDBMetadata(data) -> str: location_str = f"{data.instrument['city']}, {data.instrument['country']}" elif data.instrument.get('city'): location_str = data.instrument['city'] - + # Also check meta_data for instrument info (search all keys for dict values) if not instrument_str: for key, val in meta.items(): @@ -735,12 +735,12 @@ def _formatSASBDBMetadata(data) -> str: elif key in ['instrument', 'source', 'beamline'] and isinstance(val, str): instrument_str = val break - + if instrument_str: header_parts.append(f"Instrument: {instrument_str}") if location_str: header_parts.append(location_str) - + # Extract detector info if available (search all keys for detector dicts) detector_info = None for key, val in meta.items(): @@ -752,26 +752,26 @@ def _formatSASBDBMetadata(data) -> str: elif 'detector' in key.lower() and isinstance(val, str): detector_info = val break - + if detector_info: header_parts.append(f"Detector: {detector_info}") - + if header_parts: text += " | ".join(header_parts) + "\n" - + # Sample info (consolidated) if hasattr(data, 'sample') and data.sample: sample = data.sample sample_parts = [] - + if hasattr(sample, 'name') and sample.name: sample_parts.append(sample.name) - + # Add temperature if available if hasattr(sample, 'temperature') and sample.temperature is not None: temp_unit = getattr(sample, 'temperature_unit', 'K') or 'K' sample_parts.append(f"T = {sample.temperature} {temp_unit}") - + # Add concentration and buffer from details (if present) if hasattr(sample, 'details') and sample.details: for detail in sample.details: @@ -779,17 +779,17 @@ def _formatSASBDBMetadata(data) -> str: sample_parts.append(detail.replace("Concentration: ", "")) elif detail.startswith("Buffer:"): sample_parts.append(detail.replace("Buffer: ", "")) - + if sample_parts: text += f"Sample: {' | '.join(sample_parts)}\n" - + # Source wavelength (if available) if hasattr(data, 'source') and data.source: source = data.source if hasattr(source, 'wavelength') and source.wavelength is not None: wl_unit = getattr(source, 'wavelength_unit', 'Å') or 'Å' text += f"Wavelength: {source.wavelength} {wl_unit}\n" - + # Structural parameters (compact format) structural_parts = [] if meta.get('SASBDB_Rg') is not None: @@ -798,28 +798,28 @@ def _formatSASBDBMetadata(data) -> str: rg_str += f" ± {meta['SASBDB_Rg_error']:.2f}" rg_str += " Å" structural_parts.append(rg_str) - + if meta.get('SASBDB_I0') is not None: i0_str = f"I(0) = {meta['SASBDB_I0']:.4e}" if meta.get('SASBDB_I0_error') is not None: i0_str += f" ± {meta['SASBDB_I0_error']:.4e}" structural_parts.append(i0_str) - + if meta.get('SASBDB_Dmax') is not None: structural_parts.append(f"Dmax = {meta['SASBDB_Dmax']:.2f} Å") - + if meta.get('SASBDB_MW') is not None: mw_str = f"MW = {meta['SASBDB_MW']:.2f} kDa" if meta.get('SASBDB_MW_method'): mw_str += f" ({meta['SASBDB_MW_method']})" structural_parts.append(mw_str) - + if meta.get('SASBDB_Porod_volume') is not None: structural_parts.append(f"Porod Volume = {meta['SASBDB_Porod_volume']:.0f} ų") - + if structural_parts: text += "Structural: " + " | ".join(structural_parts) + "\n" - + # Publication info (compact, only if available) pub_parts = [] if meta.get('SASBDB_authors'): @@ -832,12 +832,12 @@ def _formatSASBDBMetadata(data) -> str: pub_parts.append(f"DOI: {meta['SASBDB_DOI']}") if meta.get('SASBDB_PMID'): pub_parts.append(f"PMID: {meta['SASBDB_PMID']}") - + if pub_parts: text += "Publication: " + " | ".join(pub_parts) + "\n" - + text += "=" * 50 + "\n\n" - + return text @@ -859,7 +859,7 @@ def retrieveData1d(data): raise ValueError(msg) text = data.__str__() - + # Format dictionary representations instead of removing them import re lines = text.split('\n') @@ -867,8 +867,8 @@ def retrieveData1d(data): for line in lines: stripped = line.strip() # Check if line contains a dictionary representation - if ("{" in stripped and "'" in stripped and ":" in stripped and - (stripped.startswith("{'") or + if ("{" in stripped and "'" in stripped and ":" in stripped and + (stripped.startswith("{'") or re.search(r':\s*\{', stripped) or # Matches "Instrument: {" re.search(r"\{'[^']+':", stripped))): # Matches "{'key':" # Count opening and closing braces to detect dict structures @@ -884,10 +884,10 @@ def retrieveData1d(data): else: cleaned_lines.append(line) text = '\n'.join(cleaned_lines) - + # Add SASBDB metadata if present text += _formatSASBDBMetadata(data) - + text += 'Data Min Max:\n' text += 'X_min = %s: X_max = %s\n' % (xmin, max(data.x)) text += 'Y_min = %s: Y_max = %s\n' % (ymin, max(data.y)) @@ -932,7 +932,7 @@ def retrieveData2d(data): raise AttributeError(msg) text = data.__str__() - + # Format dictionary representations instead of removing them import re lines = text.split('\n') @@ -940,8 +940,8 @@ def retrieveData2d(data): for line in lines: stripped = line.strip() # Check if line contains a dictionary representation - if ("{" in stripped and "'" in stripped and ":" in stripped and - (stripped.startswith("{'") or + if ("{" in stripped and "'" in stripped and ":" in stripped and + (stripped.startswith("{'") or re.search(r':\s*\{', stripped) or # Matches "Instrument: {" re.search(r"\{'[^']+':", stripped))): # Matches "{'key':" # Count opening and closing braces to detect dict structures @@ -957,10 +957,10 @@ def retrieveData2d(data): else: cleaned_lines.append(line) text = '\n'.join(cleaned_lines) - + # Add SASBDB metadata if present text += _formatSASBDBMetadata(data) - + text += 'Data Min Max:\n' text += 'I_min = %s\n' % min(data.data) text += 'I_max = %s\n\n' % max(data.data) diff --git a/src/sas/qtgui/Utilities/SASBDB/SASBDBDownloadDialog.py b/src/sas/qtgui/Utilities/SASBDB/SASBDBDownloadDialog.py index 9b0ef622ad..87250f6dde 100644 --- a/src/sas/qtgui/Utilities/SASBDB/SASBDBDownloadDialog.py +++ b/src/sas/qtgui/Utilities/SASBDB/SASBDBDownloadDialog.py @@ -8,11 +8,10 @@ import logging import os import tempfile -from typing import Optional from PySide6 import QtWidgets -from .sasbdb_api import downloadDataset, SASBDBDatasetInfo +from .sasbdb_api import SASBDBDatasetInfo, downloadDataset from .UI.SASBDBDownloadDialogUI import Ui_SASBDBDownloadDialogUI logger = logging.getLogger(__name__) @@ -25,7 +24,7 @@ class SASBDBDownloadDialog(QtWidgets.QDialog, Ui_SASBDBDownloadDialogUI): Allows users to enter a SASBDB dataset identifier, downloads the dataset, and loads it into SasView. """ - + def __init__(self, parent=None): """ Initialize the download dialog. @@ -34,20 +33,20 @@ def __init__(self, parent=None): """ super().__init__(parent) self.setupUi(self) - + # Store downloaded file path and metadata - self.downloaded_filepath: Optional[str] = None - self.dataset_info: Optional[SASBDBDatasetInfo] = None - + self.downloaded_filepath: str | None = None + self.dataset_info: SASBDBDatasetInfo | None = None + # Connect signals self.cmdDownload.clicked.connect(self.onDownload) self.cmdCancel.clicked.connect(self.reject) self.cmdHelp.clicked.connect(self.onHelp) self.txtDatasetId.returnPressed.connect(self.onDownload) - + # Set focus on dataset ID input self.txtDatasetId.setFocus() - + def onDownload(self): """ Handle download button click. @@ -56,11 +55,11 @@ def onDownload(self): and loads it into SasView. """ dataset_id = self.txtDatasetId.text().strip() - + if not dataset_id: self._showError("Please enter a dataset identifier.") return - + # Disable download button during download self.cmdDownload.setEnabled(False) self.cmdCancel.setEnabled(False) @@ -68,32 +67,32 @@ def onDownload(self): self.progressBar.setRange(0, 0) # Indeterminate progress self.lblStatus.setText("Downloading dataset...") QtWidgets.QApplication.processEvents() - + try: # Download the dataset output_dir = tempfile.gettempdir() filepath, dataset_info = downloadDataset(dataset_id, output_dir) - + # Store the metadata self.dataset_info = dataset_info - + if filepath and os.path.exists(filepath): self.downloaded_filepath = filepath - + # Build success message with metadata summary success_msg = f"Successfully downloaded dataset {dataset_id}" if dataset_info: details = self._formatMetadataSummary(dataset_info) if details: success_msg += f"\n{details}" - + self.lblStatus.setText(success_msg) self.progressBar.setVisible(False) - + # Log detailed metadata if dataset_info: self._logMetadata(dataset_info) - + self.accept() # Close dialog and return success else: self._showError(f"Failed to download dataset {dataset_id}.\n" @@ -101,14 +100,14 @@ def onDownload(self): self.progressBar.setVisible(False) self.cmdDownload.setEnabled(True) self.cmdCancel.setEnabled(True) - + except Exception as e: logger.error(f"Error downloading dataset {dataset_id}: {e}", exc_info=True) self._showError(f"Error downloading dataset:\n{str(e)}") self.progressBar.setVisible(False) self.cmdDownload.setEnabled(True) self.cmdCancel.setEnabled(True) - + def _formatMetadataSummary(self, info: SASBDBDatasetInfo) -> str: """ Format a brief summary of the dataset metadata. @@ -117,7 +116,7 @@ def _formatMetadataSummary(self, info: SASBDBDatasetInfo) -> str: :return: Formatted summary string """ parts = [] - + if info.title: parts.append(f"Title: {info.title}") if info.sample_name: @@ -130,9 +129,9 @@ def _formatMetadataSummary(self, info: SASBDBDatasetInfo) -> str: parts.append(rg_str) if info.molecular_weight is not None: parts.append(f"MW: {info.molecular_weight:.1f} kDa") - + return "\n".join(parts) - + def _logMetadata(self, info: SASBDBDatasetInfo): """ Log detailed metadata information. @@ -166,7 +165,7 @@ def _logMetadata(self, info: SASBDBDatasetInfo): logger.info(f" MW: {info.molecular_weight} kDa") if info.publication_doi: logger.info(f" DOI: {info.publication_doi}") - + def _showError(self, message: str): """ Display an error message to the user. @@ -175,23 +174,23 @@ def _showError(self, message: str): """ self.lblStatus.setText(f"{message}") QtWidgets.QMessageBox.warning(self, "Download Error", message) - - def getDownloadedFilepath(self) -> Optional[str]: + + def getDownloadedFilepath(self) -> str | None: """ Get the path to the downloaded file. :return: Path to downloaded file, or None if download failed """ return self.downloaded_filepath - - def getDatasetInfo(self) -> Optional[SASBDBDatasetInfo]: + + def getDatasetInfo(self) -> SASBDBDatasetInfo | None: """ Get the parsed dataset metadata. :return: SASBDBDatasetInfo object, or None if metadata not available """ return self.dataset_info - + def onHelp(self): """ Show the SASBDB download help documentation. @@ -210,7 +209,7 @@ def onHelp(self): elif hasattr(parent, 'showHelp'): parent.showHelp(help_location) return - + # Fallback to GuiUtils GuiUtils.showHelp(help_location) except Exception as e: diff --git a/src/sas/qtgui/Utilities/SASBDB/sasbdb_api.py b/src/sas/qtgui/Utilities/SASBDB/sasbdb_api.py index 08c69a0329..7c4b0f5b79 100644 --- a/src/sas/qtgui/Utilities/SASBDB/sasbdb_api.py +++ b/src/sas/qtgui/Utilities/SASBDB/sasbdb_api.py @@ -9,7 +9,6 @@ import os import tempfile from dataclasses import dataclass, field -from typing import Optional import requests @@ -30,13 +29,13 @@ class SASBDBDatasetInfo: entry_id: str = "" code: str = "" title: str = "" - + # Sample information sample_name: str = "" sample_description: str = "" - concentration: Optional[float] = None + concentration: float | None = None concentration_unit: str = "mg/mL" - + # Molecule information molecule_name: str = "" molecule_short_name: str = "" @@ -46,44 +45,44 @@ class SASBDBDatasetInfo: source_organism: str = "" number_of_molecules: str = "" oligomerization: str = "" - molecular_weight: Optional[float] = None # Experimental MW in kDa + molecular_weight: float | None = None # Experimental MW in kDa molecular_weight_method: str = "" oligomeric_state: str = "" - + # Buffer information buffer_description: str = "" - ph: Optional[float] = None - + ph: float | None = None + # Experimental parameters instrument: str = "" detector: str = "" - wavelength: Optional[float] = None # in Angstrom + wavelength: float | None = None # in Angstrom wavelength_unit: str = "Å" - temperature: Optional[float] = None # in Kelvin or Celsius + temperature: float | None = None # in Kelvin or Celsius temperature_unit: str = "K" - + # Analysis results - rg: Optional[float] = None # Radius of gyration in Angstrom - rg_error: Optional[float] = None - i0: Optional[float] = None # I(0) - i0_error: Optional[float] = None - dmax: Optional[float] = None # Maximum dimension in Angstrom - porod_volume: Optional[float] = None - + rg: float | None = None # Radius of gyration in Angstrom + rg_error: float | None = None + i0: float | None = None # I(0) + i0_error: float | None = None + dmax: float | None = None # Maximum dimension in Angstrom + porod_volume: float | None = None + # Q-range - q_min: Optional[float] = None - q_max: Optional[float] = None - + q_min: float | None = None + q_max: float | None = None + # Publication publication_title: str = "" publication_doi: str = "" publication_pmid: str = "" authors: list = field(default_factory=list) - + # Data files intensities_data_url: str = "" pddf_data_url: str = "" - + # Raw metadata for additional fields raw_metadata: dict = field(default_factory=dict) @@ -96,27 +95,27 @@ def parseMetadata(metadata: dict) -> SASBDBDatasetInfo: :return: Parsed SASBDBDatasetInfo object """ info = SASBDBDatasetInfo() - + if not isinstance(metadata, dict): return info - + # Store raw metadata for reference info.raw_metadata = metadata - + # Log top-level keys for debugging logger.debug(f"SASBDB API response keys: {list(metadata.keys())}") - + # Identifiers info.entry_id = _get_str(metadata, 'id', 'entry_id', 'sasbdb_id') info.code = _get_str(metadata, 'code', 'accession_code', 'entry_code') info.title = _get_str(metadata, 'title', 'entry_title', 'sample_title') - + # Sample information - try more variations info.sample_name = _get_str(metadata, 'sample_name', 'sample', 'name', 'sample_name_full') info.sample_description = _get_str(metadata, 'sample_description', 'description', 'sample_description_full') info.concentration = _get_float(metadata, 'concentration', 'sample_concentration', 'conc', 'sample_conc') info.concentration_unit = _get_str(metadata, 'concentration_unit', 'conc_unit', 'concentration_units') or "mg/mL" - + # Molecule information - try more variations info.molecule_name = _get_str( metadata, @@ -167,53 +166,53 @@ def parseMetadata(metadata: dict) -> SASBDBDatasetInfo: metadata, 'oligomerization', 'oligomeric_state', 'oligomer_state', 'complex_state' ) - info.molecular_weight = _get_float(metadata, 'molecular_weight', 'mw', 'exp_mw', + info.molecular_weight = _get_float(metadata, 'molecular_weight', 'mw', 'exp_mw', 'experimental_mw', 'mw_kda') info.molecular_weight_method = _get_str(metadata, 'mw_method', 'molecular_weight_method') info.oligomeric_state = info.oligomerization or _get_str( metadata, 'oligomeric_state', 'oligomer_state', 'oligomerization' ) - + # Buffer information info.buffer_description = _get_str(metadata, 'buffer', 'buffer_description', 'buffer_composition') info.ph = _get_float(metadata, 'ph', 'buffer_ph') - + # Experimental parameters info.instrument = _get_str(metadata, 'instrument', 'beamline', 'instrument_name') info.detector = _get_str(metadata, 'detector', 'detector_name', 'detector_type') info.wavelength = _get_float(metadata, 'wavelength', 'xray_wavelength', 'neutron_wavelength') info.temperature = _get_float(metadata, 'temperature', 'sample_temperature', 'temp') - + # Analysis results - Guinier info.rg = _get_float(metadata, 'rg', 'radius_of_gyration', 'guinier_rg') info.rg_error = _get_float(metadata, 'rg_error', 'rg_err', 'guinier_rg_error') info.i0 = _get_float(metadata, 'i0', 'i_zero', 'guinier_i0') info.i0_error = _get_float(metadata, 'i0_error', 'i0_err', 'guinier_i0_error') - + # Analysis results - P(r) info.dmax = _get_float(metadata, 'dmax', 'd_max', 'maximum_dimension') info.porod_volume = _get_float(metadata, 'porod_volume', 'volume', 'porod_vol') - + # Q-range info.q_min = _get_float(metadata, 'q_min', 'qmin', 's_min', 'smin') info.q_max = _get_float(metadata, 'q_max', 'qmax', 's_max', 'smax') - + # Publication info.publication_title = _get_str(metadata, 'publication_title', 'pub_title', 'paper_title') info.publication_doi = _get_str(metadata, 'doi', 'publication_doi', 'pub_doi') info.publication_pmid = _get_str(metadata, 'pmid', 'publication_pmid', 'pubmed_id') - + # Authors authors = metadata.get('authors') or metadata.get('author_list') or [] if isinstance(authors, list): info.authors = [str(a) for a in authors] elif isinstance(authors, str): info.authors = [authors] - + # Data file URLs info.intensities_data_url = _get_str(metadata, 'intensities_data', 'data_url', 'intensities_url') info.pddf_data_url = _get_str(metadata, 'pddf_data', 'pddf_url', 'pr_data') - + return info @@ -230,7 +229,7 @@ def _get_str(data: dict, *keys: str) -> str: for key in keys: if key in data and data[key] is not None: return str(data[key]) - + # Then search in common nested structures nested_paths = ['sample', 'molecule', 'experiment', 'experimental', 'metadata', 'info'] for path in nested_paths: @@ -238,7 +237,7 @@ def _get_str(data: dict, *keys: str) -> str: for key in keys: if key in data[path] and data[path][key] is not None: return str(data[path][key]) - + return "" @@ -345,7 +344,7 @@ def _search(obj) -> str: return _search(data) -def _get_float(data: dict, *keys: str) -> Optional[float]: +def _get_float(data: dict, *keys: str) -> float | None: """ Get float value from dictionary, trying multiple possible keys. Also searches in nested dictionaries (sample, molecule, experiment, etc.). @@ -361,7 +360,7 @@ def _get_float(data: dict, *keys: str) -> Optional[float]: return float(data[key]) except (ValueError, TypeError): continue - + # Then search in common nested structures nested_paths = ['sample', 'molecule', 'experiment', 'experimental', 'metadata', 'info'] for path in nested_paths: @@ -372,11 +371,11 @@ def _get_float(data: dict, *keys: str) -> Optional[float]: return float(data[path][key]) except (ValueError, TypeError): continue - + return None -def getDatasetMetadata(dataset_id: str) -> Optional[dict]: +def getDatasetMetadata(dataset_id: str) -> dict | None: """ Fetch dataset metadata from SASBDB API. @@ -388,21 +387,21 @@ def getDatasetMetadata(dataset_id: str) -> Optional[dict]: if not normalized_id: logger.error(f"Invalid dataset ID format: {dataset_id}") return None - + # Use the correct REST API endpoint: /rest-api/entry/summary/{id}/ endpoint = f"{SASBDB_API_BASE}/entry/summary/{normalized_id}/" - + try: logger.info(f"Fetching dataset metadata from: {endpoint}") headers = {"accept": "application/json"} response = requests.get(endpoint, headers=headers, timeout=30) response.raise_for_status() - + # Parse JSON response metadata = response.json() logger.info(f"Successfully retrieved metadata for dataset {normalized_id}") return metadata - + except requests.exceptions.HTTPError as e: if e.response.status_code == 404: logger.error(f"Dataset {normalized_id} not found (404)") @@ -417,7 +416,7 @@ def getDatasetMetadata(dataset_id: str) -> Optional[dict]: return None -def getDataFileUrl(metadata: dict) -> Optional[str]: +def getDataFileUrl(metadata: dict) -> str | None: """ Extract data file URL from dataset metadata. @@ -429,7 +428,7 @@ def getDataFileUrl(metadata: dict) -> Optional[str]: """ if not isinstance(metadata, dict): return None - + # SASBDB-specific field names that might contain data file URLs # Priority: intensities_data is the primary field for SASBDB possible_fields = [ @@ -461,7 +460,7 @@ def getDataFileUrl(metadata: dict) -> Optional[str]: 'scattering_data', 'scatteringData', ] - + # Check top-level fields for field in possible_fields: if field in metadata: @@ -489,14 +488,14 @@ def getDataFileUrl(metadata: dict) -> Optional[str]: return f"https://www.sasbdb.org{url}" elif url.startswith('http'): return url - + # Check nested structures (common in REST APIs) for nested_key in ['entry', 'data', 'files', 'experimental_data', 'scattering_data']: if nested_key in metadata and isinstance(metadata[nested_key], dict): result = getDataFileUrl(metadata[nested_key]) if result: return result - + # If we have an entry ID, try constructing a download URL # Format might be: /rest-api/entry/{id}/download/ or similar entry_id = metadata.get('entry_id') or metadata.get('id') or metadata.get('sasbdb_id') @@ -509,7 +508,7 @@ def getDataFileUrl(metadata: dict) -> Optional[str]: ] # Return first endpoint (we'll try them in downloadDataFile if needed) return download_endpoints[0] - + logger.warning("Could not find data file URL in metadata") logger.debug(f"Metadata keys: {list(metadata.keys())}") return None @@ -527,28 +526,28 @@ def downloadDataFile(url: str, filepath: str) -> bool: logger.info(f"Downloading data file from: {url}") response = requests.get(url, timeout=60, stream=True) response.raise_for_status() - + # Create directory if it doesn't exist os.makedirs(os.path.dirname(filepath), exist_ok=True) - + # Write file in chunks to handle large files with open(filepath, 'wb') as f: for chunk in response.iter_content(chunk_size=8192): if chunk: f.write(chunk) - + logger.info(f"Successfully downloaded data file to: {filepath}") return True - + except requests.exceptions.RequestException as e: logger.error(f"Error downloading data file from {url}: {e}") return False - except IOError as e: + except OSError as e: logger.error(f"Error writing data file to {filepath}: {e}") return False -def _normalizeDatasetId(dataset_id: str) -> Optional[str]: +def _normalizeDatasetId(dataset_id: str) -> str | None: """ Normalize a SASBDB dataset identifier. @@ -564,21 +563,21 @@ def _normalizeDatasetId(dataset_id: str) -> Optional[str]: """ if not dataset_id: return None - + # Remove whitespace and convert to uppercase normalized = dataset_id.strip().upper() - + # SASBDB identifiers are typically 7 characters # Accept identifiers that are 4-10 characters to be flexible if len(normalized) < 4 or len(normalized) > 10: logger.warning(f"Dataset ID length unusual: {len(normalized)} characters for '{normalized}'") # Still try it, but warn - + # Return the normalized identifier as-is (API expects full identifier) return normalized -def downloadDataset(dataset_id: str, output_dir: Optional[str] = None) -> tuple[Optional[str], Optional[SASBDBDatasetInfo]]: +def downloadDataset(dataset_id: str, output_dir: str | None = None) -> tuple[str | None, SASBDBDatasetInfo | None]: """ Download a complete dataset from SASBDB. @@ -596,35 +595,35 @@ def downloadDataset(dataset_id: str, output_dir: Optional[str] = None) -> tuple[ metadata = getDatasetMetadata(dataset_id) if not metadata: return None, None - + # Parse metadata into structured object dataset_info = parseMetadata(metadata) - + # Get data file URL data_url = getDataFileUrl(metadata) if not data_url: logger.error(f"Could not find data file URL in metadata for dataset {dataset_id}") return None, dataset_info # Return metadata even if download fails - + # Store the data URL in the info object if not dataset_info.intensities_data_url: dataset_info.intensities_data_url = data_url - + # Determine output directory if output_dir is None: output_dir = tempfile.gettempdir() - + # Generate filename normalized_id = _normalizeDatasetId(dataset_id) # Try to determine file extension from URL or metadata file_extension = _guessFileExtension(data_url, metadata) filename = f"SASBDB_{normalized_id}{file_extension}" filepath = os.path.join(output_dir, filename) - + # Download the file if downloadDataFile(data_url, filepath): return filepath, dataset_info - + return None, dataset_info @@ -643,7 +642,7 @@ def _guessFileExtension(url: str, metadata: dict) -> str: ext = '.' + parts[-1].split('?')[0] # Remove query parameters if ext.lower() in ['.dat', '.txt', '.csv', '.out', '.asc']: return ext - + # Check metadata for file type hints if isinstance(metadata, dict): file_type_fields = ['file_type', 'fileType', 'format', 'data_format'] @@ -656,7 +655,7 @@ def _guessFileExtension(url: str, metadata: dict) -> str: return '.txt' elif 'dat' in file_type or 'data' in file_type: return '.dat' - + # Default to .dat for scattering data return '.dat' From 90b5f0f499b3f263a00b6d0286ad4747b244bc4b Mon Sep 17 00:00:00 2001 From: Wojtek Potrzebowski Date: Mon, 13 Jul 2026 14:52:28 +0200 Subject: [PATCH 05/12] Removing icon from file menu to make it consistent with other items --- src/sas/qtgui/MainWindow/UI/MainWindowUI.ui | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/sas/qtgui/MainWindow/UI/MainWindowUI.ui b/src/sas/qtgui/MainWindow/UI/MainWindowUI.ui index 61aee0e236..9a32105729 100755 --- a/src/sas/qtgui/MainWindow/UI/MainWindowUI.ui +++ b/src/sas/qtgui/MainWindow/UI/MainWindowUI.ui @@ -312,10 +312,6 @@ - - - :/res/file_send-128.png:/res/file_send-128.png - Load from SASBDB... From 3230efcb5bcbb6e19e3c6895dd24a082b38da3b1 Mon Sep 17 00:00:00 2001 From: Wojtek Potrzebowski Date: Mon, 13 Jul 2026 15:25:59 +0200 Subject: [PATCH 06/12] Cleaned up and separated code from GuiManager --- src/sas/qtgui/MainWindow/GuiManager.py | 230 +----------------- .../Utilities/SASBDB/SASBDBDownloadDialog.py | 190 +++------------ src/sas/qtgui/Utilities/SASBDB/__init__.py | 2 + .../qtgui/Utilities/SASBDB/sasbdb_loader.py | 185 ++++++++++++++ 4 files changed, 224 insertions(+), 383 deletions(-) create mode 100644 src/sas/qtgui/Utilities/SASBDB/sasbdb_loader.py diff --git a/src/sas/qtgui/MainWindow/GuiManager.py b/src/sas/qtgui/MainWindow/GuiManager.py index c9f8af2164..37d74c865c 100644 --- a/src/sas/qtgui/MainWindow/GuiManager.py +++ b/src/sas/qtgui/MainWindow/GuiManager.py @@ -769,234 +769,20 @@ def actionLoad_Data_Folder(self): def actionLoad_SASBDB(self): """ Menu File/Load from SASBDB - + Opens a dialog to download and load a dataset from SASBDB. """ from sas.qtgui.Utilities.SASBDB.SASBDBDownloadDialog import SASBDBDownloadDialog + from sas.qtgui.Utilities.SASBDB.sasbdb_loader import load_downloaded_dataset dialog = SASBDBDownloadDialog(parent=self._workspace) if dialog.exec(): - # Get the downloaded file path and metadata - filepath = dialog.getDownloadedFilepath() - dataset_info = dialog.getDatasetInfo() - - if filepath and os.path.exists(filepath): - try: - # Load the downloaded file into SasView - # readData returns (output_dict, message) - loaded_data, load_message = self.filesWidget.readData([filepath]) - - # Populate additional metadata from SASBDB into loaded data - if dataset_info and loaded_data: - self._populateSASBDBMetadata(loaded_data, dataset_info) - - # Log metadata summary - if dataset_info: - entry_id = dataset_info.code or dataset_info.entry_id - logger.info(f"Successfully loaded SASBDB dataset {entry_id} from {filepath}") - if dataset_info.title: - logger.info(f" Title: {dataset_info.title}") - if dataset_info.rg is not None: - logger.info(f" Rg: {dataset_info.rg:.2f} Å") - if dataset_info.molecular_weight is not None: - logger.info(f" MW: {dataset_info.molecular_weight:.1f} kDa") - else: - logger.info(f"Successfully loaded SASBDB dataset from {filepath}") - - except Exception as e: - logger.error(f"Error loading downloaded SASBDB dataset: {e}", exc_info=True) - QMessageBox.warning( - self._workspace, - "Load Error", - f"Failed to load downloaded dataset:\n{str(e)}" - ) - - def _populateSASBDBMetadata(self, loaded_data: dict, dataset_info): - """ - Populate SASBDB metadata into loaded data objects. - - Updates the sample, source, and other metadata properties of loaded - data with information from SASBDB. - - :param loaded_data: Dictionary of loaded data objects (keyed by id) - :param dataset_info: SASBDBDatasetInfo object with parsed metadata - """ - from sasdata.dataloader.data_info import Sample, Source - - for data_id, data in loaded_data.items(): - try: - # Debug: log what we have in dataset_info - logger.debug(f"Populating metadata for {data_id}: molecule_name={dataset_info.molecule_name}, " - f"sample_name={dataset_info.sample_name}, temperature={dataset_info.temperature}, " - f"concentration={dataset_info.concentration}, buffer={dataset_info.buffer_description}") - - # Update title - if dataset_info.title and not data.title: - data.title = dataset_info.title - - # Update instrument from SASBDB instrument metadata - if dataset_info.instrument and not data.instrument: - data.instrument = dataset_info.instrument - - # Update run info with SASBDB code - if dataset_info.code: - if not data.run: - data.run = [] - if dataset_info.code not in data.run: - data.run.append(dataset_info.code) - - # Populate sample information from Molecule metadata - if data.sample is None: - data.sample = Sample() - - # Ensure sample.details is initialized - if not hasattr(data.sample, 'details') or data.sample.details is None: - data.sample.details = [] - - # Map molecule short name into Sample ID when available. - sample_id = "" - if dataset_info.molecule_short_name: - sample_id = dataset_info.molecule_short_name - elif dataset_info.sample_name: - # Fallback for entries without molecule short name. - sample_id = dataset_info.sample_name - elif dataset_info.code: - sample_id = dataset_info.code - if sample_id: - data.sample.ID = sample_id - logger.debug(f"Set sample.ID to: {sample_id}") - - # Build human-readable sample details from remaining metadata. - sample_details = [] - if dataset_info.molecule_name: - molecule_str = f"Molecule: {dataset_info.molecule_name}" - if dataset_info.molecule_type: - molecule_str += f" ({dataset_info.molecule_type})" - sample_details.append(molecule_str) - elif dataset_info.sample_name: - sample_details.append(f"Sample: {dataset_info.sample_name}") - - if dataset_info.sample_description: - sample_details.append( - f"Description: {dataset_info.sample_description}" - ) - - if dataset_info.sequence: - sample_details.append(f"Sequence: {dataset_info.sequence}") - - if dataset_info.uniprot_code: - sample_details.append(f"UniProt: {dataset_info.uniprot_code}") - - oligomerization = ( - dataset_info.oligomerization or dataset_info.oligomeric_state - ) - if oligomerization: - sample_details.append(f"Oligomerization: {oligomerization}") - - if dataset_info.number_of_molecules: - sample_details.append( - f"Number of molecules: {dataset_info.number_of_molecules}" - ) - - if dataset_info.source_organism: - sample_details.append( - f"Source organism: {dataset_info.source_organism}" - ) - - # Set temperature - if dataset_info.temperature is not None: - data.sample.temperature = dataset_info.temperature - if dataset_info.temperature_unit: - data.sample.temperature_unit = dataset_info.temperature_unit - logger.debug(f"Set sample.temperature to: {dataset_info.temperature} {dataset_info.temperature_unit}") - temp_unit = dataset_info.temperature_unit or "" - temp_str = f"Temperature: {dataset_info.temperature}" - if temp_unit: - temp_str += f" {temp_unit}" - sample_details.append(temp_str) - - # Store concentration in sample details - if dataset_info.concentration is not None: - conc_str = f"Concentration: {dataset_info.concentration}" - if dataset_info.concentration_unit: - conc_str += f" {dataset_info.concentration_unit}" - sample_details.append(conc_str) - - # Add buffer info to sample details - if dataset_info.buffer_description: - buffer_str = f"Buffer: {dataset_info.buffer_description}" - if dataset_info.ph is not None: - buffer_str += f" (pH {dataset_info.ph})" - sample_details.append(buffer_str) - - for detail in sample_details: - if detail and detail not in data.sample.details: - data.sample.details.append(detail) - logger.debug(f"Added to sample.details: {detail}") - - # Log sample info for debugging - logger.debug(f"Sample populated for {data_id}: name={getattr(data.sample, 'name', None)}, " - f"temperature={getattr(data.sample, 'temperature', None)}, " - f"details={getattr(data.sample, 'details', [])}") - - # Populate source information - if data.source is None: - data.source = Source() - - if dataset_info.wavelength is not None: - data.source.wavelength = dataset_info.wavelength - data.source.wavelength_unit = dataset_info.wavelength_unit - - # Store additional SASBDB metadata in meta_data dictionary - if not hasattr(data, 'meta_data') or data.meta_data is None: - data.meta_data = {} - - # SASBDB-specific metadata - data.meta_data['SASBDB_code'] = dataset_info.code or dataset_info.entry_id - - if dataset_info.rg is not None: - data.meta_data['SASBDB_Rg'] = dataset_info.rg - if dataset_info.rg_error is not None: - data.meta_data['SASBDB_Rg_error'] = dataset_info.rg_error - - if dataset_info.i0 is not None: - data.meta_data['SASBDB_I0'] = dataset_info.i0 - if dataset_info.i0_error is not None: - data.meta_data['SASBDB_I0_error'] = dataset_info.i0_error - - if dataset_info.dmax is not None: - data.meta_data['SASBDB_Dmax'] = dataset_info.dmax - - if dataset_info.molecular_weight is not None: - data.meta_data['SASBDB_MW'] = dataset_info.molecular_weight - if dataset_info.molecular_weight_method: - data.meta_data['SASBDB_MW_method'] = dataset_info.molecular_weight_method - - if dataset_info.porod_volume is not None: - data.meta_data['SASBDB_Porod_volume'] = dataset_info.porod_volume - - if dataset_info.molecule_name: - data.meta_data['SASBDB_molecule'] = dataset_info.molecule_name - - if dataset_info.molecule_type: - data.meta_data['SASBDB_molecule_type'] = dataset_info.molecule_type - - if dataset_info.oligomeric_state: - data.meta_data['SASBDB_oligomeric_state'] = dataset_info.oligomeric_state - - if dataset_info.publication_doi: - data.meta_data['SASBDB_DOI'] = dataset_info.publication_doi - - if dataset_info.publication_pmid: - data.meta_data['SASBDB_PMID'] = dataset_info.publication_pmid - - if dataset_info.authors: - data.meta_data['SASBDB_authors'] = ', '.join(dataset_info.authors) - - logger.debug(f"Populated SASBDB metadata for data: {data_id}") - - except Exception as e: - logger.warning(f"Error populating metadata for data {data_id}: {e}") + load_downloaded_dataset( + self.filesWidget, + self._workspace, + dialog.getDownloadedFilepath(), + dialog.getDatasetInfo(), + ) def actionOpen_Project(self): """ diff --git a/src/sas/qtgui/Utilities/SASBDB/SASBDBDownloadDialog.py b/src/sas/qtgui/Utilities/SASBDB/SASBDBDownloadDialog.py index 87250f6dde..3fe1f90db9 100644 --- a/src/sas/qtgui/Utilities/SASBDB/SASBDBDownloadDialog.py +++ b/src/sas/qtgui/Utilities/SASBDB/SASBDBDownloadDialog.py @@ -1,8 +1,5 @@ """ SASBDB Dataset Download Dialog. - -Provides a dialog interface for downloading datasets from SASBDB -and loading them into SasView. """ import logging @@ -12,211 +9,82 @@ from PySide6 import QtWidgets from .sasbdb_api import SASBDBDatasetInfo, downloadDataset +from .sasbdb_loader import metadata_summary from .UI.SASBDBDownloadDialogUI import Ui_SASBDBDownloadDialogUI logger = logging.getLogger(__name__) class SASBDBDownloadDialog(QtWidgets.QDialog, Ui_SASBDBDownloadDialogUI): - """ - Dialog for downloading datasets from SASBDB. - - Allows users to enter a SASBDB dataset identifier, downloads - the dataset, and loads it into SasView. - """ + """Dialog for downloading datasets from SASBDB.""" def __init__(self, parent=None): - """ - Initialize the download dialog. - - :param parent: Parent widget - """ super().__init__(parent) self.setupUi(self) - - # Store downloaded file path and metadata self.downloaded_filepath: str | None = None self.dataset_info: SASBDBDatasetInfo | None = None - # Connect signals self.cmdDownload.clicked.connect(self.onDownload) self.cmdCancel.clicked.connect(self.reject) self.cmdHelp.clicked.connect(self.onHelp) self.txtDatasetId.returnPressed.connect(self.onDownload) - - # Set focus on dataset ID input self.txtDatasetId.setFocus() def onDownload(self): - """ - Handle download button click. - - Validates the dataset ID, downloads the dataset, - and loads it into SasView. - """ dataset_id = self.txtDatasetId.text().strip() - if not dataset_id: self._showError("Please enter a dataset identifier.") return - # Disable download button during download - self.cmdDownload.setEnabled(False) - self.cmdCancel.setEnabled(False) - self.progressBar.setVisible(True) - self.progressBar.setRange(0, 0) # Indeterminate progress + self._set_busy(True) self.lblStatus.setText("Downloading dataset...") - QtWidgets.QApplication.processEvents() try: - # Download the dataset - output_dir = tempfile.gettempdir() - filepath, dataset_info = downloadDataset(dataset_id, output_dir) - - # Store the metadata + filepath, dataset_info = downloadDataset(dataset_id, tempfile.gettempdir()) self.dataset_info = dataset_info if filepath and os.path.exists(filepath): self.downloaded_filepath = filepath - - # Build success message with metadata summary - success_msg = f"Successfully downloaded dataset {dataset_id}" - if dataset_info: - details = self._formatMetadataSummary(dataset_info) - if details: - success_msg += f"\n{details}" - - self.lblStatus.setText(success_msg) - self.progressBar.setVisible(False) - - # Log detailed metadata - if dataset_info: - self._logMetadata(dataset_info) - - self.accept() # Close dialog and return success + summary = metadata_summary(dataset_info) if dataset_info else "" + status = f"Successfully downloaded dataset {dataset_id}" + if summary: + status += f"\n{summary}" + self.lblStatus.setText(status) + self.accept() else: - self._showError(f"Failed to download dataset {dataset_id}.\n" - "Please check the dataset identifier and try again.") - self.progressBar.setVisible(False) - self.cmdDownload.setEnabled(True) - self.cmdCancel.setEnabled(True) - + self._showError( + f"Failed to download dataset {dataset_id}.\n" + "Please check the dataset identifier and try again." + ) except Exception as e: logger.error(f"Error downloading dataset {dataset_id}: {e}", exc_info=True) - self._showError(f"Error downloading dataset:\n{str(e)}") - self.progressBar.setVisible(False) - self.cmdDownload.setEnabled(True) - self.cmdCancel.setEnabled(True) - - def _formatMetadataSummary(self, info: SASBDBDatasetInfo) -> str: - """ - Format a brief summary of the dataset metadata. - - :param info: Parsed dataset info - :return: Formatted summary string - """ - parts = [] - - if info.title: - parts.append(f"Title: {info.title}") - if info.sample_name: - parts.append(f"Sample: {info.sample_name}") - if info.rg is not None: - rg_str = f"Rg: {info.rg:.2f}" - if info.rg_error: - rg_str += f" ± {info.rg_error:.2f}" - rg_str += " Å" - parts.append(rg_str) - if info.molecular_weight is not None: - parts.append(f"MW: {info.molecular_weight:.1f} kDa") - - return "\n".join(parts) - - def _logMetadata(self, info: SASBDBDatasetInfo): - """ - Log detailed metadata information. - - :param info: Parsed dataset info - """ - logger.info(f"SASBDB Dataset: {info.code or info.entry_id}") - if info.title: - logger.info(f" Title: {info.title}") - if info.sample_name: - logger.info(f" Sample: {info.sample_name}") - if info.molecule_name: - logger.info(f" Molecule: {info.molecule_name}") - if info.concentration: - logger.info(f" Concentration: {info.concentration} {info.concentration_unit}") - if info.buffer_description: - logger.info(f" Buffer: {info.buffer_description}") - if info.instrument: - logger.info(f" Instrument: {info.instrument}") - if info.wavelength: - logger.info(f" Wavelength: {info.wavelength} {info.wavelength_unit}") - if info.temperature: - logger.info(f" Temperature: {info.temperature} {info.temperature_unit}") - if info.rg is not None: - logger.info(f" Rg: {info.rg} ± {info.rg_error or 0} Å") - if info.i0 is not None: - logger.info(f" I(0): {info.i0} ± {info.i0_error or 0}") - if info.dmax is not None: - logger.info(f" Dmax: {info.dmax} Å") - if info.molecular_weight is not None: - logger.info(f" MW: {info.molecular_weight} kDa") - if info.publication_doi: - logger.info(f" DOI: {info.publication_doi}") + self._showError(f"Error downloading dataset:\n{e}") + finally: + self._set_busy(False) + + def _set_busy(self, busy: bool): + self.cmdDownload.setEnabled(not busy) + self.cmdCancel.setEnabled(not busy) + self.progressBar.setVisible(busy) + if busy: + self.progressBar.setRange(0, 0) + QtWidgets.QApplication.processEvents() def _showError(self, message: str): - """ - Display an error message to the user. - - :param message: Error message to display - """ self.lblStatus.setText(f"{message}") QtWidgets.QMessageBox.warning(self, "Download Error", message) def getDownloadedFilepath(self) -> str | None: - """ - Get the path to the downloaded file. - - :return: Path to downloaded file, or None if download failed - """ return self.downloaded_filepath def getDatasetInfo(self) -> SASBDBDatasetInfo | None: - """ - Get the parsed dataset metadata. - - :return: SASBDBDatasetInfo object, or None if metadata not available - """ return self.dataset_info def onHelp(self): - """ - Show the SASBDB download help documentation. - """ from sas.qtgui.Utilities import GuiUtils help_location = "user/qtgui/Utilities/SASBDB/sasbdb_download_help.html" - try: - # Try to get guiManager from parent workspace - parent = self.parent() - if parent: - # Check if parent has guiManager attribute - if hasattr(parent, 'guiManager') and hasattr(parent.guiManager, 'showHelp'): - parent.guiManager.showHelp(help_location) - return - # Check if parent itself has showHelp - elif hasattr(parent, 'showHelp'): - parent.showHelp(help_location) - return - - # Fallback to GuiUtils + parent = self.parent() + if parent and hasattr(parent, "guiManager"): + parent.guiManager.showHelp(help_location) + else: GuiUtils.showHelp(help_location) - except Exception as e: - logger.warning(f"Could not display help: {e}") - # Final fallback to GuiUtils - try: - GuiUtils.showHelp(help_location) - except Exception as e2: - logger.error(f"Failed to display help: {e2}") - diff --git a/src/sas/qtgui/Utilities/SASBDB/__init__.py b/src/sas/qtgui/Utilities/SASBDB/__init__.py index 5ce0e3c888..278e774c32 100644 --- a/src/sas/qtgui/Utilities/SASBDB/__init__.py +++ b/src/sas/qtgui/Utilities/SASBDB/__init__.py @@ -5,3 +5,5 @@ including downloading datasets and exporting data to SASBDB format. """ +from .sasbdb_loader import load_downloaded_dataset, metadata_summary, populate_metadata + diff --git a/src/sas/qtgui/Utilities/SASBDB/sasbdb_loader.py b/src/sas/qtgui/Utilities/SASBDB/sasbdb_loader.py new file mode 100644 index 0000000000..893bf8e412 --- /dev/null +++ b/src/sas/qtgui/Utilities/SASBDB/sasbdb_loader.py @@ -0,0 +1,185 @@ +""" +Load SASBDB datasets into SasView. +""" + +import logging +import os + +from PySide6.QtWidgets import QMessageBox + +from sasdata.dataloader.data_info import Sample, Source + +from .sasbdb_api import SASBDBDatasetInfo + +logger = logging.getLogger(__name__) + +_META_FIELDS = ( + ("SASBDB_Rg", "rg", ("SASBDB_Rg_error", "rg_error")), + ("SASBDB_I0", "i0", ("SASBDB_I0_error", "i0_error")), + ("SASBDB_Dmax", "dmax", None), + ("SASBDB_MW", "molecular_weight", ("SASBDB_MW_method", "molecular_weight_method")), + ("SASBDB_Porod_volume", "porod_volume", None), + ("SASBDB_molecule", "molecule_name", None), + ("SASBDB_molecule_type", "molecule_type", None), + ("SASBDB_oligomeric_state", "oligomeric_state", None), + ("SASBDB_DOI", "publication_doi", None), + ("SASBDB_PMID", "publication_pmid", None), +) + + +def metadata_summary(info: SASBDBDatasetInfo) -> str: + """Format dataset metadata for display or logging.""" + lines = [] + if info.title: + lines.append(f"Title: {info.title}") + if info.sample_name: + lines.append(f"Sample: {info.sample_name}") + if info.molecule_name: + lines.append(f"Molecule: {info.molecule_name}") + if info.concentration: + lines.append(f"Concentration: {info.concentration} {info.concentration_unit}") + if info.buffer_description: + lines.append(f"Buffer: {info.buffer_description}") + if info.instrument: + lines.append(f"Instrument: {info.instrument}") + if info.wavelength: + lines.append(f"Wavelength: {info.wavelength} {info.wavelength_unit}") + if info.temperature: + lines.append(f"Temperature: {info.temperature} {info.temperature_unit}") + if info.rg is not None: + rg = f"Rg: {info.rg:.2f}" + if info.rg_error: + rg += f" ± {info.rg_error:.2f}" + lines.append(f"{rg} Å") + if info.i0 is not None: + i0 = f"I(0): {info.i0}" + if info.i0_error: + i0 += f" ± {info.i0_error}" + lines.append(i0) + if info.dmax is not None: + lines.append(f"Dmax: {info.dmax} Å") + if info.molecular_weight is not None: + lines.append(f"MW: {info.molecular_weight:.1f} kDa") + if info.publication_doi: + lines.append(f"DOI: {info.publication_doi}") + return "\n".join(lines) + + +def load_downloaded_dataset(files_widget, parent, filepath: str | None, + dataset_info: SASBDBDatasetInfo | None) -> None: + """Load a downloaded SASBDB file and apply metadata.""" + if not filepath or not os.path.exists(filepath): + return + + try: + loaded_data, _ = files_widget.readData([filepath]) + if dataset_info and loaded_data: + populate_metadata(loaded_data, dataset_info) + + entry_id = dataset_info.code or dataset_info.entry_id if dataset_info else None + logger.info( + "Successfully loaded SASBDB dataset %s from %s", + entry_id or "unknown", + filepath, + ) + if dataset_info: + for line in metadata_summary(dataset_info).splitlines(): + logger.info(" %s", line) + except Exception as e: + logger.error(f"Error loading downloaded SASBDB dataset: {e}", exc_info=True) + QMessageBox.warning(parent, "Load Error", f"Failed to load downloaded dataset:\n{e}") + + +def populate_metadata(loaded_data: dict, dataset_info: SASBDBDatasetInfo) -> None: + """Populate SASBDB metadata into loaded data objects.""" + for data_id, data in loaded_data.items(): + try: + _apply_metadata(data, dataset_info) + except Exception as e: + logger.warning(f"Error populating metadata for data {data_id}: {e}") + + +def _apply_metadata(data, info: SASBDBDatasetInfo) -> None: + if info.title and not data.title: + data.title = info.title + if info.instrument and not data.instrument: + data.instrument = info.instrument + if info.code: + data.run = data.run or [] + if info.code not in data.run: + data.run.append(info.code) + + if data.sample is None: + data.sample = Sample() + if not getattr(data.sample, "details", None): + data.sample.details = [] + + sample_id = info.molecule_short_name or info.sample_name or info.code + if sample_id: + data.sample.ID = sample_id + + details = [] + if info.molecule_name: + detail = f"Molecule: {info.molecule_name}" + if info.molecule_type: + detail += f" ({info.molecule_type})" + details.append(detail) + elif info.sample_name: + details.append(f"Sample: {info.sample_name}") + if info.sample_description: + details.append(f"Description: {info.sample_description}") + if info.sequence: + details.append(f"Sequence: {info.sequence}") + if info.uniprot_code: + details.append(f"UniProt: {info.uniprot_code}") + oligomerization = info.oligomerization or info.oligomeric_state + if oligomerization: + details.append(f"Oligomerization: {oligomerization}") + if info.number_of_molecules: + details.append(f"Number of molecules: {info.number_of_molecules}") + if info.source_organism: + details.append(f"Source organism: {info.source_organism}") + if info.temperature is not None: + data.sample.temperature = info.temperature + if info.temperature_unit: + data.sample.temperature_unit = info.temperature_unit + temp = f"Temperature: {info.temperature}" + if info.temperature_unit: + temp += f" {info.temperature_unit}" + details.append(temp) + if info.concentration is not None: + conc = f"Concentration: {info.concentration}" + if info.concentration_unit: + conc += f" {info.concentration_unit}" + details.append(conc) + if info.buffer_description: + buffer = f"Buffer: {info.buffer_description}" + if info.ph is not None: + buffer += f" (pH {info.ph})" + details.append(buffer) + + for detail in details: + if detail not in data.sample.details: + data.sample.details.append(detail) + + if data.source is None: + data.source = Source() + if info.wavelength is not None: + data.source.wavelength = info.wavelength + data.source.wavelength_unit = info.wavelength_unit + + if not getattr(data, "meta_data", None): + data.meta_data = {} + data.meta_data["SASBDB_code"] = info.code or info.entry_id + for key, attr, extra in _META_FIELDS: + value = getattr(info, attr, None) + if value is None: + continue + data.meta_data[key] = value + if extra: + extra_key, extra_attr = extra + extra_value = getattr(info, extra_attr, None) + if extra_value is not None: + data.meta_data[extra_key] = extra_value + if info.authors: + data.meta_data["SASBDB_authors"] = ", ".join(info.authors) From 2d13e53beb8e1675e3b9a0f0e9834d2361d0fd0e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci-lite[bot]" <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Date: Mon, 13 Jul 2026 13:27:55 +0000 Subject: [PATCH 07/12] [pre-commit.ci lite] apply automatic fixes for ruff linting errors --- src/sas/qtgui/MainWindow/GuiManager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sas/qtgui/MainWindow/GuiManager.py b/src/sas/qtgui/MainWindow/GuiManager.py index 37d74c865c..f0502df3e2 100644 --- a/src/sas/qtgui/MainWindow/GuiManager.py +++ b/src/sas/qtgui/MainWindow/GuiManager.py @@ -772,8 +772,8 @@ def actionLoad_SASBDB(self): Opens a dialog to download and load a dataset from SASBDB. """ - from sas.qtgui.Utilities.SASBDB.SASBDBDownloadDialog import SASBDBDownloadDialog from sas.qtgui.Utilities.SASBDB.sasbdb_loader import load_downloaded_dataset + from sas.qtgui.Utilities.SASBDB.SASBDBDownloadDialog import SASBDBDownloadDialog dialog = SASBDBDownloadDialog(parent=self._workspace) if dialog.exec(): From 8556ec7e6dd8f59228ec2856b61880839c5fed9b Mon Sep 17 00:00:00 2001 From: Wojtek Potrzebowski Date: Mon, 13 Jul 2026 15:41:20 +0200 Subject: [PATCH 08/12] Fixing ruff issues --- src/sas/qtgui/MainWindow/GuiManager.py | 2 +- src/sas/qtgui/Utilities/SASBDB/__init__.py | 2 -- src/sas/qtgui/Utilities/SASBDB/sasbdb_api.py | 34 ++++++++++---------- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/src/sas/qtgui/MainWindow/GuiManager.py b/src/sas/qtgui/MainWindow/GuiManager.py index 37d74c865c..f0502df3e2 100644 --- a/src/sas/qtgui/MainWindow/GuiManager.py +++ b/src/sas/qtgui/MainWindow/GuiManager.py @@ -772,8 +772,8 @@ def actionLoad_SASBDB(self): Opens a dialog to download and load a dataset from SASBDB. """ - from sas.qtgui.Utilities.SASBDB.SASBDBDownloadDialog import SASBDBDownloadDialog from sas.qtgui.Utilities.SASBDB.sasbdb_loader import load_downloaded_dataset + from sas.qtgui.Utilities.SASBDB.SASBDBDownloadDialog import SASBDBDownloadDialog dialog = SASBDBDownloadDialog(parent=self._workspace) if dialog.exec(): diff --git a/src/sas/qtgui/Utilities/SASBDB/__init__.py b/src/sas/qtgui/Utilities/SASBDB/__init__.py index 278e774c32..5ce0e3c888 100644 --- a/src/sas/qtgui/Utilities/SASBDB/__init__.py +++ b/src/sas/qtgui/Utilities/SASBDB/__init__.py @@ -5,5 +5,3 @@ including downloading datasets and exporting data to SASBDB format. """ -from .sasbdb_loader import load_downloaded_dataset, metadata_summary, populate_metadata - diff --git a/src/sas/qtgui/Utilities/SASBDB/sasbdb_api.py b/src/sas/qtgui/Utilities/SASBDB/sasbdb_api.py index 7c4b0f5b79..b474fae86e 100644 --- a/src/sas/qtgui/Utilities/SASBDB/sasbdb_api.py +++ b/src/sas/qtgui/Utilities/SASBDB/sasbdb_api.py @@ -22,7 +22,7 @@ class SASBDBDatasetInfo: """ Parsed metadata from a SASBDB dataset entry. - + Contains key information extracted from the SASBDB REST API response. """ # Identifiers @@ -90,7 +90,7 @@ class SASBDBDatasetInfo: def parseMetadata(metadata: dict) -> SASBDBDatasetInfo: """ Parse SASBDB API response into a structured SASBDBDatasetInfo object. - + :param metadata: Raw JSON dictionary from SASBDB API :return: Parsed SASBDBDatasetInfo object """ @@ -220,7 +220,7 @@ def _get_str(data: dict, *keys: str) -> str: """ Get string value from dictionary, trying multiple possible keys. Also searches in nested dictionaries (sample, molecule, experiment, etc.). - + :param data: Dictionary to search :param keys: Possible key names to try :return: String value or empty string if not found @@ -348,7 +348,7 @@ def _get_float(data: dict, *keys: str) -> float | None: """ Get float value from dictionary, trying multiple possible keys. Also searches in nested dictionaries (sample, molecule, experiment, etc.). - + :param data: Dictionary to search :param keys: Possible key names to try :return: Float value or None if not found @@ -378,7 +378,7 @@ def _get_float(data: dict, *keys: str) -> float | None: def getDatasetMetadata(dataset_id: str) -> dict | None: """ Fetch dataset metadata from SASBDB API. - + :param dataset_id: SASBDB dataset identifier (e.g., "SASDN24" - full 7-character ID) :return: Dictionary containing dataset metadata, or None if error """ @@ -419,10 +419,10 @@ def getDatasetMetadata(dataset_id: str) -> dict | None: def getDataFileUrl(metadata: dict) -> str | None: """ Extract data file URL from dataset metadata. - + Looks for common field names in the metadata JSON that might contain the data file URL or path. Also checks for SASBDB-specific fields. - + :param metadata: Dictionary containing dataset metadata :return: URL string for the data file, or None if not found """ @@ -462,9 +462,9 @@ def getDataFileUrl(metadata: dict) -> str | None: ] # Check top-level fields - for field in possible_fields: - if field in metadata: - value = metadata[field] + for field_name in possible_fields: + if field_name in metadata: + value = metadata[field_name] if isinstance(value, str) and (value.startswith('http') or value.startswith('/')): # If relative URL, make it absolute if value.startswith('/'): @@ -517,7 +517,7 @@ def getDataFileUrl(metadata: dict) -> str | None: def downloadDataFile(url: str, filepath: str) -> bool: """ Download a data file from the given URL to the specified filepath. - + :param url: URL of the data file to download :param filepath: Local filepath where the file should be saved :return: True if download successful, False otherwise @@ -550,14 +550,14 @@ def downloadDataFile(url: str, filepath: str) -> bool: def _normalizeDatasetId(dataset_id: str) -> str | None: """ Normalize a SASBDB dataset identifier. - + SASBDB identifiers are 7 characters long, typically in format: - "SASDN24" (prefix + number) - "SASDB1234" (if 4-digit number) - etc. - + The API requires the full 7-character identifier. - + :param dataset_id: Input dataset identifier :return: Normalized identifier (uppercase, stripped), or None if invalid """ @@ -580,13 +580,13 @@ def _normalizeDatasetId(dataset_id: str) -> str | None: def downloadDataset(dataset_id: str, output_dir: str | None = None) -> tuple[str | None, SASBDBDatasetInfo | None]: """ Download a complete dataset from SASBDB. - + This is a convenience function that: 1. Fetches metadata 2. Extracts data file URL 3. Downloads the data file 4. Returns the local filepath and parsed metadata - + :param dataset_id: SASBDB dataset identifier :param output_dir: Directory to save the file (defaults to temp directory) :return: Tuple of (path to downloaded file, parsed metadata info) or (None, None) if error @@ -630,7 +630,7 @@ def downloadDataset(dataset_id: str, output_dir: str | None = None) -> tuple[str def _guessFileExtension(url: str, metadata: dict) -> str: """ Guess the file extension from URL or metadata. - + :param url: Data file URL :param metadata: Dataset metadata :return: File extension (e.g., ".dat", ".txt", ".csv") From 1b0f127dfb9f47ca863999ab8095551a8c6a5f37 Mon Sep 17 00:00:00 2001 From: Wojtek Potrzebowski Date: Mon, 13 Jul 2026 16:13:26 +0200 Subject: [PATCH 09/12] Fixing bugs after automatic code review --- src/sas/qtgui/Utilities/GuiUtils.py | 167 ++++++------------ .../Utilities/SASBDB/SASBDBDownloadDialog.py | 22 +-- .../SASBDB/UI/SASBDBDownloadDialogUI.ui | 2 +- src/sas/qtgui/Utilities/SASBDB/sasbdb_api.py | 52 +++--- .../qtgui/Utilities/SASBDB/sasbdb_loader.py | 16 +- 5 files changed, 100 insertions(+), 159 deletions(-) diff --git a/src/sas/qtgui/Utilities/GuiUtils.py b/src/sas/qtgui/Utilities/GuiUtils.py index 4aecb69ded..1afa460899 100644 --- a/src/sas/qtgui/Utilities/GuiUtils.py +++ b/src/sas/qtgui/Utilities/GuiUtils.py @@ -1,6 +1,7 @@ """ Global defaults and various utility functions usable by the general GUI """ +import ast import json import logging import numbers @@ -532,66 +533,13 @@ def openLink(url): raise AttributeError(msg) -def _formatDictValue(value, max_depth=2) -> str: - """ - Format a dictionary value for display, extracting useful information. - - :param value: Dictionary or other value to format - :param max_depth: Maximum nesting depth to process - :return: Formatted string representation - """ - if not isinstance(value, dict): - return str(value) - - if max_depth <= 0: - return "{...}" - - # Extract common useful fields from dictionaries - parts = [] - - # Instrument/source dictionaries - if 'name' in value: - parts.append(value['name']) - if 'beamline_name' in value: - parts.append(f"Beamline: {value['beamline_name']}") - if 'type' in value: - parts.append(f"Type: {value['type']}") - if 'city' in value and 'country' in value: - parts.append(f"{value['city']}, {value['country']}") - elif 'city' in value: - parts.append(value['city']) - elif 'country' in value: - parts.append(value['country']) - - # Detector dictionaries - if 'detector' in str(value).lower() or 'type' in value: - if 'name' in value: - parts.append(f"Detector: {value['name']}") - if 'type' in value and 'name' not in value: - parts.append(f"Detector: {value['type']}") - if 'resolution' in value: - parts.append(f"Resolution: {value['resolution']}") - - if parts: - return " | ".join(parts) - - # Fallback: show key-value pairs for shallow dicts - if len(value) <= 3: - return ", ".join(f"{k}: {v}" for k, v in value.items() if not isinstance(v, dict)) - - return "{...}" - - def _formatDictLine(line: str) -> str: """ Extract and format dictionary information from a line. - + :param line: Line that may contain a dictionary representation :return: Formatted line with dictionary info extracted, or original line if no dict found """ - import ast - import re - stripped = line.strip() # Check if line contains a dictionary @@ -676,13 +624,60 @@ def _formatDictLine(line: str) -> str: return line +def _isSASBDBData(data) -> bool: + meta = getattr(data, 'meta_data', None) or {} + return 'SASBDB_code' in meta + + +def _cleanDataSummaryText(text: str, data) -> str: + """Reformat dict lines in data summaries for SASBDB datasets only.""" + if not _isSASBDBData(data): + return text + + cleaned_lines = [] + for line in text.split('\n'): + stripped = line.strip() + if ("{" in stripped and "'" in stripped and ":" in stripped and + (stripped.startswith("{'") or + re.search(r':\s*\{', stripped) or + re.search(r"\{'[^']+':", stripped))): + if stripped.count('{') > 0 and stripped.count('}') > 0: + cleaned_lines.append(_formatDictLine(line)) + else: + cleaned_lines.append(line) + else: + cleaned_lines.append(line) + return '\n'.join(cleaned_lines) + + +def _truncateAuthors(authors: str, max_len: int = 50) -> str: + """Truncate an author list at complete names, appending 'et al.' if needed.""" + if len(authors) <= max_len: + return authors + + names = [name.strip() for name in authors.split(',') if name.strip()] + if not names: + return authors + + shown = names[0] + for name in names[1:]: + candidate = f"{shown}, {name}" + if len(candidate) + len(' et al.') > max_len: + return f"{shown} et al." + shown = candidate + + if len(shown) > max_len: + return f"{shown[:max_len - len(' et al.')].rstrip(', ')} et al." + return shown + + def _formatSASBDBMetadata(data) -> str: """ Format SASBDB metadata from data object for display. - + Displays instrument, sample, source info and meta_data dictionary for datasets loaded from SASBDB in a clean, concise format. - + :param data: Data1D or Data2D object :return: Formatted string with SASBDB metadata, or empty string if none """ @@ -822,11 +817,7 @@ def _formatSASBDBMetadata(data) -> str: # Publication info (compact, only if available) pub_parts = [] if meta.get('SASBDB_authors'): - # Truncate authors if too long - authors = meta['SASBDB_authors'] - if len(authors) > 50: - authors = authors[:47] + "..." - pub_parts.append(f"Authors: {authors}") + pub_parts.append(f"Authors: {_truncateAuthors(meta['SASBDB_authors'])}") if meta.get('SASBDB_DOI'): pub_parts.append(f"DOI: {meta['SASBDB_DOI']}") if meta.get('SASBDB_PMID'): @@ -857,32 +848,7 @@ def retrieveData1d(data): #logger.error(msg) raise ValueError(msg) - text = data.__str__() - - # Format dictionary representations instead of removing them - import re - lines = text.split('\n') - cleaned_lines = [] - for line in lines: - stripped = line.strip() - # Check if line contains a dictionary representation - if ("{" in stripped and "'" in stripped and ":" in stripped and - (stripped.startswith("{'") or - re.search(r':\s*\{', stripped) or # Matches "Instrument: {" - re.search(r"\{'[^']+':", stripped))): # Matches "{'key':" - # Count opening and closing braces to detect dict structures - open_braces = stripped.count('{') - close_braces = stripped.count('}') - # If it looks like a dictionary, format it instead of skipping - if open_braces > 0 and close_braces > 0: - formatted_line = _formatDictLine(line) - if formatted_line != line: # Only add if it was successfully formatted - cleaned_lines.append(formatted_line) - else: - cleaned_lines.append(line) - else: - cleaned_lines.append(line) - text = '\n'.join(cleaned_lines) + text = _cleanDataSummaryText(data.__str__(), data) # Add SASBDB metadata if present text += _formatSASBDBMetadata(data) @@ -930,32 +896,7 @@ def retrieveData2d(data): msg = "Incorrect type passed to retrieveData2d" raise AttributeError(msg) - text = data.__str__() - - # Format dictionary representations instead of removing them - import re - lines = text.split('\n') - cleaned_lines = [] - for line in lines: - stripped = line.strip() - # Check if line contains a dictionary representation - if ("{" in stripped and "'" in stripped and ":" in stripped and - (stripped.startswith("{'") or - re.search(r':\s*\{', stripped) or # Matches "Instrument: {" - re.search(r"\{'[^']+':", stripped))): # Matches "{'key':" - # Count opening and closing braces to detect dict structures - open_braces = stripped.count('{') - close_braces = stripped.count('}') - # If it looks like a dictionary, format it instead of skipping - if open_braces > 0 and close_braces > 0: - formatted_line = _formatDictLine(line) - if formatted_line != line: # Only add if it was successfully formatted - cleaned_lines.append(formatted_line) - else: - cleaned_lines.append(line) - else: - cleaned_lines.append(line) - text = '\n'.join(cleaned_lines) + text = _cleanDataSummaryText(data.__str__(), data) # Add SASBDB metadata if present text += _formatSASBDBMetadata(data) diff --git a/src/sas/qtgui/Utilities/SASBDB/SASBDBDownloadDialog.py b/src/sas/qtgui/Utilities/SASBDB/SASBDBDownloadDialog.py index 3fe1f90db9..6e33e65a5b 100644 --- a/src/sas/qtgui/Utilities/SASBDB/SASBDBDownloadDialog.py +++ b/src/sas/qtgui/Utilities/SASBDB/SASBDBDownloadDialog.py @@ -8,7 +8,7 @@ from PySide6 import QtWidgets -from .sasbdb_api import SASBDBDatasetInfo, downloadDataset +from .sasbdb_api import SASBDBDatasetInfo, downloadDataset, validateDatasetId from .sasbdb_loader import metadata_summary from .UI.SASBDBDownloadDialogUI import Ui_SASBDBDownloadDialogUI @@ -36,28 +36,33 @@ def onDownload(self): self._showError("Please enter a dataset identifier.") return + normalized_id, validation_error = validateDatasetId(dataset_id) + if validation_error: + self._showError(validation_error) + return + self._set_busy(True) self.lblStatus.setText("Downloading dataset...") try: - filepath, dataset_info = downloadDataset(dataset_id, tempfile.gettempdir()) + filepath, dataset_info = downloadDataset(normalized_id, tempfile.gettempdir()) self.dataset_info = dataset_info if filepath and os.path.exists(filepath): self.downloaded_filepath = filepath summary = metadata_summary(dataset_info) if dataset_info else "" - status = f"Successfully downloaded dataset {dataset_id}" + status = f"Successfully downloaded dataset {normalized_id}" if summary: status += f"\n{summary}" self.lblStatus.setText(status) self.accept() else: self._showError( - f"Failed to download dataset {dataset_id}.\n" + f"Failed to download dataset {normalized_id}.\n" "Please check the dataset identifier and try again." ) except Exception as e: - logger.error(f"Error downloading dataset {dataset_id}: {e}", exc_info=True) + logger.error(f"Error downloading dataset {normalized_id}: {e}", exc_info=True) self._showError(f"Error downloading dataset:\n{e}") finally: self._set_busy(False) @@ -82,9 +87,4 @@ def getDatasetInfo(self) -> SASBDBDatasetInfo | None: def onHelp(self): from sas.qtgui.Utilities import GuiUtils - help_location = "user/qtgui/Utilities/SASBDB/sasbdb_download_help.html" - parent = self.parent() - if parent and hasattr(parent, "guiManager"): - parent.guiManager.showHelp(help_location) - else: - GuiUtils.showHelp(help_location) + GuiUtils.showHelp("user/qtgui/Utilities/SASBDB/sasbdb_download_help.html") diff --git a/src/sas/qtgui/Utilities/SASBDB/UI/SASBDBDownloadDialogUI.ui b/src/sas/qtgui/Utilities/SASBDB/UI/SASBDBDownloadDialogUI.ui index 8dd7f45f81..9a2ad0a09e 100644 --- a/src/sas/qtgui/Utilities/SASBDB/UI/SASBDBDownloadDialogUI.ui +++ b/src/sas/qtgui/Utilities/SASBDB/UI/SASBDBDownloadDialogUI.ui @@ -24,7 +24,7 @@ - e.g., 1234 or SASDB1234 + e.g., SASDN24 diff --git a/src/sas/qtgui/Utilities/SASBDB/sasbdb_api.py b/src/sas/qtgui/Utilities/SASBDB/sasbdb_api.py index b474fae86e..283ca249c8 100644 --- a/src/sas/qtgui/Utilities/SASBDB/sasbdb_api.py +++ b/src/sas/qtgui/Utilities/SASBDB/sasbdb_api.py @@ -7,6 +7,7 @@ import logging import os +import re import tempfile from dataclasses import dataclass, field @@ -16,6 +17,8 @@ # Base API URL - SASBDB REST API SASBDB_API_BASE = "https://www.sasbdb.org/rest-api" +_SASBDB_ID_PATTERN = re.compile(r'^SAS[A-Z]{2}\d+$') +INVALID_DATASET_ID_MESSAGE = "Enter the full 7-character SASBDB code (e.g. SASDN24)." @dataclass @@ -496,19 +499,6 @@ def getDataFileUrl(metadata: dict) -> str | None: if result: return result - # If we have an entry ID, try constructing a download URL - # Format might be: /rest-api/entry/{id}/download/ or similar - entry_id = metadata.get('entry_id') or metadata.get('id') or metadata.get('sasbdb_id') - if entry_id: - # Try common download endpoint patterns - download_endpoints = [ - f"{SASBDB_API_BASE}/entry/{entry_id}/download/", - f"{SASBDB_API_BASE}/entry/{entry_id}/data/", - f"{SASBDB_API_BASE}/entry/{entry_id}/file/", - ] - # Return first endpoint (we'll try them in downloadDataFile if needed) - return download_endpoints[0] - logger.warning("Could not find data file URL in metadata") logger.debug(f"Metadata keys: {list(metadata.keys())}") return None @@ -551,12 +541,7 @@ def _normalizeDatasetId(dataset_id: str) -> str | None: """ Normalize a SASBDB dataset identifier. - SASBDB identifiers are 7 characters long, typically in format: - - "SASDN24" (prefix + number) - - "SASDB1234" (if 4-digit number) - - etc. - - The API requires the full 7-character identifier. + SASBDB identifiers are 7 characters long, e.g. "SASDN24". :param dataset_id: Input dataset identifier :return: Normalized identifier (uppercase, stripped), or None if invalid @@ -564,19 +549,26 @@ def _normalizeDatasetId(dataset_id: str) -> str | None: if not dataset_id: return None - # Remove whitespace and convert to uppercase normalized = dataset_id.strip().upper() + if len(normalized) != 7 or not _SASBDB_ID_PATTERN.match(normalized): + logger.warning(f"Invalid SASBDB dataset ID: {dataset_id!r}") + return None - # SASBDB identifiers are typically 7 characters - # Accept identifiers that are 4-10 characters to be flexible - if len(normalized) < 4 or len(normalized) > 10: - logger.warning(f"Dataset ID length unusual: {len(normalized)} characters for '{normalized}'") - # Still try it, but warn - - # Return the normalized identifier as-is (API expects full identifier) return normalized +def validateDatasetId(dataset_id: str) -> tuple[str | None, str | None]: + """ + Validate and normalize a SASBDB dataset identifier. + + :return: Tuple of (normalized_id, error_message) + """ + normalized = _normalizeDatasetId(dataset_id) + if normalized: + return normalized, None + return None, INVALID_DATASET_ID_MESSAGE + + def downloadDataset(dataset_id: str, output_dir: str | None = None) -> tuple[str | None, SASBDBDatasetInfo | None]: """ Download a complete dataset from SASBDB. @@ -600,14 +592,12 @@ def downloadDataset(dataset_id: str, output_dir: str | None = None) -> tuple[str dataset_info = parseMetadata(metadata) # Get data file URL - data_url = getDataFileUrl(metadata) + data_url = dataset_info.intensities_data_url or getDataFileUrl(metadata) if not data_url: logger.error(f"Could not find data file URL in metadata for dataset {dataset_id}") return None, dataset_info # Return metadata even if download fails - # Store the data URL in the info object - if not dataset_info.intensities_data_url: - dataset_info.intensities_data_url = data_url + dataset_info.intensities_data_url = data_url # Determine output directory if output_dir is None: diff --git a/src/sas/qtgui/Utilities/SASBDB/sasbdb_loader.py b/src/sas/qtgui/Utilities/SASBDB/sasbdb_loader.py index 893bf8e412..4e5e2bdefa 100644 --- a/src/sas/qtgui/Utilities/SASBDB/sasbdb_loader.py +++ b/src/sas/qtgui/Utilities/SASBDB/sasbdb_loader.py @@ -72,8 +72,17 @@ def load_downloaded_dataset(files_widget, parent, filepath: str | None, return try: - loaded_data, _ = files_widget.readData([filepath]) - if dataset_info and loaded_data: + loaded_data, load_message = files_widget.readData([filepath]) + if not loaded_data: + logger.error("Failed to load SASBDB dataset from %s: %s", filepath, load_message) + QMessageBox.warning( + parent, + "Load Error", + f"Failed to load downloaded dataset:\n{load_message}", + ) + return + + if dataset_info: populate_metadata(loaded_data, dataset_info) entry_id = dataset_info.code or dataset_info.entry_id if dataset_info else None @@ -105,7 +114,8 @@ def _apply_metadata(data, info: SASBDBDatasetInfo) -> None: if info.instrument and not data.instrument: data.instrument = info.instrument if info.code: - data.run = data.run or [] + if not data.run: + data.run = [] if info.code not in data.run: data.run.append(info.code) From 6c0573c49e2bef02454487383d220a433fa9c1a7 Mon Sep 17 00:00:00 2001 From: Wojtek Potrzebowski Date: Mon, 13 Jul 2026 16:37:28 +0200 Subject: [PATCH 10/12] Moving out SASBDB-related parts out of GuiUtils --- src/sas/qtgui/Utilities/GuiUtils.py | 310 +--------------- .../Utilities/SASBDB/SASBDBDownloadDialog.py | 2 +- .../qtgui/Utilities/SASBDB/sasbdb_display.py | 332 ++++++++++++++++++ .../qtgui/Utilities/SASBDB/sasbdb_loader.py | 39 +- 4 files changed, 337 insertions(+), 346 deletions(-) create mode 100644 src/sas/qtgui/Utilities/SASBDB/sasbdb_display.py diff --git a/src/sas/qtgui/Utilities/GuiUtils.py b/src/sas/qtgui/Utilities/GuiUtils.py index 1afa460899..94e3d47dc6 100644 --- a/src/sas/qtgui/Utilities/GuiUtils.py +++ b/src/sas/qtgui/Utilities/GuiUtils.py @@ -1,7 +1,6 @@ """ Global defaults and various utility functions usable by the general GUI """ -import ast import json import logging import numbers @@ -40,6 +39,7 @@ from sas.qtgui.Plotting.Plottables import Chisq, Plottable, PlottableFit1D, PlottableTheory1D, Text, View from sas.qtgui.Plotting.PlotterData import Data1D, Data2D, DataRole from sas.qtgui.Utilities.BackgroundColor import BG_DEFAULT, BG_WARNING +from sas.qtgui.Utilities.SASBDB.sasbdb_display import append_sasbdb_data_summary from sas.sascalc.fit.AbstractFitEngine import FitData1D, FitData2D, FResult from sas.system import HELP_SYSTEM from sas.system.user import PATH_LIKE @@ -533,304 +533,6 @@ def openLink(url): raise AttributeError(msg) -def _formatDictLine(line: str) -> str: - """ - Extract and format dictionary information from a line. - - :param line: Line that may contain a dictionary representation - :return: Formatted line with dictionary info extracted, or original line if no dict found - """ - stripped = line.strip() - - # Check if line contains a dictionary - if not ("{" in stripped and "'" in stripped and ":" in stripped): - return line - - # Try to extract the label (e.g., "Instrument:", "Detector:") - label_match = re.match(r'^(\w+)\s*:\s*(.+)', stripped) - if label_match: - label = label_match.group(1) - dict_str = label_match.group(2) - elif stripped.startswith("{'") or stripped.startswith("{"): - label = None - dict_str = stripped - else: - return line - - # Try to parse the dictionary string - try: - # Use ast.literal_eval to safely parse the dictionary - parsed_dict = ast.literal_eval(dict_str) - if not isinstance(parsed_dict, dict): - return line - - # Format the dictionary - parts = [] - - # Extract instrument/source info - if 'name' in parsed_dict and parsed_dict['name']: - parts.append(parsed_dict['name']) - if 'beamline_name' in parsed_dict and parsed_dict['beamline_name']: - parts.append(f"Beamline: {parsed_dict['beamline_name']}") - if 'type_of_source' in parsed_dict and parsed_dict['type_of_source']: - parts.append(f"Source: {parsed_dict['type_of_source']}") - if 'city' in parsed_dict and parsed_dict['city']: - city = parsed_dict['city'] - if 'country' in parsed_dict and parsed_dict['country']: - parts.append(f"{city}, {parsed_dict['country']}") - else: - parts.append(city) - elif 'country' in parsed_dict and parsed_dict['country']: - parts.append(parsed_dict['country']) - - # Extract detector info (nested or direct) - detector_info = None - if 'detector' in parsed_dict and isinstance(parsed_dict['detector'], dict): - det = parsed_dict['detector'] - if 'name' in det and det['name']: - detector_info = det['name'] - elif 'type' in det and det['type']: - detector_info = det['type'] - elif 'detector' in parsed_dict: - detector_info = str(parsed_dict['detector']) - - if detector_info: - parts.append(f"Detector: {detector_info}") - - # If we extracted useful info, format it - if parts: - formatted = " | ".join(parts) - if label: - return f"{label}: {formatted}" - else: - return formatted - - # Fallback: show key-value pairs for simple dicts - simple_parts = [] - for k, v in parsed_dict.items(): - if v is not None and not isinstance(v, dict): - simple_parts.append(f"{k}: {v}") - if simple_parts and len(simple_parts) <= 5: - formatted = " | ".join(simple_parts) - if label: - return f"{label}: {formatted}" - else: - return formatted - - except (ValueError, SyntaxError): - # If parsing fails, return original line - pass - - return line - - -def _isSASBDBData(data) -> bool: - meta = getattr(data, 'meta_data', None) or {} - return 'SASBDB_code' in meta - - -def _cleanDataSummaryText(text: str, data) -> str: - """Reformat dict lines in data summaries for SASBDB datasets only.""" - if not _isSASBDBData(data): - return text - - cleaned_lines = [] - for line in text.split('\n'): - stripped = line.strip() - if ("{" in stripped and "'" in stripped and ":" in stripped and - (stripped.startswith("{'") or - re.search(r':\s*\{', stripped) or - re.search(r"\{'[^']+':", stripped))): - if stripped.count('{') > 0 and stripped.count('}') > 0: - cleaned_lines.append(_formatDictLine(line)) - else: - cleaned_lines.append(line) - else: - cleaned_lines.append(line) - return '\n'.join(cleaned_lines) - - -def _truncateAuthors(authors: str, max_len: int = 50) -> str: - """Truncate an author list at complete names, appending 'et al.' if needed.""" - if len(authors) <= max_len: - return authors - - names = [name.strip() for name in authors.split(',') if name.strip()] - if not names: - return authors - - shown = names[0] - for name in names[1:]: - candidate = f"{shown}, {name}" - if len(candidate) + len(' et al.') > max_len: - return f"{shown} et al." - shown = candidate - - if len(shown) > max_len: - return f"{shown[:max_len - len(' et al.')].rstrip(', ')} et al." - return shown - - -def _formatSASBDBMetadata(data) -> str: - """ - Format SASBDB metadata from data object for display. - - Displays instrument, sample, source info and meta_data dictionary - for datasets loaded from SASBDB in a clean, concise format. - - :param data: Data1D or Data2D object - :return: Formatted string with SASBDB metadata, or empty string if none - """ - text = "" - - # Check if meta_data exists and contains SASBDB info - meta = getattr(data, 'meta_data', None) or {} - - # Check if this is SASBDB data - if 'SASBDB_code' not in meta: - return text - - text += "\n" + "=" * 50 + "\n" - text += "SASBDB Metadata\n" - text += "=" * 50 + "\n" - - # Entry and instrument info (compact header) - header_parts = [] - if meta.get('SASBDB_code'): - header_parts.append(f"Code: {meta['SASBDB_code']}") - - # Extract instrument info from metadata (handle both string and dict formats) - instrument_str = None - location_str = None - - if hasattr(data, 'instrument') and data.instrument: - if isinstance(data.instrument, str): - instrument_str = data.instrument - elif isinstance(data.instrument, dict): - # Extract from dict - instrument_str = data.instrument.get('name') or data.instrument.get('beamline_name') - if data.instrument.get('city') and data.instrument.get('country'): - location_str = f"{data.instrument['city']}, {data.instrument['country']}" - elif data.instrument.get('city'): - location_str = data.instrument['city'] - - # Also check meta_data for instrument info (search all keys for dict values) - if not instrument_str: - for key, val in meta.items(): - if isinstance(val, dict): - # Check if this looks like an instrument dict - if 'beamline_name' in val or 'name' in val or 'type_of_source' in val: - instrument_str = val.get('beamline_name') or val.get('name') - if val.get('city') and val.get('country'): - location_str = f"{val['city']}, {val['country']}" - elif val.get('city'): - location_str = val['city'] - break - elif key in ['instrument', 'source', 'beamline'] and isinstance(val, str): - instrument_str = val - break - - if instrument_str: - header_parts.append(f"Instrument: {instrument_str}") - if location_str: - header_parts.append(location_str) - - # Extract detector info if available (search all keys for detector dicts) - detector_info = None - for key, val in meta.items(): - if isinstance(val, dict) and ('detector' in key.lower() or 'type' in val): - detector_name = val.get('name') or val.get('type') - if detector_name: - detector_info = detector_name - break - elif 'detector' in key.lower() and isinstance(val, str): - detector_info = val - break - - if detector_info: - header_parts.append(f"Detector: {detector_info}") - - if header_parts: - text += " | ".join(header_parts) + "\n" - - # Sample info (consolidated) - if hasattr(data, 'sample') and data.sample: - sample = data.sample - sample_parts = [] - - if hasattr(sample, 'name') and sample.name: - sample_parts.append(sample.name) - - # Add temperature if available - if hasattr(sample, 'temperature') and sample.temperature is not None: - temp_unit = getattr(sample, 'temperature_unit', 'K') or 'K' - sample_parts.append(f"T = {sample.temperature} {temp_unit}") - - # Add concentration and buffer from details (if present) - if hasattr(sample, 'details') and sample.details: - for detail in sample.details: - if detail.startswith("Concentration:"): - sample_parts.append(detail.replace("Concentration: ", "")) - elif detail.startswith("Buffer:"): - sample_parts.append(detail.replace("Buffer: ", "")) - - if sample_parts: - text += f"Sample: {' | '.join(sample_parts)}\n" - - # Source wavelength (if available) - if hasattr(data, 'source') and data.source: - source = data.source - if hasattr(source, 'wavelength') and source.wavelength is not None: - wl_unit = getattr(source, 'wavelength_unit', 'Å') or 'Å' - text += f"Wavelength: {source.wavelength} {wl_unit}\n" - - # Structural parameters (compact format) - structural_parts = [] - if meta.get('SASBDB_Rg') is not None: - rg_str = f"Rg = {meta['SASBDB_Rg']:.2f}" - if meta.get('SASBDB_Rg_error') is not None: - rg_str += f" ± {meta['SASBDB_Rg_error']:.2f}" - rg_str += " Å" - structural_parts.append(rg_str) - - if meta.get('SASBDB_I0') is not None: - i0_str = f"I(0) = {meta['SASBDB_I0']:.4e}" - if meta.get('SASBDB_I0_error') is not None: - i0_str += f" ± {meta['SASBDB_I0_error']:.4e}" - structural_parts.append(i0_str) - - if meta.get('SASBDB_Dmax') is not None: - structural_parts.append(f"Dmax = {meta['SASBDB_Dmax']:.2f} Å") - - if meta.get('SASBDB_MW') is not None: - mw_str = f"MW = {meta['SASBDB_MW']:.2f} kDa" - if meta.get('SASBDB_MW_method'): - mw_str += f" ({meta['SASBDB_MW_method']})" - structural_parts.append(mw_str) - - if meta.get('SASBDB_Porod_volume') is not None: - structural_parts.append(f"Porod Volume = {meta['SASBDB_Porod_volume']:.0f} ų") - - if structural_parts: - text += "Structural: " + " | ".join(structural_parts) + "\n" - - # Publication info (compact, only if available) - pub_parts = [] - if meta.get('SASBDB_authors'): - pub_parts.append(f"Authors: {_truncateAuthors(meta['SASBDB_authors'])}") - if meta.get('SASBDB_DOI'): - pub_parts.append(f"DOI: {meta['SASBDB_DOI']}") - if meta.get('SASBDB_PMID'): - pub_parts.append(f"PMID: {meta['SASBDB_PMID']}") - - if pub_parts: - text += "Publication: " + " | ".join(pub_parts) + "\n" - - text += "=" * 50 + "\n\n" - - return text - - def retrieveData1d(data): """ Retrieve 1D data from file and construct its text @@ -848,10 +550,7 @@ def retrieveData1d(data): #logger.error(msg) raise ValueError(msg) - text = _cleanDataSummaryText(data.__str__(), data) - - # Add SASBDB metadata if present - text += _formatSASBDBMetadata(data) + text = append_sasbdb_data_summary(data.__str__(), data) text += 'Data Min Max:\n' text += 'X_min = %s: X_max = %s\n' % (xmin, max(data.x)) @@ -896,10 +595,7 @@ def retrieveData2d(data): msg = "Incorrect type passed to retrieveData2d" raise AttributeError(msg) - text = _cleanDataSummaryText(data.__str__(), data) - - # Add SASBDB metadata if present - text += _formatSASBDBMetadata(data) + text = append_sasbdb_data_summary(data.__str__(), data) text += 'Data Min Max:\n' text += 'I_min = %s\n' % min(data.data) diff --git a/src/sas/qtgui/Utilities/SASBDB/SASBDBDownloadDialog.py b/src/sas/qtgui/Utilities/SASBDB/SASBDBDownloadDialog.py index 6e33e65a5b..ac84c65631 100644 --- a/src/sas/qtgui/Utilities/SASBDB/SASBDBDownloadDialog.py +++ b/src/sas/qtgui/Utilities/SASBDB/SASBDBDownloadDialog.py @@ -9,7 +9,7 @@ from PySide6 import QtWidgets from .sasbdb_api import SASBDBDatasetInfo, downloadDataset, validateDatasetId -from .sasbdb_loader import metadata_summary +from .sasbdb_display import metadata_summary from .UI.SASBDBDownloadDialogUI import Ui_SASBDBDownloadDialogUI logger = logging.getLogger(__name__) diff --git a/src/sas/qtgui/Utilities/SASBDB/sasbdb_display.py b/src/sas/qtgui/Utilities/SASBDB/sasbdb_display.py new file mode 100644 index 0000000000..385b2e53a5 --- /dev/null +++ b/src/sas/qtgui/Utilities/SASBDB/sasbdb_display.py @@ -0,0 +1,332 @@ +""" +Format SASBDB metadata for dialogs, logs, and data summary panels. +""" + +import ast +import re + +from .sasbdb_api import SASBDBDatasetInfo + + +def is_sasbdb_data(data) -> bool: + meta = getattr(data, "meta_data", None) or {} + return "SASBDB_code" in meta + + +def metadata_summary(info: SASBDBDatasetInfo) -> str: + """Format dataset metadata for dialog status and logging.""" + lines = [] + if info.title: + lines.append(f"Title: {info.title}") + if info.sample_name: + lines.append(f"Sample: {info.sample_name}") + if info.molecule_name: + lines.append(f"Molecule: {info.molecule_name}") + if info.concentration: + lines.append(f"Concentration: {info.concentration} {info.concentration_unit}") + if info.buffer_description: + lines.append(f"Buffer: {info.buffer_description}") + if info.instrument: + lines.append(f"Instrument: {info.instrument}") + if info.wavelength: + lines.append(f"Wavelength: {info.wavelength} {info.wavelength_unit}") + if info.temperature: + lines.append(f"Temperature: {info.temperature} {info.temperature_unit}") + if info.rg is not None: + lines.append(_format_rg_line(info.rg, info.rg_error, style="label")) + if info.i0 is not None: + lines.append(_format_i0_line(info.i0, info.i0_error, style="label")) + if info.dmax is not None: + lines.append(f"Dmax: {info.dmax} Å") + if info.molecular_weight is not None: + lines.append(f"MW: {info.molecular_weight:.1f} kDa") + if info.publication_doi: + lines.append(f"DOI: {info.publication_doi}") + return "\n".join(lines) + + +def append_sasbdb_data_summary(text: str, data) -> str: + """Clean dict repr lines and append SASBDB metadata for data summary panels.""" + if not is_sasbdb_data(data): + return text + return _clean_dict_lines(text) + format_data_panel(data) + + +def format_data_panel(data) -> str: + """Format SASBDB metadata from a loaded data object.""" + meta = getattr(data, "meta_data", None) or {} + if "SASBDB_code" not in meta: + return "" + + lines = [ + "\n" + "=" * 50, + "SASBDB Metadata", + "=" * 50, + ] + + header_parts = [] + if meta.get("SASBDB_code"): + header_parts.append(f"Code: {meta['SASBDB_code']}") + + instrument_str, location_str = _instrument_from_data(data, meta) + if instrument_str: + header_parts.append(f"Instrument: {instrument_str}") + if location_str: + header_parts.append(location_str) + + detector_info = _detector_from_meta(meta) + if detector_info: + header_parts.append(f"Detector: {detector_info}") + + if header_parts: + lines.append(" | ".join(header_parts)) + + sample_line = _sample_line_from_data(data) + if sample_line: + lines.append(sample_line) + + if hasattr(data, "source") and data.source: + source = data.source + if hasattr(source, "wavelength") and source.wavelength is not None: + wl_unit = getattr(source, "wavelength_unit", "Å") or "Å" + lines.append(f"Wavelength: {source.wavelength} {wl_unit}") + + structural = _structural_lines_from_meta(meta) + if structural: + lines.append("Structural: " + " | ".join(structural)) + + publication = _publication_lines_from_meta(meta) + if publication: + lines.append("Publication: " + " | ".join(publication)) + + lines.extend(["=" * 50, ""]) + return "\n".join(line + "\n" for line in lines) + + +def _format_rg_line(rg, rg_error=None, style="label") -> str: + if style == "panel": + text = f"Rg = {rg:.2f}" + if rg_error is not None: + text += f" ± {rg_error:.2f}" + return text + " Å" + rg_text = f"Rg: {rg:.2f}" + if rg_error: + rg_text += f" ± {rg_error:.2f}" + return f"{rg_text} Å" + + +def _format_i0_line(i0, i0_error=None, style="label") -> str: + if style == "panel": + text = f"I(0) = {i0:.4e}" + if i0_error is not None: + text += f" ± {i0_error:.4e}" + return text + text = f"I(0): {i0}" + if i0_error: + text += f" ± {i0_error}" + return text + + +def _instrument_parts_from_dict(parsed_dict: dict) -> list[str]: + parts = [] + if parsed_dict.get("name"): + parts.append(parsed_dict["name"]) + if parsed_dict.get("beamline_name"): + parts.append(f"Beamline: {parsed_dict['beamline_name']}") + if parsed_dict.get("type_of_source"): + parts.append(f"Source: {parsed_dict['type_of_source']}") + if parsed_dict.get("city"): + city = parsed_dict["city"] + if parsed_dict.get("country"): + parts.append(f"{city}, {parsed_dict['country']}") + else: + parts.append(city) + elif parsed_dict.get("country"): + parts.append(parsed_dict["country"]) + + detector_info = None + detector = parsed_dict.get("detector") + if isinstance(detector, dict): + detector_info = detector.get("name") or detector.get("type") + elif detector: + detector_info = str(detector) + if detector_info: + parts.append(f"Detector: {detector_info}") + return parts + + +def _instrument_from_data(data, meta: dict) -> tuple[str | None, str | None]: + instrument_str = None + location_str = None + + if hasattr(data, "instrument") and data.instrument: + if isinstance(data.instrument, str): + instrument_str = data.instrument + elif isinstance(data.instrument, dict): + instrument_str = data.instrument.get("name") or data.instrument.get("beamline_name") + if data.instrument.get("city") and data.instrument.get("country"): + location_str = f"{data.instrument['city']}, {data.instrument['country']}" + elif data.instrument.get("city"): + location_str = data.instrument["city"] + + if not instrument_str: + for key, val in meta.items(): + if isinstance(val, dict) and ( + "beamline_name" in val or "name" in val or "type_of_source" in val + ): + instrument_str = val.get("beamline_name") or val.get("name") + if val.get("city") and val.get("country"): + location_str = f"{val['city']}, {val['country']}" + elif val.get("city"): + location_str = val["city"] + break + if key in ("instrument", "source", "beamline") and isinstance(val, str): + instrument_str = val + break + + return instrument_str, location_str + + +def _detector_from_meta(meta: dict) -> str | None: + for key, val in meta.items(): + if isinstance(val, dict) and ("detector" in key.lower() or "type" in val): + detector_name = val.get("name") or val.get("type") + if detector_name: + return detector_name + if "detector" in key.lower() and isinstance(val, str): + return val + return None + + +def _sample_line_from_data(data) -> str | None: + if not hasattr(data, "sample") or not data.sample: + return None + + sample = data.sample + sample_parts = [] + if hasattr(sample, "name") and sample.name: + sample_parts.append(sample.name) + if hasattr(sample, "temperature") and sample.temperature is not None: + temp_unit = getattr(sample, "temperature_unit", "K") or "K" + sample_parts.append(f"T = {sample.temperature} {temp_unit}") + if hasattr(sample, "details") and sample.details: + for detail in sample.details: + if detail.startswith("Concentration:"): + sample_parts.append(detail.replace("Concentration: ", "")) + elif detail.startswith("Buffer:"): + sample_parts.append(detail.replace("Buffer: ", "")) + + if sample_parts: + return f"Sample: {' | '.join(sample_parts)}" + return None + + +def _structural_lines_from_meta(meta: dict) -> list[str]: + lines = [] + if meta.get("SASBDB_Rg") is not None: + lines.append(_format_rg_line(meta["SASBDB_Rg"], meta.get("SASBDB_Rg_error"), style="panel")) + if meta.get("SASBDB_I0") is not None: + lines.append(_format_i0_line(meta["SASBDB_I0"], meta.get("SASBDB_I0_error"), style="panel")) + if meta.get("SASBDB_Dmax") is not None: + lines.append(f"Dmax = {meta['SASBDB_Dmax']:.2f} Å") + if meta.get("SASBDB_MW") is not None: + mw = f"MW = {meta['SASBDB_MW']:.2f} kDa" + if meta.get("SASBDB_MW_method"): + mw += f" ({meta['SASBDB_MW_method']})" + lines.append(mw) + if meta.get("SASBDB_Porod_volume") is not None: + lines.append(f"Porod Volume = {meta['SASBDB_Porod_volume']:.0f} ų") + return lines + + +def _publication_lines_from_meta(meta: dict) -> list[str]: + lines = [] + if meta.get("SASBDB_authors"): + lines.append(f"Authors: {_truncate_authors(meta['SASBDB_authors'])}") + if meta.get("SASBDB_DOI"): + lines.append(f"DOI: {meta['SASBDB_DOI']}") + if meta.get("SASBDB_PMID"): + lines.append(f"PMID: {meta['SASBDB_PMID']}") + return lines + + +def _truncate_authors(authors: str, max_len: int = 50) -> str: + if len(authors) <= max_len: + return authors + + names = [name.strip() for name in authors.split(",") if name.strip()] + if not names: + return authors + + shown = names[0] + for name in names[1:]: + candidate = f"{shown}, {name}" + if len(candidate) + len(" et al.") > max_len: + return f"{shown} et al." + shown = candidate + + if len(shown) > max_len: + return f"{shown[: max_len - len(' et al.')].rstrip(', ')} et al." + return shown + + +def _format_dict_line(line: str) -> str: + stripped = line.strip() + if not ("{" in stripped and "'" in stripped and ":" in stripped): + return line + + label_match = re.match(r"^(\w+)\s*:\s*(.+)", stripped) + if label_match: + label = label_match.group(1) + dict_str = label_match.group(2) + elif stripped.startswith("{'") or stripped.startswith("{"): + label = None + dict_str = stripped + else: + return line + + try: + parsed_dict = ast.literal_eval(dict_str) + if not isinstance(parsed_dict, dict): + return line + + parts = _instrument_parts_from_dict(parsed_dict) + if parts: + formatted = " | ".join(parts) + return f"{label}: {formatted}" if label else formatted + + simple_parts = [ + f"{key}: {value}" + for key, value in parsed_dict.items() + if value is not None and not isinstance(value, dict) + ] + if simple_parts and len(simple_parts) <= 5: + formatted = " | ".join(simple_parts) + return f"{label}: {formatted}" if label else formatted + except (ValueError, SyntaxError): + pass + + return line + + +def _clean_dict_lines(text: str) -> str: + cleaned_lines = [] + for line in text.split("\n"): + stripped = line.strip() + if ( + "{" in stripped + and "'" in stripped + and ":" in stripped + and ( + stripped.startswith("{'") + or re.search(r":\s*\{", stripped) + or re.search(r"\{'[^']+':", stripped) + ) + ): + if stripped.count("{") > 0 and stripped.count("}") > 0: + cleaned_lines.append(_format_dict_line(line)) + else: + cleaned_lines.append(line) + else: + cleaned_lines.append(line) + return "\n".join(cleaned_lines) diff --git a/src/sas/qtgui/Utilities/SASBDB/sasbdb_loader.py b/src/sas/qtgui/Utilities/SASBDB/sasbdb_loader.py index 4e5e2bdefa..e560f26bef 100644 --- a/src/sas/qtgui/Utilities/SASBDB/sasbdb_loader.py +++ b/src/sas/qtgui/Utilities/SASBDB/sasbdb_loader.py @@ -10,6 +10,7 @@ from sasdata.dataloader.data_info import Sample, Source from .sasbdb_api import SASBDBDatasetInfo +from .sasbdb_display import metadata_summary logger = logging.getLogger(__name__) @@ -27,44 +28,6 @@ ) -def metadata_summary(info: SASBDBDatasetInfo) -> str: - """Format dataset metadata for display or logging.""" - lines = [] - if info.title: - lines.append(f"Title: {info.title}") - if info.sample_name: - lines.append(f"Sample: {info.sample_name}") - if info.molecule_name: - lines.append(f"Molecule: {info.molecule_name}") - if info.concentration: - lines.append(f"Concentration: {info.concentration} {info.concentration_unit}") - if info.buffer_description: - lines.append(f"Buffer: {info.buffer_description}") - if info.instrument: - lines.append(f"Instrument: {info.instrument}") - if info.wavelength: - lines.append(f"Wavelength: {info.wavelength} {info.wavelength_unit}") - if info.temperature: - lines.append(f"Temperature: {info.temperature} {info.temperature_unit}") - if info.rg is not None: - rg = f"Rg: {info.rg:.2f}" - if info.rg_error: - rg += f" ± {info.rg_error:.2f}" - lines.append(f"{rg} Å") - if info.i0 is not None: - i0 = f"I(0): {info.i0}" - if info.i0_error: - i0 += f" ± {info.i0_error}" - lines.append(i0) - if info.dmax is not None: - lines.append(f"Dmax: {info.dmax} Å") - if info.molecular_weight is not None: - lines.append(f"MW: {info.molecular_weight:.1f} kDa") - if info.publication_doi: - lines.append(f"DOI: {info.publication_doi}") - return "\n".join(lines) - - def load_downloaded_dataset(files_widget, parent, filepath: str | None, dataset_info: SASBDBDatasetInfo | None) -> None: """Load a downloaded SASBDB file and apply metadata.""" From 2415149053778c7b57567440a38cb412eb6b0b60 Mon Sep 17 00:00:00 2001 From: Wojtek Potrzebowski Date: Wed, 15 Jul 2026 08:33:29 +0200 Subject: [PATCH 11/12] Updated documentation --- .../SASBDB/media/sasbdb_download_help.rst | 28 +------------------ 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/src/sas/qtgui/Utilities/SASBDB/media/sasbdb_download_help.rst b/src/sas/qtgui/Utilities/SASBDB/media/sasbdb_download_help.rst index 00d7a30d43..054edcf1ab 100644 --- a/src/sas/qtgui/Utilities/SASBDB/media/sasbdb_download_help.rst +++ b/src/sas/qtgui/Utilities/SASBDB/media/sasbdb_download_help.rst @@ -23,7 +23,7 @@ Dataset Identifier Enter a valid SASBDB dataset identifier in the input field. The identifier can be: -- A 7-character SASBDB code (e.g., ``SASDN24``, ``SASDB1234``) +- A 7-character SASBDB code (e.g., ``SASDN24``) - The identifier is case-insensitive and will be automatically normalized Examples of valid identifiers: @@ -41,15 +41,6 @@ When you click **Download and Load**: 4. **Data Loading**: The downloaded data is automatically loaded into SasView 5. **Metadata Population**: Metadata from SASBDB is automatically populated into the loaded dataset -Progress Indicator ------------------- - -During the download process, you will see: - -- A progress bar indicating the download is in progress -- Status messages showing the current operation -- A success message with a summary of the loaded dataset - Metadata Population -------------------- @@ -106,23 +97,6 @@ The metadata is displayed in a compact format with key information organized by - Structural parameters (Rg, I(0), Dmax, MW, etc.) - Publication information -Error Handling --------------- - -If an error occurs during download: - -- An error message will be displayed explaining the issue -- Common errors include: - - Invalid dataset identifier format - - Dataset not found in SASBDB - - Network connection issues - - API service unavailable - -If you encounter errors: - -1. Verify the dataset identifier is correct -2. Check your internet connection -3. Try again after a few moments if the SASBDB service is temporarily unavailable Tips ---- From e100ebcca7d3eceb27a30681bc5884aec29a7816 Mon Sep 17 00:00:00 2001 From: Wojtek Potrzebowski Date: Wed, 15 Jul 2026 08:44:26 +0200 Subject: [PATCH 12/12] Further separation and code clean up --- src/sas/qtgui/Utilities/SASBDB/sasbdb_api.py | 685 +++--------------- .../qtgui/Utilities/SASBDB/sasbdb_parse.py | 230 ++++++ 2 files changed, 339 insertions(+), 576 deletions(-) create mode 100644 src/sas/qtgui/Utilities/SASBDB/sasbdb_parse.py diff --git a/src/sas/qtgui/Utilities/SASBDB/sasbdb_api.py b/src/sas/qtgui/Utilities/SASBDB/sasbdb_api.py index 283ca249c8..0e912d7157 100644 --- a/src/sas/qtgui/Utilities/SASBDB/sasbdb_api.py +++ b/src/sas/qtgui/Utilities/SASBDB/sasbdb_api.py @@ -1,651 +1,184 @@ """ SASBDB REST API client module. - -Provides functions to interact with the SASBDB REST API for downloading -datasets and retrieving metadata. """ import logging import os import re import tempfile -from dataclasses import dataclass, field import requests +from .sasbdb_parse import SASBDBDatasetInfo, parseMetadata + logger = logging.getLogger(__name__) -# Base API URL - SASBDB REST API SASBDB_API_BASE = "https://www.sasbdb.org/rest-api" -_SASBDB_ID_PATTERN = re.compile(r'^SAS[A-Z]{2}\d+$') +SASBDB_SITE_BASE = "https://www.sasbdb.org" +_SASBDB_ID_PATTERN = re.compile(r"^SAS[A-Z]{2}\d+$") INVALID_DATASET_ID_MESSAGE = "Enter the full 7-character SASBDB code (e.g. SASDN24)." - -@dataclass -class SASBDBDatasetInfo: - """ - Parsed metadata from a SASBDB dataset entry. - - Contains key information extracted from the SASBDB REST API response. - """ - # Identifiers - entry_id: str = "" - code: str = "" - title: str = "" - - # Sample information - sample_name: str = "" - sample_description: str = "" - concentration: float | None = None - concentration_unit: str = "mg/mL" - - # Molecule information - molecule_name: str = "" - molecule_short_name: str = "" - molecule_type: str = "" - sequence: str = "" - uniprot_code: str = "" - source_organism: str = "" - number_of_molecules: str = "" - oligomerization: str = "" - molecular_weight: float | None = None # Experimental MW in kDa - molecular_weight_method: str = "" - oligomeric_state: str = "" - - # Buffer information - buffer_description: str = "" - ph: float | None = None - - # Experimental parameters - instrument: str = "" - detector: str = "" - wavelength: float | None = None # in Angstrom - wavelength_unit: str = "Å" - temperature: float | None = None # in Kelvin or Celsius - temperature_unit: str = "K" - - # Analysis results - rg: float | None = None # Radius of gyration in Angstrom - rg_error: float | None = None - i0: float | None = None # I(0) - i0_error: float | None = None - dmax: float | None = None # Maximum dimension in Angstrom - porod_volume: float | None = None - - # Q-range - q_min: float | None = None - q_max: float | None = None - - # Publication - publication_title: str = "" - publication_doi: str = "" - publication_pmid: str = "" - authors: list = field(default_factory=list) - - # Data files - intensities_data_url: str = "" - pddf_data_url: str = "" - - # Raw metadata for additional fields - raw_metadata: dict = field(default_factory=dict) - - -def parseMetadata(metadata: dict) -> SASBDBDatasetInfo: - """ - Parse SASBDB API response into a structured SASBDBDatasetInfo object. - - :param metadata: Raw JSON dictionary from SASBDB API - :return: Parsed SASBDBDatasetInfo object - """ - info = SASBDBDatasetInfo() - - if not isinstance(metadata, dict): - return info - - # Store raw metadata for reference - info.raw_metadata = metadata - - # Log top-level keys for debugging - logger.debug(f"SASBDB API response keys: {list(metadata.keys())}") - - # Identifiers - info.entry_id = _get_str(metadata, 'id', 'entry_id', 'sasbdb_id') - info.code = _get_str(metadata, 'code', 'accession_code', 'entry_code') - info.title = _get_str(metadata, 'title', 'entry_title', 'sample_title') - - # Sample information - try more variations - info.sample_name = _get_str(metadata, 'sample_name', 'sample', 'name', 'sample_name_full') - info.sample_description = _get_str(metadata, 'sample_description', 'description', 'sample_description_full') - info.concentration = _get_float(metadata, 'concentration', 'sample_concentration', 'conc', 'sample_conc') - info.concentration_unit = _get_str(metadata, 'concentration_unit', 'conc_unit', 'concentration_units') or "mg/mL" - - # Molecule information - try more variations - info.molecule_name = _get_str( - metadata, - 'molecule_name', - 'macromolecule_name', - 'protein_name', - 'molecule', - 'macromolecule', - 'protein', - 'name', - ) or _get_deep_str( - metadata, 'long_name', 'molecule_name', 'macromolecule_name', 'protein_name' - ) - info.molecule_short_name = _get_str( - metadata, 'short_name', 'molecule_short_name', 'shortName', 'short' - ) or _get_deep_str(metadata, 'short_name', 'molecule_short_name', 'shortName') - info.molecule_type = _get_str( - metadata, 'molecule_type', 'macromolecule_type', 'sample_type', - 'type', 'molecule_type_full' - ) or _get_deep_str( - metadata, 'molecule_type', 'macromolecule_type', 'molecular_type' - ) - info.sequence = _get_sequence(metadata) - info.uniprot_code = _get_str( - metadata, - 'uniprot_code', - 'uniprot', - 'uniprot_id', - 'uniprot_accession', - 'uniprot_ac', - ) or _get_deep_str( - metadata, 'uniprot_code', 'uniprot', 'uniprot_id', 'uniprot_accession' - ) - info.source_organism = _get_str( - metadata, 'source_organism', 'organism', 'organism_name' - ) or _get_deep_str( - metadata, 'source_organism', 'organism', 'organism_name' - ) - info.number_of_molecules = _get_str( - metadata, 'number_of_molecules', 'num_molecules', 'copy_number' - ) or _get_deep_str( - metadata, 'number_of_molecules', 'number_molecules', 'num_molecules', - 'copy_number' - ) - info.oligomerization = _get_str( - metadata, 'oligomerization', 'oligomeric_state', 'oligomer_state' - ) or _get_deep_str( - metadata, 'oligomerization', 'oligomeric_state', 'oligomer_state', - 'complex_state' - ) - info.molecular_weight = _get_float(metadata, 'molecular_weight', 'mw', 'exp_mw', - 'experimental_mw', 'mw_kda') - info.molecular_weight_method = _get_str(metadata, 'mw_method', 'molecular_weight_method') - info.oligomeric_state = info.oligomerization or _get_str( - metadata, 'oligomeric_state', 'oligomer_state', 'oligomerization' - ) - - # Buffer information - info.buffer_description = _get_str(metadata, 'buffer', 'buffer_description', 'buffer_composition') - info.ph = _get_float(metadata, 'ph', 'buffer_ph') - - # Experimental parameters - info.instrument = _get_str(metadata, 'instrument', 'beamline', 'instrument_name') - info.detector = _get_str(metadata, 'detector', 'detector_name', 'detector_type') - info.wavelength = _get_float(metadata, 'wavelength', 'xray_wavelength', 'neutron_wavelength') - info.temperature = _get_float(metadata, 'temperature', 'sample_temperature', 'temp') - - # Analysis results - Guinier - info.rg = _get_float(metadata, 'rg', 'radius_of_gyration', 'guinier_rg') - info.rg_error = _get_float(metadata, 'rg_error', 'rg_err', 'guinier_rg_error') - info.i0 = _get_float(metadata, 'i0', 'i_zero', 'guinier_i0') - info.i0_error = _get_float(metadata, 'i0_error', 'i0_err', 'guinier_i0_error') - - # Analysis results - P(r) - info.dmax = _get_float(metadata, 'dmax', 'd_max', 'maximum_dimension') - info.porod_volume = _get_float(metadata, 'porod_volume', 'volume', 'porod_vol') - - # Q-range - info.q_min = _get_float(metadata, 'q_min', 'qmin', 's_min', 'smin') - info.q_max = _get_float(metadata, 'q_max', 'qmax', 's_max', 'smax') - - # Publication - info.publication_title = _get_str(metadata, 'publication_title', 'pub_title', 'paper_title') - info.publication_doi = _get_str(metadata, 'doi', 'publication_doi', 'pub_doi') - info.publication_pmid = _get_str(metadata, 'pmid', 'publication_pmid', 'pubmed_id') - - # Authors - authors = metadata.get('authors') or metadata.get('author_list') or [] - if isinstance(authors, list): - info.authors = [str(a) for a in authors] - elif isinstance(authors, str): - info.authors = [authors] - - # Data file URLs - info.intensities_data_url = _get_str(metadata, 'intensities_data', 'data_url', 'intensities_url') - info.pddf_data_url = _get_str(metadata, 'pddf_data', 'pddf_url', 'pr_data') - - return info - - -def _get_str(data: dict, *keys: str) -> str: - """ - Get string value from dictionary, trying multiple possible keys. - Also searches in nested dictionaries (sample, molecule, experiment, etc.). - - :param data: Dictionary to search - :param keys: Possible key names to try - :return: String value or empty string if not found - """ - # First try top-level keys - for key in keys: - if key in data and data[key] is not None: - return str(data[key]) - - # Then search in common nested structures - nested_paths = ['sample', 'molecule', 'experiment', 'experimental', 'metadata', 'info'] - for path in nested_paths: - if path in data and isinstance(data[path], dict): - for key in keys: - if key in data[path] and data[path][key] is not None: - return str(data[path][key]) - - return "" - - -def _get_sequence(data: dict) -> str: - """ - Extract a protein/nucleotide sequence from nested SASBDB metadata. - - This helper searches recursively through dictionaries and lists since - sequence information may appear under molecule lists or nested blocks. - - :param data: Dictionary to search - :return: Sequence string or empty string if not found - """ - sequence_keys = { - 'sequence', - 'fasta_sequence', - 'fasta', - 'primary_sequence', - 'sequence_string', - } - - def _from_value(value) -> str: - """Normalize potential sequence values to a plain string.""" - if value is None: - return "" - if isinstance(value, str): - return value.strip() - if isinstance(value, list): - parts = [] - for item in value: - item_str = _from_value(item) - if item_str: - parts.append(item_str) - if parts: - return " ".join(parts) - return "" - if isinstance(value, dict): - # Common wrappers for sequence strings. - for nested_key in ("value", "text", "seq"): - nested = value.get(nested_key) - nested_str = _from_value(nested) - if nested_str: - return nested_str - return "" - - def _search(obj) -> str: - if isinstance(obj, dict): - for key in sequence_keys: - if key in obj: - sequence = _from_value(obj[key]) - if sequence: - return sequence - for value in obj.values(): - sequence = _search(value) - if sequence: - return sequence - elif isinstance(obj, list): - for item in obj: - sequence = _search(item) - if sequence: - return sequence - return "" - - return _search(data) - - -def _get_deep_str(data: dict, *keys: str) -> str: - """ - Recursively search nested dict/list structures for the first key match. - - :param data: Dictionary to search - :param keys: Candidate field names to locate - :return: String value or empty string if not found - """ - key_set = set(keys) - - def _normalize(value) -> str: - if value is None: - return "" - if isinstance(value, str): - return value.strip() - if isinstance(value, (int, float, bool)): - return str(value) - return "" - - def _search(obj) -> str: - if isinstance(obj, dict): - for key, value in obj.items(): - if key in key_set: - normalized = _normalize(value) - if normalized: - return normalized - for value in obj.values(): - found = _search(value) - if found: - return found - elif isinstance(obj, list): - for item in obj: - found = _search(item) - if found: - return found - return "" - - return _search(data) - - -def _get_float(data: dict, *keys: str) -> float | None: - """ - Get float value from dictionary, trying multiple possible keys. - Also searches in nested dictionaries (sample, molecule, experiment, etc.). - - :param data: Dictionary to search - :param keys: Possible key names to try - :return: Float value or None if not found - """ - # First try top-level keys - for key in keys: - if key in data and data[key] is not None: - try: - return float(data[key]) - except (ValueError, TypeError): - continue - - # Then search in common nested structures - nested_paths = ['sample', 'molecule', 'experiment', 'experimental', 'metadata', 'info'] - for path in nested_paths: - if path in data and isinstance(data[path], dict): - for key in keys: - if key in data[path] and data[path][key] is not None: - try: - return float(data[path][key]) - except (ValueError, TypeError): - continue - - return None +_URL_FIELDS = ( + "intensities_data", "intensitiesData", "data_file_url", "dataFileUrl", + "data_file", "dataFile", "scattering_data_url", "scatteringDataUrl", + "experimental_data_url", "experimentalDataUrl", "download_url", "downloadUrl", + "file_url", "fileUrl", "files", "data_files", "dataFiles", + "experimental_files", "scattering_files", "experimental_data", + "experimentalData", "scattering_data", "scatteringData", +) +_URL_NESTED = ("entry", "data", "files", "experimental_data", "scattering_data") +_URL_ITEM_KEYS = ("url", "path", "file", "file_url", "download_url") +_KNOWN_EXTENSIONS = {".dat", ".txt", ".csv", ".out", ".asc"} def getDatasetMetadata(dataset_id: str) -> dict | None: - """ - Fetch dataset metadata from SASBDB API. - - :param dataset_id: SASBDB dataset identifier (e.g., "SASDN24" - full 7-character ID) - :return: Dictionary containing dataset metadata, or None if error - """ - # Normalize dataset ID (uppercase, strip whitespace, ensure 7 characters) + """Fetch dataset metadata from the SASBDB API.""" normalized_id = _normalizeDatasetId(dataset_id) if not normalized_id: - logger.error(f"Invalid dataset ID format: {dataset_id}") + logger.error("Invalid dataset ID format: %s", dataset_id) return None - # Use the correct REST API endpoint: /rest-api/entry/summary/{id}/ endpoint = f"{SASBDB_API_BASE}/entry/summary/{normalized_id}/" - try: - logger.info(f"Fetching dataset metadata from: {endpoint}") - headers = {"accept": "application/json"} - response = requests.get(endpoint, headers=headers, timeout=30) + logger.info("Fetching dataset metadata from: %s", endpoint) + response = requests.get(endpoint, headers={"accept": "application/json"}, timeout=30) response.raise_for_status() - - # Parse JSON response - metadata = response.json() - logger.info(f"Successfully retrieved metadata for dataset {normalized_id}") - return metadata - - except requests.exceptions.HTTPError as e: - if e.response.status_code == 404: - logger.error(f"Dataset {normalized_id} not found (404)") + logger.info("Successfully retrieved metadata for dataset %s", normalized_id) + return response.json() + except requests.exceptions.HTTPError as error: + if error.response is not None and error.response.status_code == 404: + logger.error("Dataset %s not found (404)", normalized_id) else: - logger.error(f"HTTP error fetching dataset {normalized_id}: {e}") - return None - except requests.exceptions.RequestException as e: - logger.error(f"Network error fetching dataset {normalized_id}: {e}") - return None - except ValueError as e: - logger.error(f"Invalid JSON response for dataset {normalized_id}: {e}") - return None + logger.error("HTTP error fetching dataset %s: %s", normalized_id, error) + except requests.exceptions.RequestException as error: + logger.error("Network error fetching dataset %s: %s", normalized_id, error) + except ValueError as error: + logger.error("Invalid JSON response for dataset %s: %s", normalized_id, error) + return None def getDataFileUrl(metadata: dict) -> str | None: - """ - Extract data file URL from dataset metadata. - - Looks for common field names in the metadata JSON that might contain - the data file URL or path. Also checks for SASBDB-specific fields. - - :param metadata: Dictionary containing dataset metadata - :return: URL string for the data file, or None if not found - """ + """Extract a data file URL from dataset metadata.""" if not isinstance(metadata, dict): return None - # SASBDB-specific field names that might contain data file URLs - # Priority: intensities_data is the primary field for SASBDB - possible_fields = [ - # SASBDB primary data field - 'intensities_data', - 'intensitiesData', - # Direct URL fields - 'data_file_url', - 'dataFileUrl', - 'data_file', - 'dataFile', - 'scattering_data_url', - 'scatteringDataUrl', - 'experimental_data_url', - 'experimentalDataUrl', - 'download_url', - 'downloadUrl', - 'file_url', - 'fileUrl', - # File list fields - 'files', - 'data_files', - 'dataFiles', - 'experimental_files', - 'scattering_files', - # SASBDB API specific fields - 'experimental_data', - 'experimentalData', - 'scattering_data', - 'scatteringData', - ] - - # Check top-level fields - for field_name in possible_fields: + for field_name in _URL_FIELDS: if field_name in metadata: - value = metadata[field_name] - if isinstance(value, str) and (value.startswith('http') or value.startswith('/')): - # If relative URL, make it absolute - if value.startswith('/'): - return f"https://www.sasbdb.org{value}" - return value - elif isinstance(value, list) and len(value) > 0: - # If it's a list, take the first item - first_item = value[0] - if isinstance(first_item, str): - if first_item.startswith('/'): - return f"https://www.sasbdb.org{first_item}" - elif first_item.startswith('http'): - return first_item - elif isinstance(first_item, dict): - # Check for 'url' or 'path' in the item - for url_field in ['url', 'path', 'file', 'file_url', 'download_url']: - if url_field in first_item: - url = first_item[url_field] - if isinstance(url, str): - if url.startswith('/'): - return f"https://www.sasbdb.org{url}" - elif url.startswith('http'): - return url - - # Check nested structures (common in REST APIs) - for nested_key in ['entry', 'data', 'files', 'experimental_data', 'scattering_data']: - if nested_key in metadata and isinstance(metadata[nested_key], dict): - result = getDataFileUrl(metadata[nested_key]) - if result: - return result + url = _resolve_url_value(metadata[field_name]) + if url: + return url + + for nested_key in _URL_NESTED: + nested = metadata.get(nested_key) + if isinstance(nested, dict): + url = getDataFileUrl(nested) + if url: + return url logger.warning("Could not find data file URL in metadata") - logger.debug(f"Metadata keys: {list(metadata.keys())}") + logger.debug("Metadata keys: %s", list(metadata.keys())) return None def downloadDataFile(url: str, filepath: str) -> bool: - """ - Download a data file from the given URL to the specified filepath. - - :param url: URL of the data file to download - :param filepath: Local filepath where the file should be saved - :return: True if download successful, False otherwise - """ + """Download a data file from the given URL.""" try: - logger.info(f"Downloading data file from: {url}") + logger.info("Downloading data file from: %s", url) response = requests.get(url, timeout=60, stream=True) response.raise_for_status() - - # Create directory if it doesn't exist os.makedirs(os.path.dirname(filepath), exist_ok=True) - - # Write file in chunks to handle large files - with open(filepath, 'wb') as f: + with open(filepath, "wb") as handle: for chunk in response.iter_content(chunk_size=8192): if chunk: - f.write(chunk) - - logger.info(f"Successfully downloaded data file to: {filepath}") + handle.write(chunk) + logger.info("Successfully downloaded data file to: %s", filepath) return True - - except requests.exceptions.RequestException as e: - logger.error(f"Error downloading data file from {url}: {e}") - return False - except OSError as e: - logger.error(f"Error writing data file to {filepath}: {e}") - return False - - -def _normalizeDatasetId(dataset_id: str) -> str | None: - """ - Normalize a SASBDB dataset identifier. - - SASBDB identifiers are 7 characters long, e.g. "SASDN24". - - :param dataset_id: Input dataset identifier - :return: Normalized identifier (uppercase, stripped), or None if invalid - """ - if not dataset_id: - return None - - normalized = dataset_id.strip().upper() - if len(normalized) != 7 or not _SASBDB_ID_PATTERN.match(normalized): - logger.warning(f"Invalid SASBDB dataset ID: {dataset_id!r}") - return None - - return normalized + except requests.exceptions.RequestException as error: + logger.error("Error downloading data file from %s: %s", url, error) + except OSError as error: + logger.error("Error writing data file to %s: %s", filepath, error) + return False def validateDatasetId(dataset_id: str) -> tuple[str | None, str | None]: - """ - Validate and normalize a SASBDB dataset identifier. - - :return: Tuple of (normalized_id, error_message) - """ + """Validate and normalize a SASBDB dataset identifier.""" normalized = _normalizeDatasetId(dataset_id) if normalized: return normalized, None return None, INVALID_DATASET_ID_MESSAGE -def downloadDataset(dataset_id: str, output_dir: str | None = None) -> tuple[str | None, SASBDBDatasetInfo | None]: - """ - Download a complete dataset from SASBDB. - - This is a convenience function that: - 1. Fetches metadata - 2. Extracts data file URL - 3. Downloads the data file - 4. Returns the local filepath and parsed metadata - - :param dataset_id: SASBDB dataset identifier - :param output_dir: Directory to save the file (defaults to temp directory) - :return: Tuple of (path to downloaded file, parsed metadata info) or (None, None) if error - """ - # Get metadata +def downloadDataset( + dataset_id: str, output_dir: str | None = None, +) -> tuple[str | None, SASBDBDatasetInfo | None]: + """Fetch metadata, download the intensity file, and return path + parsed info.""" metadata = getDatasetMetadata(dataset_id) if not metadata: return None, None - # Parse metadata into structured object dataset_info = parseMetadata(metadata) - - # Get data file URL data_url = dataset_info.intensities_data_url or getDataFileUrl(metadata) if not data_url: - logger.error(f"Could not find data file URL in metadata for dataset {dataset_id}") - return None, dataset_info # Return metadata even if download fails + logger.error("Could not find data file URL in metadata for dataset %s", dataset_id) + return None, dataset_info dataset_info.intensities_data_url = data_url - - # Determine output directory - if output_dir is None: - output_dir = tempfile.gettempdir() - - # Generate filename + output_dir = output_dir or tempfile.gettempdir() normalized_id = _normalizeDatasetId(dataset_id) - # Try to determine file extension from URL or metadata - file_extension = _guessFileExtension(data_url, metadata) - filename = f"SASBDB_{normalized_id}{file_extension}" + filename = f"SASBDB_{normalized_id}{_guessFileExtension(data_url, metadata)}" filepath = os.path.join(output_dir, filename) - # Download the file if downloadDataFile(data_url, filepath): return filepath, dataset_info - return None, dataset_info +def _normalizeDatasetId(dataset_id: str) -> str | None: + if not dataset_id: + return None + normalized = dataset_id.strip().upper() + if len(normalized) != 7 or not _SASBDB_ID_PATTERN.match(normalized): + logger.warning("Invalid SASBDB dataset ID: %r", dataset_id) + return None + return normalized + + +def _resolve_url_value(value) -> str | None: + if isinstance(value, str): + return _absolute_url(value) + if isinstance(value, list) and value: + return _resolve_url_value(value[0]) + if isinstance(value, dict): + for key in _URL_ITEM_KEYS: + if key in value: + return _resolve_url_value(value[key]) + return None + + +def _absolute_url(url: str) -> str | None: + if url.startswith("/"): + return f"{SASBDB_SITE_BASE}{url}" + if url.startswith("http"): + return url + return None + + def _guessFileExtension(url: str, metadata: dict) -> str: - """ - Guess the file extension from URL or metadata. - - :param url: Data file URL - :param metadata: Dataset metadata - :return: File extension (e.g., ".dat", ".txt", ".csv") - """ - # Check URL for extension - if '.' in url: - parts = url.split('.') - if len(parts) > 1: - ext = '.' + parts[-1].split('?')[0] # Remove query parameters - if ext.lower() in ['.dat', '.txt', '.csv', '.out', '.asc']: - return ext - - # Check metadata for file type hints - if isinstance(metadata, dict): - file_type_fields = ['file_type', 'fileType', 'format', 'data_format'] - for field in file_type_fields: - if field in metadata: - file_type = str(metadata[field]).lower() - if 'csv' in file_type: - return '.csv' - elif 'txt' in file_type or 'text' in file_type: - return '.txt' - elif 'dat' in file_type or 'data' in file_type: - return '.dat' - - # Default to .dat for scattering data - return '.dat' + if "." in url: + ext = "." + url.split(".")[-1].split("?")[0] + if ext.lower() in _KNOWN_EXTENSIONS: + return ext + if isinstance(metadata, dict): + for field_name in ("file_type", "fileType", "format", "data_format"): + file_type = metadata.get(field_name) + if file_type is None: + continue + file_type = str(file_type).lower() + if "csv" in file_type: + return ".csv" + if "txt" in file_type or "text" in file_type: + return ".txt" + if "dat" in file_type or "data" in file_type: + return ".dat" + return ".dat" diff --git a/src/sas/qtgui/Utilities/SASBDB/sasbdb_parse.py b/src/sas/qtgui/Utilities/SASBDB/sasbdb_parse.py new file mode 100644 index 0000000000..1b393ab8e6 --- /dev/null +++ b/src/sas/qtgui/Utilities/SASBDB/sasbdb_parse.py @@ -0,0 +1,230 @@ +"""Parse SASBDB REST API metadata into structured objects.""" + +import logging +from dataclasses import dataclass, field + +logger = logging.getLogger(__name__) + +_NESTED_PATHS = ("sample", "molecule", "experiment", "experimental", "metadata", "info") +_SEQUENCE_KEYS = frozenset( + {"sequence", "fasta_sequence", "fasta", "primary_sequence", "sequence_string"} +) + +_STR_FIELDS = { + "entry_id": ("id", "entry_id", "sasbdb_id"), + "code": ("code", "accession_code", "entry_code"), + "title": ("title", "entry_title", "sample_title"), + "sample_name": ("sample_name", "sample", "name", "sample_name_full"), + "sample_description": ("sample_description", "description", "sample_description_full"), + "concentration_unit": ("concentration_unit", "conc_unit", "concentration_units"), + "molecule_name": ( + "molecule_name", "macromolecule_name", "protein_name", "molecule", + "macromolecule", "protein", "name", "long_name", + ), + "molecule_short_name": ("short_name", "molecule_short_name", "shortName", "short"), + "molecule_type": ( + "molecule_type", "macromolecule_type", "sample_type", "type", + "molecule_type_full", "molecular_type", + ), + "uniprot_code": ("uniprot_code", "uniprot", "uniprot_id", "uniprot_accession", "uniprot_ac"), + "source_organism": ("source_organism", "organism", "organism_name"), + "number_of_molecules": ( + "number_of_molecules", "num_molecules", "copy_number", "number_molecules", + ), + "oligomerization": ( + "oligomerization", "oligomeric_state", "oligomer_state", "complex_state", + ), + "molecular_weight_method": ("mw_method", "molecular_weight_method"), + "buffer_description": ("buffer", "buffer_description", "buffer_composition"), + "instrument": ("instrument", "beamline", "instrument_name"), + "detector": ("detector", "detector_name", "detector_type"), + "publication_title": ("publication_title", "pub_title", "paper_title"), + "publication_doi": ("doi", "publication_doi", "pub_doi"), + "publication_pmid": ("pmid", "publication_pmid", "pubmed_id"), + "intensities_data_url": ("intensities_data", "data_url", "intensities_url"), + "pddf_data_url": ("pddf_data", "pddf_url", "pr_data"), +} + +_FLOAT_FIELDS = { + "concentration": ("concentration", "sample_concentration", "conc", "sample_conc"), + "molecular_weight": ("molecular_weight", "mw", "exp_mw", "experimental_mw", "mw_kda"), + "ph": ("ph", "buffer_ph"), + "wavelength": ("wavelength", "xray_wavelength", "neutron_wavelength"), + "temperature": ("temperature", "sample_temperature", "temp"), + "rg": ("rg", "radius_of_gyration", "guinier_rg"), + "rg_error": ("rg_error", "rg_err", "guinier_rg_error"), + "i0": ("i0", "i_zero", "guinier_i0"), + "i0_error": ("i0_error", "i0_err", "guinier_i0_error"), + "dmax": ("dmax", "d_max", "maximum_dimension"), + "porod_volume": ("porod_volume", "volume", "porod_vol"), + "q_min": ("q_min", "qmin", "s_min", "smin"), + "q_max": ("q_max", "qmax", "s_max", "smax"), +} + + +@dataclass +class SASBDBDatasetInfo: + """Parsed metadata from a SASBDB dataset entry.""" + entry_id: str = "" + code: str = "" + title: str = "" + sample_name: str = "" + sample_description: str = "" + concentration: float | None = None + concentration_unit: str = "mg/mL" + molecule_name: str = "" + molecule_short_name: str = "" + molecule_type: str = "" + sequence: str = "" + uniprot_code: str = "" + source_organism: str = "" + number_of_molecules: str = "" + oligomerization: str = "" + molecular_weight: float | None = None + molecular_weight_method: str = "" + oligomeric_state: str = "" + buffer_description: str = "" + ph: float | None = None + instrument: str = "" + detector: str = "" + wavelength: float | None = None + wavelength_unit: str = "Å" + temperature: float | None = None + temperature_unit: str = "K" + rg: float | None = None + rg_error: float | None = None + i0: float | None = None + i0_error: float | None = None + dmax: float | None = None + porod_volume: float | None = None + q_min: float | None = None + q_max: float | None = None + publication_title: str = "" + publication_doi: str = "" + publication_pmid: str = "" + authors: list = field(default_factory=list) + intensities_data_url: str = "" + pddf_data_url: str = "" + raw_metadata: dict = field(default_factory=dict) + + +def parseMetadata(metadata: dict) -> SASBDBDatasetInfo: + """Parse SASBDB API response into a structured SASBDBDatasetInfo object.""" + info = SASBDBDatasetInfo() + if not isinstance(metadata, dict): + return info + + info.raw_metadata = metadata + logger.debug("SASBDB API response keys: %s", list(metadata.keys())) + + for attr, keys in _STR_FIELDS.items(): + setattr(info, attr, _get_str(metadata, *keys)) + for attr, keys in _FLOAT_FIELDS.items(): + setattr(info, attr, _get_float(metadata, *keys)) + + info.sequence = _get_sequence(metadata) + info.concentration_unit = info.concentration_unit or "mg/mL" + info.oligomeric_state = info.oligomerization or _get_str( + metadata, "oligomeric_state", "oligomer_state", "oligomerization" + ) + + authors = metadata.get("authors") or metadata.get("author_list") or [] + if isinstance(authors, list): + info.authors = [str(author) for author in authors] + elif isinstance(authors, str): + info.authors = [authors] + + return info + + +def _get_str(data: dict, *keys: str) -> str: + value = _find_shallow(data, keys, as_float=False) + return value if isinstance(value, str) else _find_deep(data, keys, as_float=False) or "" + + +def _get_float(data: dict, *keys: str) -> float | None: + value = _find_shallow(data, keys, as_float=True) + return value if isinstance(value, float) else _find_deep(data, keys, as_float=True) + + +def _find_shallow(data: dict, keys: tuple[str, ...], *, as_float: bool): + for key in keys: + if key in data and data[key] is not None: + converted = _convert_value(data[key], as_float=as_float) + if _is_present(converted, as_float): + return converted + + for path in _NESTED_PATHS: + nested = data.get(path) + if isinstance(nested, dict): + for key in keys: + if key in nested and nested[key] is not None: + converted = _convert_value(nested[key], as_float=as_float) + if _is_present(converted, as_float): + return converted + return None + + +def _find_deep(data, keys: tuple[str, ...], *, as_float: bool): + key_set = set(keys) + + def walk(obj): + if isinstance(obj, dict): + for key, value in obj.items(): + if key in key_set: + converted = _convert_value(value, as_float=as_float) + if _is_present(converted, as_float): + return converted + for value in obj.values(): + found = walk(value) + if found is not None: + return found + elif isinstance(obj, list): + for item in obj: + found = walk(item) + if found is not None: + return found + return None + + return walk(data) + + +def _convert_value(value, *, as_float: bool): + if value is None: + return None + if as_float: + try: + return float(value) + except (ValueError, TypeError): + return None + if isinstance(value, str): + return value.strip() + if isinstance(value, (int, float, bool)): + return str(value) + return "" + + +def _is_present(value, as_float: bool) -> bool: + if as_float: + return value is not None + return bool(value) + + +def _get_sequence(data: dict) -> str: + def walk(obj) -> str: + if isinstance(obj, dict): + for key in _SEQUENCE_KEYS: + if key in obj and isinstance(obj[key], str) and obj[key].strip(): + return obj[key].strip() + for value in obj.values(): + found = walk(value) + if found: + return found + elif isinstance(obj, list): + for item in obj: + found = walk(item) + if found: + return found + return "" + + return walk(data) if isinstance(data, dict) else ""