From 9e933173b6a36937514cda5fe6a6495630092c80 Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Thu, 13 Nov 2025 07:52:50 +0800 Subject: [PATCH 001/110] Add "scan_maven_package" pipeline (Working-in-progress) #1763 Signed-off-by: Chin Yeung Li --- pyproject.toml | 1 + scanpipe/pipelines/scan_maven_package.py | 162 ++++++++++++++++++++++ scanpipe/pipes/resolve.py | 168 +++++++++++++++++++++++ 3 files changed, 331 insertions(+) create mode 100644 scanpipe/pipelines/scan_maven_package.py diff --git a/pyproject.toml b/pyproject.toml index f0ae21f332..3fea1404fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -161,6 +161,7 @@ resolve_dependencies = "scanpipe.pipelines.resolve_dependencies:ResolveDependenc scan_codebase = "scanpipe.pipelines.scan_codebase:ScanCodebase" scan_for_virus = "scanpipe.pipelines.scan_for_virus:ScanForVirus" scan_single_package = "scanpipe.pipelines.scan_single_package:ScanSinglePackage" +scan_maven_package = "scanpipe.pipelines.scan_maven_package:ScanMavenPackage" [tool.setuptools.packages.find] where = ["."] diff --git a/scanpipe/pipelines/scan_maven_package.py b/scanpipe/pipelines/scan_maven_package.py new file mode 100644 index 0000000000..841899ff88 --- /dev/null +++ b/scanpipe/pipelines/scan_maven_package.py @@ -0,0 +1,162 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + +import json + +from django.core.serializers.json import DjangoJSONEncoder + +from commoncode.hash import multi_checksums + +from scanpipe.pipelines import Pipeline +from scanpipe.pipes import input +from scanpipe.pipes import scancode +from scanpipe.pipes.input import copy_input +from scanpipe.pipes.input import is_archive + +from scanpipe.pipes.resolve import get_pom_url_list +from scanpipe.pipes.resolve import download_and_scan_pom_file + + +class ScanMavenPackage(Pipeline): + """ + Scan a single package archive (or package manifest file). + + This pipeline scans a single package for package metadata, + declared dependencies, licenses, license clarity score and copyrights. + + The output is a summary of the scan results in JSON format. + """ + + @classmethod + def steps(cls): + return ( + cls.get_package_input, + cls.collect_input_information, + cls.extract_input_to_codebase_directory, + cls.extract_archives, + cls.run_scan, + cls.fetch_and_scan_remote_pom, + cls.load_inventory_from_toolkit_scan, + cls.make_summary_from_scan_results, + ) + + scancode_run_scan_args = { + "copyright": True, + "email": True, + "info": True, + "license": True, + "license_text": True, + "license_diagnostics": True, + "license_text_diagnostics": True, + "license_references": True, + "package": True, + "url": True, + "classify": True, + "summary": True, + "todo": True, + } + + def get_package_input(self): + """Locate the package input in the project's input/ directory.""" + # Using the input_sources model property as it includes input sources instances + # as well as any files manually copied into the input/ directory. + input_sources = self.project.input_sources + inputs = list(self.project.inputs("*")) + + if len(inputs) != 1 or len(input_sources) != 1: + raise Exception("Only 1 input file supported") + + self.input_path = inputs[0] + + def collect_input_information(self): + """Collect and store information about the project input.""" + self.project.update_extra_data( + { + "filename": self.input_path.name, + "size": self.input_path.stat().st_size, + **multi_checksums(self.input_path), + } + ) + + def extract_input_to_codebase_directory(self): + """Copy or extract input to project codebase/ directory.""" + if not is_archive(self.input_path): + copy_input(self.input_path, self.project.codebase_path) + return + + self.extract_archive(self.input_path, self.project.codebase_path) + + # Reload the project env post-extraction as the scancode-config.yml file + # may be located in one of the extracted archives. + self.env = self.project.get_env() + + def run_scan(self): + """Scan extracted codebase/ content.""" + scan_output_path = self.project.get_output_file_path("scancode", "json") + self.scan_output_location = str(scan_output_path.absolute()) + + scanning_errors = scancode.run_scan( + location=str(self.project.codebase_path), + output_file=self.scan_output_location, + run_scan_args=self.scancode_run_scan_args.copy(), + ) + + for resource_path, errors in scanning_errors.items(): + self.project.add_error( + description="\n".join(errors), + model=self.pipeline_name, + details={"resource_path": resource_path.removeprefix("codebase/")}, + ) + + if not scan_output_path.exists(): + raise FileNotFoundError("ScanCode output not available.") + + def fetch_and_scan_remote_pom(self): + """Fetch the pom.xml file from from maven.org if not present in codebase.""" + # TODO Verify if the following filter actually work + if not self.project.codebaseresources.files().filter(name="pom.xml").exists(): + with open(self.scan_output_location, 'r') as file: + data = json.load(file) + packages = data.get("packages", []) + + pom_url_list = get_pom_url_list(self.project.input_sources[0], packages) + scanned_pom_packages, scanned_dependencies = download_and_scan_pom_file(pom_url_list) + + updated_pacakges = packages + scanned_pom_packages + # Replace/Update the package and dependencies section + data['packages'] = updated_pacakges + # Need to update the dependencies + # data['dependencies'] = scanned_dependencies + with open(self.scan_output_location, 'w') as file: + json.dump(data, file, indent=2) + + def load_inventory_from_toolkit_scan(self): + """Process a JSON Scan results to populate codebase resources and packages.""" + input.load_inventory_from_toolkit_scan(self.project, self.scan_output_location) + + def make_summary_from_scan_results(self): + """Build a summary in JSON format from the generated scan results.""" + summary = scancode.make_results_summary(self.project, self.scan_output_location) + output_file = self.project.get_output_file_path("summary", "json") + + with output_file.open("w") as summary_file: + summary_file.write(json.dumps(summary, indent=2, cls=DjangoJSONEncoder)) diff --git a/scanpipe/pipes/resolve.py b/scanpipe/pipes/resolve.py index 0a409dd88c..f9ea444301 100644 --- a/scanpipe/pipes/resolve.py +++ b/scanpipe/pipes/resolve.py @@ -22,6 +22,8 @@ import json import logging +import re +import requests import sys import uuid from pathlib import Path @@ -44,6 +46,9 @@ from scanpipe.pipes import update_or_create_dependency from scanpipe.pipes import update_or_create_package +from scanpipe.pipes import fetch +from scanpipe.pipes import scancode + """ Resolve packages from manifest, lockfile, and SBOM. """ @@ -521,3 +526,166 @@ def extract_headers(input_location, extract_fields): return extracted_headers return {} + + +def parse_maven_filename(filename): + """Parse a Maven's jar filename to extract artifactId and version.""" + # Remove the .jar extension + base = filename.rsplit('.', 1)[0] + + # Common classifiers pattern + common_classifiers = { + 'sources', 'javadoc', 'tests', 'test', 'test-sources', + 'src', 'bin', 'docs', 'javadocs', 'client', 'server', + 'linux', 'windows', 'macos', 'linux-x86_64', 'windows-x86_64' + } + + # Remove known classifier if present + for classifier in common_classifiers: + if base.endswith(f"-{classifier}"): + base = base[:-(len(classifier) + 1)] + break + + # Match artifactId and version + match = re.match(r'^(.*)-(\d[\w.\-]+)$', base) + if match: + artifact_id = match.group(1) + version = match.group(2) + return artifact_id, version + else: + return None, None + + +def get_pom_url_list(input_source, packages): + pom_url_list = [] + if packages: + for package in packages: + package_ns = package.get("namespace", "") + package_name = package.get("name", "") + package_version = package.get("version", "") + pom_url = f"https://repo1.maven.org/maven2/{package_ns.replace('.', '/')}/{package_name}/{package_version}/{package_name}-{package_version}.pom".lower() + pom_url_list.append(pom_url) + else: + # Check what's the input source + input_source_url = input_source.get("download_url", "") + + if input_source_url and "maven.org/" in input_source_url: + base_url = input_source_url.rsplit('/', 1)[0] + pom_url = base_url + "/" + "-".join(base_url.rstrip("/").split("/")[-2:]) + ".pom" + pom_url_list.append(pom_url) + else: + # Construct a pom_url from filename + input_filename = input_source.get("filename", "") + if input_filename.endswith(".jar"): + artifact_id, version = parse_maven_filename(input_filename) + if not artifact_id or not version: + return [] + pom_url_list = construct_pom_url_from_filename(artifact_id, version) + else: + # Only work with input that's a .jar file + return [] + + return pom_url_list + + +def construct_pom_url_from_filename(artifact_id, version): + """Construct a pom.xml URL from the given Maven filename.""" + # Search Maven Central for the artifact to get its groupId + url = f"https://search.maven.org/solrsearch/select?q=a:{artifact_id}&wt=json" + pom_url_list = [] + group_ids = [] + try: + response = requests.get(url) + response.raise_for_status() + data = response.json() + # Extract all 'g' fields from the docs array that represent + # groupIds + group_ids = [doc['g'] for doc in data['response']['docs']] + except requests.RequestException as e: + print(f"Error fetching data: {e}") + return [] + except KeyError as e: + print(f"Error parsing JSON: {e}") + return [] + + for group_id in group_ids: + pom_url = f"https://repo1.maven.org/maven2/{group_id.replace('.', '/')}/{artifact_id}/{version}/{artifact_id}-{version}.pom".lower() + if is_maven_pom_url(pom_url): + pom_url_list.append(pom_url) + if len(pom_url_list) > 1: + # If multiple valid POM URLs are found, it means the same + # artifactId and version exist under different groupIds. Since we + # can't confidently determine the correct groupId, we return an + # empty list to avoid fetching the wrong POM. + return [] + + return pom_url_list + + +def is_maven_pom_url(url): + """ + Return True if the url is a accessible, False otherwise + Maven Central has a fallback mechanism that serves a generic/error page + instead of returning a proper 404. + """ + try: + response = requests.get(url, timeout=5) + if response.status_code != 200: + return False + # Check content-type + content_type = response.headers.get('content-type', '').lower() + is_xml = 'xml' in content_type or 'text/xml' in content_type + + # Check content + content = response.text.strip() + is_pom = content.startswith(' Date: Thu, 13 Nov 2025 09:58:45 +0800 Subject: [PATCH 002/110] Use pom_url as the datafile_path for the dependenies #1763 Signed-off-by: Chin Yeung Li --- scanpipe/pipes/resolve.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scanpipe/pipes/resolve.py b/scanpipe/pipes/resolve.py index f9ea444301..431108a7a6 100644 --- a/scanpipe/pipes/resolve.py +++ b/scanpipe/pipes/resolve.py @@ -685,7 +685,7 @@ def download_and_scan_pom_file(pom_url_list): scanned_pom_packages.append(scanned_package) if scanned_dependencies: for scanned_dep in scanned_dependencies: - # Replace the 'datafile_path' with the empty list - scanned_dep['datafile_path'] = scanned_pom_output_path + # Replace the 'datafile_path' with the pom_url + scanned_dep['datafile_path'] = pom_url scanned_pom_deps.append(scanned_dep) return scanned_pom_packages, scanned_pom_deps From d63a1e539094239244280e85e4ec7ba3de6de82f Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Thu, 13 Nov 2025 17:36:50 +0800 Subject: [PATCH 003/110] Use empty string for datafile_path in dependencies #1763 Signed-off-by: Chin Yeung Li --- scanpipe/pipes/resolve.py | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/scanpipe/pipes/resolve.py b/scanpipe/pipes/resolve.py index 431108a7a6..c5a32918ef 100644 --- a/scanpipe/pipes/resolve.py +++ b/scanpipe/pipes/resolve.py @@ -557,6 +557,7 @@ def parse_maven_filename(filename): def get_pom_url_list(input_source, packages): + """Generate Maven POM URLs from package metadata or input source.""" pom_url_list = [] if packages: for package in packages: @@ -623,11 +624,9 @@ def construct_pom_url_from_filename(artifact_id, version): def is_maven_pom_url(url): - """ - Return True if the url is a accessible, False otherwise - Maven Central has a fallback mechanism that serves a generic/error page - instead of returning a proper 404. - """ + """Return True if the url is a accessible, False otherwise""" + # Maven Central has a fallback mechanism that serves a generic/error + # page instead of returning a proper 404. try: response = requests.get(url, timeout=5) if response.status_code != 200: @@ -650,6 +649,7 @@ def is_maven_pom_url(url): def download_and_scan_pom_file(pom_url_list): + """Fetch and scan the pom file from the input pom_url_list""" scanned_pom_packages = [] scanned_pom_deps = [] for pom_url in pom_url_list: @@ -661,16 +661,7 @@ def download_and_scan_pom_file(pom_url_list): location=str(downloaded_pom.path), output_file=scanned_pom_output_path, run_scan_args={ - "copyright": True, - "email": True, - "info": True, - "license": True, - "license_text": True, - "license_diagnostics": True, - "license_text_diagnostics": True, - "license_references": True, "package": True, - "url": True, }, ) @@ -685,7 +676,8 @@ def download_and_scan_pom_file(pom_url_list): scanned_pom_packages.append(scanned_package) if scanned_dependencies: for scanned_dep in scanned_dependencies: - # Replace the 'datafile_path' with the pom_url - scanned_dep['datafile_path'] = pom_url + # Replace the 'datafile_path' with empty string + # See https://github.com/aboutcode-org/scancode.io/issues/1763#issuecomment-3525165830 + scanned_dep['datafile_path'] = "" scanned_pom_deps.append(scanned_dep) return scanned_pom_packages, scanned_pom_deps From 9812129f0f1a63d496d514d38710a62a93bedb65 Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Thu, 13 Nov 2025 17:37:18 +0800 Subject: [PATCH 004/110] Removed dup code that's already present in ScanSinglePackage #1763 Signed-off-by: Chin Yeung Li --- scanpipe/pipelines/scan_maven_package.py | 128 ++++------------------- 1 file changed, 19 insertions(+), 109 deletions(-) diff --git a/scanpipe/pipelines/scan_maven_package.py b/scanpipe/pipelines/scan_maven_package.py index 841899ff88..a06697ba6f 100644 --- a/scanpipe/pipelines/scan_maven_package.py +++ b/scanpipe/pipelines/scan_maven_package.py @@ -22,21 +22,13 @@ import json -from django.core.serializers.json import DjangoJSONEncoder - -from commoncode.hash import multi_checksums - -from scanpipe.pipelines import Pipeline -from scanpipe.pipes import input -from scanpipe.pipes import scancode -from scanpipe.pipes.input import copy_input -from scanpipe.pipes.input import is_archive +from scanpipe.pipelines.scan_single_package import ScanSinglePackage from scanpipe.pipes.resolve import get_pom_url_list from scanpipe.pipes.resolve import download_and_scan_pom_file -class ScanMavenPackage(Pipeline): +class ScanMavenPackage(ScanSinglePackage): """ Scan a single package archive (or package manifest file). @@ -59,104 +51,22 @@ def steps(cls): cls.make_summary_from_scan_results, ) - scancode_run_scan_args = { - "copyright": True, - "email": True, - "info": True, - "license": True, - "license_text": True, - "license_diagnostics": True, - "license_text_diagnostics": True, - "license_references": True, - "package": True, - "url": True, - "classify": True, - "summary": True, - "todo": True, - } - - def get_package_input(self): - """Locate the package input in the project's input/ directory.""" - # Using the input_sources model property as it includes input sources instances - # as well as any files manually copied into the input/ directory. - input_sources = self.project.input_sources - inputs = list(self.project.inputs("*")) - - if len(inputs) != 1 or len(input_sources) != 1: - raise Exception("Only 1 input file supported") - - self.input_path = inputs[0] - - def collect_input_information(self): - """Collect and store information about the project input.""" - self.project.update_extra_data( - { - "filename": self.input_path.name, - "size": self.input_path.stat().st_size, - **multi_checksums(self.input_path), - } - ) - - def extract_input_to_codebase_directory(self): - """Copy or extract input to project codebase/ directory.""" - if not is_archive(self.input_path): - copy_input(self.input_path, self.project.codebase_path) - return - - self.extract_archive(self.input_path, self.project.codebase_path) - - # Reload the project env post-extraction as the scancode-config.yml file - # may be located in one of the extracted archives. - self.env = self.project.get_env() - - def run_scan(self): - """Scan extracted codebase/ content.""" - scan_output_path = self.project.get_output_file_path("scancode", "json") - self.scan_output_location = str(scan_output_path.absolute()) - - scanning_errors = scancode.run_scan( - location=str(self.project.codebase_path), - output_file=self.scan_output_location, - run_scan_args=self.scancode_run_scan_args.copy(), - ) - - for resource_path, errors in scanning_errors.items(): - self.project.add_error( - description="\n".join(errors), - model=self.pipeline_name, - details={"resource_path": resource_path.removeprefix("codebase/")}, - ) - - if not scan_output_path.exists(): - raise FileNotFoundError("ScanCode output not available.") - def fetch_and_scan_remote_pom(self): """Fetch the pom.xml file from from maven.org if not present in codebase.""" - # TODO Verify if the following filter actually work - if not self.project.codebaseresources.files().filter(name="pom.xml").exists(): - with open(self.scan_output_location, 'r') as file: - data = json.load(file) - packages = data.get("packages", []) - - pom_url_list = get_pom_url_list(self.project.input_sources[0], packages) - scanned_pom_packages, scanned_dependencies = download_and_scan_pom_file(pom_url_list) - - updated_pacakges = packages + scanned_pom_packages - # Replace/Update the package and dependencies section - data['packages'] = updated_pacakges - # Need to update the dependencies - # data['dependencies'] = scanned_dependencies - with open(self.scan_output_location, 'w') as file: - json.dump(data, file, indent=2) - - def load_inventory_from_toolkit_scan(self): - """Process a JSON Scan results to populate codebase resources and packages.""" - input.load_inventory_from_toolkit_scan(self.project, self.scan_output_location) - - def make_summary_from_scan_results(self): - """Build a summary in JSON format from the generated scan results.""" - summary = scancode.make_results_summary(self.project, self.scan_output_location) - output_file = self.project.get_output_file_path("summary", "json") - - with output_file.open("w") as summary_file: - summary_file.write(json.dumps(summary, indent=2, cls=DjangoJSONEncoder)) + with open(self.scan_output_location, 'r') as file: + data = json.load(file) + # Return and do nothing if data has pom.xml + for file in data['files']: + if 'pom.xml' in file['path']: + return + packages = data.get("packages", []) + + pom_url_list = get_pom_url_list(self.project.input_sources[0], packages) + scanned_pom_packages, scanned_dependencies = download_and_scan_pom_file(pom_url_list) + + updated_pacakges = packages + scanned_pom_packages + # Replace/Update the package and dependencies section + data['packages'] = updated_pacakges + data['dependencies'] = scanned_dependencies + with open(self.scan_output_location, 'w') as file: + json.dump(data, file, indent=2) From 114eb75f8bf77d5cf5212facb0e01d726428d93e Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Thu, 13 Nov 2025 18:38:51 +0800 Subject: [PATCH 005/110] Update the matching regex for parse_maven_filename and added test #1763 - Update format Signed-off-by: Chin Yeung Li --- scanpipe/pipelines/scan_maven_package.py | 19 +++---- scanpipe/pipes/resolve.py | 68 ++++++++++++++++-------- scanpipe/tests/pipes/test_resolve.py | 27 ++++++++++ 3 files changed, 83 insertions(+), 31 deletions(-) diff --git a/scanpipe/pipelines/scan_maven_package.py b/scanpipe/pipelines/scan_maven_package.py index a06697ba6f..00ddc5e500 100644 --- a/scanpipe/pipelines/scan_maven_package.py +++ b/scanpipe/pipelines/scan_maven_package.py @@ -23,9 +23,8 @@ import json from scanpipe.pipelines.scan_single_package import ScanSinglePackage - -from scanpipe.pipes.resolve import get_pom_url_list from scanpipe.pipes.resolve import download_and_scan_pom_file +from scanpipe.pipes.resolve import get_pom_url_list class ScanMavenPackage(ScanSinglePackage): @@ -53,20 +52,22 @@ def steps(cls): def fetch_and_scan_remote_pom(self): """Fetch the pom.xml file from from maven.org if not present in codebase.""" - with open(self.scan_output_location, 'r') as file: + with open(self.scan_output_location) as file: data = json.load(file) # Return and do nothing if data has pom.xml - for file in data['files']: - if 'pom.xml' in file['path']: + for file in data["files"]: + if "pom.xml" in file["path"]: return packages = data.get("packages", []) pom_url_list = get_pom_url_list(self.project.input_sources[0], packages) - scanned_pom_packages, scanned_dependencies = download_and_scan_pom_file(pom_url_list) + scanned_pom_packages, scanned_dependencies = download_and_scan_pom_file( + pom_url_list + ) updated_pacakges = packages + scanned_pom_packages # Replace/Update the package and dependencies section - data['packages'] = updated_pacakges - data['dependencies'] = scanned_dependencies - with open(self.scan_output_location, 'w') as file: + data["packages"] = updated_pacakges + data["dependencies"] = scanned_dependencies + with open(self.scan_output_location, "w") as file: json.dump(data, file, indent=2) diff --git a/scanpipe/pipes/resolve.py b/scanpipe/pipes/resolve.py index c5a32918ef..e94de32e41 100644 --- a/scanpipe/pipes/resolve.py +++ b/scanpipe/pipes/resolve.py @@ -23,7 +23,6 @@ import json import logging import re -import requests import sys import uuid from pathlib import Path @@ -32,6 +31,7 @@ from django.core.exceptions import ObjectDoesNotExist import python_inspector.api as python_inspector +import requests import saneyaml from attributecode.model import About from packagedcode import APPLICATION_PACKAGE_DATAFILE_HANDLERS @@ -41,14 +41,13 @@ from scanpipe.models import DiscoveredDependency from scanpipe.models import DiscoveredPackage from scanpipe.pipes import cyclonedx +from scanpipe.pipes import fetch from scanpipe.pipes import flag +from scanpipe.pipes import scancode from scanpipe.pipes import spdx from scanpipe.pipes import update_or_create_dependency from scanpipe.pipes import update_or_create_package -from scanpipe.pipes import fetch -from scanpipe.pipes import scancode - """ Resolve packages from manifest, lockfile, and SBOM. """ @@ -531,26 +530,42 @@ def extract_headers(input_location, extract_fields): def parse_maven_filename(filename): """Parse a Maven's jar filename to extract artifactId and version.""" # Remove the .jar extension - base = filename.rsplit('.', 1)[0] + base = filename.rsplit(".", 1)[0] # Common classifiers pattern common_classifiers = { - 'sources', 'javadoc', 'tests', 'test', 'test-sources', - 'src', 'bin', 'docs', 'javadocs', 'client', 'server', - 'linux', 'windows', 'macos', 'linux-x86_64', 'windows-x86_64' + "sources", + "javadoc", + "tests", + "test", + "test-sources", + "src", + "bin", + "docs", + "javadocs", + "client", + "server", + "linux", + "windows", + "macos", + "linux-x86_64", + "windows-x86_64", } # Remove known classifier if present for classifier in common_classifiers: if base.endswith(f"-{classifier}"): - base = base[:-(len(classifier) + 1)] + base = base[: -(len(classifier) + 1)] break # Match artifactId and version - match = re.match(r'^(.*)-(\d[\w.\-]+)$', base) + match = re.match(r"^(.*?)-((\d[\w.\-]*))$", base) + if match: artifact_id = match.group(1) version = match.group(2) + print("artifact_id", artifact_id) + print("version", version) return artifact_id, version else: return None, None @@ -564,15 +579,21 @@ def get_pom_url_list(input_source, packages): package_ns = package.get("namespace", "") package_name = package.get("name", "") package_version = package.get("version", "") - pom_url = f"https://repo1.maven.org/maven2/{package_ns.replace('.', '/')}/{package_name}/{package_version}/{package_name}-{package_version}.pom".lower() + pom_url = ( + f"https://repo1.maven.org/maven2/{package_ns.replace('.', '/')}/" + f"{package_name}/{package_version}/" + f"{package_name}-{package_version}.pom".lower() + ) pom_url_list.append(pom_url) else: # Check what's the input source input_source_url = input_source.get("download_url", "") if input_source_url and "maven.org/" in input_source_url: - base_url = input_source_url.rsplit('/', 1)[0] - pom_url = base_url + "/" + "-".join(base_url.rstrip("/").split("/")[-2:]) + ".pom" + base_url = input_source_url.rsplit("/", 1)[0] + pom_url = ( + base_url + "/" + "-".join(base_url.rstrip("/").split("/")[-2:]) + ".pom" + ) pom_url_list.append(pom_url) else: # Construct a pom_url from filename @@ -596,12 +617,12 @@ def construct_pom_url_from_filename(artifact_id, version): pom_url_list = [] group_ids = [] try: - response = requests.get(url) + response = requests.get(url, timeout=5) response.raise_for_status() data = response.json() # Extract all 'g' fields from the docs array that represent # groupIds - group_ids = [doc['g'] for doc in data['response']['docs']] + group_ids = [doc["g"] for doc in data["response"]["docs"]] except requests.RequestException as e: print(f"Error fetching data: {e}") return [] @@ -610,7 +631,10 @@ def construct_pom_url_from_filename(artifact_id, version): return [] for group_id in group_ids: - pom_url = f"https://repo1.maven.org/maven2/{group_id.replace('.', '/')}/{artifact_id}/{version}/{artifact_id}-{version}.pom".lower() + pom_url = ( + f"https://repo1.maven.org/maven2/{group_id.replace('.', '/')}/" + f"{artifact_id}/{version}/{artifact_id}-{version}.pom".lower() + ) if is_maven_pom_url(pom_url): pom_url_list.append(pom_url) if len(pom_url_list) > 1: @@ -632,12 +656,12 @@ def is_maven_pom_url(url): if response.status_code != 200: return False # Check content-type - content_type = response.headers.get('content-type', '').lower() - is_xml = 'xml' in content_type or 'text/xml' in content_type + content_type = response.headers.get("content-type", "").lower() + is_xml = "xml" in content_type or "text/xml" in content_type # Check content content = response.text.strip() - is_pom = content.startswith(' Date: Fri, 14 Nov 2025 15:21:15 +0800 Subject: [PATCH 006/110] Refactor code and add tests #1763 Signed-off-by: Chin Yeung Li --- scanpipe/pipelines/scan_maven_package.py | 10 +- scanpipe/pipes/resolve.py | 30 +++- scanpipe/tests/pipes/test_resolve.py | 186 +++++++++++++++++++++++ 3 files changed, 215 insertions(+), 11 deletions(-) diff --git a/scanpipe/pipelines/scan_maven_package.py b/scanpipe/pipelines/scan_maven_package.py index 00ddc5e500..7438d781c0 100644 --- a/scanpipe/pipelines/scan_maven_package.py +++ b/scanpipe/pipelines/scan_maven_package.py @@ -23,8 +23,9 @@ import json from scanpipe.pipelines.scan_single_package import ScanSinglePackage -from scanpipe.pipes.resolve import download_and_scan_pom_file +from scanpipe.pipes.resolve import download_pom_files from scanpipe.pipes.resolve import get_pom_url_list +from scanpipe.pipes.resolve import scan_pom_files class ScanMavenPackage(ScanSinglePackage): @@ -51,7 +52,7 @@ def steps(cls): ) def fetch_and_scan_remote_pom(self): - """Fetch the pom.xml file from from maven.org if not present in codebase.""" + """Fetch the .pom file from from maven.org if not present in codebase.""" with open(self.scan_output_location) as file: data = json.load(file) # Return and do nothing if data has pom.xml @@ -61,9 +62,8 @@ def fetch_and_scan_remote_pom(self): packages = data.get("packages", []) pom_url_list = get_pom_url_list(self.project.input_sources[0], packages) - scanned_pom_packages, scanned_dependencies = download_and_scan_pom_file( - pom_url_list - ) + pom_file_list = download_pom_files(pom_url_list) + scanned_pom_packages, scanned_dependencies = scan_pom_files(pom_file_list) updated_pacakges = packages + scanned_pom_packages # Replace/Update the package and dependencies section diff --git a/scanpipe/pipes/resolve.py b/scanpipe/pipes/resolve.py index e94de32e41..b672df13bc 100644 --- a/scanpipe/pipes/resolve.py +++ b/scanpipe/pipes/resolve.py @@ -586,10 +586,13 @@ def get_pom_url_list(input_source, packages): ) pom_url_list.append(pom_url) else: + from urllib.parse import urlparse + # Check what's the input source input_source_url = input_source.get("download_url", "") - if input_source_url and "maven.org/" in input_source_url: + parsed_url = urlparse(input_source_url) + if input_source_url and parsed_url.netloc.endswith("maven.org"): base_url = input_source_url.rsplit("/", 1)[0] pom_url = ( base_url + "/" + "-".join(base_url.rstrip("/").split("/")[-2:]) + ".pom" @@ -672,17 +675,32 @@ def is_maven_pom_url(url): return False -def download_and_scan_pom_file(pom_url_list): +def download_pom_files(pom_url_list): + """Fetch the pom file from the input pom_url_list""" + pom_file_list = [] + for pom_url in pom_url_list: + pom_file_dict = {} + downloaded_pom = fetch.fetch_http(pom_url) + print("download_pom.path", str(downloaded_pom.path)) + pom_file_dict["pom_file_path"] = str(downloaded_pom.path) + pom_file_dict["output_path"] = str(downloaded_pom.path) + "-output.json" + pom_file_dict["pom_url"] = pom_url + pom_file_list.append(pom_file_dict) + return pom_file_list + + +def scan_pom_files(pom_file_list): """Fetch and scan the pom file from the input pom_url_list""" scanned_pom_packages = [] scanned_pom_deps = [] - for pom_url in pom_url_list: - downloaded_pom = fetch.fetch_http(pom_url) - scanned_pom_output_path = str(downloaded_pom.path) + "-output.json" + for pom_file_dict in pom_file_list: + pom_file_path = pom_file_dict.get("pom_file_path", "") + scanned_pom_output_path = pom_file_dict.get("output_path", "") + pom_url = pom_file_dict.get("pom_url", "") # Run a package scan on the fetched pom.xml _scanning_errors = scancode.run_scan( - location=str(downloaded_pom.path), + location=pom_file_path, output_file=scanned_pom_output_path, run_scan_args={ "package": True, diff --git a/scanpipe/tests/pipes/test_resolve.py b/scanpipe/tests/pipes/test_resolve.py index f60608ee04..016d00bd7c 100644 --- a/scanpipe/tests/pipes/test_resolve.py +++ b/scanpipe/tests/pipes/test_resolve.py @@ -400,3 +400,189 @@ def test_scanpipe_resolve_parse_maven_filename(self): self.assertEqual(result3_version, expected3_version) self.assertEqual(result4_name, expected2_name) self.assertEqual(result4_version, expected2_version) + + @mock.patch("requests.get") + def test_scanpipe_resolve_is_maven_pom_url_valid(self, mock_get): + mock_response = mock.Mock() + mock_response.status_code = 200 + mock_response.headers = {"content-type": "application/xml"} + mock_response.text = '' + mock_get.return_value = mock_response + + result = resolve.is_maven_pom_url( + "https://repo1.maven.org/maven2/example/example.pom" + ) + self.assertTrue(result) + + @mock.patch("requests.get") + def test_scanpipe_resolve_is_maven_pom_url_404(self, mock_get): + mock_response = mock.Mock() + mock_response.status_code = 404 + mock_get.return_value = mock_response + + result = resolve.is_maven_pom_url( + "https://repo.maven.apache.org/maven2/example/404.pom" + ) + self.assertFalse(result) + + @mock.patch("requests.get") + def test_scanpipe_resolve_is_maven_pom_url_error(self, mock_get): + mock_response = mock.Mock() + mock_response.status_code = 200 + mock_response.headers = {"content-type": "text/html"} + mock_response.text = "Error page" + mock_get.return_value = mock_response + + result = resolve.is_maven_pom_url( + "https://repo.maven.apache.org/maven2/example/error.pom" + ) + self.assertFalse(result) + + @mock.patch("scanpipe.pipes.resolve.fetch.fetch_http") + def test_scanpipe_resolve_download_pom_files(self, mock_fetch_http): + mock_response = mock.Mock() + mock_response.path = "/safe/example1.pom" + mock_fetch_http.return_value = mock_response + + pom_urls = ["https://repo1.maven.org/maven2/example/example1.pom"] + + expected = [ + { + "pom_file_path": "/safe/example1.pom", + "output_path": "/safe/example1.pom-output.json", + "pom_url": "https://repo1.maven.org/maven2/example/example1.pom", + } + ] + + result = resolve.download_pom_files(pom_urls) + self.assertEqual(result, expected) + + @mock.patch("scanpipe.pipes.resolve.scancode.run_scan") + @mock.patch("builtins.open", new_callable=mock.mock_open) + @mock.patch("json.load") + def test_scanpipe_resolve_scan_pom_files( + self, mock_json_load, mock_open, mock_run_scan + ): + mock_json_load.return_value = { + "packages": [ + { + "name": "example-package", + "version": "1.0.0", + "datafile_paths": ["/safe/mock_pom.xml"], + } + ], + "dependencies": [ + { + "name": "example-dep", + "version": "2.0.0", + "datafile_path": "/safe/mock_pom.xml", + } + ], + } + + pom_file_list = [ + { + "pom_file_path": "/safe/mock.pom", + "output_path": "/safe/mock.pom-output.json", + "pom_url": "https://repo1.maven.org/maven2/example/example.pom", + } + ] + + expected_packages = [ + { + "name": "example-package", + "version": "1.0.0", + "datafile_paths": [ + "https://repo1.maven.org/maven2/example/example.pom" + ], + } + ] + expected_deps = [ + {"name": "example-dep", "version": "2.0.0", "datafile_path": ""} + ] + + packages, deps = resolve.scan_pom_files(pom_file_list) + + self.assertEqual(packages, expected_packages) + self.assertEqual(deps, expected_deps) + + mock_run_scan.assert_called_once_with( + location="/safe/mock.pom", + output_file="/safe/mock.pom-output.json", + run_scan_args={"package": True}, + ) + mock_open.assert_called_once_with("/safe/mock.pom-output.json") + mock_json_load.assert_called_once() + + @mock.patch("scanpipe.pipes.resolve.is_maven_pom_url") + @mock.patch("scanpipe.pipes.resolve.requests.get") + def test_scanpipe_resolve_construct_pom_url_from_filename( + self, mock_get, mock_is_maven_pom_url + ): + # Setup mock response from Maven Central + mock_response = mock.Mock() + mock_response.raise_for_status.return_value = None + mock_response.json.return_value = { + "response": {"docs": [{"g": "org.apache.commons"}]} + } + mock_get.return_value = mock_response + mock_is_maven_pom_url.return_value = True + + # Inputs + artifact_id = "commons-lang3" + version = "3.12.0" + + expected_url = [ + "https://repo1.maven.org/maven2/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.pom" + ] + + result = resolve.construct_pom_url_from_filename(artifact_id, version) + + self.assertEqual(result, expected_url) + mock_get.assert_called_once_with( + "https://search.maven.org/solrsearch/select?q=a:commons-lang3&wt=json", + timeout=5, + ) + mock_is_maven_pom_url.assert_called_once_with(expected_url[0]) + + def test_scanpipe_resolve_get_pom_url_list_with_packages(self): + packages = [ + { + "namespace": "org.apache.commons", + "name": "commons-lang3", + "version": "3.12.0", + } + ] + result = resolve.get_pom_url_list({}, packages) + expected = [ + "https://repo1.maven.org/maven2/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.pom" + ] + self.assertEqual(result, expected) + + def test_scanpipe_resolve_get_pom_url_list_with_maven_download_url(self): + input_source = { + "download_url": "https://repo1.maven.org/maven2/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.jar" + } + result = resolve.get_pom_url_list(input_source, []) + expected = [ + "https://repo1.maven.org/maven2/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.pom" + ] + self.assertEqual(result, expected) + + @mock.patch("scanpipe.pipes.resolve.construct_pom_url_from_filename") + @mock.patch("scanpipe.pipes.resolve.parse_maven_filename") + def test_scanpipe_resolve_get_pom_url_list_with_jar_filename( + self, mock_parse, mock_construct + ): + input_source = {"filename": "commons-lang3-3.12.0.jar"} + mock_parse.return_value = ("commons-lang3", "3.12.0") + mock_construct.return_value = [ + "https://repo1.maven.org/maven2/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.pom" + ] + result = resolve.get_pom_url_list(input_source, []) + self.assertEqual(result, mock_construct.return_value) + + def test_scanpipe_resolve_get_pom_url_list_with_invalid_filename(self): + input_source = {"filename": "not-a-jar.txt"} + result = resolve.get_pom_url_list(input_source, []) + self.assertEqual(result, []) From cb623c1c43aea551f6442dfe01b561affd6621cf Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Mon, 17 Nov 2025 18:08:24 +0800 Subject: [PATCH 007/110] Implement "update_package_license_from_resource_if_missing" function #1763 - Update package's license if missing while the same package has license detected in RESOURCES Signed-off-by: Chin Yeung Li --- scanpipe/pipelines/scan_maven_package.py | 5 ++- scanpipe/pipelines/scan_single_package.py | 19 ++++++++ scanpipe/pipes/resolve.py | 29 +++++++++++++ scanpipe/tests/pipes/test_resolve.py | 53 +++++++++++++++++++++++ 4 files changed, 104 insertions(+), 2 deletions(-) diff --git a/scanpipe/pipelines/scan_maven_package.py b/scanpipe/pipelines/scan_maven_package.py index 7438d781c0..6b86b06791 100644 --- a/scanpipe/pipelines/scan_maven_package.py +++ b/scanpipe/pipelines/scan_maven_package.py @@ -47,6 +47,7 @@ def steps(cls): cls.extract_archives, cls.run_scan, cls.fetch_and_scan_remote_pom, + cls.update_package_license_from_resource_if_missing, cls.load_inventory_from_toolkit_scan, cls.make_summary_from_scan_results, ) @@ -65,9 +66,9 @@ def fetch_and_scan_remote_pom(self): pom_file_list = download_pom_files(pom_url_list) scanned_pom_packages, scanned_dependencies = scan_pom_files(pom_file_list) - updated_pacakges = packages + scanned_pom_packages + updated_packages = packages + scanned_pom_packages # Replace/Update the package and dependencies section - data["packages"] = updated_pacakges + data["packages"] = updated_packages data["dependencies"] = scanned_dependencies with open(self.scan_output_location, "w") as file: json.dump(data, file, indent=2) diff --git a/scanpipe/pipelines/scan_single_package.py b/scanpipe/pipelines/scan_single_package.py index 605ef0ea5d..7f0bf8b909 100644 --- a/scanpipe/pipelines/scan_single_package.py +++ b/scanpipe/pipelines/scan_single_package.py @@ -31,6 +31,7 @@ from scanpipe.pipes import scancode from scanpipe.pipes.input import copy_input from scanpipe.pipes.input import is_archive +from scanpipe.pipes.resolve import update_package_license_from_resource_if_missing class ScanSinglePackage(Pipeline): @@ -51,6 +52,7 @@ def steps(cls): cls.extract_input_to_codebase_directory, cls.extract_archives, cls.run_scan, + cls.update_package_license_from_resource_if_missing, cls.load_inventory_from_toolkit_scan, cls.make_summary_from_scan_results, ) @@ -126,6 +128,23 @@ def run_scan(self): if not scan_output_path.exists(): raise FileNotFoundError("ScanCode output not available.") + def update_package_license_from_resource_if_missing(self): + """Update PACKAGE license from the license detected in RESOURCES if missing.""" + with open(self.scan_output_location) as file: + data = json.load(file) + packages = data.get("packages", []) + resources = data.get("files", []) + if not packages or not resources: + return + + updated_packages = update_package_license_from_resource_if_missing( + packages, resources + ) + # Update the package section + data["packages"] = updated_packages + with open(self.scan_output_location, "w") as file: + json.dump(data, file, indent=2) + def load_inventory_from_toolkit_scan(self): """Process a JSON Scan results to populate codebase resources and packages.""" input.load_inventory_from_toolkit_scan(self.project, self.scan_output_location) diff --git a/scanpipe/pipes/resolve.py b/scanpipe/pipes/resolve.py index b672df13bc..4f7ed272c5 100644 --- a/scanpipe/pipes/resolve.py +++ b/scanpipe/pipes/resolve.py @@ -723,3 +723,32 @@ def scan_pom_files(pom_file_list): scanned_dep["datafile_path"] = "" scanned_pom_deps.append(scanned_dep) return scanned_pom_packages, scanned_pom_deps + + +def update_package_license_from_resource_if_missing(packages, resources): + """Populate missing licenses to packages based on resource data.""" + from license_expression import Licensing + + updated_packages = [] + for package in packages: + if not package.get("declared_license_expression"): + package_uid = package.get("package_uid") + detected_lic_list = [] + for resource in resources: + if ( + resource.get("detected_license_expression") + and package_uid in resource["for_packages"] + ): + if ( + resource.get("detected_license_expression") + not in detected_lic_list + ): + detected_lic_list.append( + resource.get("detected_license_expression") + ) + license_expression = " AND ".join(detected_lic_list) + if license_expression: + declared_license_expression = str(Licensing().dedup(license_expression)) + package["declared_license_expression"] = declared_license_expression + updated_packages.append(package) + return updated_packages diff --git a/scanpipe/tests/pipes/test_resolve.py b/scanpipe/tests/pipes/test_resolve.py index 016d00bd7c..89be9dd4f4 100644 --- a/scanpipe/tests/pipes/test_resolve.py +++ b/scanpipe/tests/pipes/test_resolve.py @@ -586,3 +586,56 @@ def test_scanpipe_resolve_get_pom_url_list_with_invalid_filename(self): input_source = {"filename": "not-a-jar.txt"} result = resolve.get_pom_url_list(input_source, []) self.assertEqual(result, []) + + def test_scanpipe_resolve_update_package_license_from_resource_if_missing(self): + packages = [ + {"package_uid": "pkg1", "declared_license_expression": ""}, + {"package_uid": "pkg2", "declared_license_expression": None}, + {"package_uid": "pkg3", "declared_license_expression": "MIT"}, + ] + resources = [ + { + "for_packages": ["pkg1", "pkg2"], + "detected_license_expression": "GPL-2.0", + }, + {"for_packages": ["pkg1"], "detected_license_expression": "MIT"}, + ] + + expected_pkg1_expr = "GPL-2.0 AND MIT" + expected_pkg2_expr = "GPL-2.0" + + updated = resolve.update_package_license_from_resource_if_missing( + packages, resources + ) + + self.assertEqual(updated[0]["declared_license_expression"], expected_pkg1_expr) + self.assertEqual(updated[1]["declared_license_expression"], expected_pkg2_expr) + self.assertEqual(updated[2]["declared_license_expression"], "MIT") + + def test_scanpipe_resolve_update_package_license_from_resource_if_missing_no_match( + self, + ): + packages = [{"package_uid": "pkgX", "declared_license_expression": None}] + resources = [{"for_packages": ["pkgY"], "detected_license_expression": "MIT"}] + + updated = resolve.update_package_license_from_resource_if_missing( + packages, resources + ) + self.assertEqual(updated[0]["declared_license_expression"], None) + + def test_scanpipe_resolve_update_package_license_from_resource_if_missing_no_change( + self, + ): + packages = [ + {"package_uid": "pkg1", "declared_license_expression": "GPL-2.0"}, + {"package_uid": "pkg2", "declared_license_expression": "Apache-2.0"}, + ] + resources = [ + {"for_packages": ["pkg1", "pkg2"], "detected_license_expression": "MIT"}, + ] + + updated = resolve.update_package_license_from_resource_if_missing( + packages, resources + ) + self.assertEqual(updated[0]["declared_license_expression"], "GPL-2.0") + self.assertEqual(updated[1]["declared_license_expression"], "Apache-2.0") From b937cffac5185cd097b3a914d0af1c72e833bdfa Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Tue, 18 Nov 2025 15:27:31 +0800 Subject: [PATCH 008/110] Only work with "maven" package type #1763 Signed-off-by: Chin Yeung Li --- scanpipe/pipes/resolve.py | 21 +++++++++++---------- scanpipe/tests/pipes/test_resolve.py | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/scanpipe/pipes/resolve.py b/scanpipe/pipes/resolve.py index 4f7ed272c5..81a6d7efee 100644 --- a/scanpipe/pipes/resolve.py +++ b/scanpipe/pipes/resolve.py @@ -576,16 +576,17 @@ def get_pom_url_list(input_source, packages): pom_url_list = [] if packages: for package in packages: - package_ns = package.get("namespace", "") - package_name = package.get("name", "") - package_version = package.get("version", "") - pom_url = ( - f"https://repo1.maven.org/maven2/{package_ns.replace('.', '/')}/" - f"{package_name}/{package_version}/" - f"{package_name}-{package_version}.pom".lower() - ) - pom_url_list.append(pom_url) - else: + if package.get("type") == "maven": + package_ns = package.get("namespace", "") + package_name = package.get("name", "") + package_version = package.get("version", "") + pom_url = ( + f"https://repo1.maven.org/maven2/{package_ns.replace('.', '/')}/" + f"{package_name}/{package_version}/" + f"{package_name}-{package_version}.pom".lower() + ) + pom_url_list.append(pom_url) + if not pom_url_list: from urllib.parse import urlparse # Check what's the input source diff --git a/scanpipe/tests/pipes/test_resolve.py b/scanpipe/tests/pipes/test_resolve.py index 89be9dd4f4..7249763253 100644 --- a/scanpipe/tests/pipes/test_resolve.py +++ b/scanpipe/tests/pipes/test_resolve.py @@ -548,6 +548,7 @@ def test_scanpipe_resolve_construct_pom_url_from_filename( def test_scanpipe_resolve_get_pom_url_list_with_packages(self): packages = [ { + "type": "maven", "namespace": "org.apache.commons", "name": "commons-lang3", "version": "3.12.0", @@ -559,6 +560,19 @@ def test_scanpipe_resolve_get_pom_url_list_with_packages(self): ] self.assertEqual(result, expected) + def test_scanpipe_resolve_get_pom_url_list_with_non_maven_packages(self): + packages = [ + { + "type": "jar", + "namespace": "", + "name": "spring-context", + "version": "7.0.0", + } + ] + result = resolve.get_pom_url_list({}, packages) + expected = [] + self.assertEqual(result, expected) + def test_scanpipe_resolve_get_pom_url_list_with_maven_download_url(self): input_source = { "download_url": "https://repo1.maven.org/maven2/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.jar" From 0bbe97969bb57c251cbd0a20c251e18f0a280181 Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Tue, 18 Nov 2025 15:39:17 +0800 Subject: [PATCH 009/110] Update docstring description #1763 Signed-off-by: Chin Yeung Li --- scanpipe/pipelines/scan_maven_package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scanpipe/pipelines/scan_maven_package.py b/scanpipe/pipelines/scan_maven_package.py index 6b86b06791..5b0e3d340b 100644 --- a/scanpipe/pipelines/scan_maven_package.py +++ b/scanpipe/pipelines/scan_maven_package.py @@ -30,9 +30,9 @@ class ScanMavenPackage(ScanSinglePackage): """ - Scan a single package archive (or package manifest file). + Scan a single maven package archive. - This pipeline scans a single package for package metadata, + This pipeline scans a single maven package for package metadata, declared dependencies, licenses, license clarity score and copyrights. The output is a summary of the scan results in JSON format. From 2b3c9bbe2f4d6f8d2a2ecf1f342cd08458ab58a7 Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Tue, 18 Nov 2025 18:34:06 +0800 Subject: [PATCH 010/110] Add test for the "scan_maven_package" pipeline #1763 Signed-off-by: Chin Yeung Li --- .../data/jvm/scancodeio_wisp-logging.json | 1220 +++++++++++++++++ ...wisp-logging-2025.11.11.195957-97a44b0.jar | Bin 0 -> 24585 bytes scanpipe/tests/test_pipelines.py | 23 + 3 files changed, 1243 insertions(+) create mode 100644 scanpipe/tests/data/jvm/scancodeio_wisp-logging.json create mode 100644 scanpipe/tests/data/jvm/wisp-logging-2025.11.11.195957-97a44b0.jar diff --git a/scanpipe/tests/data/jvm/scancodeio_wisp-logging.json b/scanpipe/tests/data/jvm/scancodeio_wisp-logging.json new file mode 100644 index 0000000000..a01499e8c0 --- /dev/null +++ b/scanpipe/tests/data/jvm/scancodeio_wisp-logging.json @@ -0,0 +1,1220 @@ +{ + "headers": [ + { + "tool_name": "scancode-toolkit", + "tool_version": "32.4.0", + "options": { + "--copyright": true, + "--email": true, + "--info": true, + "--license": true, + "--license-text": true, + "--license-diagnostics": true, + "--license-text-diagnostics": true, + "--license-references": true, + "--package": true, + "--url": true, + "--classify": true, + "--summary": true, + "--todo": true + }, + "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "start_timestamp": "2025-11-18T103015.577081", + "end_timestamp": "2025-11-18T103020.058228", + "output_format_version": "4.1.0", + "duration": 4.481181859970093, + "message": null, + "errors": [], + "warnings": [], + "extra_data": { + "system_environment": { + "operating_system": "linux", + "cpu_architecture": "64", + "platform": "Linux-6.6.87.2-microsoft-standard-WSL2-x86_64-with-glibc2.39", + "platform_version": "#1 SMP PREEMPT_DYNAMIC Thu Jun 5 18:30:46 UTC 2025", + "python_version": "3.13.0 (main, Nov 11 2025, 11:11:38) [GCC 13.3.0]" + }, + "spdx_license_list_version": "3.26", + "files_count": 10 + } + } + ], + "summary": { + "declared_license_expression": null, + "license_clarity_score": { + "score": 0, + "declared_license": false, + "identification_precision": false, + "has_license_text": false, + "declared_copyrights": false, + "conflicting_license_categories": false, + "ambiguous_compound_licensing": true + }, + "declared_holder": "", + "primary_language": null, + "other_license_expressions": [], + "other_holders": [ + { + "value": null, + "count": 10 + } + ], + "other_languages": [] + }, + "todo": [], + "packages": [ + { + "type": "maven", + "namespace": "app.cash.wisp", + "name": "wisp-logging", + "version": "2025.11.11.195957-97a44b0", + "qualifiers": {}, + "subpath": null, + "primary_language": "Java", + "description": "wisp-logging\na module containing logging helpers", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "developer", + "name": "Square, Inc.", + "email": null, + "url": null + } + ], + "keywords": [], + "homepage_url": "https://github.com/cashapp/misk/", + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": "https://github.com/cashapp/misk/", + "vcs_url": "git://github.com/cashapp/misk.git", + "copyright": null, + "holder": null, + "declared_license_expression": "apache-2.0", + "declared_license_expression_spdx": "Apache-2.0", + "license_detections": [ + { + "license_expression": "apache-2.0", + "license_expression_spdx": "Apache-2.0", + "matches": [ + { + "license_expression": "apache-2.0", + "license_expression_spdx": "Apache-2.0", + "from_file": "wisp-logging-2025.11.11.195957-97a44b0.pom", + "start_line": 1, + "end_line": 2, + "matcher": "1-hash", + "score": 100.0, + "matched_length": 18, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "apache-2.0_40.RULE", + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/apache-2.0_40.RULE", + "matched_text": "name: The Apache Software License, Version 2.0\nurl: http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ], + "identifier": "apache_2_0-bfa9e97a-62d3-0076-c881-8443e5e95192" + } + ], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": "- license:\n name: The Apache Software License, Version 2.0\n url: http://www.apache.org/licenses/LICENSE-2.0.txt\n", + "notice_text": null, + "source_packages": [ + "pkg:maven/app.cash.wisp/wisp-logging@2025.11.11.195957-97a44b0?classifier=sources" + ], + "is_private": false, + "is_virtual": false, + "extra_data": {}, + "repository_homepage_url": "https://repo1.maven.org/maven2/app/cash/wisp/wisp-logging/2025.11.11.195957-97a44b0/", + "repository_download_url": "https://repo1.maven.org/maven2/app/cash/wisp/wisp-logging/2025.11.11.195957-97a44b0/wisp-logging-2025.11.11.195957-97a44b0.jar", + "api_data_url": "https://repo1.maven.org/maven2/app/cash/wisp/wisp-logging/2025.11.11.195957-97a44b0/wisp-logging-2025.11.11.195957-97a44b0.pom", + "package_uid": "pkg:maven/app.cash.wisp/wisp-logging@2025.11.11.195957-97a44b0?uuid=5b064161-83c2-45dd-acb3-77e7e728c73b", + "datafile_paths": [ + "https://repo1.maven.org/maven2/app/cash/wisp/wisp-logging/2025.11.11.195957-97a44b0/wisp-logging-2025.11.11.195957-97a44b0.pom" + ], + "datasource_ids": [ + "maven_pom" + ], + "purl": "pkg:maven/app.cash.wisp/wisp-logging@2025.11.11.195957-97a44b0" + } + ], + "dependencies": [ + { + "purl": "pkg:maven/com.squareup.misk/misk-bom@2025.11.11.195957-97a44b0", + "extracted_requirement": "2025.11.11.195957-97a44b0", + "scope": "import", + "is_runtime": false, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:maven/com.squareup.misk/misk-bom@2025.11.11.195957-97a44b0?uuid=64d1a32f-7666-444a-88dd-daa5c6f3ef3c", + "for_package_uid": "pkg:maven/app.cash.wisp/wisp-logging@2025.11.11.195957-97a44b0?uuid=5b064161-83c2-45dd-acb3-77e7e728c73b", + "datafile_path": "", + "datasource_id": "maven_pom" + }, + { + "purl": "pkg:maven/io.grpc/grpc-bom@1.75.0", + "extracted_requirement": "1.75.0", + "scope": "import", + "is_runtime": false, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:maven/io.grpc/grpc-bom@1.75.0?uuid=83843170-7694-448c-b8d2-d2d00fb53f00", + "for_package_uid": "pkg:maven/app.cash.wisp/wisp-logging@2025.11.11.195957-97a44b0?uuid=5b064161-83c2-45dd-acb3-77e7e728c73b", + "datafile_path": "", + "datasource_id": "maven_pom" + }, + { + "purl": "pkg:maven/com.google.guava/guava-bom@33.5.0-jre", + "extracted_requirement": "33.5.0-jre", + "scope": "import", + "is_runtime": false, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:maven/com.google.guava/guava-bom@33.5.0-jre?uuid=331c1df9-dba0-4216-852a-84668e55e253", + "for_package_uid": "pkg:maven/app.cash.wisp/wisp-logging@2025.11.11.195957-97a44b0?uuid=5b064161-83c2-45dd-acb3-77e7e728c73b", + "datafile_path": "", + "datasource_id": "maven_pom" + }, + { + "purl": "pkg:maven/com.google.inject/guice-bom@6.0.0", + "extracted_requirement": "6.0.0", + "scope": "import", + "is_runtime": false, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:maven/com.google.inject/guice-bom@6.0.0?uuid=81443998-920c-46fd-9f9a-455724507032", + "for_package_uid": "pkg:maven/app.cash.wisp/wisp-logging@2025.11.11.195957-97a44b0?uuid=5b064161-83c2-45dd-acb3-77e7e728c73b", + "datafile_path": "", + "datasource_id": "maven_pom" + }, + { + "purl": "pkg:maven/com.fasterxml.jackson/jackson-bom@2.20.0", + "extracted_requirement": "2.20.0", + "scope": "import", + "is_runtime": false, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:maven/com.fasterxml.jackson/jackson-bom@2.20.0?uuid=9b781ed3-222c-4248-9313-5573ac7a6c16", + "for_package_uid": "pkg:maven/app.cash.wisp/wisp-logging@2025.11.11.195957-97a44b0?uuid=5b064161-83c2-45dd-acb3-77e7e728c73b", + "datafile_path": "", + "datasource_id": "maven_pom" + }, + { + "purl": "pkg:maven/org.glassfish.jersey/jersey-bom@3.1.11", + "extracted_requirement": "3.1.11", + "scope": "import", + "is_runtime": false, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:maven/org.glassfish.jersey/jersey-bom@3.1.11?uuid=3ffd518e-15b5-48db-9f13-2eb4122a8e9f", + "for_package_uid": "pkg:maven/app.cash.wisp/wisp-logging@2025.11.11.195957-97a44b0?uuid=5b064161-83c2-45dd-acb3-77e7e728c73b", + "datafile_path": "", + "datasource_id": "maven_pom" + }, + { + "purl": "pkg:maven/org.eclipse.jetty/jetty-bom@10.0.26", + "extracted_requirement": "10.0.26", + "scope": "import", + "is_runtime": false, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:maven/org.eclipse.jetty/jetty-bom@10.0.26?uuid=51d440a9-3aa3-4e96-8e28-b1aaed86fead", + "for_package_uid": "pkg:maven/app.cash.wisp/wisp-logging@2025.11.11.195957-97a44b0?uuid=5b064161-83c2-45dd-acb3-77e7e728c73b", + "datafile_path": "", + "datasource_id": "maven_pom" + }, + { + "purl": "pkg:maven/org.jetbrains.kotlin/kotlin-bom@2.1.21", + "extracted_requirement": "2.1.21", + "scope": "import", + "is_runtime": false, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:maven/org.jetbrains.kotlin/kotlin-bom@2.1.21?uuid=b9c10eb5-a833-45f6-806f-1ccf7074b1bc", + "for_package_uid": "pkg:maven/app.cash.wisp/wisp-logging@2025.11.11.195957-97a44b0?uuid=5b064161-83c2-45dd-acb3-77e7e728c73b", + "datafile_path": "", + "datasource_id": "maven_pom" + }, + { + "purl": "pkg:maven/io.netty/netty-bom@4.2.6.Final", + "extracted_requirement": "4.2.6.Final", + "scope": "import", + "is_runtime": false, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:maven/io.netty/netty-bom@4.2.6.Final?uuid=b72a23a8-d836-4ed2-83c0-a38efe55fabd", + "for_package_uid": "pkg:maven/app.cash.wisp/wisp-logging@2025.11.11.195957-97a44b0?uuid=5b064161-83c2-45dd-acb3-77e7e728c73b", + "datafile_path": "", + "datasource_id": "maven_pom" + }, + { + "purl": "pkg:maven/io.prometheus/simpleclient_bom@0.16.0", + "extracted_requirement": "0.16.0", + "scope": "import", + "is_runtime": false, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:maven/io.prometheus/simpleclient_bom@0.16.0?uuid=cbefb54e-8898-4e0d-8858-6115d0861034", + "for_package_uid": "pkg:maven/app.cash.wisp/wisp-logging@2025.11.11.195957-97a44b0?uuid=5b064161-83c2-45dd-acb3-77e7e728c73b", + "datafile_path": "", + "datasource_id": "maven_pom" + }, + { + "purl": "pkg:maven/app.cash.tempest/tempest-bom@2025.10.27.160126-16a22a2", + "extracted_requirement": "2025.10.27.160126-16a22a2", + "scope": "import", + "is_runtime": false, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:maven/app.cash.tempest/tempest-bom@2025.10.27.160126-16a22a2?uuid=0c2d8c82-4692-403a-9f77-f3453a0af2b3", + "for_package_uid": "pkg:maven/app.cash.wisp/wisp-logging@2025.11.11.195957-97a44b0?uuid=5b064161-83c2-45dd-acb3-77e7e728c73b", + "datafile_path": "", + "datasource_id": "maven_pom" + }, + { + "purl": "pkg:maven/com.squareup.wire/wire-bom@5.4.0", + "extracted_requirement": "5.4.0", + "scope": "import", + "is_runtime": false, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:maven/com.squareup.wire/wire-bom@5.4.0?uuid=921ac16d-df4c-4bd6-b2aa-a3a2b5050bca", + "for_package_uid": "pkg:maven/app.cash.wisp/wisp-logging@2025.11.11.195957-97a44b0?uuid=5b064161-83c2-45dd-acb3-77e7e728c73b", + "datafile_path": "", + "datasource_id": "maven_pom" + }, + { + "purl": "pkg:maven/io.github.microutils/kotlin-logging-jvm@3.0.5", + "extracted_requirement": "3.0.5", + "scope": "compile", + "is_runtime": false, + "is_optional": true, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:maven/io.github.microutils/kotlin-logging-jvm@3.0.5?uuid=a6d457f3-951a-4248-9641-6206b4350fbf", + "for_package_uid": "pkg:maven/app.cash.wisp/wisp-logging@2025.11.11.195957-97a44b0?uuid=5b064161-83c2-45dd-acb3-77e7e728c73b", + "datafile_path": "", + "datasource_id": "maven_pom" + }, + { + "purl": "pkg:maven/org.slf4j/slf4j-api@2.0.17", + "extracted_requirement": "2.0.17", + "scope": "compile", + "is_runtime": false, + "is_optional": true, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:maven/org.slf4j/slf4j-api@2.0.17?uuid=bf841549-cf22-40d4-956a-75799102b974", + "for_package_uid": "pkg:maven/app.cash.wisp/wisp-logging@2025.11.11.195957-97a44b0?uuid=5b064161-83c2-45dd-acb3-77e7e728c73b", + "datafile_path": "", + "datasource_id": "maven_pom" + }, + { + "purl": "pkg:maven/app.cash.wisp/wisp-sampling@2025.11.11.195957-97a44b0", + "extracted_requirement": "2025.11.11.195957-97a44b0", + "scope": "compile", + "is_runtime": false, + "is_optional": true, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:maven/app.cash.wisp/wisp-sampling@2025.11.11.195957-97a44b0?uuid=a9bf96b2-6f38-46e2-918c-76148c663ed0", + "for_package_uid": "pkg:maven/app.cash.wisp/wisp-logging@2025.11.11.195957-97a44b0?uuid=5b064161-83c2-45dd-acb3-77e7e728c73b", + "datafile_path": "", + "datasource_id": "maven_pom" + }, + { + "purl": "pkg:maven/org.jetbrains.kotlin/kotlin-stdlib@2.1.21", + "extracted_requirement": "2.1.21", + "scope": "compile", + "is_runtime": false, + "is_optional": true, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:maven/org.jetbrains.kotlin/kotlin-stdlib@2.1.21?uuid=98b565da-b9df-40a0-8dfd-e28d393e81ef", + "for_package_uid": "pkg:maven/app.cash.wisp/wisp-logging@2025.11.11.195957-97a44b0?uuid=5b064161-83c2-45dd-acb3-77e7e728c73b", + "datafile_path": "", + "datasource_id": "maven_pom" + }, + { + "purl": "pkg:maven/com.squareup.misk/misk-api@2025.11.11.195957-97a44b0", + "extracted_requirement": "2025.11.11.195957-97a44b0", + "scope": "runtime", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:maven/com.squareup.misk/misk-api@2025.11.11.195957-97a44b0?uuid=79e6d8f9-6f53-47c5-8095-eb927342e541", + "for_package_uid": "pkg:maven/app.cash.wisp/wisp-logging@2025.11.11.195957-97a44b0?uuid=5b064161-83c2-45dd-acb3-77e7e728c73b", + "datafile_path": "", + "datasource_id": "maven_pom" + }, + { + "purl": "pkg:maven/com.squareup.misk/misk-bom@2025.11.11.195957-97a44b0", + "extracted_requirement": "2025.11.11.195957-97a44b0", + "scope": "dependencymanagement", + "is_runtime": false, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:maven/com.squareup.misk/misk-bom@2025.11.11.195957-97a44b0?uuid=c909a298-a436-4abe-bbc4-e228922374cf", + "for_package_uid": "pkg:maven/app.cash.wisp/wisp-logging@2025.11.11.195957-97a44b0?uuid=5b064161-83c2-45dd-acb3-77e7e728c73b", + "datafile_path": "", + "datasource_id": "maven_pom" + }, + { + "purl": "pkg:maven/io.grpc/grpc-bom@1.75.0", + "extracted_requirement": "1.75.0", + "scope": "dependencymanagement", + "is_runtime": false, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:maven/io.grpc/grpc-bom@1.75.0?uuid=f69d81e2-494f-42be-889d-8fe21e5d01b8", + "for_package_uid": "pkg:maven/app.cash.wisp/wisp-logging@2025.11.11.195957-97a44b0?uuid=5b064161-83c2-45dd-acb3-77e7e728c73b", + "datafile_path": "", + "datasource_id": "maven_pom" + }, + { + "purl": "pkg:maven/com.google.guava/guava-bom@33.5.0-jre", + "extracted_requirement": "33.5.0-jre", + "scope": "dependencymanagement", + "is_runtime": false, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:maven/com.google.guava/guava-bom@33.5.0-jre?uuid=31ec39a6-b6c7-4002-aac2-6c4f2ad4b3ba", + "for_package_uid": "pkg:maven/app.cash.wisp/wisp-logging@2025.11.11.195957-97a44b0?uuid=5b064161-83c2-45dd-acb3-77e7e728c73b", + "datafile_path": "", + "datasource_id": "maven_pom" + }, + { + "purl": "pkg:maven/com.google.inject/guice-bom@6.0.0", + "extracted_requirement": "6.0.0", + "scope": "dependencymanagement", + "is_runtime": false, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:maven/com.google.inject/guice-bom@6.0.0?uuid=535397a8-0a10-41c7-a694-1c973cb42b01", + "for_package_uid": "pkg:maven/app.cash.wisp/wisp-logging@2025.11.11.195957-97a44b0?uuid=5b064161-83c2-45dd-acb3-77e7e728c73b", + "datafile_path": "", + "datasource_id": "maven_pom" + }, + { + "purl": "pkg:maven/com.fasterxml.jackson/jackson-bom@2.20.0", + "extracted_requirement": "2.20.0", + "scope": "dependencymanagement", + "is_runtime": false, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:maven/com.fasterxml.jackson/jackson-bom@2.20.0?uuid=06344079-eb43-41e9-afd2-0008406bde50", + "for_package_uid": "pkg:maven/app.cash.wisp/wisp-logging@2025.11.11.195957-97a44b0?uuid=5b064161-83c2-45dd-acb3-77e7e728c73b", + "datafile_path": "", + "datasource_id": "maven_pom" + }, + { + "purl": "pkg:maven/org.glassfish.jersey/jersey-bom@3.1.11", + "extracted_requirement": "3.1.11", + "scope": "dependencymanagement", + "is_runtime": false, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:maven/org.glassfish.jersey/jersey-bom@3.1.11?uuid=05aa4554-9b63-4f42-9e89-4c08cad2d15f", + "for_package_uid": "pkg:maven/app.cash.wisp/wisp-logging@2025.11.11.195957-97a44b0?uuid=5b064161-83c2-45dd-acb3-77e7e728c73b", + "datafile_path": "", + "datasource_id": "maven_pom" + }, + { + "purl": "pkg:maven/org.eclipse.jetty/jetty-bom@10.0.26", + "extracted_requirement": "10.0.26", + "scope": "dependencymanagement", + "is_runtime": false, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:maven/org.eclipse.jetty/jetty-bom@10.0.26?uuid=07f9b132-f5b9-41aa-9aea-5a4318df81a9", + "for_package_uid": "pkg:maven/app.cash.wisp/wisp-logging@2025.11.11.195957-97a44b0?uuid=5b064161-83c2-45dd-acb3-77e7e728c73b", + "datafile_path": "", + "datasource_id": "maven_pom" + }, + { + "purl": "pkg:maven/org.jetbrains.kotlin/kotlin-bom@2.1.21", + "extracted_requirement": "2.1.21", + "scope": "dependencymanagement", + "is_runtime": false, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:maven/org.jetbrains.kotlin/kotlin-bom@2.1.21?uuid=1c9f883a-8bbe-4c30-baa8-1e31de8e8c5b", + "for_package_uid": "pkg:maven/app.cash.wisp/wisp-logging@2025.11.11.195957-97a44b0?uuid=5b064161-83c2-45dd-acb3-77e7e728c73b", + "datafile_path": "", + "datasource_id": "maven_pom" + }, + { + "purl": "pkg:maven/io.netty/netty-bom@4.2.6.Final", + "extracted_requirement": "4.2.6.Final", + "scope": "dependencymanagement", + "is_runtime": false, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:maven/io.netty/netty-bom@4.2.6.Final?uuid=7bb260f7-6ce3-48b8-ab97-9ee96fa432f7", + "for_package_uid": "pkg:maven/app.cash.wisp/wisp-logging@2025.11.11.195957-97a44b0?uuid=5b064161-83c2-45dd-acb3-77e7e728c73b", + "datafile_path": "", + "datasource_id": "maven_pom" + }, + { + "purl": "pkg:maven/io.prometheus/simpleclient_bom@0.16.0", + "extracted_requirement": "0.16.0", + "scope": "dependencymanagement", + "is_runtime": false, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:maven/io.prometheus/simpleclient_bom@0.16.0?uuid=36a8cdd2-e49f-4e92-a866-e8b0dc31971d", + "for_package_uid": "pkg:maven/app.cash.wisp/wisp-logging@2025.11.11.195957-97a44b0?uuid=5b064161-83c2-45dd-acb3-77e7e728c73b", + "datafile_path": "", + "datasource_id": "maven_pom" + }, + { + "purl": "pkg:maven/app.cash.tempest/tempest-bom@2025.10.27.160126-16a22a2", + "extracted_requirement": "2025.10.27.160126-16a22a2", + "scope": "dependencymanagement", + "is_runtime": false, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:maven/app.cash.tempest/tempest-bom@2025.10.27.160126-16a22a2?uuid=9eaa91c9-a9c7-4c11-9c1d-f4de87ceaf50", + "for_package_uid": "pkg:maven/app.cash.wisp/wisp-logging@2025.11.11.195957-97a44b0?uuid=5b064161-83c2-45dd-acb3-77e7e728c73b", + "datafile_path": "", + "datasource_id": "maven_pom" + }, + { + "purl": "pkg:maven/com.squareup.wire/wire-bom@5.4.0", + "extracted_requirement": "5.4.0", + "scope": "dependencymanagement", + "is_runtime": false, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:maven/com.squareup.wire/wire-bom@5.4.0?uuid=f83233dc-a6da-4bea-848d-21bf0315e365", + "for_package_uid": "pkg:maven/app.cash.wisp/wisp-logging@2025.11.11.195957-97a44b0?uuid=5b064161-83c2-45dd-acb3-77e7e728c73b", + "datafile_path": "", + "datasource_id": "maven_pom" + } + ], + "license_detections": [], + "license_references": [], + "license_rule_references": [], + "files": [ + { + "path": "codebase", + "type": "directory", + "name": "codebase", + "base_name": "codebase", + "extension": "", + "size": 0, + "date": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha1_git": null, + "mime_type": null, + "file_type": null, + "programming_language": null, + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "for_todo": [], + "is_legal": false, + "is_manifest": false, + "is_readme": false, + "is_top_level": true, + "is_key_file": false, + "is_community": false, + "package_data": [], + "for_packages": [], + "detected_license_expression": null, + "detected_license_expression_spdx": null, + "license_detections": [], + "license_clues": [], + "percentage_of_license_text": 0, + "copyrights": [], + "holders": [], + "authors": [], + "emails": [], + "urls": [], + "files_count": 10, + "dirs_count": 3, + "size_count": 78414, + "scan_errors": [] + }, + { + "path": "codebase/META-INF", + "type": "directory", + "name": "META-INF", + "base_name": "META-INF", + "extension": "", + "size": 0, + "date": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha1_git": null, + "mime_type": null, + "file_type": null, + "programming_language": null, + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "for_todo": [], + "is_legal": false, + "is_manifest": false, + "is_readme": false, + "is_top_level": true, + "is_key_file": false, + "is_community": false, + "package_data": [], + "for_packages": [], + "detected_license_expression": null, + "detected_license_expression_spdx": null, + "license_detections": [], + "license_clues": [], + "percentage_of_license_text": 0, + "copyrights": [], + "holders": [], + "authors": [], + "emails": [], + "urls": [], + "files_count": 2, + "dirs_count": 0, + "size_count": 76, + "scan_errors": [] + }, + { + "path": "codebase/META-INF/MANIFEST.MF", + "type": "file", + "name": "MANIFEST.MF", + "base_name": "MANIFEST", + "extension": ".MF", + "size": 25, + "date": "1980-02-01", + "sha1": "79e33dd52ebdf615e6696ae69add91cb990d81e2", + "md5": "92d04d6bd8a0235843240bba30d2f091", + "sha256": "566ad1a80220026d05099562645ce968ff0e7c36cde22634332605bb34cc3eff", + "sha1_git": "58630c02ef423cffd6dd6aafd946eb8512040c37", + "mime_type": "text/plain", + "file_type": "ASCII text, with CRLF line terminators", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "for_todo": [], + "is_legal": false, + "is_manifest": true, + "is_readme": false, + "is_top_level": false, + "is_key_file": false, + "is_community": false, + "package_data": [], + "for_packages": [], + "detected_license_expression": null, + "detected_license_expression_spdx": null, + "license_detections": [], + "license_clues": [], + "percentage_of_license_text": 0, + "copyrights": [], + "holders": [], + "authors": [], + "emails": [], + "urls": [], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "codebase/META-INF/wisp-logging.kotlin_module", + "type": "file", + "name": "wisp-logging.kotlin_module", + "base_name": "wisp-logging", + "extension": ".kotlin_module", + "size": 51, + "date": "1980-02-01", + "sha1": "d42d499d7a0dc593b505e40e5cf88c134ba369cd", + "md5": "5c21479c92f9a0c17d94cb32332f0ac1", + "sha256": "7e326e0855b42d282ee50be30c1f18b56a57f6e61f1ed28a415f04b006e855a5", + "sha1_git": "d0e1fcf3513e6931d604e6f9d2b69ec83cd79e07", + "mime_type": "application/octet-stream", + "file_type": "data", + "programming_language": null, + "is_binary": true, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "for_todo": [], + "is_legal": false, + "is_manifest": false, + "is_readme": false, + "is_top_level": false, + "is_key_file": false, + "is_community": false, + "package_data": [], + "for_packages": [], + "detected_license_expression": null, + "detected_license_expression_spdx": null, + "license_detections": [], + "license_clues": [], + "percentage_of_license_text": 0, + "copyrights": [], + "holders": [], + "authors": [], + "emails": [], + "urls": [], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "codebase/wisp", + "type": "directory", + "name": "wisp", + "base_name": "wisp", + "extension": "", + "size": 0, + "date": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha1_git": null, + "mime_type": null, + "file_type": null, + "programming_language": null, + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "for_todo": [], + "is_legal": false, + "is_manifest": false, + "is_readme": false, + "is_top_level": true, + "is_key_file": false, + "is_community": false, + "package_data": [], + "for_packages": [], + "detected_license_expression": null, + "detected_license_expression_spdx": null, + "license_detections": [], + "license_clues": [], + "percentage_of_license_text": 0, + "copyrights": [], + "holders": [], + "authors": [], + "emails": [], + "urls": [], + "files_count": 8, + "dirs_count": 1, + "size_count": 78338, + "scan_errors": [] + }, + { + "path": "codebase/wisp/logging", + "type": "directory", + "name": "logging", + "base_name": "logging", + "extension": "", + "size": 0, + "date": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha1_git": null, + "mime_type": null, + "file_type": null, + "programming_language": null, + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "for_todo": [], + "is_legal": false, + "is_manifest": false, + "is_readme": false, + "is_top_level": false, + "is_key_file": false, + "is_community": false, + "package_data": [], + "for_packages": [], + "detected_license_expression": null, + "detected_license_expression_spdx": null, + "license_detections": [], + "license_clues": [], + "percentage_of_license_text": 0, + "copyrights": [], + "holders": [], + "authors": [], + "emails": [], + "urls": [], + "files_count": 8, + "dirs_count": 0, + "size_count": 78338, + "scan_errors": [] + }, + { + "path": "codebase/wisp/logging/Copyable.class", + "type": "file", + "name": "Copyable.class", + "base_name": "Copyable", + "extension": ".class", + "size": 867, + "date": "1980-02-01", + "sha1": "0ce1f2d5366ef4355fcadf58976cfe3208211a40", + "md5": "53b285a2396218fe8dbbe673b845fbfd", + "sha256": "095e8170334c7473e12953ab787579ccb55a463a706b6b94ca0a163d5ccfc84d", + "sha1_git": "95b097be0b417e3a76e6af15e46bc69f08bf07d6", + "mime_type": "application/x-java-applet", + "file_type": "compiled Java class data, version 55.0", + "programming_language": null, + "is_binary": true, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "for_todo": [], + "is_legal": false, + "is_manifest": false, + "is_readme": false, + "is_top_level": false, + "is_key_file": false, + "is_community": false, + "package_data": [], + "for_packages": [], + "detected_license_expression": null, + "detected_license_expression_spdx": null, + "license_detections": [], + "license_clues": [], + "percentage_of_license_text": 0, + "copyrights": [], + "holders": [], + "authors": [], + "emails": [], + "urls": [], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "codebase/wisp/logging/LoggingKt$WhenMappings.class", + "type": "file", + "name": "LoggingKt$WhenMappings.class", + "base_name": "LoggingKt$WhenMappings", + "extension": ".class", + "size": 844, + "date": "1980-02-01", + "sha1": "66717009f241737925e9321b90fe7fa5d29b0fc1", + "md5": "9eea8f9f6c7655fc42b960994ee2cc70", + "sha256": "4aa63a9173b1019987790fc5d5e57a71f49bcf700c6b74d7b9f1f75b7a3492cf", + "sha1_git": "2c6afacbef5ad804bfb6d76943d99ae6ddf13ef3", + "mime_type": "application/x-java-applet", + "file_type": "compiled Java class data, version 55.0", + "programming_language": null, + "is_binary": true, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "for_todo": [], + "is_legal": false, + "is_manifest": false, + "is_readme": false, + "is_top_level": false, + "is_key_file": false, + "is_community": false, + "package_data": [], + "for_packages": [], + "detected_license_expression": null, + "detected_license_expression_spdx": null, + "license_detections": [], + "license_clues": [], + "percentage_of_license_text": 0, + "copyrights": [], + "holders": [], + "authors": [], + "emails": [], + "urls": [], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "codebase/wisp/logging/LoggingKt.class", + "type": "file", + "name": "LoggingKt.class", + "base_name": "LoggingKt", + "extension": ".class", + "size": 15451, + "date": "1980-02-01", + "sha1": "bf541eedf40dddfe8db4682b8ec38b8d30ca0c81", + "md5": "20c86a38f69b6e4780f8737f2778f284", + "sha256": "f49e83050b5c410067d859b27bf54331b631eeae6d70873501953b03d42d2502", + "sha1_git": "e82a1f8c7df5d3120c1832c1c14e8d57efd6988d", + "mime_type": "application/x-java-applet", + "file_type": "compiled Java class data, version 55.0", + "programming_language": null, + "is_binary": true, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "for_todo": [], + "is_legal": false, + "is_manifest": false, + "is_readme": false, + "is_top_level": false, + "is_key_file": false, + "is_community": false, + "package_data": [], + "for_packages": [], + "detected_license_expression": null, + "detected_license_expression_spdx": null, + "license_detections": [], + "license_clues": [], + "percentage_of_license_text": 0, + "copyrights": [], + "holders": [], + "authors": [], + "emails": [], + "urls": [], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "codebase/wisp/logging/SampledLogger.class", + "type": "file", + "name": "SampledLogger.class", + "base_name": "SampledLogger", + "extension": ".class", + "size": 29813, + "date": "1980-02-01", + "sha1": "78d0e86dd4b0c58d077385690f773669885fb351", + "md5": "1893dbb7d487ab03390c2408ae087bec", + "sha256": "9aaf17a8a076ae1726631933f9c42de9d5f810678fbb4510195c6058274eb64a", + "sha1_git": "44c400f6cd0186de0f1944d9ed399608e36b317a", + "mime_type": "application/x-java-applet", + "file_type": "compiled Java class data, version 55.0", + "programming_language": null, + "is_binary": true, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "for_todo": [], + "is_legal": false, + "is_manifest": false, + "is_readme": false, + "is_top_level": false, + "is_key_file": false, + "is_community": false, + "package_data": [], + "for_packages": [], + "detected_license_expression": null, + "detected_license_expression_spdx": null, + "license_detections": [], + "license_clues": [], + "percentage_of_license_text": 0, + "copyrights": [], + "holders": [], + "authors": [], + "emails": [], + "urls": [], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "codebase/wisp/logging/SmartTagsThreadLocalHandler$ThreadLocalSmartTagsMdcContext.class", + "type": "file", + "name": "SmartTagsThreadLocalHandler$ThreadLocalSmartTagsMdcContext.class", + "base_name": "SmartTagsThreadLocalHandler$ThreadLocalSmartTagsMdcContext", + "extension": ".class", + "size": 4388, + "date": "1980-02-01", + "sha1": "46cba6cb951d5de1df125b28fae7cde0e469a020", + "md5": "108702fde924e63718d51253a50c577c", + "sha256": "9526e2eccc1b81acf753a9944cb01491e3c2fead64f4295dfe9fa41eb36efad4", + "sha1_git": "41bc1e3823d4b948af7c6a14b6bb5d49585dbbd9", + "mime_type": "application/x-java-applet", + "file_type": "compiled Java class data, version 55.0", + "programming_language": null, + "is_binary": true, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "for_todo": [], + "is_legal": false, + "is_manifest": false, + "is_readme": false, + "is_top_level": false, + "is_key_file": false, + "is_community": false, + "package_data": [], + "for_packages": [], + "detected_license_expression": null, + "detected_license_expression_spdx": null, + "license_detections": [], + "license_clues": [], + "percentage_of_license_text": 0, + "copyrights": [], + "holders": [], + "authors": [], + "emails": [], + "urls": [], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "codebase/wisp/logging/SmartTagsThreadLocalHandler.class", + "type": "file", + "name": "SmartTagsThreadLocalHandler.class", + "base_name": "SmartTagsThreadLocalHandler", + "extension": ".class", + "size": 5243, + "date": "1980-02-01", + "sha1": "e37b7c99f2d98351ea4e5ffbacfeaa9983b71f63", + "md5": "9383cba7363b982a6b9588f3bb676bce", + "sha256": "5beb3803e2319fd1be60952b584bf4194cfcf6d789397baa8c4dd5ec492e1b6d", + "sha1_git": "087781b46b7e46a87e8d619021b45c8f11e3ed43", + "mime_type": "application/x-java-applet", + "file_type": "compiled Java class data, version 55.0", + "programming_language": null, + "is_binary": true, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "for_todo": [], + "is_legal": false, + "is_manifest": false, + "is_readme": false, + "is_top_level": false, + "is_key_file": false, + "is_community": false, + "package_data": [], + "for_packages": [], + "detected_license_expression": null, + "detected_license_expression_spdx": null, + "license_detections": [], + "license_clues": [], + "percentage_of_license_text": 0, + "copyrights": [], + "holders": [], + "authors": [], + "emails": [], + "urls": [], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "codebase/wisp/logging/TaggedLogger$Companion.class", + "type": "file", + "name": "TaggedLogger$Companion.class", + "base_name": "TaggedLogger$Companion", + "extension": ".class", + "size": 1965, + "date": "1980-02-01", + "sha1": "801bfe25e9ef34ca4cca8a1c4edbc606057e1729", + "md5": "ac85af0645dea85627dc40b308581d83", + "sha256": "bdb0b40d552d9edbe7dd1319f3501f9c7d3830d83538770633e3ab71fade1758", + "sha1_git": "430bcef605dc5fcce11ecbab4d4005b2782f191d", + "mime_type": "application/x-java-applet", + "file_type": "compiled Java class data, version 55.0", + "programming_language": null, + "is_binary": true, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "for_todo": [], + "is_legal": false, + "is_manifest": false, + "is_readme": false, + "is_top_level": false, + "is_key_file": false, + "is_community": false, + "package_data": [], + "for_packages": [], + "detected_license_expression": null, + "detected_license_expression_spdx": null, + "license_detections": [], + "license_clues": [], + "percentage_of_license_text": 0, + "copyrights": [], + "holders": [], + "authors": [], + "emails": [], + "urls": [], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "codebase/wisp/logging/TaggedLogger.class", + "type": "file", + "name": "TaggedLogger.class", + "base_name": "TaggedLogger", + "extension": ".class", + "size": 19767, + "date": "1980-02-01", + "sha1": "23460c4f2cde2d8cd859b08dc73975fa25558a38", + "md5": "ff588da8e47c28b06a5a8d62f15bb954", + "sha256": "38d4eefe889cb9f3bc74e85e1a146e19e6f0649c42199ecd81d9baf1cf91084f", + "sha1_git": "ed7589eb6619a43d530fe85f2ab26bd239333a4a", + "mime_type": "application/x-java-applet", + "file_type": "compiled Java class data, version 55.0", + "programming_language": null, + "is_binary": true, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "for_todo": [], + "is_legal": false, + "is_manifest": false, + "is_readme": false, + "is_top_level": false, + "is_key_file": false, + "is_community": false, + "package_data": [], + "for_packages": [], + "detected_license_expression": null, + "detected_license_expression_spdx": null, + "license_detections": [], + "license_clues": [], + "percentage_of_license_text": 0, + "copyrights": [], + "holders": [], + "authors": [], + "emails": [], + "urls": [], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + } + ] +} \ No newline at end of file diff --git a/scanpipe/tests/data/jvm/wisp-logging-2025.11.11.195957-97a44b0.jar b/scanpipe/tests/data/jvm/wisp-logging-2025.11.11.195957-97a44b0.jar new file mode 100644 index 0000000000000000000000000000000000000000..8021647e366700e8ec944f194cf3cc3d1c881279 GIT binary patch literal 24585 zcmb5V1C%9Avo74z#v= z{SCnUrti!DKcI|&tc0kDk}{o)=)KIuxU>{4-7KsWE!EV-biLxQ1*VR<@?5<~F8umbOk-<~DlPwnolY#)kU(`X=85Y#<6wu@ir= zM>$PHH#0seCpSGIg}(RT|EohP5u*X)ziIjw{(r^?_RasJ;D3zmzk*=@6Z9W<@jt*Z z2euR_-zGBz0RsBRApcw9e}aWpLORscswXGICyd43chCJF1|P3?J7nY_dR z`SsEjW$|3j@}O9G5He+w03kZ*_+II~l_b;nIV~R;avu-~l*f@6L2gsTP4;Kn?e#AA z>F%%hcQBx7Kt|74T5O3d284eY@Ue=Q#}pRfvOj{GYB?_6Yx9uQpynjkUGzKaITN`N zhR3$qeeoTn`Z5D<9Vl9c`m01~HL8!Qp+8AYJyX>cB_gLvRmbMKDPX*LSy4))8@WPW zcnS(x;uI;eT(V|~dW$5)s#C{EW+xAy+<*S4Jnk2wM5u^DWhxTGT9Ueo2OvyCY98^r z#elXL!=Pb(dS}m!N}S3M=#=17rZ78F^_qMSiLvC>=>?A2{dnQy6BPtTe}PHEH3@L}CZ z+aE^XX0bS=^D#4#-0|yK^JSB;AR+xDHcvgFw?#s>q9(-v)!T})o}=$3B@D84H+wvq z5KRJ^s5c@(+KM%*5C6qyvI*w0gIP4^p)3#GmU*d@u$`A2OT%Rwc*n6Nt5$f z%6@IkqwyM`$kJY_exwy=0;H=R2V4`4(RRTbCSgaa@R(?yfH&Pg+h*60>*lIn+xMsxKkKu%=Rl4>j>2u=t<=uQ!V3sS z5%BFwY$2S)Y#ME341&+ZGFWt2F)ZVB> zh<$lqYkdDRj&o5{fHfTQifV+ha59?Q+~_r%);&Q=m!zC4Y>t-*pbr27IS;~T-W`EJ z^qDnwOi(Mz?a9^IKo9irX`79E$gNv$z$~_mXX6~exiJjwFlni}Ou@wi-JMxXU;c~z z6b(7esJfk$Z;2nlkD%?KBj-TL%_fd|LRnigs7&UyTC)mTj6m)!c!iuV47B8`&HGk% zWpD{o<&G5sb_!9q@oZz^m8#|5Gh$gttOh= z{D+8DLRY|#gv4FS$JpT4P`lzC9}J{lpf)I33Q6QViOj) z3Luu?co=SB&LG$`um@kj{~mHdV3b-|-%oHYZXh6>|KE`NSCn09x_e3u z%p3|1Kq#d}p)=OBu9Ms(wQekj_GH!U1hane_F+O|X|oL3WB4i2t7yf?kneme~b2(csd(7u(7PHFh6t6wro^AZv) zP%`4T5_3?#yoi%grAf){Mij!v$|gz{=bx9cvVJtN(1fy<^wQPxC<{qxS;~(Eswj#! zI(RBb&*tdgLn=#n@?xUs;;@wd zwLzS@b;E`Q(AZs4i=h>W-06&1DTb0U=~#m5@k_ambxFzyNd<+mRS6X7aRON71ef9o zg~@ZHCd5$%=JjVZj&7LZQ%lT+K#>R=m$*8cw77q)A-hf%5GRpnLP)`O0*fG5E<+9> zD`XboVqz9b!xQsK58PxtiHgdgbbg-#KTAN!>{7VKP+>Js&}m?saMHjun$q&4XE3j5 z*Xji*s#6L=Ri^vk(O~xWgM-N^tqH{DtU8)RglbtzM6@=h0RKoyum0Ec%2VO zJdjo@l7*nmY^Bs)9VFBmv^7yfA1ie0C={@j7{yOU{@y4|l8TSy=^m84l}w}yHSmL4 zH)wz*?^?sL9d5-)5eGSRO-4Mx(!oes_Qt7}wb@VK1hREW_k_leXC}DDB)S*)g7|`_ zR%8*8Ma0kyQ;>_wgr86VUYoejhn=9;c!nYX7LJg8VSpbAkT3`rI=9;Z6=y2$Cu_F_f8*da?LZkRmf4;!(V3g8(>V4n<^|Na_K@ z57(*%`*lN}e|)%4JO;j!PrD#$UQ`jeNJ|@Nj4Apl+PXbW?hr>%6gMaj7z2o<6EnJK z!A&NzcyyS;6*ggiIKE)m1)e}T2u?<9MmMfNAplW43;|-UexwZzc*9ShoC0!Fb$Rm7 zwwIbBFs|YR%Misy)GwwB)Kp8lw6n0vxP)qx=J%BT)PocA(~H=h{SXGeI7FKVPE;8w z#bnL?>-yU#4Y!Q>OzW(yb}rJE#T_5)%XRR-PQKl9Y@%p7{nsz`Y$Y7cP1dWfji5O! zE53C7yqh_5rCtP2@7a8wj_LL7^+k;`0iHjf^hRpf-njnw{5qA{6w$`H zB%hDOJ22)RQ4)&lp?Rt=3fZ_J2uPPRqX{cg@8F);J475K;^2rs<0d$g@7%DvY!$`C z<6NfbpFP5G{@ZasETVTn(CR}m1v*{cx{O4|jmh~!!^X|p$jJQSxr=g{#LSG5BKG=v zs80)@iOEg8PUqOl@6p3cWlQ1sh3A#l+JBASz5`#n1F7-Wc_VfJpRiUIetS@(hMy&Gy`O{cr2j_*DeSJ4j7Tx4&9FCKd z9`>lQr}Zbpw)i`iR~GF6+ep8zaLeu9c-G{4*C|L)3gVEd+nc?;YHNE2Ewzq@ibw;#IaX{ZFM9Z#9(4Z3ZMrnBo=rf#(ZJx{iz!N%=7kZZ%hns_u zC1bX`<&9e6LY*D-&9d^z;0bqv^i05l#EmyQY2Wz`$an^6KPd3K%+rnsl#kKRPX=5_ z@)VclCDk9F$reVO@%Q~E(FI=7f5CkreChi#Zs48DaZ~mUp{ZBsM<2%smJfUbe9?Ac z-69L4ql=1hic5}nD*GSSZ{U#MsvDG2T~6@|`qQU$t3S9ZOiS-eT-`-5!zU<`#zCv<5cUas~WU&$~O=vE@)VDznZ+`vQGMw@r1G(=1Qhed0 z%;PCCqwxT3;iptJmMbIi{cEHdJ8sUloj={R9YRwK#ruGAZtrz)3S8DUF`z}rqRlquX_Xvtq_>=G9gA2F6;WESa?m6nbN|!{>VwQ)9X|d$5H@39T(ESj zW10SFE02<6ReIRq@GI(41J|qzb8%Figpqp|((Odne8OY?1Xdc?xrfQ0K>RMRUP#s1 zs_?}@x=Mi+F5hfN+Yt3*VyH(*QYN8dWxLo+QV^#5KASHuZ=&Pzc5P>D)n7Z3H)uQ?y8Dg{2G?qQwSWTovXdUrR(&J-&vh@(VChSa=guMegNG z(`%rE-kF93cov6C2X7wa3rt0%_Jdgkk2$an!CFi!si@yhaVE8d^?_LdR-tR88xhuE zD-r7y<7ACKW3={kX&z06?T{cXfuO;69B}xu@-cpWHFReK*?b( z@NZdKT}aZ#w3s=1ABahlKBJ@{SA-71bV>JE-1O_h^pmpvLd`~t7knab*MQ$Kbj7p; zV%6$TEjm)v_OJAhJPTp-;8h>&OBulPa()~v0D(-F5rQWvg<>kpG9-YnqDv8;SKKwE z#M+5m#gG{~Y_RLjHi^9nvZr6$MZbjrBQsl`7Chz zZa>@9N4C}>@*u(2a(T3A?ryTFd*3v2vfD}L(PTPlFpc-5T{ohoZcsr><=8rn{}-*g z=aN3tE`{daL&GqY_tA!}HlKXkCGWt`l|ezu&9lJOd{lB9n(a0D8HZQv&*bqPhpPUc zPBi%SDm~ryN*Ys7@3>_RB9k>$D|B7A1>^hIaMw|h104@U_v((Y4mBO-Er%t|wvLK$ z-TEFzV3L9Z;#h@O#1ZwEZ+iTu4AO$AlRL0vL>kN~!hRXIm6l-9hG?RLc%=2GJw^yQ zJ^a+pgU*J5NslIU)6`H&1DO6y;~D5WA@-6~$v%WuGmJinO<-#*Box5s)JNCrKY;SpVneZl7h#_ zoZK^w6~=N)PP5NYAsAQCg=N+T#<_H{EF`W);ss|QokMKNR*7Z8G>#DqlhsVF#U7rX z^CXiNkPotWET-w{ECJKa+>yJb>T0^^=B4X1%H&Ini_5kY)wUL;?Exp_SlDCd#IQ23 zkBtVKAw$$-*hC8|+rPqN*GYU+Nh-RkYCEFu*~(tm!~@m!Y-dRLe>R6hMlTuhC4~$s zu;e*K{eGa-WTS>&g|2t7?5-@kShb=#QG_`C#aoO>EBmPl)w|7+EUpmVt+%LxwHmLp zl#=d!8Geh?*<03`re~oZg6dLqu=2}2p)7nMTbu&xYc;AbbKEY=n(ACDemv%czm_yj z`jk=zk7aA)Tpoheu4WB3+4Exu6rKI%{=tc5W%Txk>S~5uKVxsR0kG;D4l)8a3?K*u zS>7A`EJyzke@jLU9kxk;a!aVeXl4fJoYJkUXUNa*?;Y?hZ(C)a4_hPZ2|<7<)9+@1 zJU9_~FGzU^`sY=6w4Tw&6_K^olPmbQbn=~YNj(^yX@PTpyTf_AM|ZR4GlE`|s(wa8 zqzk_+oJ@RHArw0sS~|AI6~Dh3G}XCK@CZ6)On6Zh&3?+~1f$ZUGDb8e1EkcrY%U^g5p zb|Zn@TVf()N5lu}dKAdtNYHZLPyE~vzQgp5Ov4ZGycfvTLfH7bi|Dl_Z1>j+vMTsZ z29)hK6M80A^aeq!$4uK?*d-Vg?NDd_tpeV&NE<$4aPkR+yzpjh$559+U~9B6P?yWJ3VhTJ`}h7Xp4l^3 zwfyadgn1t!q_`8%mjhn8n`z_?Vl*>Xn&M{N=A+2ME*W4N-q4WNY&DQZ zDrcO;(`IZ)WK!uC&CR;Gz%;>Cj9bny-N5yMv(M9n2fMKJe#r5oW=;yK!cJ$5lTl~L_K40- zK^#4j+d1J)yrwlI`e`?QDlpW&7p4batBV zqO+c~PW~>T%Y15|>ZdRB`NIv+pel;0n6QIB z)qgHy<7w!>Z7qq4`G_p){+68q8=sO?dWk42Q{{uJv%OkxhRELfJ@l=lENqV`0RnZ`M3 zOJ<@6Ldf=eXiH^+xdFA+MwNtm=0S`zhy%LEGO729jNr-x_?S#N z$Lr`0IlDngQLOYLL))FHNK%-QCGur(<#ia4CE6ado7O$}qncD*kS@>||AEa2{w0SG zVB$#*A)eL;k&9M>ALN@Z4_Gcn4Q_0~qu+P8r&U|63}E0c8B;e`Xl2&{Gld&o2}{Wc z-G?PfI^5q=k>$kUy2S1;ii7ItbD(oE&5GP6%JP+D+?LFVtPMw78FaraMx^mf(Y!NV z1BxE|x&6~=BlVFtml>qfDcdq=5?tTFv zVvSyMJi^eS=fdbuM|Ll)>MNc=u8Qu7)~kI+f9&Q=UX_Q|OkUcDRSiszNL*HSFxaTI z{!(aBXGROyf;P;?M1?oZLypj895*a@USBNF9C32_2ns^Qo=;GlImS7x9f>e`d6qsk zeI_qdmmZ}VSW}9boWcXfxI^>MTMtRf9?G;Qg9By!duHHgmbF?~Kil+bxp*ZYyiDL& z%+_>Xtu%6F(ekN$1M*4w;{yU>;!I`C*5ri#DD9z9#J5Tck*Ad3ci2I9B8(?eHDFy5 z8HeL`ut|6f7Wg4*brGdT;Kq!BjNwrAQ{oOCE))R1#$UJ5vrP5VXYrx>tL6(#!z&}0 zI3<&`ea(jwkDxWQwguW#N%!`7*K?2PHJL8ADmS{xqiXwz(B?Y}3in~{STXhq?Kc9v z+oEafAvyO%7bVgiDZm$mcucN){YKYczVlwQTg>$%z7bwKqLRmKxWJxVJ{`Dxv(aq( zw8DG0F7G5h5#)Yaw@pIwfA%jqxAOL(p5Se=N?>rgImJ2U&^b|*>SlV&`ekvG&+XFS z7BCifHF=9-rau~@b7prraidS|v>`0`%K8JndOkQ$IJt}a5pY>b$(DCB;i}JWDx=R| zHlT4coZe~VJVXj#bsjW^RJg-qQgvNJcw_qU`|vXsq|Bi$cW=Ky{(Vi6l;@SH?z=MN z>;wcv@ZZ-Il>pXuR>nsEtY#ZK{KvYYj=8?_vU8Vf18l($Q6xwb5(YCNY;h)A)gFvr z`UnOL8P+|lfy_OAXpFWb>k?-MY!V3z&N|EX^^`Ij1NJ&>_BipWB+e8H3Ct3kWadxa z&aB4U=<(wj2G7cW{a0pMd!PC7eWJ+6F^(r3$ym|O@uGJ3wvbKtKFSzX2huFf8a$U1RiI3Z zu#x)17euEt*I$e2|6CVLInSOtU5`FqhH_Y*VM}D6@gmD*i4P(5d$?M;xsjksk&4c2 zF~zSjak97I6~w^74a#i48pEUp(jHrBpgq{Aj#>myLQh2~l^}DPI(x6y`)QPNqRQcp zr9w`jH8ze5IjS-u@G*EmE3Ge)%gl_IW-R={R4nOYFHqQRb)WL9D( zSVh)EH3lFKWzKrx+Sju#=}xKE!?9_NcF_4T`m zKdNYJ5Cv=yMI^C!B!*Iwkg_Sx0nMs_;juTtpJ}t*aJ1gQpNjvQc7q<<=En)vj}uK& zek?O)VGsvZ4yG`gyh+x<=1B$9)*f5bon~r(c!C7$rzCUKb;{RMl{0dRBO6gR%-~<% zE+#uecAf)vo=G7vgr?+T@j?v0YAKSrjd6K4sZAA%X6pwX#RnYKDYJh?(DdD4LeRvq zX2iv;!49kfyG6kbhAOos;3{Fk!ZeN*Gfi}Qds{J3cW!cf9ulq{PV)P`{c7oMgV zpM)lD4!tX!ac<(yLJK5a02c~znRKYJ>W7cm4P6L)?~~nAbQ%glic16qxkClZ@OW zWP#{T$2?33d+}g_K&WSD@NyWK9D>#D3a=m1_UTifaoET`)KZ@K=*v`pQAKr>NkyjI zt}?(wI2sroR-Op#%cOX*r8*)6$VPby4sur-pdfl9;nSC*ASdx^8tsrA> zR~yN#k}!U@$9L+DM%0Nbzr&MyDiA4+s{3wClKrImG{xhqL(|Uu)$+_h>e*dG&gdgJ zs*C(IFl3r|0n(R!@=QtkmRm#4`A&e+?UPv zh_h3u7?sG!1YwGHkOp*|F2Uj;!viio8q!0e`9 zhS*?1w_;axw&_I6i#Z=KOqYPVc7NKwHvD6a%0^Erj}@cGN>nny9oG~F`BnB}r+AXJ z--JsuWD>5f6|s=T+5p88?;RKF7MY(5+{7txx+;THXmRv4I zZ$7nYP@03gajxaiWcqZZs5L#@hV!nflGR0A5PHdFq-yd!1^YAt$9|1a@2YHkEQb^9 zbAj@Hc#DzE_sr^9DJ6lAse0|6e4dWa_srpr9Gf$ZZ`JDJC1HCloA2fQ?w%Yz`RC_? z>7iUFumJx&&x{! zgcP_hR^45RfIlDX*n4gB#AFUQ_9yef!SqDMH?CoPIwvIaeKbV&NRAcT^Yfs&=~#}p z+hTf*`+YQ!G>~tr^8J?TX1HPk-*bH9{dVnUI3tj6ld0-xU!+3D*TubTJn9>@Iy3q{ zYGOE@^S<;g^f;ya^Pty2plhN&qH;5PNYm8Vo^)+w^UyQ1Q&%pbW$wgDm-E{E%8p>L zC_7nmIQj@T1%HoOox0eUrPY-9G0_;ixJBogwJqxE=Rr|6oK$F5Crpq=j8x|7pBtye zwhqgYCTI?XV>Gv=JrA}<%P7{^r&JeI9#nI#9HZlLz;VjOJI4j{sO&fZJ@x`v4ng~e}8~h$-1_FVt=E1qy?t!^p0mQpry0z-OOBlBkZNMg{LC3 z?RUa90VTTaMrt+wO<*Q1Q{_@NCbnj~?XP5{?l)eyV~!2V_Q7!m%em`^9h2ERX5&kZ zjZW8x)y@$oeIw>oe6}mUYw^KQ=P25(E6hmP=ppyjF_3n)Ye!?3qcstpR!%s#63um; zEZrQ}5Wo}LZ*5#}I=zJI@$38Q`_^|Jg<0L>zvhZu6pj6gT%4OvQe8fy%k(d=q>p5<;>adw?D9B@}Nfeg`Yn1sG+c zc?luit~tPt^CFJ76~6l|-i0{IM)e{}x?OsJo#a(8-=!HNXJPp-wOV(N5?x5u%;9S3 z1Jp8mNo2Q@32sFa9djhQ=Kq(&upNwQB!5m;>7;xU!ao8`EFK(WOmU=n*Hox z;sy9yz{*0uxj1Y7mf)XSyJ$q~x&QR!TY^VfDK`3;?f&6tzqtc6Bpbv!Sg2-5%%C8x zAWj&lW{O7yBpU@rNSJ>(G#iCF*m>!vSo_rCy5X8=#f%AJUW)|pp!l7FS>gjr5^}*V zQ5T{E%L5Y0E>Xq&1It39Z-y@aph6)>;jU;vzf6Hk*qZo-XMt?n4)D>e+K*d)JWaR?nx)N!BkL(vXnGfLMlS&Z3KKF&dz9^_^ZEJtuQ%eoN8ZiHwGr9TW_h4-n%U%Mug4Q0QSXjBXpQAZ$~LVcC}%W<=Jb zzKMk@h37p_?bZfmvom09bALFHnhq}i{;BV(yODwseT|9Bi71DMBIk1o+yA+;v#ZWX z=p&nBHFYuWm9#Q<+6Pjm{uBB#27y!(VpujyvPNepw4IHBp1g8-mAP0L%E3*n z-xR^2&hQ{k8J)EEXfX6bVEByLzoSm6OFqT!3OdYqWCX_>l(Tn(BA2G-q?*h(NYzGF zakF~MtGYS+f~Pt^`-rFdm+sas9bU;_HcYOOyv=jV}~N=X(7W#h(ALJ7y>#yOg+)r39O$| zfT5+SaU@V(yP4rKn5t`>v%?dossaktl^ea*ynCE8mp*AM>Mi=Nx@MZBZ_#1^3W7*)jLqHJ?Wd8aK4SutuakP$&e zAVV2uOk?c`-qh&N%aeSR;U+rbSDrA}NvG!|kvt^m?sK9@w8^M$7UEBl%1m#(OOdd* z`>;J2D;6!`Tiby}O>LdfQYpL2g6BxYFUbf!r!AAA?V+W5SmqDwpn*71q;y$Jb;)xo z7nxl*R?3}Qf;sy9%OBbFz|$#y!-l=}8i>tFIIWF`t?BN{aFE)lz>u3$B{t_(n7b#gk4pKkXWEl3U4U<#(oGn+XIEm1~<0(PTnx{YL;_lSj=tV zC(04}u$-ub@sym6sRWwR2(%X_KLRwxW~k~);+{LOb#P?@J-D-ypQ^^Rtdf0IOdJ^f z|4gY4qELk~3D|oPn!RH|HB#6z%lEM;P!qtr> zf-q`=cp~L95Y(9nxFZ$1=z-()TUN`Mqdbs@oQTq;F=R|;Er|Fv0GYD=Zmu3>4M&vS zrWa$jj^OdC1Fs;J;frBiu!!$K4DSdJ770t^f-bC`x~l2t#%=Fo>+dM>szfP-z;Fk; z^uVk7SgWBkjmX1I(axm&ib{;XE88fPlSNgPyy9R}R`e|=dzSpee1o33{L5fceS*w< zeirHkS@VUIn?$ZcB7_e%M${w$YcTWWv;4wbFhQ--Ll zi<%F%C9NIPT9c}b_neD3!$w&l>s=bSR^T;wYc~eT`T1-!>RUhm^B3c8JHc%P5q0D z=JBzqU4G7>q(e>k>e3hU_-dilbz3w0PZ8?p0w^`t_m5Co{ zfR8n5%F1CA%F2H{>yKw-(*yQQH&gC#z;UCR;X`CY@05XN@1t?Y zOoZ0pPOX^6Qiec6@{x13BkGI|HE3`uvLM-p7*=miB`o&wW)8mdF*HQv~^Q zj2_upZ!89U=ZGrteF^(KI(`8@;J`NRu-X8a5549mA;vSX0l|;$ z-#3nla)5=h$yA9rOr{%o%4LYSBsW#^t^IdRla%by9saG*9{7~~~6N&*nh%y2kb^DHVkpec?q?P* zXQqz*OL51mKx#9a&CX(`f{U|Le;0q!2g{t`SqWyDZ3h-Dh8Hac7dgimBfma-nVmr? zDM@uBH1!jI;oT)rbtCv-0{s!#p%)|4{lRa!V$p>r(Ajqfv-_p`9(;qcxb{7)0h-{) zuhNBJwR}1!VyJ51WVHf1rvkc%T)K`mMfA_Vl91=&fBC!U0r-Uv%R z1HRQF>b^(cE)V}Uy(y|wzxNC*)Edc-A{(a}nC_*~GSSsx%4+9)Ih0U6I8#rZDgls=FA@2Eiq z^WcH>5D_SV5U8d&X@tA~%njDn%aQ`{D5TbWwgCr(`swo!c|$@jf%JkCxQzttV1Ij^ z92#wDTM==o8g-k|Kk0@u>eHh#*0QGG7PMK<7IA4iTMod(00a2YK?Qt>>bIla#u(~u z2PHYss*aIWCbE5J%-^cQn(P|nrS0i%|Ft7VM-Xo&)|pBDbuEPH{uO>ULy*k?*oS`- zGZ3AiGG~(fTcwY|2Qg}Lmx@J>2d;V>DvkF*-yHt*G4lCn-{PYj>cV4z z{PKEkK0a8vH9x5j#7n7&&X*P4!sCI}!XrJ3Oa5>TvQEsCp}O9(q4#CatH;E5f;#a2 zb3R-BOY_I{eosB}<*>gg`OO3H^mm4Kr^S#jCDfxVbt{#@T=VO!1iNjAt7reiw9SOJ z-2{E-=;yaDN56fE|Lx0!(zbTO+?=4AEm%MVpW;*zh&caqk9n>#bWrUhuT2K{vh z*r{LlB*c5golf1U|K5?j@)|rqqa*b_bIkk1vsT##sPu215l?WD1IU%o-#`Po(Z6|N zws3mV+055SD9z|s@~gUAF_(uxO|kI$RD5B3?m)rM1oGbUTe@y>dryqPg^L|3W|OWJ z1hL<9fO|ObY&Q5ToWHzWpjjw?LqEaVJmaj}skY?BXaEVTD-}TtC@34zoD{zuF~SBY zXbKG^+CC5X2q+tY18F|VBYqlBxpIwYyl!$p}K6qZ+6z(eeqeQy96X%7T@VA>KCMm*dG=^ z$HHCTZ!?^x4(*?HjHubU>@!Cw4t)(IV<8NDW$)?d7)Y&_Omr zSDLlmN67BGOrLl@PIOU!e@|Q@O~u`BdD5S8@<{>jAEifA}U`o3kVlhbyFXYx^>pk^d%p|Z7KtLpj-y4DcN4>`y;NYYTFm+Tmb1(*c*L@5D zR^k8~BP(MElK*l@=|5pIMutMRHcrNFPX8(nHL7i^VGpBznmk$}0w;rkqV@6t+r$bJ zVKz77)+D525EF$NLYB!Sc*L^K5t*y4Y5=&b4Rg33s9xo z+oz-qIxjFfQk!8q+6Mj@EZP;XDgQC29_$OSZXF_K10Kq?zQ@fja}f0KwOpr&W$i)8 zW}}_&?7dlY`qc0HBL-G*mEOd%tjwxipRU2Q(Ru*hPfni3ku#-c)oh`e#xQeU;oVBW2WLyh@iMDr~aoqK3VAhSQI*n*iUK%wYZ%Jy?;JNjeE#Pe62Hj6s?BTW}Dfy&qSX$;{aO`+o@6;R28)WGmY zxLuCrIQsUJ=gLQIPy{@QYv|83R>$ky%}e^h2ZnBc!%w~VyVA%s$hjaXF1;OCfYY4g zj?B;M{ofQOv4F6~5|qKNNNPqdI)Exx7fwG(aX!jv4$f2ph+T+TZ`Gppc+G~?WtPU+ z?*JY0_=^PGHKmE%>aXh1uLYc>l3z5dopc~V+A^rW<=hL-7~ef#7b-J@-nCPh*$1t= z!Z}(eT`6>Q+DGa7Z}OZTIZRu8xYwLg0z9%NGgA&PIi<>527fsNS!!~*4BpH_<@aog z=-OEy+k-Zrjx4whnw9f0ZZb;r9VSaTAGlsp9u!n!*sRjAy3!WkHR+fyw4-d{Yqos0 z*f^zit^R%^Mu6$i)M3(bPj^(0Hh{W@4T>{u@#vd(9;gDoO zE!}GPMXVmV3$pRs02hC3d|#+74OhqwYTQg=B^FHfonIp_o!xmOO9E`Pnu2hhcHC2z zFQV_}>NN4L_rn6)vEmD32`XKKi_!ZMw*0YFB$b9^ zxq^q|(T9Zmr*gi9ZL3Ad(ay;{Am{#t@&&51NczRvV>#0#ns#@M7z;_FOjs*w_<&ap z+C=dadvA64GfCCOeh<4SV+~fp0-n{INWP#z3*~d2tL28Ec-%}_OU_22aqqPo-l4g_ z-B2`oGosP`w@2FZdkx7OVP#|fF!je}u9e6QskHFs1=%3wtjRZ`DP3RI`mJJILTjL- zQ3v+%DwNx}96cqprNRupQ4z�K-Z8`+V1MmiKTDjFc#775( zLcvjEP?Uy>f7`0RUr#ju6PYV(+q^G@BEt^8`x{yYqkFwZyWZrW5?3iT@$nib6V!vV zGL1B{9X2%Du*RJV;;z`ZlGQ>hq@GSLd?Oijb0?bet;?Ytx%tt8+bGv|@-TP|y83P0 zfMc97Gnfj!FL(DOebWwlD4%k8>Nyr$2WQb{t%+JYbU6c6mO#pwmtq>h>BX-QqjdpB zsY0=0Je2r1O9y5*JM)qhtxhPUey9id?{Ox!?4^R~+{qJ8Z>4PmlB7>?!?HBnU!Umk z2YC?Kg~MaCX6gfwHpBvV#B({ef;*_vBmL5f^Q4rGTu8l!7H=rq2}iC7ruNa`MQoTF zkfpFipAofu%@-j&hxw+|OPM?C7TYTXFZR9Ls$4T}!%b$+_XYqU|_jtxYLo8q7O0p zcZU*I%KM^9YzRacovL=27Y`a^p`Uj9P8iblX=dwK+q15YVtP`_Eo*YixZEvR+q>GF z{1G@M5J)affLohgUzYB* z9bxo<^s@aHEpMdfw`3xZ)kKWt@feadJU&k<{Y z=w4K>D7m%z^J6@*?!sy#iC$AX3K8E!Lg@GOHF}9JKZ@fF$b*_BA;M6ipBA^~aHNTB zy`a)owg!SD-4XmT{MxxxzfPPjx0aD0p@P7spfi!#>Fosc=djyOysM$Lk=>Mbx_X9z z@!iJ2eqigx;#0lU3U&{E{rLBJSDg`Y?BILjuIKmuExP}QdG}uv>!RAJ6816X*QKPm z0c}leuQpt=030JQvGP8980XLm=qfNi(Q~d$op|+NpGDDCd*2K-vvtBl*E&h(X1_V# z;Aqt45)-Yn=|eHMEPA-J zU~O3@hrrVg3r!}A=xLf%>B(YfOqCqv-<4C+^dwOJBF-AP3o1mNzq&^llhNs*vK1z0 zDie^jo!8*}ZUyvsJJB*e{V(IQl^Q4OGE;)p4=d`IUVdYS9PJPh8} z%q+_EPtzIP6c#6D7m7R0$4L&u7hK?x$WO`h$W?wh21ZV#v#Ct(1eOC(Xcp(7nX~EC z=Sg+Q%bSP{r5Y>m4e8&ZjX+7B8yUa?=02@RNCl>In)Z5to3fGm%(%!w{rhBAZrbWzzjc(TMh&F2kO2}KB22}M`hy(L$V$koUDfK-4?c^I2oAP_k`#n(7 zb(y5;R86{J>13WxNq20C#atPj)~jJiB;YXxs9u@)!oi6;=JGRn*sx)?Vc=6Gq_avv z&BB3Q3l(e!q;=}$n0XAfUm;BSdN}B4((;P-gn6WQE#vvs(nOSkj?u4@3iJvT(IQO} zn^{LF#ll3=Qa7g1gNt^_oAOeel%H}&%fHVkPt#XqqDd@UUM7Ct>=V&JsnaVd=~H7h zi|ZFXCrnGn1bH;=5(HNjZ|@P*Sdv#_Vs<7P3l4!52=?^%r7z=Hl4M3tsGy4&7}m99 z;&&)8W};-;iPj4L#v=i^JDC)Uo?4-Hu6uY!fECQ9VJaj%Y$zt7!goGL!K;xvsf^A- z^3TgZ_7)N{JJR-dy8Zq&K~pmk?|7uGEWVSZHvk>9LyRdIMrr|!<>8_i7>(3viQyL||%OtJ!Pa*>K{wtdyn$hnlMq=*jkIU%jy zLfWJ)7uHdqz(hWOGW_@Z}QO z@c1%zmBybeVl^6*48eQw`IJOmJJ-c6gGow z_07dl+ctACf|nbR=^sDbw2=U2E2>_-e-;H5n$U6c&l2(*UV4|Yk z^Sp8$I-D+J&Ban^?DfhPi|2#m(*UL`UX-pbg9hOA*xJpNXQWh}zJ(=2bW~iCs#EI@ zNnK86hRk`wLPB{ZHJ43`v6XHoA#DSoO0@W7X4f4>4@HT2*?FJH{lr4dVKJHPRS#Jz zrFQfwKO@s+=hT;4MD^qhGnxrFPv$kJu>>=_y_101JR_cz6k~lq|u+w29vW2eOx;$?- z|5qhv0Tsv6<#8mq1qs1PaCf&raJS&@&fxAE2oi<__rZd@JAnxVcXwxSf(I7f+xNY< z?6-S%YxfgAJ$2^H>ArRQU)krs|8ktnesugOYF`>{Bt)=gtty$lhCIr0uUQt2 ziEoqd8Z+f6oAbsqiI|t0rSRTFz4H`On zhTD;fo^UEh9oMLp8mhU?&RB&{2S8Fd4Ix;TRa@9#$*0@(rauN zuTk$gSr%mSW+~R4vF-O5-%#Tt?Nwe<88-JNb*s@3Ep!%UHDK{qpz~IE*@0s05bEk> za;GuzaY%8+C}$qTU0`M!BNfMCV@G&i>VZGGpX^?aN)uv?cr(tm$0Q!ELj+rVUT*_H z{=pl9L}Wl{s{lg>OUO1{y$h*TPOECCXy86}4K_(>T+!!WA`7f@yHmU&AiO-WMg@@0 zMNH}ilM#IROvf-U`o~d*vKfWJ)gt0vHU(y-jxtPtUPFvsMvm{GPK_h-dD0dU#q$R^ zmvecHs0Kaoe!o>nAco|E@4?}YFV0b?t@}f|=`F2<;8lqUe`oT^nB2^7f!S3sYh*vm z^)XrA6rwLQnks5|1i6T#Pecn|wJ(o!8u1H3YK(oU`1kCe?)c;uC{INKNXxHYQ3Dd} z+Sd#lv`%c$+Y8)-Ar!*byWKLBXz&m@cx@8lHaCo7QqVOt`smIX>G}$Bsu?*gj`76` zhMT~SPBPvyW$L}_b z3D%Fn^%fXl{plFG4kpsz_3JuO^-g!Zi&`fDyT-WlE>+X>OG)^~ExjxFc-J*`{qy<# zlK01>Gt?JQP^FKtivK3B@ED@7_$w`yLKI-bQ`>8qNaDxn&l-w@U8B7=^! zsBGX=Wau0e5QKN(f zRmS4i@-Eb&4&GZoful?B)5~}M_i!srZDx2kOQ-cHb@R5`3e`&6gZYhGZ~M=~WbM6E z`4jeO3{jIA+qzM>s7u2Mb@DN3q*#7bCz!xFqed)9iAo|@g;C&ynI12k;@fL58MgoU zY@Z_Xol353%n5eh2|43>fJ!m+D{#=aokzvW9hp9k!02#r%7U%{L^qTkoKZUc$+?wSC*PwM~DhE7;(;p08;W zXpDj#$lY3M$C((8YDDXceY{i8)eEa&QTfzNlYuDWsG#lFn}shhCBbMdrK^1QY%ZBt zbaa`~_2y0=IgKOJ0d46(VOh!QOIuu#e+VY%yq*&6gV?c!Jak)#-pb10U(i39tT zT@_87u6_K(dT6=_O|Eqm4873nQOqP~jv0fyd@@&?P8K+{)wm1YY`sr<>MU-AI6<8U z=$`HI-qn|YF!C_A*q|^+xsjT=laFXyLrQ`|1bq3&ffB7ZpQ+xy?+I;kQy|5wA#Vwa z{bstrLbI`I0%}cb9ud+ty?1pHnSNuG@RpOvZhm9+b6Vkk&~=nMBx{4uZ^bpLUll!BULZ#`7Uy%Agu8mwYb&JV*lgC5VAcx+FFe1l=;e`j z7yL`J-3NkE0z+e$8ze$EaIN7%p-BSiazFIcAz7m0;(EW)bY|fX7FEX#zq|EZ~kaHib3$H!rCr{31m84u5I%4n4w0+Z4GlhZ#v>wqB5pM> zG*rwrG%?pPt!5Wk6Fr`rY(A%80ZgtNx3U15SElFUM_U|0bSw_!%<~7@j%@h1bp>nM zZfnG@0em z;n@)PhY%ZR?g`lD%-@SZ+jqV^)N^G5l-brvWh(tI+{-wt86EOiq z2{{@uW+RCwi(pDU0Pav&^DC|h!h|S$eW^vmnO|Uqx}dvd$AKOFTQr+$=+mm(5vdO9 z5M^tQP$tFBY_j|!wl}TuGh8ztl~}Q~HP7K_H4q6WyHCDP(x9hiN3V}4BoAkc^16&M zT*@ojy!Y6ukt!iJDq*B{j>M}g$hSkWtrdOkLf(L7LYlXWMUTxKqytVyhr^Rp7>Wz= z`M6gWYc5K`QHisvfMXdcb^1XkkbHiZeO+P3KHRP%m|8tA-MAXXD7wozC5QtblW%^A zVmnT0kV@6R$gNBZK%J01hGLzVZwd{`m$rHNan)hQUA)&Y36TU z;5Qa^_In-CQt-$$m;Hn}pEUw%7mcJdYBnFICZc3CGzt?sexXG0j8d4H zDZ1(`GkiiJq`o#!K45{Wk7B_toZR6{p;-#zUG6)ldj|$)M7!P32ffAcopPOyF-WhR zTOFmo75W-eh?RKplBLV9&CTXz36;P$O;oR0If=atLnuI1|Cw!TSVN}JXQ_Y*KQyO`%g6tURTDERV{}@Td+ZPbFY(y6hZGTrOfNTovn`K&6MO> z)e3R`is;SelA*WxAf{e6yAlb@)RiYg6OPcw-R<}IdiGo+Wz}4YFB{g~5TttG^M0XC zUr1Rs`h0V1eVj_BzOP&MvUv{V@%a4nX@ow)iQ(!-IxYzaRld&}0 zwRv~FZp+$aSycG(i*wnJZg@72dL}PcHI7k=M%$8o^_2t)+QTeaU^3c1Ey>h4hx0sJ zVq{Wpf#$l_i=W0A7<{7i2Vo!Dt%Yr{4!rRX*rT@O#Xqqhm8Fe*xXQG+JF>W2`d$P? zH4|&7+D2J8L%_5~<9Nw8&?;m868@Gf*jGBNmm!ok@Sg4R8WTT&=%z62L`~vaIYbmL zr_jN`bRm(a!q zvKuhHigtK~>y8>OYJ?zW;&P46Abol*onSinz#&^D=a)dKh5l|BnAYFk0Zemg{|-zO zYwrT4LAUn;2lfFK6wYTl41JdOT(WGq?LUD;14rX}32w8bjve(`fy{SryQqQ@gt}7` z%zNZfn03yL2I}$86hHQMrD!6*Y}NkVIr|wl!36q>J;4;}K8O-Opxsy+eg&Wjgudc` zG6g=G{GUt)PbTgsQ}Cnd;mPFxWI~rEf?D&b=-X6d8rO?7)2^sFtUnGz=ShQ#LR z5j`eXeu&P9aS^`SLC0^80JYuK!a3%Ao>#ZQk1zFH1J`PT?V-S$tDkk#**=oxvvv}M zQ+^j}1CPBW;=$kd;g(g|#aryT>7g0CXc#(48bV6Q9isi_3{-hJ%}YGa+YOv(R3w-m zZo$;xBUEk5r2OU*V}7#YBJ9t;Dc_j=sr~42=lG@&epgvstV?!0PRYZ}@-7UX(g?@h24NOWnm=W@c-n2N=f!W)=~wO(s+3 zSD0QR*j?O`vkR=^^~?4*srty_-vBF-w`me#i$->=!qRDuw?XXojzTp!NsjT1)~?J* zL)L@Hry;l}gI8oiU9}9l4A4+0{%`%2f0GM3XfoYiN7?f-u*8be9A`-jS8^dm$G8z}RDbFjY;r5$~y}fj6?oTt5cbT1p zFaWHbCY$#f8o`Hj&RT>O;4xKJKx2Cq*zNY3t$1*?WUjee{m6-@IF({xXs0$qjn~aJ zCYr}4g@M1)y?K1nCA_#ZERv*_F>D$MyjDWEKcSq0ByQ(gJ5ffebEG?N$~!Y+T&tCa z){1UnGH?=%CMd6Ba*#Cdi2!G)_bHl_=JQ~y+0d_r?Ue%y<-GFk_yYHIC%+`-+MMgj;!BHub+j7pi`~jF-<2#bvvGFg=-Urd-cc1BXqhfs zFqYg-$3NlCG+9ocxJ>UUE`IY)W6fe>`ptHj2@xZMkV_jStWCUg@%JKkH*Q{CE&|_@ z+B$KPbVR7S9jzR54A~sCINky_)qrYVnJ)Vv`=uMFi^~&5Bdfj0*}= z7H)70tWPi5l{tAgb(J~cdcYOd+G(v$#8p4lB0;r^V;mm&o9LQ>WAj-LKmf!QM zmtH&?0v)nDPPhUrRh2J$n|i+8Li{N9{M;2>0YsIyu9>P39M?>p<+Taldq1i@p@KB} zy|%0?yeVIhbF4HYg!Nmfpy$N<#|6QCToCKd#|5eV{J0>DkiKuYQRE4wxK5GjIk-;y zI%(QJmWRa{H@-NFGveTog0y+GiM>6Y_bx!{Bwq!+4tA){5?T9-n6_DdEqU)fsxY~H zE9tW)bUbYSomT~W=T`h08m!!Q$=Ba(9(5`B=0lzxl%pAhXq6E`bs>wM>fyYV%FS;7 zt|zoK9@@Z^3rBy2bOe*mlUn%>L2g^6H&tI_%Uk~i0^xEFp`_;57Du!FP{jA)tMtPr z*?Gq`xZbMXyg6{Xu-1^cw_W1rp1 z`RkO8A>UsVr$C_jyt&FeX^fBX>b-SZ^_KJj?bA#3pxXN7`FWe8`K7t}Nl9aaE_&X= zJ3SLvLBUe}h-R{1u9(Mu{9SV(+0KvtB7|uA=P2*wLsq{lZXvz`-#LpIn9|Bw)x?CD zb1c65Yc75An;Tre&dG&)nABHw6n7|_z{14FB_lXil?2u z_l(x=a^t5T5SNV3Gt9lWPqfz716=HjNd=rS>P);G|Zn&$NAFS-T;;2YBT3U!AL zOouLtlCb7%q?i@(ZWBK|G+|b#fqXoL9#92HzS5yMQrVy5U3F>~uD+NG2V-3LE4@q} z3D5>loX(wHK#$zYZM2QHqv-KLv2(RmLCt|^Z{@{oDw#KRe2X^erL~}K2=wmc4Nkd$ zguWnLu)}N3JPib?zOL>gmjfgJAIY%FJXZunUaW_RlJ%%VQr*8;4;qYKL&6;Ul?E1+ zoVaqWu5CyNOIOovn?{jNGmHpaLB!<&&8iYBGd2!v$O$g<7V);+y>{Hd169X)Z1cLK zz;Q2v45~>sJ}-W~%St;o`E^0vs>PPXFpGR4U&^N-ABH@7&X;c^#}#^K^~RI$Njf%F^G$QPR~km6$pstXVj1S7@d z9dP5aRt^Gof%x9!y1P+J5X8J`8SosVbZ^~4kVT2~6rVAjqfPDO;q)1owsH|`@sK)iaSqt9 z_+dYj%&=~$V7o+RcoprjMHP9Ty*}F< zEJOm_JFT|htDS9?Z~aVF<^Yjg+r^KQb_`Y2uf(z-jbqx3vNlf7^UddL%w}q@h{x02a=}@JVm~AWBvdOwGTUJz z^@b!W{k6pYJnKE3Wh(t|nN%QEK;s7??npg#Rmi(7*~*_(IJd*0s*tJ_;|ETIKg{jh zk7n!0-@`t%ZnGiA56upddR%!k4~+0VOsPr`ca(w|GtM3|T)D~?I4ysC<15Qpv*a(| zJZdD1Z+Z|;d(GLIdLp|Tztqd5juEQk#>#qf*ELoatvS*HuDG^UbblQ_*4VhWR7(0H zWIX{*K`$$~6qz;`5Bjg$n+%yYOHb)uPjB?)wan^=P9{+&p1S$Inwk@6{K9=lFIr#r zIO2gs$(go{87A?xosvdyS&1d_^nr)QNC#zGMG-e<{v4fWC_Z|9F7k(ChAq*wvRKTS z>1IQjd)&H!Ow4==A=q<*XgZ!0EZFtH1nSxwOpFqs@@XtpYBSZO1ltWKt_xVl%qKI0 zJ>_`H+8tuflAIO{S$T#`x*SDAozuV}>lZjfj@=C!{tXJg6ky*Z7f{lvRL1E~n$rhu ze4&YT5lR?6!XDOj{dGhG)?Lo*!b9?eTziR0Up@89+TeD*Z{;a zZ`Rw1-CNu2LZ7|5;H&v^YU*q?U9}KyayuL3a;eUv)1p(kPXHoso}JPLm? z1NQ-WhQ?4>xHv!ry15bQ5V#D2ImGl(>u7}`#dXl`5TA`p**OA2S$m0f6u{|YALj<_ z!|a|r!|1;}ZV1?hYav@PgDyw$Q0*Xu$$@F3T$zVi!fi0Gz&0)LEc zJ(ibH2Y-l!|4SGAx!ZGH@IT$4pmGDB0`C8E`-e*SbHZ~m@V^Oe|0Mj+(%{d5Pk_HP z&7U;C|13}6WaY8?hs5&#=JKQf{%3h|frEag%KY5*xgPjm&bfhX&t3nkI{0(Yb1mq< zLH>{G(En#;=;zqydcS{TPahZO-(&xbT|Os0SNZsxBth_u^lvmjp3|P^kp4~M;&@K` zD^>I#>7)OjjMC>*KTo6id+IX&|NGScevzJUIsMIE5_`t}XSDeF2GsLQejaxI8_O#F YzhM8mMmYE<2eik}{^M^6QTM6!9}|u<1ONa4 literal 0 HcmV?d00001 diff --git a/scanpipe/tests/test_pipelines.py b/scanpipe/tests/test_pipelines.py index 3512f6ac21..15cbee1ef7 100644 --- a/scanpipe/tests/test_pipelines.py +++ b/scanpipe/tests/test_pipelines.py @@ -840,6 +840,29 @@ def mock_make_git_directory(**kwargs): self.assertEqual(2, project1.codebaseresources.count()) self.assertEqual(0, project1.discoveredpackages.count()) + def test_scanpipe_scan_maven_package_single_file(self): + pipeline_name = "scan_maven_package" + project1 = make_project() + + input_location = ( + self.data / "jvm" / "wisp-logging-2025.11.11.195957-97a44b0.jar" + ) + project1.copy_input_from(input_location) + + run = project1.add_pipeline(pipeline_name) + pipeline = run.make_pipeline_instance() + + exitcode, out = pipeline.execute() + self.assertEqual(0, exitcode, msg=out) + + self.assertEqual(13, project1.codebaseresources.count()) + self.assertEqual(1, project1.discoveredpackages.count()) + self.assertEqual(29, project1.discovereddependencies.count()) + + scancode_file = project1.get_latest_output(filename="scancode") + expected_file = self.data / "jvm" / "scancodeio_wisp-logging.json" + self.assertPipelineResultEqual(expected_file, scancode_file) + def test_scanpipe_scan_codebase_pipeline_integration(self): pipeline_name = "scan_codebase" project1 = make_project() From 9a11dd80261da3e19c36f8f6e109179c465a0e49 Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Wed, 19 Nov 2025 07:12:22 +0800 Subject: [PATCH 011/110] Fix the Incomplete URL substring sanitization #1763 Signed-off-by: Chin Yeung Li --- scanpipe/pipes/resolve.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scanpipe/pipes/resolve.py b/scanpipe/pipes/resolve.py index 81a6d7efee..52447286c7 100644 --- a/scanpipe/pipes/resolve.py +++ b/scanpipe/pipes/resolve.py @@ -593,7 +593,11 @@ def get_pom_url_list(input_source, packages): input_source_url = input_source.get("download_url", "") parsed_url = urlparse(input_source_url) - if input_source_url and parsed_url.netloc.endswith("maven.org"): + maven_hosts = { + "repo1.maven.org", + "repo.maven.apache.org", + } + if input_source_url and parsed_url.netloc in maven_hosts: base_url = input_source_url.rsplit("/", 1)[0] pom_url = ( base_url + "/" + "-".join(base_url.rstrip("/").split("/")[-2:]) + ".pom" From f303430d80c8cbc11317689365f6024ef13ba79f Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Wed, 19 Nov 2025 13:28:44 +0800 Subject: [PATCH 012/110] Add "maven.google.com" as a maven_hosts and accept .aar file #1763 Signed-off-by: Chin Yeung Li --- scanpipe/pipes/resolve.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scanpipe/pipes/resolve.py b/scanpipe/pipes/resolve.py index 52447286c7..2bce4d71e7 100644 --- a/scanpipe/pipes/resolve.py +++ b/scanpipe/pipes/resolve.py @@ -596,6 +596,7 @@ def get_pom_url_list(input_source, packages): maven_hosts = { "repo1.maven.org", "repo.maven.apache.org", + "maven.google.com", } if input_source_url and parsed_url.netloc in maven_hosts: base_url = input_source_url.rsplit("/", 1)[0] @@ -606,13 +607,13 @@ def get_pom_url_list(input_source, packages): else: # Construct a pom_url from filename input_filename = input_source.get("filename", "") - if input_filename.endswith(".jar"): + if input_filename.endswith(".jar") or input_filename.endswith(".aar"): artifact_id, version = parse_maven_filename(input_filename) if not artifact_id or not version: return [] pom_url_list = construct_pom_url_from_filename(artifact_id, version) else: - # Only work with input that's a .jar file + # Only work with input that's a .jar or .aar file return [] return pom_url_list From ab7175155e7736651d9979c28de15e25036eac46 Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Wed, 26 Nov 2025 18:37:09 +0800 Subject: [PATCH 013/110] Reorganize code structure and code enhancement #1763 - Create a new maven pipe module - Use database queries for update_package_license_from_resource_if_missing() - Add tests Signed-off-by: Chin Yeung Li --- scanpipe/pipelines/scan_maven_package.py | 44 ++- scanpipe/pipelines/scan_single_package.py | 19 -- scanpipe/pipes/maven.py | 264 ++++++++++++++++ scanpipe/pipes/resolve.py | 237 -------------- scanpipe/tests/data/maven/lic_in_package.json | 153 ++++++++++ .../data/maven/missing_lic_in_package.json | 153 ++++++++++ .../missing_lic_in_package_pid_not_match.json | 153 ++++++++++ scanpipe/tests/pipes/test_maven.py | 288 ++++++++++++++++++ scanpipe/tests/pipes/test_resolve.py | 280 ----------------- 9 files changed, 1031 insertions(+), 560 deletions(-) create mode 100644 scanpipe/pipes/maven.py create mode 100644 scanpipe/tests/data/maven/lic_in_package.json create mode 100644 scanpipe/tests/data/maven/missing_lic_in_package.json create mode 100644 scanpipe/tests/data/maven/missing_lic_in_package_pid_not_match.json create mode 100644 scanpipe/tests/pipes/test_maven.py diff --git a/scanpipe/pipelines/scan_maven_package.py b/scanpipe/pipelines/scan_maven_package.py index 5b0e3d340b..5c7e3fe4ab 100644 --- a/scanpipe/pipelines/scan_maven_package.py +++ b/scanpipe/pipelines/scan_maven_package.py @@ -20,12 +20,9 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. -import json - from scanpipe.pipelines.scan_single_package import ScanSinglePackage -from scanpipe.pipes.resolve import download_pom_files -from scanpipe.pipes.resolve import get_pom_url_list -from scanpipe.pipes.resolve import scan_pom_files +from scanpipe.pipes.maven import fetch_and_scan_remote_pom +from scanpipe.pipes.maven import update_package_license_from_resource_if_missing class ScanMavenPackage(ScanSinglePackage): @@ -47,28 +44,27 @@ def steps(cls): cls.extract_archives, cls.run_scan, cls.fetch_and_scan_remote_pom, - cls.update_package_license_from_resource_if_missing, cls.load_inventory_from_toolkit_scan, + cls.update_package_license_from_resource_if_missing, cls.make_summary_from_scan_results, ) def fetch_and_scan_remote_pom(self): - """Fetch the .pom file from from maven.org if not present in codebase.""" - with open(self.scan_output_location) as file: - data = json.load(file) - # Return and do nothing if data has pom.xml - for file in data["files"]: - if "pom.xml" in file["path"]: - return - packages = data.get("packages", []) - - pom_url_list = get_pom_url_list(self.project.input_sources[0], packages) - pom_file_list = download_pom_files(pom_url_list) - scanned_pom_packages, scanned_dependencies = scan_pom_files(pom_file_list) + """Fetch and scan remote POM files.""" + scanning_errors = fetch_and_scan_remote_pom( + self.project, self.scan_output_location + ) + if scanning_errors: + for scanning_error in scanning_errors: + for resource_path, errors in scanning_error.items(): + self.project.add_error( + description="\n".join(errors), + model=self.pipeline_name, + details={ + "resource_path": resource_path.removeprefix("codebase/") + }, + ) - updated_packages = packages + scanned_pom_packages - # Replace/Update the package and dependencies section - data["packages"] = updated_packages - data["dependencies"] = scanned_dependencies - with open(self.scan_output_location, "w") as file: - json.dump(data, file, indent=2) + def update_package_license_from_resource_if_missing(self): + """Update PACKAGE license from the license detected in RESOURCES if missing.""" + update_package_license_from_resource_if_missing(self.project) diff --git a/scanpipe/pipelines/scan_single_package.py b/scanpipe/pipelines/scan_single_package.py index 7f0bf8b909..605ef0ea5d 100644 --- a/scanpipe/pipelines/scan_single_package.py +++ b/scanpipe/pipelines/scan_single_package.py @@ -31,7 +31,6 @@ from scanpipe.pipes import scancode from scanpipe.pipes.input import copy_input from scanpipe.pipes.input import is_archive -from scanpipe.pipes.resolve import update_package_license_from_resource_if_missing class ScanSinglePackage(Pipeline): @@ -52,7 +51,6 @@ def steps(cls): cls.extract_input_to_codebase_directory, cls.extract_archives, cls.run_scan, - cls.update_package_license_from_resource_if_missing, cls.load_inventory_from_toolkit_scan, cls.make_summary_from_scan_results, ) @@ -128,23 +126,6 @@ def run_scan(self): if not scan_output_path.exists(): raise FileNotFoundError("ScanCode output not available.") - def update_package_license_from_resource_if_missing(self): - """Update PACKAGE license from the license detected in RESOURCES if missing.""" - with open(self.scan_output_location) as file: - data = json.load(file) - packages = data.get("packages", []) - resources = data.get("files", []) - if not packages or not resources: - return - - updated_packages = update_package_license_from_resource_if_missing( - packages, resources - ) - # Update the package section - data["packages"] = updated_packages - with open(self.scan_output_location, "w") as file: - json.dump(data, file, indent=2) - def load_inventory_from_toolkit_scan(self): """Process a JSON Scan results to populate codebase resources and packages.""" input.load_inventory_from_toolkit_scan(self.project, self.scan_output_location) diff --git a/scanpipe/pipes/maven.py b/scanpipe/pipes/maven.py new file mode 100644 index 0000000000..3f8d442317 --- /dev/null +++ b/scanpipe/pipes/maven.py @@ -0,0 +1,264 @@ +import json +import re + +import requests + +from scanpipe.pipes import fetch +from scanpipe.pipes import scancode + + +def fetch_and_scan_remote_pom(project, scan_output_location): + """Fetch the .pom file from from maven.org if not present in codebase.""" + with open(scan_output_location) as file: + data = json.load(file) + # Return and do nothing if data has pom.xml + for file in data["files"]: + if "pom.xml" in file["path"]: + return + packages = data.get("packages", []) + + pom_url_list = get_pom_url_list(project.input_sources[0], packages) + pom_file_list = download_pom_files(pom_url_list) + scanning_errors = scan_pom_files(pom_file_list) + + scanned_pom_packages, scanned_dependencies = update_datafile_paths(pom_file_list) + + updated_packages = packages + scanned_pom_packages + # Replace/Update the package and dependencies section + data["packages"] = updated_packages + data["dependencies"] = scanned_dependencies + with open(scan_output_location, "w") as file: + json.dump(data, file, indent=2) + return scanning_errors + + +def parse_maven_filename(filename): + """Parse a Maven's jar filename to extract artifactId and version.""" + # Remove the .jar extension + base = filename.rsplit(".", 1)[0] + + # Common classifiers pattern + common_classifiers = { + "sources", + "javadoc", + "tests", + "test", + "test-sources", + "src", + "bin", + "docs", + "javadocs", + "client", + "server", + "linux", + "windows", + "macos", + "linux-x86_64", + "windows-x86_64", + } + + # Remove known classifier if present + for classifier in common_classifiers: + if base.endswith(f"-{classifier}"): + base = base[: -(len(classifier) + 1)] + break + + # Match artifactId and version + match = re.match(r"^(.*?)-((\d[\w.\-]*))$", base) + + if match: + artifact_id = match.group(1) + version = match.group(2) + return artifact_id, version + else: + return None, None + + +def get_pom_url_list(input_source, packages): + """Generate Maven POM URLs from package metadata or input source.""" + pom_url_list = [] + if packages: + for package in packages: + if package.get("type") == "maven": + package_ns = package.get("namespace", "") + package_name = package.get("name", "") + package_version = package.get("version", "") + pom_url = ( + f"https://repo1.maven.org/maven2/{package_ns.replace('.', '/')}/" + f"{package_name}/{package_version}/" + f"{package_name}-{package_version}.pom".lower() + ) + pom_url_list.append(pom_url) + if not pom_url_list: + from urllib.parse import urlparse + + # Check what's the input source + input_source_url = input_source.get("download_url", "") + + parsed_url = urlparse(input_source_url) + maven_hosts = { + "repo1.maven.org", + "repo.maven.apache.org", + "maven.google.com", + } + if input_source_url and parsed_url.netloc in maven_hosts: + base_url = input_source_url.rsplit("/", 1)[0] + pom_url = ( + base_url + "/" + "-".join(base_url.rstrip("/").split("/")[-2:]) + ".pom" + ) + pom_url_list.append(pom_url) + else: + # Construct a pom_url from filename + input_filename = input_source.get("filename", "") + if input_filename.endswith(".jar") or input_filename.endswith(".aar"): + artifact_id, version = parse_maven_filename(input_filename) + if not artifact_id or not version: + return [] + pom_url_list = construct_pom_url_from_filename(artifact_id, version) + else: + # Only work with input that's a .jar or .aar file + return [] + + return pom_url_list + + +def construct_pom_url_from_filename(artifact_id, version): + """Construct a pom.xml URL from the given Maven filename.""" + # Search Maven Central for the artifact to get its groupId + url = f"https://search.maven.org/solrsearch/select?q=a:{artifact_id}&wt=json" + pom_url_list = [] + group_ids = [] + try: + response = requests.get(url, timeout=5) + response.raise_for_status() + data = response.json() + # Extract all 'g' fields from the docs array that represent + # groupIds + group_ids = [doc["g"] for doc in data["response"]["docs"]] + except requests.RequestException as e: + print(f"Error fetching data: {e}") + return [] + except KeyError as e: + print(f"Error parsing JSON: {e}") + return [] + + for group_id in group_ids: + pom_url = ( + f"https://repo1.maven.org/maven2/{group_id.replace('.', '/')}/" + f"{artifact_id}/{version}/{artifact_id}-{version}.pom".lower() + ) + if is_maven_pom_url(pom_url): + pom_url_list.append(pom_url) + if len(pom_url_list) > 1: + # If multiple valid POM URLs are found, it means the same + # artifactId and version exist under different groupIds. Since we + # can't confidently determine the correct groupId, we return an + # empty list to avoid fetching the wrong POM. + return [] + + return pom_url_list + + +def is_maven_pom_url(url): + """Return True if the url is a accessible, False otherwise""" + # Maven Central has a fallback mechanism that serves a generic/error + # page instead of returning a proper 404. + try: + response = requests.get(url, timeout=5) + if response.status_code != 200: + return False + # Check content-type + content_type = response.headers.get("content-type", "").lower() + is_xml = "xml" in content_type or "text/xml" in content_type + + # Check content + content = response.text.strip() + is_pom = content.startswith(" 1: - # If multiple valid POM URLs are found, it means the same - # artifactId and version exist under different groupIds. Since we - # can't confidently determine the correct groupId, we return an - # empty list to avoid fetching the wrong POM. - return [] - - return pom_url_list - - -def is_maven_pom_url(url): - """Return True if the url is a accessible, False otherwise""" - # Maven Central has a fallback mechanism that serves a generic/error - # page instead of returning a proper 404. - try: - response = requests.get(url, timeout=5) - if response.status_code != 200: - return False - # Check content-type - content_type = response.headers.get("content-type", "").lower() - is_xml = "xml" in content_type or "text/xml" in content_type - - # Check content - content = response.text.strip() - is_pom = content.startswith("' + mock_get.return_value = mock_response + + result = maven.is_maven_pom_url( + "https://repo1.maven.org/maven2/example/example.pom" + ) + self.assertTrue(result) + + @mock.patch("requests.get") + def test_scanpipe_maven_is_maven_pom_url_404(self, mock_get): + mock_response = mock.Mock() + mock_response.status_code = 404 + mock_get.return_value = mock_response + + result = maven.is_maven_pom_url( + "https://repo.maven.apache.org/maven2/example/404.pom" + ) + self.assertFalse(result) + + @mock.patch("requests.get") + def test_scanpipe_maven_is_maven_pom_url_error(self, mock_get): + mock_response = mock.Mock() + mock_response.status_code = 200 + mock_response.headers = {"content-type": "text/html"} + mock_response.text = "Error page" + mock_get.return_value = mock_response + + result = maven.is_maven_pom_url( + "https://repo.maven.apache.org/maven2/example/error.pom" + ) + self.assertFalse(result) + + @mock.patch("scanpipe.pipes.maven.fetch.fetch_http") + def test_scanpipe_maven_download_pom_files(self, mock_fetch_http): + mock_response = mock.Mock() + mock_response.path = "/safe/example1.pom" + mock_fetch_http.return_value = mock_response + + pom_urls = ["https://repo1.maven.org/maven2/example/example1.pom"] + + expected = [ + { + "pom_file_path": "/safe/example1.pom", + "output_path": "/safe/example1.pom-output.json", + "pom_url": "https://repo1.maven.org/maven2/example/example1.pom", + } + ] + + result = maven.download_pom_files(pom_urls) + self.assertEqual(result, expected) + + @mock.patch("scanpipe.pipes.maven.scancode.run_scan") + @mock.patch("builtins.open", new_callable=mock.mock_open) + @mock.patch("json.load") + def test_scanpipe_maven_update_datafile_paths( + self, mock_json_load, mock_open, mock_run_scan + ): + mock_json_load.return_value = { + "packages": [ + { + "name": "example-package", + "version": "1.0.0", + "datafile_paths": ["/safe/mock_pom.xml"], + } + ], + "dependencies": [ + { + "name": "example-dep", + "version": "2.0.0", + "datafile_path": "/safe/mock_pom.xml", + } + ], + } + + pom_file_list = [ + { + "pom_file_path": "/safe/mock.pom", + "output_path": "/safe/mock.pom-output.json", + "pom_url": "https://repo1.maven.org/maven2/example/example.pom", + } + ] + + expected_packages = [ + { + "name": "example-package", + "version": "1.0.0", + "datafile_paths": [ + "https://repo1.maven.org/maven2/example/example.pom" + ], + } + ] + expected_deps = [ + {"name": "example-dep", "version": "2.0.0", "datafile_path": ""} + ] + + packages, deps = maven.update_datafile_paths(pom_file_list) + + self.assertEqual(packages, expected_packages) + self.assertEqual(deps, expected_deps) + + @mock.patch("scanpipe.pipes.maven.is_maven_pom_url") + @mock.patch("scanpipe.pipes.maven.requests.get") + def test_scanpipe_maven_construct_pom_url_from_filename( + self, mock_get, mock_is_maven_pom_url + ): + # Setup mock response from Maven Central + mock_response = mock.Mock() + mock_response.raise_for_status.return_value = None + mock_response.json.return_value = { + "response": {"docs": [{"g": "org.apache.commons"}]} + } + mock_get.return_value = mock_response + mock_is_maven_pom_url.return_value = True + + # Inputs + artifact_id = "commons-lang3" + version = "3.12.0" + + expected_url = [ + "https://repo1.maven.org/maven2/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.pom" + ] + + result = maven.construct_pom_url_from_filename(artifact_id, version) + + self.assertEqual(result, expected_url) + mock_get.assert_called_once_with( + "https://search.maven.org/solrsearch/select?q=a:commons-lang3&wt=json", + timeout=5, + ) + mock_is_maven_pom_url.assert_called_once_with(expected_url[0]) + + def test_scanpipe_maven_get_pom_url_list_with_packages(self): + packages = [ + { + "type": "maven", + "namespace": "org.apache.commons", + "name": "commons-lang3", + "version": "3.12.0", + } + ] + result = maven.get_pom_url_list({}, packages) + expected = [ + "https://repo1.maven.org/maven2/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.pom" + ] + self.assertEqual(result, expected) + + def test_scanpipe_maven_get_pom_url_list_with_non_maven_packages(self): + packages = [ + { + "type": "jar", + "namespace": "", + "name": "spring-context", + "version": "7.0.0", + } + ] + result = maven.get_pom_url_list({}, packages) + expected = [] + self.assertEqual(result, expected) + + def test_scanpipe_maven_get_pom_url_list_with_maven_download_url(self): + input_source = { + "download_url": "https://repo1.maven.org/maven2/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.jar" + } + result = maven.get_pom_url_list(input_source, []) + expected = [ + "https://repo1.maven.org/maven2/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.pom" + ] + self.assertEqual(result, expected) + + @mock.patch("scanpipe.pipes.maven.construct_pom_url_from_filename") + @mock.patch("scanpipe.pipes.maven.parse_maven_filename") + def test_scanpipe_maven_get_pom_url_list_with_jar_filename( + self, mock_parse, mock_construct + ): + input_source = {"filename": "commons-lang3-3.12.0.jar"} + mock_parse.return_value = ("commons-lang3", "3.12.0") + mock_construct.return_value = [ + "https://repo1.maven.org/maven2/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.pom" + ] + result = maven.get_pom_url_list(input_source, []) + self.assertEqual(result, mock_construct.return_value) + + def test_scanpipe_maven_get_pom_url_list_with_invalid_filename(self): + input_source = {"filename": "not-a-jar.txt"} + result = maven.get_pom_url_list(input_source, []) + self.assertEqual(result, []) + + def test_scanpipe_maven_update_package_license_from_resource_if_missing(self): + project1 = Project.objects.create(name="Analysis") + input_location = self.data / "maven" / "missing_lic_in_package.json" + project1.copy_input_from(input_location) + copy_inputs(project1.inputs(), project1.codebase_path) + + load_inventory_from_toolkit_scan(project1, str(input_location)) + + for package in project1.discoveredpackages.all(): + self.assertEqual(package.get_declared_license_expression(), "") + + maven.update_package_license_from_resource_if_missing(project1) + + for package in project1.discoveredpackages.all(): + self.assertEqual(package.get_declared_license_expression(), "apache-2.0") + + def test_scanpipe_maven_update_package_license_from_resource_if_missing_no_change( + self, + ): + project1 = Project.objects.create(name="Analysis") + input_location = self.data / "maven" / "lic_in_package.json" + project1.copy_input_from(input_location) + copy_inputs(project1.inputs(), project1.codebase_path) + + load_inventory_from_toolkit_scan(project1, str(input_location)) + + for package in project1.discoveredpackages.all(): + self.assertEqual(package.get_declared_license_expression(), "custom") + + maven.update_package_license_from_resource_if_missing(project1) + + for package in project1.discoveredpackages.all(): + self.assertEqual(package.get_declared_license_expression(), "custom") diff --git a/scanpipe/tests/pipes/test_resolve.py b/scanpipe/tests/pipes/test_resolve.py index 7249763253..aa16edaafc 100644 --- a/scanpipe/tests/pipes/test_resolve.py +++ b/scanpipe/tests/pipes/test_resolve.py @@ -373,283 +373,3 @@ def test_scanpipe_resolve_get_manifest_headers(self): ] headers = resolve.get_manifest_headers(resource) self.assertEqual(expected, list(headers.keys())) - - def test_scanpipe_resolve_parse_maven_filename(self): - test1 = "wisp-logging-2025.11.11.195957-97a44b0-sources.jar" - test2 = "guava-33.5.0-jre-javadoc.jar" - test3 = "junit-4.13.2.jar" - test4 = "guava-33.5.0-jre.jar" - - expected1_name = "wisp-logging" - expected1_version = "2025.11.11.195957-97a44b0" - expected2_name = "guava" - expected2_version = "33.5.0-jre" - expected3_name = "junit" - expected3_version = "4.13.2" - - result1_name, result1_version = resolve.parse_maven_filename(test1) - result2_name, result2_version = resolve.parse_maven_filename(test2) - result3_name, result3_version = resolve.parse_maven_filename(test3) - result4_name, result4_version = resolve.parse_maven_filename(test4) - - self.assertEqual(result1_name, expected1_name) - self.assertEqual(result1_version, expected1_version) - self.assertEqual(result2_name, expected2_name) - self.assertEqual(result2_version, expected2_version) - self.assertEqual(result3_name, expected3_name) - self.assertEqual(result3_version, expected3_version) - self.assertEqual(result4_name, expected2_name) - self.assertEqual(result4_version, expected2_version) - - @mock.patch("requests.get") - def test_scanpipe_resolve_is_maven_pom_url_valid(self, mock_get): - mock_response = mock.Mock() - mock_response.status_code = 200 - mock_response.headers = {"content-type": "application/xml"} - mock_response.text = '' - mock_get.return_value = mock_response - - result = resolve.is_maven_pom_url( - "https://repo1.maven.org/maven2/example/example.pom" - ) - self.assertTrue(result) - - @mock.patch("requests.get") - def test_scanpipe_resolve_is_maven_pom_url_404(self, mock_get): - mock_response = mock.Mock() - mock_response.status_code = 404 - mock_get.return_value = mock_response - - result = resolve.is_maven_pom_url( - "https://repo.maven.apache.org/maven2/example/404.pom" - ) - self.assertFalse(result) - - @mock.patch("requests.get") - def test_scanpipe_resolve_is_maven_pom_url_error(self, mock_get): - mock_response = mock.Mock() - mock_response.status_code = 200 - mock_response.headers = {"content-type": "text/html"} - mock_response.text = "Error page" - mock_get.return_value = mock_response - - result = resolve.is_maven_pom_url( - "https://repo.maven.apache.org/maven2/example/error.pom" - ) - self.assertFalse(result) - - @mock.patch("scanpipe.pipes.resolve.fetch.fetch_http") - def test_scanpipe_resolve_download_pom_files(self, mock_fetch_http): - mock_response = mock.Mock() - mock_response.path = "/safe/example1.pom" - mock_fetch_http.return_value = mock_response - - pom_urls = ["https://repo1.maven.org/maven2/example/example1.pom"] - - expected = [ - { - "pom_file_path": "/safe/example1.pom", - "output_path": "/safe/example1.pom-output.json", - "pom_url": "https://repo1.maven.org/maven2/example/example1.pom", - } - ] - - result = resolve.download_pom_files(pom_urls) - self.assertEqual(result, expected) - - @mock.patch("scanpipe.pipes.resolve.scancode.run_scan") - @mock.patch("builtins.open", new_callable=mock.mock_open) - @mock.patch("json.load") - def test_scanpipe_resolve_scan_pom_files( - self, mock_json_load, mock_open, mock_run_scan - ): - mock_json_load.return_value = { - "packages": [ - { - "name": "example-package", - "version": "1.0.0", - "datafile_paths": ["/safe/mock_pom.xml"], - } - ], - "dependencies": [ - { - "name": "example-dep", - "version": "2.0.0", - "datafile_path": "/safe/mock_pom.xml", - } - ], - } - - pom_file_list = [ - { - "pom_file_path": "/safe/mock.pom", - "output_path": "/safe/mock.pom-output.json", - "pom_url": "https://repo1.maven.org/maven2/example/example.pom", - } - ] - - expected_packages = [ - { - "name": "example-package", - "version": "1.0.0", - "datafile_paths": [ - "https://repo1.maven.org/maven2/example/example.pom" - ], - } - ] - expected_deps = [ - {"name": "example-dep", "version": "2.0.0", "datafile_path": ""} - ] - - packages, deps = resolve.scan_pom_files(pom_file_list) - - self.assertEqual(packages, expected_packages) - self.assertEqual(deps, expected_deps) - - mock_run_scan.assert_called_once_with( - location="/safe/mock.pom", - output_file="/safe/mock.pom-output.json", - run_scan_args={"package": True}, - ) - mock_open.assert_called_once_with("/safe/mock.pom-output.json") - mock_json_load.assert_called_once() - - @mock.patch("scanpipe.pipes.resolve.is_maven_pom_url") - @mock.patch("scanpipe.pipes.resolve.requests.get") - def test_scanpipe_resolve_construct_pom_url_from_filename( - self, mock_get, mock_is_maven_pom_url - ): - # Setup mock response from Maven Central - mock_response = mock.Mock() - mock_response.raise_for_status.return_value = None - mock_response.json.return_value = { - "response": {"docs": [{"g": "org.apache.commons"}]} - } - mock_get.return_value = mock_response - mock_is_maven_pom_url.return_value = True - - # Inputs - artifact_id = "commons-lang3" - version = "3.12.0" - - expected_url = [ - "https://repo1.maven.org/maven2/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.pom" - ] - - result = resolve.construct_pom_url_from_filename(artifact_id, version) - - self.assertEqual(result, expected_url) - mock_get.assert_called_once_with( - "https://search.maven.org/solrsearch/select?q=a:commons-lang3&wt=json", - timeout=5, - ) - mock_is_maven_pom_url.assert_called_once_with(expected_url[0]) - - def test_scanpipe_resolve_get_pom_url_list_with_packages(self): - packages = [ - { - "type": "maven", - "namespace": "org.apache.commons", - "name": "commons-lang3", - "version": "3.12.0", - } - ] - result = resolve.get_pom_url_list({}, packages) - expected = [ - "https://repo1.maven.org/maven2/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.pom" - ] - self.assertEqual(result, expected) - - def test_scanpipe_resolve_get_pom_url_list_with_non_maven_packages(self): - packages = [ - { - "type": "jar", - "namespace": "", - "name": "spring-context", - "version": "7.0.0", - } - ] - result = resolve.get_pom_url_list({}, packages) - expected = [] - self.assertEqual(result, expected) - - def test_scanpipe_resolve_get_pom_url_list_with_maven_download_url(self): - input_source = { - "download_url": "https://repo1.maven.org/maven2/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.jar" - } - result = resolve.get_pom_url_list(input_source, []) - expected = [ - "https://repo1.maven.org/maven2/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.pom" - ] - self.assertEqual(result, expected) - - @mock.patch("scanpipe.pipes.resolve.construct_pom_url_from_filename") - @mock.patch("scanpipe.pipes.resolve.parse_maven_filename") - def test_scanpipe_resolve_get_pom_url_list_with_jar_filename( - self, mock_parse, mock_construct - ): - input_source = {"filename": "commons-lang3-3.12.0.jar"} - mock_parse.return_value = ("commons-lang3", "3.12.0") - mock_construct.return_value = [ - "https://repo1.maven.org/maven2/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.pom" - ] - result = resolve.get_pom_url_list(input_source, []) - self.assertEqual(result, mock_construct.return_value) - - def test_scanpipe_resolve_get_pom_url_list_with_invalid_filename(self): - input_source = {"filename": "not-a-jar.txt"} - result = resolve.get_pom_url_list(input_source, []) - self.assertEqual(result, []) - - def test_scanpipe_resolve_update_package_license_from_resource_if_missing(self): - packages = [ - {"package_uid": "pkg1", "declared_license_expression": ""}, - {"package_uid": "pkg2", "declared_license_expression": None}, - {"package_uid": "pkg3", "declared_license_expression": "MIT"}, - ] - resources = [ - { - "for_packages": ["pkg1", "pkg2"], - "detected_license_expression": "GPL-2.0", - }, - {"for_packages": ["pkg1"], "detected_license_expression": "MIT"}, - ] - - expected_pkg1_expr = "GPL-2.0 AND MIT" - expected_pkg2_expr = "GPL-2.0" - - updated = resolve.update_package_license_from_resource_if_missing( - packages, resources - ) - - self.assertEqual(updated[0]["declared_license_expression"], expected_pkg1_expr) - self.assertEqual(updated[1]["declared_license_expression"], expected_pkg2_expr) - self.assertEqual(updated[2]["declared_license_expression"], "MIT") - - def test_scanpipe_resolve_update_package_license_from_resource_if_missing_no_match( - self, - ): - packages = [{"package_uid": "pkgX", "declared_license_expression": None}] - resources = [{"for_packages": ["pkgY"], "detected_license_expression": "MIT"}] - - updated = resolve.update_package_license_from_resource_if_missing( - packages, resources - ) - self.assertEqual(updated[0]["declared_license_expression"], None) - - def test_scanpipe_resolve_update_package_license_from_resource_if_missing_no_change( - self, - ): - packages = [ - {"package_uid": "pkg1", "declared_license_expression": "GPL-2.0"}, - {"package_uid": "pkg2", "declared_license_expression": "Apache-2.0"}, - ] - resources = [ - {"for_packages": ["pkg1", "pkg2"], "detected_license_expression": "MIT"}, - ] - - updated = resolve.update_package_license_from_resource_if_missing( - packages, resources - ) - self.assertEqual(updated[0]["declared_license_expression"], "GPL-2.0") - self.assertEqual(updated[1]["declared_license_expression"], "Apache-2.0") From 5579db8617aa20e7afc9632a2469dd70cf2f0fc8 Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Thu, 27 Nov 2025 07:40:55 +0800 Subject: [PATCH 014/110] Update build-in-pipelines.rst #1763 Signed-off-by: Chin Yeung Li --- docs/built-in-pipelines.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/built-in-pipelines.rst b/docs/built-in-pipelines.rst index 55baefa6df..3c9f366819 100644 --- a/docs/built-in-pipelines.rst +++ b/docs/built-in-pipelines.rst @@ -265,6 +265,14 @@ Scan Single Package :members: :member-order: bysource +.. _pipeline_scan_maven_package: + +Scan Maven Package +------------------- +.. autoclass:: scanpipe.pipelines.scan_maven_package.ScanMavenPackage() + :members: + :member-order: bysource + Fetch Scores (addon) -------------------- .. warning:: From 12728e46488a38ea05572e0ab3aa4f94f9802b5a Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Mon, 29 Dec 2025 09:01:36 +0800 Subject: [PATCH 015/110] Added steps to handle D2D in the maven pipeline #1763 Signed-off-by: Chin Yeung Li --- scanpipe/pipelines/scan_maven_package.py | 94 +++++++++++++++++++++++- 1 file changed, 90 insertions(+), 4 deletions(-) diff --git a/scanpipe/pipelines/scan_maven_package.py b/scanpipe/pipelines/scan_maven_package.py index 5c7e3fe4ab..80b83973b7 100644 --- a/scanpipe/pipelines/scan_maven_package.py +++ b/scanpipe/pipelines/scan_maven_package.py @@ -20,12 +20,14 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. +from aboutcode.pipeline import optional_step +from scanpipe.pipelines.deploy_to_develop import DeployToDevelop from scanpipe.pipelines.scan_single_package import ScanSinglePackage from scanpipe.pipes.maven import fetch_and_scan_remote_pom from scanpipe.pipes.maven import update_package_license_from_resource_if_missing -class ScanMavenPackage(ScanSinglePackage): +class ScanMavenPackage(ScanSinglePackage, DeployToDevelop): """ Scan a single maven package archive. @@ -34,14 +36,17 @@ class ScanMavenPackage(ScanSinglePackage): The output is a summary of the scan results in JSON format. """ + d2d_option_enabled = False @classmethod def steps(cls): return ( - cls.get_package_input, - cls.collect_input_information, - cls.extract_input_to_codebase_directory, + cls.check_option_status, + cls.get_input, + cls.collect_input_info, + cls.extract_input, cls.extract_archives, + cls.maven_d2d_steps, cls.run_scan, cls.fetch_and_scan_remote_pom, cls.load_inventory_from_toolkit_scan, @@ -49,6 +54,87 @@ def steps(cls): cls.make_summary_from_scan_results, ) + @optional_step("deploy_to_develop") + def check_option_status(self): + """Set d2d_option_enabled to True.""" + self.d2d_option_enabled = True + + def get_input(self): + """Locate the the input.""" + if not self.d2d_option_enabled: + self.get_package_input() + else: + self.get_inputs() + + def collect_input_info(self): + """Collect information about the input.""" + if not self.d2d_option_enabled: + self.collect_input_information() + + def extract_input(self): + """Extract the input to the codebase directory.""" + if not self.d2d_option_enabled: + self.extract_input_to_codebase_directory() + else: + self.extract_inputs_to_codebase_directory() + + @optional_step("deploy_to_develop") + def maven_d2d_steps(self): + """Run D2D steps for Maven projects.""" + # The following langages will be included: + # - Java + # - Kotlin + # - Scala + # - JavaScript + from scanpipe.pipes import d2d_config + + self.collect_and_create_codebase_resources() + self.fingerprint_codebase_directories() + self.flag_empty_files() + self.flag_whitespace_files() + self.flag_ignored_resources() + + options = ["Java", "Kotlin", "Scala", "JavaScript"] + d2d_config.load_ecosystem_config(pipeline=self, options=options) + + self.map_about_files() + self.map_checksum() + self.match_archives_to_purldb() + # Java + self.find_java_packages() + self.map_java_to_class() + self.map_jar_to_java_source() + # Scala + self.find_scala_packages() + self.map_scala_to_class() + self.map_jar_to_scala_source() + # Kotlin + self.find_kotlin_packages() + self.map_kotlin_to_class() + self.map_jar_to_kotlin_source() + # JavaScript + self.map_javascript() + self.map_javascript_symbols() + self.map_javascript_strings() + self.get_symbols_from_binaries() + self.map_elf() + self.match_directories_to_purldb() + self.match_resources_to_purldb() + self.map_javascript_post_purldb_match() + self.map_javascript_path() + self.map_javascript_colocation() + self.map_path() + self.flag_mapped_resources_archives_and_ignored_directories() + self.perform_house_keeping_tasks() + self.match_purldb_resources_post_process() + self.remove_packages_without_resources() + self.scan_ignored_to_files() + self.scan_unmapped_to_files() + self.scan_mapped_from_for_files() + self.collect_and_create_license_detections() + self.flag_deployed_from_resources_with_missing_license() + self.create_local_files_packages() + def fetch_and_scan_remote_pom(self): """Fetch and scan remote POM files.""" scanning_errors = fetch_and_scan_remote_pom( From 3242b16b206c935737421630ccf1147420689361 Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Mon, 29 Dec 2025 11:18:29 +0800 Subject: [PATCH 016/110] Update "exclude_from_diff" list #1763 Signed-off-by: Chin Yeung Li --- scanpipe/tests/test_pipelines.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scanpipe/tests/test_pipelines.py b/scanpipe/tests/test_pipelines.py index 15cbee1ef7..ddfa58341d 100644 --- a/scanpipe/tests/test_pipelines.py +++ b/scanpipe/tests/test_pipelines.py @@ -533,6 +533,8 @@ class PipelinesIntegrationTest(TestCase): "--verbose", # system_environment differs between systems "system_environment", + # the version number may change over time + "spdx_license_list_version", "file_type", # mime type and is_script are inconsistent across systems "mime_type", From 97a109be5920ac581d25cc3aa6943252e698ea6c Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Mon, 29 Dec 2025 11:22:17 +0800 Subject: [PATCH 017/110] Ruff reformatted #1763 Signed-off-by: Chin Yeung Li --- scanpipe/pipelines/scan_maven_package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scanpipe/pipelines/scan_maven_package.py b/scanpipe/pipelines/scan_maven_package.py index 80b83973b7..a1c352b56e 100644 --- a/scanpipe/pipelines/scan_maven_package.py +++ b/scanpipe/pipelines/scan_maven_package.py @@ -36,6 +36,7 @@ class ScanMavenPackage(ScanSinglePackage, DeployToDevelop): The output is a summary of the scan results in JSON format. """ + d2d_option_enabled = False @classmethod From 9992d15606ba68836198c24df87c1774b671e4a0 Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Mon, 29 Dec 2025 11:45:43 +0800 Subject: [PATCH 018/110] Update class description #1763 Signed-off-by: Chin Yeung Li --- scanpipe/pipelines/scan_maven_package.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scanpipe/pipelines/scan_maven_package.py b/scanpipe/pipelines/scan_maven_package.py index a1c352b56e..d3c97c80a7 100644 --- a/scanpipe/pipelines/scan_maven_package.py +++ b/scanpipe/pipelines/scan_maven_package.py @@ -29,11 +29,19 @@ class ScanMavenPackage(ScanSinglePackage, DeployToDevelop): """ - Scan a single maven package archive. + Scan a single maven package archive, or run a D2D scan for a maven package. This pipeline scans a single maven package for package metadata, declared dependencies, licenses, license clarity score and copyrights. + In addition, if the `deploy_to_develop` option is enabled, it performs a + D2D (deploy to develop) scan for Maven projects. + + It requires a minimum of two archive files, each properly tagged with: + + - **from** for archives containing the development source code. + - **to** for archives containing the deployment compiled code. + The output is a summary of the scan results in JSON format. """ From 4f1b4d31bf389250b86cfde41bbbe2b88fec24ac Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Wed, 31 Dec 2025 11:16:35 +0800 Subject: [PATCH 019/110] Error handling if pom url not found #1763 Signed-off-by: Chin Yeung Li --- scanpipe/pipes/maven.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/scanpipe/pipes/maven.py b/scanpipe/pipes/maven.py index 3f8d442317..d6f1f6b6bc 100644 --- a/scanpipe/pipes/maven.py +++ b/scanpipe/pipes/maven.py @@ -189,11 +189,15 @@ def download_pom_files(pom_url_list): pom_file_list = [] for pom_url in pom_url_list: pom_file_dict = {} - downloaded_pom = fetch.fetch_http(pom_url) - pom_file_dict["pom_file_path"] = str(downloaded_pom.path) - pom_file_dict["output_path"] = str(downloaded_pom.path) + "-output.json" - pom_file_dict["pom_url"] = pom_url - pom_file_list.append(pom_file_dict) + try: + downloaded_pom = fetch.fetch_http(pom_url) + pom_file_dict["pom_file_path"] = str(downloaded_pom.path) + pom_file_dict["output_path"] = str(downloaded_pom.path) + "-output.json" + pom_file_dict["pom_url"] = pom_url + pom_file_list.append(pom_file_dict) + except requests.RequestException: + # Keep the process going if one pom_url fails + continue return pom_file_list From 6fffa2ce49dfc995c1a3b01ff9efeb9694a0950e Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Mon, 25 May 2026 09:50:30 +0800 Subject: [PATCH 020/110] Prevent duplicate CodebaseRelation entries #1763 Some projects encountered a unique constraint violation when a resource was already mapped: ``` duplicate key value violates unique constraint "scanpipe_codebaserelation_unique_relation" DETAIL: Key (from_resource_id, to_resource_id, map_type)=(1512780, 1512790, jar_to_source) already exists. ``` Signed-off-by: Chin Yeung Li --- scanpipe/pipes/d2d.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanpipe/pipes/d2d.py b/scanpipe/pipes/d2d.py index a428400129..7c01564043 100644 --- a/scanpipe/pipes/d2d.py +++ b/scanpipe/pipes/d2d.py @@ -353,7 +353,7 @@ def map_jar_to_jvm_source(project, jvm_lang: jvm.JvmLanguage, logger=None): """Map .jar files to their related source directory.""" project_files = project.codebaseresources.files() # Include the directories to map on the common source - from_resources = project.codebaseresources.from_codebase() + from_resources = project.codebaseresources.from_codebase().has_no_relation() to_resources = project_files.to_codebase() to_jars = to_resources.filter(extension=".jar") From ee673f13b85daaf7b18558350a0bc8f65cfac114 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Fri, 26 Dec 2025 17:35:11 +0530 Subject: [PATCH 021/110] Update minecode pipelines with stable release v0.1.1 (#2013) Signed-off-by: Ayan Sinha Mahapatra --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2f215c11b6..52e50bd250 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -123,7 +123,7 @@ android_analysis = [ "android_inspector==0.0.1" ] mining = [ - "minecode_pipelines==0.0.1b8" + "minecode_pipelines==0.1.1" ] [project.urls] From 257560c4b7d6cad35bf744490c24692830876a13 Mon Sep 17 00:00:00 2001 From: pranavkeerti66-hue Date: Fri, 26 Dec 2025 18:38:48 +0530 Subject: [PATCH 022/110] Fix typos in docstrings for pathmap.py (#1985) Signed-off-by: pranavkeerti66-hue --- scanpipe/pipes/pathmap.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/scanpipe/pipes/pathmap.py b/scanpipe/pipes/pathmap.py index 9a0cdab9a1..44978b367d 100644 --- a/scanpipe/pipes/pathmap.py +++ b/scanpipe/pipes/pathmap.py @@ -164,11 +164,12 @@ def get_reversed_path_segments(path): Return reversed segments list given a POSIX ``path`` string. We reverse based on path segments separated by a "/". - Note that the inputh ``path`` is assumed to be normalized, not relative and + Note that the input ``path`` is assumed to be normalized, not relative and not containing double slash. - For example:: - >>> assert get_reversed_path_segments("a/b/c.js") == ["c.js", "b", "a"] + For example: + >>> get_reversed_path_segments("a/b/c.js") + ['c.js', 'b', 'a'] """ # [::-1] does the list reversing reversed_segments = path.strip("/").split("/")[::-1] @@ -177,13 +178,14 @@ def get_reversed_path_segments(path): def convert_segments_to_path(segments): """ - Return a path string is suitable for indexing or matching given a + Return a path string suitable for indexing or matching given a ``segments`` sequence of path segment strings. The resulting reversed path is prefixed and suffixed by a "/" irrespective of whether the original path is a file or directory and had such prefix or suffix. - For example:: - >>> assert convert_segments_to_path(["c.js", "b", "a"]) == "/c.js/b/a/" + For example: + >>> convert_segments_to_path(["c.js", "b", "a"]) + '/c.js/b/a/' """ return "/" + "/".join(segments) + "/" From 312445577e46140c006ac86c061be6f2a69f5f3d Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Fri, 26 Dec 2025 18:32:56 +0400 Subject: [PATCH 023/110] Store the whole vulnerability data from cdx to local models (#2007) Signed-off-by: tdruez --- scanpipe/pipes/cyclonedx.py | 3 + .../scanpipe/tabset/tab_vulnerabilities.html | 55 ++++++++++++------- scanpipe/tests/pipes/test_cyclonedx.py | 21 +++++-- scanpipe/tests/test_pipelines.py | 21 +++++-- 4 files changed, 70 insertions(+), 30 deletions(-) diff --git a/scanpipe/pipes/cyclonedx.py b/scanpipe/pipes/cyclonedx.py index bba33e18bb..00a59bd4e2 100644 --- a/scanpipe/pipes/cyclonedx.py +++ b/scanpipe/pipes/cyclonedx.py @@ -30,6 +30,7 @@ from cyclonedx.model import license as cdx_license_model from cyclonedx.model.bom import Bom from cyclonedx.schema import SchemaVersion +from cyclonedx.schema.schema import BaseSchemaVersion from cyclonedx.validation import ValidationError from cyclonedx.validation.json import JsonStrictValidator from defusedxml import ElementTree as SafeElementTree @@ -184,10 +185,12 @@ def cyclonedx_component_to_package_data( affected_by_vulnerabilities = [] if affected_by := vulnerabilities.get(bom_ref): for cdx_vulnerability in affected_by: + cdx_vulnerability_json = cdx_vulnerability.as_json(view_=BaseSchemaVersion) affected_by_vulnerabilities.append( { "vulnerability_id": str(cdx_vulnerability.id), "summary": cdx_vulnerability.description, + "cdx_vulnerability_data": json.loads(cdx_vulnerability_json), } ) diff --git a/scanpipe/templates/scanpipe/tabset/tab_vulnerabilities.html b/scanpipe/templates/scanpipe/tabset/tab_vulnerabilities.html index c05659883f..a2ffb2b7c9 100644 --- a/scanpipe/templates/scanpipe/tabset/tab_vulnerabilities.html +++ b/scanpipe/templates/scanpipe/tabset/tab_vulnerabilities.html @@ -2,9 +2,9 @@ - + - + @@ -15,28 +15,43 @@ {{ vulnerability.vulnerability_id }} +
    + {% for alias in aliases %} +
  • + {% if alias|slice:":3" == "CVE" %} + {{ alias }} + + + {% elif alias|slice:":4" == "GHSA" %} + {{ alias }} + + + {% elif alias|slice:":3" == "NPM" %} + {{ alias }} + + + {% else %} + {{ alias }} + {% endif %} +
    • + {% endfor %} +
- + diff --git a/scanpipe/tests/pipes/test_cyclonedx.py b/scanpipe/tests/pipes/test_cyclonedx.py index 3ae61035f0..0ffaae395d 100644 --- a/scanpipe/tests/pipes/test_cyclonedx.py +++ b/scanpipe/tests/pipes/test_cyclonedx.py @@ -250,13 +250,24 @@ def test_scanpipe_cyclonedx_resolve_cyclonedx_packages_vulnerabilities(self): self.assertEqual(1, len(packages)) affected_by = packages[0]["affected_by_vulnerabilities"] + self.assertEqual("CVE-2005-2541", affected_by[0]["vulnerability_id"]) + self.assertEqual( + "Tar 1.15.1 does not properly warn the user when...", + affected_by[0]["summary"], + ) + self.assertIn("cdx_vulnerability_data", affected_by[0]) + cdx_vulnerability_data = affected_by[0]["cdx_vulnerability_data"] expected = [ - { - "vulnerability_id": "CVE-2005-2541", - "summary": "Tar 1.15.1 does not properly warn the user when...", - } + "advisories", + "affects", + "description", + "id", + "published", + "ratings", + "source", + "updated", ] - self.assertEqual(expected, affected_by) + self.assertEqual(expected, sorted(cdx_vulnerability_data.keys())) def test_scanpipe_cyclonedx_resolve_cyclonedx_packages_pre_validation(self): # This SBOM includes multiple deserialization issues that are "fixed" diff --git a/scanpipe/tests/test_pipelines.py b/scanpipe/tests/test_pipelines.py index ddfa58341d..4b72f2badd 100644 --- a/scanpipe/tests/test_pipelines.py +++ b/scanpipe/tests/test_pipelines.py @@ -1663,13 +1663,24 @@ def test_scanpipe_load_sbom_pipeline_cyclonedx_with_vulnerabilities(self): self.assertEqual(1, project1.discoveredpackages.count()) package = project1.discoveredpackages.get() + affected_by = package.affected_by_vulnerabilities[0] + cdx_vulnerability_data = affected_by.pop("cdx_vulnerability_data") + expected = { + "vulnerability_id": "CVE-2005-2541", + "summary": "Tar 1.15.1 does not properly warn the user when...", + } + self.assertEqual(expected, affected_by) expected = [ - { - "vulnerability_id": "CVE-2005-2541", - "summary": "Tar 1.15.1 does not properly warn the user when...", - } + "advisories", + "affects", + "description", + "id", + "published", + "ratings", + "source", + "updated", ] - self.assertEqual(expected, package.affected_by_vulnerabilities) + self.assertEqual(expected, sorted(cdx_vulnerability_data.keys())) @mock.patch("scanpipe.pipes.purldb.request_post") @mock.patch("uuid.uuid4") From 9e2b2bc41484a7c169fdf3fcac3455eef0365f0c Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Mon, 29 Dec 2025 13:38:57 +0400 Subject: [PATCH 024/110] Add project vulnerability list view (#2018) Signed-off-by: tdruez --- scancodeio/static/main.css | 6 +++ .../management/commands/check-compliance.py | 24 ++++------- scanpipe/models.py | 5 +++ .../includes/project_summary_level.html | 4 ++ .../scanpipe/includes/vulnerability_id.html | 30 +++++++++++++ .../includes/vulnerability_summary.html | 10 +++++ .../scanpipe/tabset/tab_vulnerabilities.html | 37 +--------------- .../scanpipe/vulnerability_list.html | 43 +++++++++++++++++++ scanpipe/tests/test_commands.py | 9 ++-- scanpipe/tests/test_models.py | 1 + scanpipe/tests/test_views.py | 27 +++++++++++- scanpipe/urls.py | 5 +++ scanpipe/views.py | 24 ++++++++++- 13 files changed, 168 insertions(+), 57 deletions(-) create mode 100644 scanpipe/templates/scanpipe/includes/vulnerability_id.html create mode 100644 scanpipe/templates/scanpipe/includes/vulnerability_summary.html create mode 100644 scanpipe/templates/scanpipe/vulnerability_list.html diff --git a/scancodeio/static/main.css b/scancodeio/static/main.css index e8f45f1bf8..a51269fcbe 100644 --- a/scancodeio/static/main.css +++ b/scancodeio/static/main.css @@ -391,6 +391,12 @@ progress.file-upload::before { #message-list th#column-severity { min-width: 110px; } +th#column-vulnerability_id { + min-width: 220px; +} +th#column-summary { + width: 40%; +} .menu.is-info .is-active { background-color: #3e8ed0; } diff --git a/scanpipe/management/commands/check-compliance.py b/scanpipe/management/commands/check-compliance.py index 216a849e67..0016177a01 100644 --- a/scanpipe/management/commands/check-compliance.py +++ b/scanpipe/management/commands/check-compliance.py @@ -104,23 +104,17 @@ def check_compliance(self, fail_level): return total_issues > 0 def check_vulnerabilities(self): - packages = self.project.discoveredpackages.vulnerable_ordered() - dependencies = self.project.discovereddependencies.vulnerable_ordered() - - vulnerable_records = list(packages) + list(dependencies) - count = len(vulnerable_records) + all_vulnerabilities = self.project.vulnerabilities + vulnerabilities_count = len(all_vulnerabilities) if self.verbosity > 0: - if count: - self.stderr.write(f"{count} vulnerable records found:") - for entry in vulnerable_records: - self.stderr.write(str(entry)) - vulnerability_ids = [ - vulnerability.get("vulnerability_id") - for vulnerability in entry.affected_by_vulnerabilities - ] - self.stderr.write(" > " + ", ".join(vulnerability_ids)) + if vulnerabilities_count: + self.stderr.write(f"{vulnerabilities_count} vulnerabilities found:") + for vulnerability_id, vulnerability_data in all_vulnerabilities.items(): + self.stderr.write(str(vulnerability_id)) + for affected_obj in vulnerability_data.get("affects", []): + self.stderr.write(f" > {affected_obj}") else: self.stdout.write("No vulnerabilities found") - return count > 0 + return vulnerabilities_count > 0 diff --git a/scanpipe/models.py b/scanpipe/models.py index 6f3c5f550c..ac8e2da155 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -1495,6 +1495,11 @@ def vulnerable_dependency_count(self): """Return the number of vulnerable dependencies related to this project.""" return self.vulnerable_dependencies.count() + @cached_property + def vulnerability_count(self): + """Return the number of vulnerabilities related to this project.""" + return self.vulnerable_package_count + self.vulnerable_dependency_count + @cached_property def dependency_count(self): """Return the number of dependencies related to this project.""" diff --git a/scanpipe/templates/scanpipe/includes/project_summary_level.html b/scanpipe/templates/scanpipe/includes/project_summary_level.html index f942ebb418..55b40ae31c 100644 --- a/scanpipe/templates/scanpipe/includes/project_summary_level.html +++ b/scanpipe/templates/scanpipe/includes/project_summary_level.html @@ -94,4 +94,8 @@ {% endif %} {% url 'project_messages' project.slug as project_messages_url %} {% include "scanpipe/includes/project_summary_level_item.html" with label="Messages" count=project.message_count url=project_messages_url only %} + {% if project.vulnerability_count %} + {% url 'project_vulnerabilities' project.slug as project_vulnerabilities_url %} + {% include "scanpipe/includes/project_summary_level_item.html" with label="Vulnerabilities" count=project.vulnerability_count url=project_vulnerabilities_url only %} + {% endif %} \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/includes/vulnerability_id.html b/scanpipe/templates/scanpipe/includes/vulnerability_id.html new file mode 100644 index 0000000000..51b3b18163 --- /dev/null +++ b/scanpipe/templates/scanpipe/includes/vulnerability_id.html @@ -0,0 +1,30 @@ +{% if vulnerability.vulnerability_id|slice:":4" == "VCID" and VULNERABLECODE_URL %} + + {{ vulnerability.vulnerability_id }} + + +{% else %} + {{ vulnerability.vulnerability_id }} +{% endif %} + +
    + {% for alias in aliases %} +
  • + {% if alias|slice:":3" == "CVE" %} + {{ alias }} + + + {% elif alias|slice:":4" == "GHSA" %} + {{ alias }} + + + {% elif alias|slice:":3" == "NPM" %} + {{ alias }} + + + {% else %} + {{ alias }} + {% endif %} +
    • + {% endfor %} +
\ No newline at end of file diff --git a/scanpipe/templates/scanpipe/includes/vulnerability_summary.html b/scanpipe/templates/scanpipe/includes/vulnerability_summary.html new file mode 100644 index 0000000000..79e249d3fd --- /dev/null +++ b/scanpipe/templates/scanpipe/includes/vulnerability_summary.html @@ -0,0 +1,10 @@ +{% if vulnerability.summary %} + {% if vulnerability.summary|length > 150 %} +
+ {{ vulnerability.summary|slice:":150" }}... + {{ vulnerability.summary|slice:"150:" }} +
+ {% else %} + {{ vulnerability.summary }} + {% endif %} +{% endif %} \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/tabset/tab_vulnerabilities.html b/scanpipe/templates/scanpipe/tabset/tab_vulnerabilities.html index a2ffb2b7c9..0d7a5e4053 100644 --- a/scanpipe/templates/scanpipe/tabset/tab_vulnerabilities.html +++ b/scanpipe/templates/scanpipe/tabset/tab_vulnerabilities.html @@ -11,43 +11,10 @@ {% for vulnerability in tab_data.fields.affected_by_vulnerabilities.value %} + + + + + + + + + + {% if parent_path %} + - Next page - - {% endif %} -
    -
  • - - Page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }} - -
    • -
- - {% endif %} -{% else %} -
-
- -
-

- {% if path %} - No resources found in this directory. - {% else %} - Select a file or folder from the tree to view its contents. + hx-push-url="{% url 'project_resource_tree' project.slug parent_path %}"> +

+ {% endif %} -

- + {% for resource in resources %} + {% include 'scanpipe/tree/resource_table_resource_row.html' %} + {% endfor %} + +
Affected byAffected by SummaryAliasesAnalysis
- {{ vulnerability.summary }} - - {% for alias in vulnerability.aliases %} - {% if alias|slice:":3" == "CVE" %} - {{ alias }} - - - {% elif alias|slice:":4" == "GHSA" %} - {{ alias }} - - - {% elif alias|slice:":3" == "NPM" %} - {{ alias }} - - + {% if vulnerability.summary %} + {% if vulnerability.summary|length > 150 %} +
+ {{ vulnerability.summary|slice:":150" }}... + {{ vulnerability.summary|slice:"150:" }} +
{% else %} - {{ alias }} + {{ vulnerability.summary }} {% endif %} -
+ {% endif %} + 		+ {% for key, value in vulnerability.cdx_vulnerability.analysis.items %} + {{ key }}: {{ value }}{% if not forloop.last %}
{% endif %} {% endfor %}
- - {{ vulnerability.vulnerability_id }} - - -
    - {% for alias in aliases %} -
  • - {% if alias|slice:":3" == "CVE" %} - {{ alias }} - - - {% elif alias|slice:":4" == "GHSA" %} - {{ alias }} - - - {% elif alias|slice:":3" == "NPM" %} - {{ alias }} - - - {% else %} - {{ alias }} - {% endif %} -
    • - {% endfor %} -
+ {% include 'scanpipe/includes/vulnerability_id.html' with vulnerability=vulnerability %} 		- {% if vulnerability.summary %} - {% if vulnerability.summary|length > 150 %} -
- {{ vulnerability.summary|slice:":150" }}... - {{ vulnerability.summary|slice:"150:" }} -
- {% else %} - {{ vulnerability.summary }} - {% endif %} - {% endif %} + {% include 'scanpipe/includes/vulnerability_summary.html' with vulnerability=vulnerability only %} 		{% for key, value in vulnerability.cdx_vulnerability.analysis.items %} diff --git a/scanpipe/templates/scanpipe/vulnerability_list.html b/scanpipe/templates/scanpipe/vulnerability_list.html new file mode 100644 index 0000000000..4ec9b7fce8 --- /dev/null +++ b/scanpipe/templates/scanpipe/vulnerability_list.html @@ -0,0 +1,43 @@ +{% extends "scanpipe/base.html" %} +{% load humanize %} + +{% block title %}ScanCode.io: Vulnerabilities{% endblock %} + +{% block content %} +
+ {% include 'scanpipe/includes/navbar_header.html' %} +
+ {% include 'scanpipe/includes/breadcrumb.html' with linked_project=True current="Vulnerabilities" %} +
{{ object_list|length|intcomma }} results
+
+
+ +
+ + {% include 'scanpipe/includes/list_view_thead.html' %} + + {% for vulnerability in object_list.values %} + + + + + + {% empty %} + + + + {% endfor %} + +
+ {% include 'scanpipe/includes/vulnerability_id.html' with vulnerability=vulnerability %} + + {% include 'scanpipe/includes/vulnerability_summary.html' with vulnerability=vulnerability only %} + + {% for obj in vulnerability.affects %} + {{ obj }}
+ {% endfor %} +
+ No Vulnerabilities found. Clear search and filters +
+
+{% endblock %} \ No newline at end of file diff --git a/scanpipe/tests/test_commands.py b/scanpipe/tests/test_commands.py index 15dc32449b..7e93bc1c64 100644 --- a/scanpipe/tests/test_commands.py +++ b/scanpipe/tests/test_commands.py @@ -1386,11 +1386,10 @@ def test_scanpipe_management_command_check_compliance_vulnerabilities(self): self.assertEqual(cm.exception.code, 1) out_value = out.getvalue().strip() expected = ( - "2 vulnerable records found:\n" - "pkg:generic/name@1.0\n" - " > VCID-cah8-awtr-aaad\n" - "dependency1\n" - " > VCID-cah8-awtr-aaad" + "1 vulnerabilities found:\n" + "VCID-cah8-awtr-aaad\n" + " > pkg:generic/name@1.0\n" + " > dependency1" ) self.assertEqual(expected, out_value) diff --git a/scanpipe/tests/test_models.py b/scanpipe/tests/test_models.py index 6f5a3acf0b..e4a5b4cb7d 100644 --- a/scanpipe/tests/test_models.py +++ b/scanpipe/tests/test_models.py @@ -678,6 +678,7 @@ def test_scanpipe_project_vulnerability_properties(self): "VCID-3": {"vulnerability_id": "VCID-3", "affects": [p2, d2]}, } self.assertEqual(expected, project.vulnerabilities) + self.assertEqual(4, project.vulnerability_count) def test_scanpipe_project_get_codebase_config_directory(self): self.assertIsNone(self.project1.get_codebase_config_directory()) diff --git a/scanpipe/tests/test_views.py b/scanpipe/tests/test_views.py index 9172c42410..63e9b9f295 100644 --- a/scanpipe/tests/test_views.py +++ b/scanpipe/tests/test_views.py @@ -821,7 +821,7 @@ def test_scanpipe_views_project_views(self): with self.assertNumQueries(7): self.client.get(url) - with self.assertNumQueries(13): + with self.assertNumQueries(15): self.client.get(self.project1.get_absolute_url()) @mock.patch("scanpipe.models.Run.execute_task_async") @@ -1276,6 +1276,31 @@ def test_scanpipe_views_project_message_views(self): with self.assertNumQueries(5): self.client.get(url) + @override_settings(VULNERABLECODE_URL="https://vcio/") + def test_scanpipe_views_vulnerability_list_view(self): + self.assertEqual(0, self.project1.vulnerability_count) + url = reverse("project_vulnerabilities", args=[self.project1.slug]) + with self.assertNumQueries(5): + response = self.client.get(url) + self.assertContains(response, "No Vulnerabilities found.") + + v1 = {"vulnerability_id": "VCID-1"} + v2 = {"vulnerability_id": "VCID-2"} + project = make_project() + make_package(project, "pkg:type/a", affected_by_vulnerabilities=[v1]) + make_dependency(project, affected_by_vulnerabilities=[v2]) + + self.assertEqual(2, project.vulnerability_count) + url = reverse("project_vulnerabilities", args=[project.slug]) + with self.assertNumQueries(5): + response = self.client.get(url) + + expected = '' + self.assertContains(response, expected) + expected = '' + self.assertContains(response, expected) + self.assertContains(response, "pkg:type/a") + def test_scanpipe_views_license_list_view(self): url = reverse("license_list") response = self.client.get(url) diff --git a/scanpipe/urls.py b/scanpipe/urls.py index 05a8a8a8ef..c0d47fbb2c 100644 --- a/scanpipe/urls.py +++ b/scanpipe/urls.py @@ -96,6 +96,11 @@ views.ProjectMessageListView.as_view(), name="project_messages", ), + path( + "project//vulnerabilities/", + views.VulnerabilityListView.as_view(), + name="project_vulnerabilities", + ), path( "project//archive/", views.ProjectArchiveView.as_view(), diff --git a/scanpipe/views.py b/scanpipe/views.py index 9913d4947f..5e657b874b 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -1626,7 +1626,7 @@ def get_queryset(self): def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - context["project"] = self.project + context["project"] = self.get_project() context["model_label"] = self.model_label return context @@ -1951,6 +1951,28 @@ def get_filterset_kwargs(self, filterset_class): return kwargs +class VulnerabilityListView( + ConditionalLoginRequired, + ProjectRelatedViewMixin, + TableColumnsMixin, + generic.ListView, +): + template_name = "scanpipe/vulnerability_list.html" + table_columns = [ + "vulnerability_id", + "summary", + "affects", + ] + + def get_queryset(self): + return [] + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context["object_list"] = self.project.vulnerabilities + return context + + class CodebaseResourceDetailsView( ConditionalLoginRequired, ProjectRelatedViewMixin, From c602e9dcaf68cc82667edbeb47452af65c50f12c Mon Sep 17 00:00:00 2001 From: Chin Yeung Date: Mon, 5 Jan 2026 13:43:26 +0800 Subject: [PATCH 025/110] feat: Improve class file mapping #2021 Signed-off-by: Chin Yeung Li --- scanpipe/pipelines/deploy_to_develop.py | 78 +++++++++++++- scanpipe/pipes/d2d.py | 4 +- scanpipe/pipes/d2d_config.py | 8 ++ scanpipe/pipes/jvm.py | 29 +++++- scanpipe/tests/pipes/test_d2d.py | 130 +++++++++++++++++++++++- 5 files changed, 239 insertions(+), 10 deletions(-) diff --git a/scanpipe/pipelines/deploy_to_develop.py b/scanpipe/pipelines/deploy_to_develop.py index 9d60ac1dc4..fa481fa95f 100644 --- a/scanpipe/pipelines/deploy_to_develop.py +++ b/scanpipe/pipelines/deploy_to_develop.py @@ -83,6 +83,15 @@ def steps(cls): cls.find_grammar_packages, cls.map_grammar_to_class, cls.map_jar_to_grammar_source, + cls.find_groovy_packages, + cls.map_groovy_to_class, + cls.map_jar_to_groovy_source, + cls.find_aspectj_packages, + cls.map_aspectj_to_class, + cls.map_jar_to_aspectj_source, + cls.find_clojure_packages, + cls.map_clojure_to_class, + cls.map_jar_to_clojure_source, cls.find_xtend_packages, cls.map_xtend_to_class, cls.map_javascript, @@ -208,7 +217,7 @@ def find_scala_packages(self): @optional_step("Scala") def map_scala_to_class(self): - """Map a .class compiled file to its .java source.""" + """Map a .class compiled file to its .scala source.""" d2d.map_jvm_to_class( project=self.project, jvm_lang=jvm.ScalaLanguage, logger=self.log ) @@ -222,14 +231,14 @@ def map_jar_to_scala_source(self): @optional_step("Kotlin") def find_kotlin_packages(self): - """Find the java package of the .java source files.""" + """Find the java package of the kotlin source files.""" d2d.find_jvm_packages( project=self.project, jvm_lang=jvm.KotlinLanguage, logger=self.log ) @optional_step("Kotlin") def map_kotlin_to_class(self): - """Map a .class compiled file to its .java source.""" + """Map a .class compiled file to its kotlin source.""" d2d.map_jvm_to_class( project=self.project, jvm_lang=jvm.KotlinLanguage, logger=self.log ) @@ -262,6 +271,69 @@ def map_jar_to_grammar_source(self): project=self.project, jvm_lang=jvm.GrammarLanguage, logger=self.log ) + @optional_step("Groovy") + def find_groovy_packages(self): + """Find the package of the .groovy source files.""" + d2d.find_jvm_packages( + project=self.project, jvm_lang=jvm.GroovyLanguage, logger=self.log + ) + + @optional_step("Groovy") + def map_groovy_to_class(self): + """Map a .class compiled file to its .groovy source.""" + d2d.map_jvm_to_class( + project=self.project, jvm_lang=jvm.GroovyLanguage, logger=self.log + ) + + @optional_step("Groovy") + def map_jar_to_groovy_source(self): + """Map .jar files to their related source directory.""" + d2d.map_jar_to_jvm_source( + project=self.project, jvm_lang=jvm.GroovyLanguage, logger=self.log + ) + + @optional_step("AspectJ") + def find_aspectj_packages(self): + """Find the package of the .aj source files.""" + d2d.find_jvm_packages( + project=self.project, jvm_lang=jvm.AspectJLanguage, logger=self.log + ) + + @optional_step("AspectJ") + def map_aspectj_to_class(self): + """Map a .class compiled file to its .aj source.""" + d2d.map_jvm_to_class( + project=self.project, jvm_lang=jvm.AspectJLanguage, logger=self.log + ) + + @optional_step("AspectJ") + def map_jar_to_aspectj_source(self): + """Map .jar files to their related source directory.""" + d2d.map_jar_to_jvm_source( + project=self.project, jvm_lang=jvm.AspectJLanguage, logger=self.log + ) + + @optional_step("Clojure") + def find_clojure_packages(self): + """Find the package of the .clj source files.""" + d2d.find_jvm_packages( + project=self.project, jvm_lang=jvm.ClojureLanguage, logger=self.log + ) + + @optional_step("Clojure") + def map_clojure_to_class(self): + """Map a .class compiled file to its .clj source.""" + d2d.map_jvm_to_class( + project=self.project, jvm_lang=jvm.ClojureLanguage, logger=self.log + ) + + @optional_step("Clojure") + def map_jar_to_clojure_source(self): + """Map .jar files to their related source directory.""" + d2d.map_jar_to_jvm_source( + project=self.project, jvm_lang=jvm.ClojureLanguage, logger=self.log + ) + @optional_step("Xtend") def find_xtend_packages(self): """Find the java package of the xtend source files.""" diff --git a/scanpipe/pipes/d2d.py b/scanpipe/pipes/d2d.py index 7c01564043..4d1f194083 100644 --- a/scanpipe/pipes/d2d.py +++ b/scanpipe/pipes/d2d.py @@ -239,8 +239,8 @@ def map_jvm_to_class(project, jvm_lang: jvm.JvmLanguage, logger=None): if logger: logger( - f"Mapping {to_resource_count:,d} .class resources to " - f"{from_resource_count:,d} {jvm_lang.source_extensions}" + f"Mapping {to_resource_count:,d} .class (or other deployed file) " + f"resources to {from_resource_count:,d} {jvm_lang.source_extensions}" ) # build an index using from-side fully qualified class file names diff --git a/scanpipe/pipes/d2d_config.py b/scanpipe/pipes/d2d_config.py index d965feae84..05ea09ca44 100644 --- a/scanpipe/pipes/d2d_config.py +++ b/scanpipe/pipes/d2d_config.py @@ -112,6 +112,14 @@ class EcosystemConfig: "*kotlin-project-structure-metadata.json", ], ), + "Groovy": EcosystemConfig( + ecosystem_option="Groovy", + matchable_package_extensions=[".jar", ".war"], + matchable_resource_extensions=[".class"], + deployed_resource_path_exclusions=[ + "*META-INF/*", + ], + ), "JavaScript": EcosystemConfig( ecosystem_option="JavaScript", matchable_resource_extensions=[ diff --git a/scanpipe/pipes/jvm.py b/scanpipe/pipes/jvm.py index b071d9dcd4..ae7b5b9203 100644 --- a/scanpipe/pipes/jvm.py +++ b/scanpipe/pipes/jvm.py @@ -192,12 +192,39 @@ class JavaLanguage(JvmLanguage): class ScalaLanguage(JvmLanguage): name = "scala" source_extensions = (".scala",) - binary_extensions = (".class",) + binary_extensions = (".class", ".tasty") source_package_attribute_name = "scala_package" package_regex = re.compile(r"^\s*package\s+([\w\.]+)\s*;?") binary_map_type = "scala_to_class" +class GroovyLanguage(JvmLanguage): + name = "groovy" + source_extensions = (".groovy",) + binary_extensions = (".class",) + source_package_attribute_name = "groovy_package" + package_regex = re.compile(r"^\s*package\s+([\w\.]+)\s*;?") + binary_map_type = "groovy_to_class" + + +class AspectJLanguage(JvmLanguage): + name = "aspectj" + source_extensions = (".aj",) + binary_extensions = (".class",) + source_package_attribute_name = "aspectj_package" + package_regex = re.compile(r"^\s*package\s+([\w\.]+)\s*;?") + binary_map_type = "aspectj_to_class" + + +class ClojureLanguage(JvmLanguage): + name = "clojure" + source_extensions = (".clj",) + binary_extensions = (".class",) + source_package_attribute_name = "clojure_package" + package_regex = re.compile(r"^\s*package\s+([\w\.]+)\s*;?") + binary_map_type = "clojure_to_class" + + class KotlinLanguage(JvmLanguage): name = "kotlin" source_extensions = (".kt", ".kts") diff --git a/scanpipe/tests/pipes/test_d2d.py b/scanpipe/tests/pipes/test_d2d.py index 4d8433498e..73db8c0c9e 100644 --- a/scanpipe/tests/pipes/test_d2d.py +++ b/scanpipe/tests/pipes/test_d2d.py @@ -372,7 +372,7 @@ def test_scanpipe_pipes_d2d_map_java_to_class(self): self.project1, logger=buffer.write, jvm_lang=jvm.JavaLanguage ) - expected = "Mapping 3 .class resources to 2 ('.java',)" + expected = "Mapping 3 .class (or other deployed file) resources to 2 ('.java',)" self.assertIn(expected, buffer.getvalue()) self.assertEqual(2, self.project1.codebaserelations.count()) @@ -432,7 +432,7 @@ def test_scanpipe_pipes_d2d_map_java_to_class_with_java_in_deploy(self): d2d.map_jvm_to_class( self.project1, logger=buffer.write, jvm_lang=jvm.JavaLanguage ) - expected = "Mapping 1 .class resources to 1 ('.java',)" + expected = "Mapping 1 .class (or other deployed file) resources to 1 ('.java',)" self.assertIn(expected, buffer.getvalue()) def test_scanpipe_pipes_d2d_map_grammar_to_class(self): @@ -452,7 +452,9 @@ def test_scanpipe_pipes_d2d_map_grammar_to_class(self): self.project1, logger=buffer.write, jvm_lang=jvm.GrammarLanguage ) - expected = "Mapping 1 .class resources to 1 ('.g', '.g4')" + expected = ( + "Mapping 1 .class (or other deployed file) resources to 1 ('.g', '.g4')" + ) self.assertIn(expected, buffer.getvalue()) self.assertEqual(1, self.project1.codebaserelations.count()) @@ -480,7 +482,9 @@ def test_scanpipe_pipes_d2d_map_xtend_to_class(self): self.project1, logger=buffer.write, jvm_lang=jvm.XtendLanguage ) - expected = "Mapping 1 .class resources to 1 ('.xtend',)" + expected = ( + "Mapping 1 .class (or other deployed file) resources to 1 ('.xtend',)" + ) self.assertIn(expected, buffer.getvalue()) self.assertEqual(1, self.project1.codebaserelations.count()) @@ -569,6 +573,124 @@ def test_scanpipe_pipes_d2d_map_jar_to_java_source(self): self.assertEqual(from2, relation.from_resource) self.assertEqual(to_jar, relation.to_resource) + def test_scanpipe_pipes_d2d_map_groovy_to_class(self): + from1 = make_resource_file( + self.project1, + path="from/project/test.groovy", + extra_data={"groovy_package": "project"}, + ) + + to1 = make_resource_file( + self.project1, + path="to/project/test.class", + ) + + buffer = io.StringIO() + d2d.map_jvm_to_class( + self.project1, logger=buffer.write, jvm_lang=jvm.GroovyLanguage + ) + + expected = ( + "Mapping 1 .class (or other deployed file) resources to 1 ('.groovy',)" + ) + self.assertIn(expected, buffer.getvalue()) + self.assertEqual(1, self.project1.codebaserelations.count()) + + r1 = self.project1.codebaserelations.get(to_resource=to1, from_resource=from1) + self.assertEqual("groovy_to_class", r1.map_type) + expected = {"from_source_root": "from/"} + self.assertEqual(expected, r1.extra_data) + + def test_scanpipe_pipes_d2d_map_aspectj_to_class(self): + from1 = make_resource_file( + self.project1, + path="from/project/test.aj", + extra_data={"aspectj_package": "project"}, + ) + + to1 = make_resource_file( + self.project1, + path="to/project/test.class", + ) + + buffer = io.StringIO() + d2d.map_jvm_to_class( + self.project1, logger=buffer.write, jvm_lang=jvm.AspectJLanguage + ) + + expected = "Mapping 1 .class (or other deployed file) resources to 1 ('.aj',)" + self.assertIn(expected, buffer.getvalue()) + self.assertEqual(1, self.project1.codebaserelations.count()) + + r1 = self.project1.codebaserelations.get(to_resource=to1, from_resource=from1) + self.assertEqual("aspectj_to_class", r1.map_type) + expected = {"from_source_root": "from/"} + self.assertEqual(expected, r1.extra_data) + + def test_scanpipe_pipes_d2d_map_clojure_to_class(self): + from1 = make_resource_file( + self.project1, + path="from/project/test.clj", + extra_data={"clojure_package": "project"}, + ) + + to1 = make_resource_file( + self.project1, + path="to/project/test.class", + ) + + buffer = io.StringIO() + d2d.map_jvm_to_class( + self.project1, logger=buffer.write, jvm_lang=jvm.ClojureLanguage + ) + + expected = "Mapping 1 .class (or other deployed file) resources to 1 ('.clj',)" + self.assertIn(expected, buffer.getvalue()) + self.assertEqual(1, self.project1.codebaserelations.count()) + + r1 = self.project1.codebaserelations.get(to_resource=to1, from_resource=from1) + self.assertEqual("clojure_to_class", r1.map_type) + expected = {"from_source_root": "from/"} + self.assertEqual(expected, r1.extra_data) + + def test_scanpipe_pipes_d2d_map_scala_to_class(self): + from1 = make_resource_file( + self.project1, + path="from/tastyquery/Annotations.scala", + extra_data={"scala_package": "tastyquery"}, + ) + + to1 = make_resource_file( + self.project1, + path="to/tastyquery/Annotations.tasty", + ) + + to2 = make_resource_file( + self.project1, + path="to/tastyquery/Annotations.class", + ) + + buffer = io.StringIO() + d2d.map_jvm_to_class( + self.project1, logger=buffer.write, jvm_lang=jvm.ScalaLanguage + ) + + expected = ( + "Mapping 2 .class (or other deployed file) resources to 1 ('.scala',)" + ) + self.assertIn(expected, buffer.getvalue()) + self.assertEqual(2, self.project1.codebaserelations.count()) + + r1 = self.project1.codebaserelations.get(to_resource=to1, from_resource=from1) + self.assertEqual("scala_to_class", r1.map_type) + expected = {"from_source_root": "from/"} + self.assertEqual(expected, r1.extra_data) + + r2 = self.project1.codebaserelations.get(to_resource=to2, from_resource=from1) + self.assertEqual("scala_to_class", r2.map_type) + expected = {"from_source_root": "from/"} + self.assertEqual(expected, r2.extra_data) + def test_scanpipe_pipes_d2d_map_jar_to_scala_source(self): from1 = make_resource_file( self.project1, From 3bd311231172b61263fa0442862d3fd20c19864a Mon Sep 17 00:00:00 2001 From: Dennis Clark Date: Tue, 30 Dec 2025 17:15:07 -0800 Subject: [PATCH 026/110] Update index.rst with new structure --- docs/index.rst | 141 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 92 insertions(+), 49 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index d1ce2299c6..ee442ee549 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,62 +1,105 @@ ScanCode.io Documentation ========================= -Welcome! This is your starting point for everything ScanCode.io. +ScanCode.io provides a Web UI and API to run and review complex scans in rich scripted +pipelines, on different kinds of containers, docker images, package archives, manifests +etc, to get information on licenses, copyrights, sources, and vulnerabilities. -In this documentation, you’ll find: +ScanCode.io provides an easy-to-use front-end to ScanCode Toolkit and other AboutCode +projects.The flexible pipeline technology supports advanced scanning tasks such as +container scanning and deploy-to-develop analysis. You can run ScanCode.io in a Docker +container or install it on a Linux server. It provides full support for generating and +consuming CycloneDX and SPDX SBOMs. -- A **QuickStart guide** to run your first scan -- An **overview** of what ScanCode.io is and what it can do -- **Installation instructions** for full access -- Step-by-step **tutorials** to dive deeper -- In-depth **reference docs** on concepts, pipelines, configuration, and APIs -- Guides on how to **contribute** to the project and community +Documentation overview +~~~~~~~~~~~~~~~~~~~~~~ -.. toctree:: - :maxdepth: 2 - :caption: Getting Started +The overview below outlines how the documentation is structured +to help you know where to look for certain things. - quickstart - introduction - installation - user-interface - faq - contributing - changelog +.. rst-class:: clearfix row -.. toctree:: - :maxdepth: 2 - :caption: Tutorials - - tutorial_web_ui_analyze_docker_image - tutorial_web_ui_review_scan_results - tutorial_cli_analyze_docker_image - tutorial_cli_analyze_codebase - tutorial_api_analyze_package_archive - tutorial_license_policies - tutorial_vulnerablecode_integration - tutorial_web_ui_symbol_and_string_collection - tutorial_cli_end_to_end_scanning_to_dejacode +.. rst-class:: column column2 top-left + +:ref:`getting-started` +~~~~~~~~~~~~~~~~~~~~~~ + +Start here if you are new to ScanCode. + +- :ref:`quickstart` + + - :ref:`introduction` + - :ref:`installation` + - :ref:`user-interface` + +- :ref:`faq` + +.. rst-class:: column column2 top-right + +:ref:`tutorials` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Learn via practical step-by-step guides. + +- :ref:`tutorial_web_ui_analyze_docker_image` +- :ref:`tutorial_web_ui_review_scan_result` +- :ref:`tutorial_cli_analyze_docker_image` +- :ref:`tutorial_api_analyze_package_archive` +- :ref:`tutorial_license_policies` +- :ref:`tutorial_vulnerablecode_integration` +- :ref:`tutorial_web_ui_symbol_and_string_collection` +- :ref:`tutorial_cli_end_to_end_scanning_to_dejacode` + +.. rst-class:: column column2 bottom-left + +:ref:`how-to-guides` +~~~~~~~~~~~~~~~~~~~~ + +Helps you accomplish things. + +- :ref:`contributing` + +.. rst-class:: column column2 bottom-right + +:ref:`reference` and :ref:`explanation` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Consult the reference to understand ScanCode.io concepts. + +- :ref:`scanpipe-concepts` +- :ref:`built-in-pipelines` +- :ref:`custom-pipelines` +- :ref:`scanpipe-pipes` +- :ref:`project-configuration` +- :ref:`inputs` +- :ref:`output-files` +- :ref:`command-line-interface` +- :ref:`rest-api` +- :ref:`policies` +- :ref:`data-models` +- :ref:`automation` +- :ref:`webhooks` +- :ref:`application-settings` +- :ref:`distros-os-images` + +.. rst-class:: row clearfix + +Improving Documentation +~~~~~~~~~~~~~~~~~~~~~~~ + +.. include:: /rst-snippets/improve-docs.rst .. toctree:: - :maxdepth: 2 - :caption: Reference - - scanpipe-concepts - built-in-pipelines - custom-pipelines - scanpipe-pipes - project-configuration - inputs - output-files - command-line-interface - rest-api - policies - data-models - automation - webhooks - application-settings - distros-os-images + :maxdepth: 2 + :hidden: + + getting-started/index + tutorials/index + how-to-guides/index + reference/index + explanation/index + + changelog Indices and tables ================== From 395a90656cfac706ae34c1e8e9daec08fd993848 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Mon, 5 Jan 2026 16:26:42 +0530 Subject: [PATCH 027/110] Fix index page issues Signed-off-by: Ayan Sinha Mahapatra --- docs/_static/theme_overrides.css | 73 ++++++++++++++++++++ docs/conf.py | 14 ++++ docs/improve-docs.rst | 8 +++ docs/index.rst | 111 +++++++++++++++++++------------ 4 files changed, 163 insertions(+), 43 deletions(-) create mode 100644 docs/_static/theme_overrides.css create mode 100644 docs/improve-docs.rst diff --git a/docs/_static/theme_overrides.css b/docs/_static/theme_overrides.css new file mode 100644 index 0000000000..5d17cb92b6 --- /dev/null +++ b/docs/_static/theme_overrides.css @@ -0,0 +1,73 @@ +/* this is the container for the pages */ +.wy-nav-content { + max-width: 100%; + padding: 0px 40px 0px 0px; + margin-top: 0px; +} + +.wy-nav-content-wrap { + border-right: solid 1px; +} + +div.rst-content { + max-width: 1300px; + border: 0; + padding: 10px 80px 10px 80px; + margin-left: 50px; +} + +@media (max-width: 768px) { + div.rst-content { + max-width: 1300px; + border: 0; + padding: 0px 10px 10px 10px; + margin-left: 0px; + } +} + +.row { + clear: both; +} + +.column img { + border: 1px solid gray; +} + +@media only screen and (min-width: 1000px), +only screen and (min-width: 500px) and (max-width: 768px) { + + .column { + padding-left: 5px; + padding-right: 5px; + float: left; + } + + .column3 { + width: calc(33.3% - 10px); + } + + .column2 { + width: calc(50% - 11px); + position: relative; + } + + .column2:before { + padding-top: 61.8%; + content: ""; + display: block; + float: left; + } + + .top-left { + border-right: 1px solid var(--color-background-border); + border-bottom: 1px solid var(--color-background-border); + } + + .top-right { + border-bottom: 1px solid var(--color-background-border); + } + + .bottom-left { + border-right: 1px solid var(--color-background-border); + } +} \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 296166cab7..1f2ae1e7d1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -65,6 +65,20 @@ # -- Options for HTML output ------------------------------------------------- +html_static_path = ["_static"] + +html_context = { + "display_github": True, + "github_user": "aboutcode-org", + "github_repo": "scancode.io", + "github_version": "main", # branch + "conf_py_path": "/docs/", # path in the checkout to the docs root +} + +html_css_files = [ + "theme_overrides.css", +] + # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = "sphinx_rtd_theme" diff --git a/docs/improve-docs.rst b/docs/improve-docs.rst new file mode 100644 index 0000000000..407f93aec8 --- /dev/null +++ b/docs/improve-docs.rst @@ -0,0 +1,8 @@ +Something Missing? +------------------ + +If something is missing in the documentation or if you found some part confusing, please file +an `issue `_ with your suggestions for +improvement. Use the ``documentation`` issue label. + +Your help makes ScanCode docs better, we love hearing from you! diff --git a/docs/index.rst b/docs/index.rst index ee442ee549..7149a5cfe7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -21,28 +21,26 @@ to help you know where to look for certain things. .. rst-class:: column column2 top-left -:ref:`getting-started` +Getting started ~~~~~~~~~~~~~~~~~~~~~~ Start here if you are new to ScanCode. - :ref:`quickstart` - - - :ref:`introduction` - - :ref:`installation` - - :ref:`user-interface` - -- :ref:`faq` +- :ref:`introduction` +- :ref:`installation` +- :ref:`contributing` +- :ref:`user_interface` .. rst-class:: column column2 top-right -:ref:`tutorials` +Tutorials ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Learn via practical step-by-step guides. - :ref:`tutorial_web_ui_analyze_docker_image` -- :ref:`tutorial_web_ui_review_scan_result` +- :ref:`tutorial_web_ui_review_scan_results` - :ref:`tutorial_cli_analyze_docker_image` - :ref:`tutorial_api_analyze_package_archive` - :ref:`tutorial_license_policies` @@ -52,57 +50,84 @@ Learn via practical step-by-step guides. .. rst-class:: column column2 bottom-left -:ref:`how-to-guides` -~~~~~~~~~~~~~~~~~~~~ +Reference Docs +~~~~~~~~~~~~~~~~~~ -Helps you accomplish things. +Reference documentation for scancode features and customizations. -- :ref:`contributing` +- :ref:`built_in_pipelines` +- :ref:`custom_pipelines` +- :ref:`project_configuration` +- :ref:`inputs` +- :ref:`output_files` +- :ref:`command_line_interface` +- :ref:`rest_api` +- :ref:`policies` +- :ref:`data_model` +- :ref:`automation` +- :ref:`webhooks` +- :ref:`scancodeio_settings` +- :ref:`recognized_distros_os_images` .. rst-class:: column column2 bottom-right -:ref:`reference` and :ref:`explanation` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Explanations +~~~~~~~~~~~~~~~~~~ Consult the reference to understand ScanCode.io concepts. -- :ref:`scanpipe-concepts` -- :ref:`built-in-pipelines` -- :ref:`custom-pipelines` -- :ref:`scanpipe-pipes` -- :ref:`project-configuration` -- :ref:`inputs` -- :ref:`output-files` -- :ref:`command-line-interface` -- :ref:`rest-api` -- :ref:`policies` -- :ref:`data-models` -- :ref:`automation` -- :ref:`webhooks` -- :ref:`application-settings` -- :ref:`distros-os-images` +- :ref:`scanpipe_concepts` +- :ref:`scanpipe_pipes` .. rst-class:: row clearfix -Improving Documentation -~~~~~~~~~~~~~~~~~~~~~~~ - -.. include:: /rst-snippets/improve-docs.rst +Misc +~~~~~~~~~~~~~~~ -.. toctree:: - :maxdepth: 2 - :hidden: +- :ref:`faq` +- :ref:`changelog` - getting-started/index - tutorials/index - how-to-guides/index - reference/index - explanation/index +.. include:: improve-docs.rst - changelog Indices and tables ================== * :ref:`genindex` * :ref:`search` + +.. toctree:: + :maxdepth: 2 + :hidden: + + quickstart + introduction + installation + user-interface + faq + contributing + changelog + tutorial_web_ui_analyze_docker_image + tutorial_web_ui_review_scan_results + tutorial_cli_analyze_docker_image + tutorial_cli_analyze_codebase + tutorial_api_analyze_package_archive + tutorial_license_policies + tutorial_vulnerablecode_integration + tutorial_web_ui_symbol_and_string_collection + tutorial_cli_end_to_end_scanning_to_dejacode + scanpipe-concepts + built-in-pipelines + custom-pipelines + scanpipe-pipes + project-configuration + inputs + output-files + command-line-interface + rest-api + policies + data-models + automation + webhooks + application-settings + distros-os-images From 6b03c2d35135caabc00ad3ad40b4797d9182ec28 Mon Sep 17 00:00:00 2001 From: Dennis Clark Date: Fri, 16 Jan 2026 10:58:40 -0800 Subject: [PATCH 028/110] Update README.rst to improve structure --- README.rst | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/README.rst b/README.rst index 5f18dcc4b2..78afe33409 100644 --- a/README.rst +++ b/README.rst @@ -1,29 +1,31 @@ +=========== ScanCode.io =========== -ScanCode.io is a server to script and automate software composition analysis -with ScanPipe pipelines. +ScanCode.io provides a Web UI and API to run and review complex scans in rich scripted pipelines, on different kinds of containers, docker images, package archives, manifests etc, to get information on licenses, copyrights, sources, and vulnerabilities. -First application is for Docker container and VM composition analysis. +Why Use ScanCode.io? +==================== -Getting started ---------------- +ScanCode.io provides an easy-to-use front-end to ScanCode Toolkit and other AboutCode projects.The flexible pipeline technology supports advanced scanning tasks such as container scanning and deploy-to-develop analysis. You can run ScanCode.io in a Docker container or install it on a Linux server. It provides full support for generating and consuming CycloneDX and SPDX SBOMs. -The ScanCode.io documentation is available here: https://scancodeio.readthedocs.org/ +Getting Started +=============== -If you have questions that are not covered by our -`Documentation `_ or -`FAQs `_, -please ask them in `Discussions `_. +Instructions to get you up and running on your local machine are at `Getting Started `_ -If you want to contribute to ScanCode.io, start with our -`Contributing `_ page. +The ScanCode.io documentation also provides: -A new GitHub action is now available at -`scancode-action `_ -to run ScanCode.io pipelines from your GitHub Workflows. -Visit https://scancodeio.readthedocs.io/en/latest/automation.html to learn more -about automation. +- prerequisites for installing the software. +- instructions guiding you to start scanning code. +- tutorials that provide hands-on guidance to ScanCode features. +- how to customize your own pipelines. +- how to use a GitHub action to run ScanCode.io pipelines from your GitHub Workflows. +- references explaining integration with other AboutCode projects. +- guidelines for contributing to code development. + +If you have questions that are not covered by our documentation, +please ask them in `Discussions `_. Build and tests status ---------------------- From aeb6f3ca8074005989e8e74d2bdc6552e4d05b04 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 22 Jan 2026 17:58:00 +0530 Subject: [PATCH 029/110] Update to scancode-toolkit v32.5.0 (#2000) Signed-off-by: Ayan Sinha Mahapatra --- ...ublish-pypi-release-aboutcode-pipeline.yml | 2 +- .github/workflows/publish-pypi-release.yml | 2 +- .github/workflows/run-unit-tests-macos.yml | 2 +- .github/workflows/run-unit-tests.yml | 2 +- CHANGELOG.rst | 40 +++++++++ pyproject.toml | 24 +++-- scancodeio/__init__.py | 2 +- scanpipe/pipelines/analyze_root_filesystem.py | 6 +- scanpipe/pipelines/inspect_packages.py | 12 +++ scanpipe/pipes/docker.py | 2 + scanpipe/pipes/rootfs.py | 10 ++- scanpipe/pipes/scancode.py | 13 ++- .../data/asgiref/asgiref-3.3.0.spdx.json | 44 +++++----- .../data/asgiref/asgiref-3.3.0_fixtures.json | 88 +++++++++---------- .../asgiref-3.3.0_scanpipe_output.json | 66 +++++++------- .../asgiref/asgiref-3.3.0_toolkit_scan.json | 66 +++++++------- .../asgiref-3.3.0_walk_test_fixtures.json | 88 +++++++++---------- .../data/cyclonedx/asgiref-3.3.0.cdx.json | 36 ++++---- ...ved_dependencies_npm_inspect_packages.json | 35 ++------ ..._dependencies_poetry_inspect_packages.json | 41 ++------- .../scancode/package_assembly_codebase.json | 20 ++--- scanpipe/tests/pipes/test_scancode.py | 2 +- 22 files changed, 315 insertions(+), 288 deletions(-) diff --git a/.github/workflows/publish-pypi-release-aboutcode-pipeline.yml b/.github/workflows/publish-pypi-release-aboutcode-pipeline.yml index cdadc18dc9..cef72ed191 100644 --- a/.github/workflows/publish-pypi-release-aboutcode-pipeline.yml +++ b/.github/workflows/publish-pypi-release-aboutcode-pipeline.yml @@ -17,7 +17,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.13 + python-version: 3.14 - name: Install flot run: python -m pip install flot --user diff --git a/.github/workflows/publish-pypi-release.yml b/.github/workflows/publish-pypi-release.yml index f5a217d736..7d13564a9c 100644 --- a/.github/workflows/publish-pypi-release.yml +++ b/.github/workflows/publish-pypi-release.yml @@ -17,7 +17,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.13 + python-version: 3.14 - name: Install pypa/build run: python -m pip install build --user diff --git a/.github/workflows/run-unit-tests-macos.yml b/.github/workflows/run-unit-tests-macos.yml index b55ddd3ef3..59e2ccbf27 100644 --- a/.github/workflows/run-unit-tests-macos.yml +++ b/.github/workflows/run-unit-tests-macos.yml @@ -21,7 +21,7 @@ jobs: strategy: matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - name: Checkout code diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 59d30d1152..96b191eaa9 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -39,7 +39,7 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - name: Checkout code diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1caba618d3..891b6398fd 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,46 @@ Docker Compose users with existing data: run `./migrate-pg13-to-17.sh` before st the stack. Fresh installations require no action. +v36.1.0 (2026-01-22) +-------------------- + +- Bump to latest scancode-toolkit v32.5.0 with: + * package and license detection performance improvement + * python3.14 support with updated dependencies + * improved copyright, license and package detection + For more details see https://github.com/aboutcode-org/scancode-toolkit/releases/tag/v32.5.0 + https://github.com/aboutcode-org/scancode.io/pull/2000 + +- Support python3.14 + https://github.com/aboutcode-org/scancode.io/pull/2000 + +- Update to scancode-toolkit v32.4.1 + https://github.com/aboutcode-org/scancode.io/pull/1984 + For more details see https://github.com/aboutcode-org/scancode-toolkit/releases/tag/v32.4.1 + +- Store the whole vulnerability data from cdx to local models + https://github.com/aboutcode-org/scancode.io/pull/2007 + +- Add project vulnerability list view + https://github.com/aboutcode-org/scancode.io/pull/2018 + +- Update minecode-pipelines to latest v0.1.1 + https://github.com/aboutcode-org/scancode.io/pull/2013 + +- Refine d2d pipelines with misc improvements + https://github.com/aboutcode-org/scancode.io/pull/1996 + https://github.com/aboutcode-org/scancode.io/pull/1995 + https://github.com/aboutcode-org/scancode.io/pull/1999 + https://github.com/aboutcode-org/scancode.io/pull/2021 + +- Sanitize ORT package IDs to handle colons in versions + https://github.com/aboutcode-org/scancode.io/pull/2005 + +- Restructure docs and README + https://github.com/aboutcode-org/scancode.io/pull/2032 + + + v36.0.1 (2025-12-09) -------------------- diff --git a/pyproject.toml b/pyproject.toml index 52e50bd250..3038428c5b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,10 +4,10 @@ build-backend = "setuptools.build_meta" [project] name = "scancodeio" -version = "36.0.1" +version = "36.1.0" description = "Automate software composition analysis pipelines" readme = "README.rst" -requires-python = ">=3.10,<3.14" +requires-python = ">=3.10" license = "Apache-2.0" license-files = ["LICENSE", "NOTICE", "scan.NOTICE"] authors = [ @@ -30,6 +30,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Topic :: Utilities" ] @@ -56,26 +57,23 @@ dependencies = [ # WSGI server "gunicorn==23.0.0", # Docker - "container-inspector==33.0.0", + "container-inspector==33.1.0", # ScanCode-toolkit - "scancode-toolkit[packages]==32.4.1", + "scancode-toolkit[packages]==32.5.0", "extractcode[full]==31.0.0", - "commoncode==32.3.0", + "commoncode==32.4.2", "Beautifulsoup4[chardet]==4.13.4", "packageurl-python==0.17.6", - # Workaround issue https://github.com/aboutcode-org/scancode.io/issues/1795 - "fingerprints==1.2.3", - "normality==2.6.1", # FetchCode "fetchcode==0.8.0", "fetchcode-container==1.2.3.210512; sys_platform == 'linux'", # Inspectors "elf-inspector==0.0.3", "go-inspector==0.5.0", - "rust-inspector==0.1.0", - "binary-inspector==0.1.2", + "rust-inspector==0.2.1", + "binary-inspector==0.2.0", "python-inspector==0.15.0", - "source-inspector==0.7.0; sys_platform != 'darwin' and platform_machine != 'arm64'", + "source-inspector==0.7.1; sys_platform != 'darwin' and platform_machine != 'arm64'", "aboutcode-toolkit==11.1.1", # Utilities "XlsxWriter==3.2.9", @@ -101,9 +99,7 @@ dependencies = [ # AboutCode pipeline "aboutcode.pipeline==0.2.1", # ScoreCode - "scorecode==0.0.4", - # Workaround issue https://github.com/aboutcode-org/scancode.io/issues/1885 - "click==8.2.1" + "scorecode==0.0.4" ] [project.optional-dependencies] diff --git a/scancodeio/__init__.py b/scancodeio/__init__.py index 621add5c6d..5b1b055ffb 100644 --- a/scancodeio/__init__.py +++ b/scancodeio/__init__.py @@ -28,7 +28,7 @@ import git -VERSION = "36.0.1" +VERSION = "36.1.0" PROJECT_DIR = Path(__file__).resolve().parent ROOT_DIR = PROJECT_DIR.parent diff --git a/scanpipe/pipelines/analyze_root_filesystem.py b/scanpipe/pipelines/analyze_root_filesystem.py index 558b28c8df..69f2a21d8e 100644 --- a/scanpipe/pipelines/analyze_root_filesystem.py +++ b/scanpipe/pipelines/analyze_root_filesystem.py @@ -94,7 +94,11 @@ def flag_uninteresting_codebase_resources(self): def scan_for_application_packages(self): """Scan unknown resources for packages information.""" - scancode.scan_for_application_packages(self.project, progress_logger=self.log) + scancode.scan_for_application_packages( + project=self.project, + compiled=True, + progress_logger=self.log, + ) def match_not_analyzed_to_system_packages(self): """ diff --git a/scanpipe/pipelines/inspect_packages.py b/scanpipe/pipelines/inspect_packages.py index 7674f7f25f..0e4346118c 100644 --- a/scanpipe/pipelines/inspect_packages.py +++ b/scanpipe/pipelines/inspect_packages.py @@ -41,6 +41,8 @@ class InspectPackages(ScanCodebase): https://scancode-toolkit.readthedocs.io/en/stable/reference/available_package_parsers.html """ + scan_binaries = False + @classmethod def steps(cls): return ( @@ -49,10 +51,19 @@ def steps(cls): cls.collect_and_create_codebase_resources, cls.flag_empty_files, cls.flag_ignored_resources, + cls.scan_binaries_for_package, cls.scan_for_application_packages, cls.resolve_dependencies, ) + @optional_step("Compiled") + def scan_binaries_for_package(self): + """ + Scan compiled binaries for package and dependency related data' + Currently supported compiled binaries: Go, Rust. + """ + self.scan_binaries = True + def scan_for_application_packages(self): """ Scan resources for package information to add DiscoveredPackage @@ -61,6 +72,7 @@ def scan_for_application_packages(self): scancode.scan_for_application_packages( project=self.project, assemble=True, + compiled=self.scan_binaries, package_only=True, progress_logger=self.log, ) diff --git a/scanpipe/pipes/docker.py b/scanpipe/pipes/docker.py index aacfe44bd6..2229bf4a9c 100644 --- a/scanpipe/pipes/docker.py +++ b/scanpipe/pipes/docker.py @@ -82,6 +82,7 @@ def extract_image_from_tarball(input_tarball, extract_target, verify=False): target_dir=extract_target, skip_symlinks=False, as_events=False, + tar_filter="tar", ) images = Image.get_images_from_dir( extracted_location=str(extract_target), @@ -126,6 +127,7 @@ def extract_layers_from_images_to_base_path(base_path, images): target_dir=extract_target, skip_symlinks=False, as_events=False, + tar_filter="tar", ) errors.extend(extract_errors) layer.extracted_location = str(extract_target) diff --git a/scanpipe/pipes/rootfs.py b/scanpipe/pipes/rootfs.py index 95325d38d3..6d0c4675b6 100644 --- a/scanpipe/pipes/rootfs.py +++ b/scanpipe/pipes/rootfs.py @@ -30,6 +30,7 @@ import attr from commoncode.ignore import default_ignores +from commoncode.system import py314 from container_inspector.distro import Distro from packagedcode import plugin_package @@ -377,12 +378,19 @@ def flag_ignorable_codebase_resources(project): """ lookups = Q() for pattern in default_ignores.keys(): + # These are not patterns + if "*" not in pattern: + lookups |= Q(rootfs_path__icontains=pattern) + continue + # Translate glob pattern to regex translated_pattern = fnmatch.translate(pattern) # PostgreSQL does not like parts of Python regex if translated_pattern.startswith("(?s"): translated_pattern = translated_pattern.replace("(?s", "(?") - lookups |= Q(rootfs_path__icontains=pattern) + if py314: + translated_pattern = translated_pattern.replace("\\z", "\\Z") + lookups |= Q(rootfs_path__iregex=translated_pattern) qs = project.codebaseresources.no_status() diff --git a/scanpipe/pipes/scancode.py b/scanpipe/pipes/scancode.py index 609e86b69c..cd496cd793 100644 --- a/scanpipe/pipes/scancode.py +++ b/scanpipe/pipes/scancode.py @@ -250,7 +250,13 @@ def scan_file(location, with_threading=True, min_license_score=0, **kwargs): return _scan_resource(location, scanners, with_threading=with_threading) -def scan_for_package_data(location, with_threading=True, package_only=False, **kwargs): +def scan_for_package_data( + location, + with_threading=True, + package_only=False, + compiled=False, + **kwargs, +): """ Run a package scan on provided `location` using the scancode-toolkit direct API. @@ -259,6 +265,7 @@ def scan_for_package_data(location, with_threading=True, package_only=False, **k scancode_get_packages = partial( scancode_api.get_package_data, package_only=package_only, + compiled=compiled, ) scanners = [ Scanner("package_data", scancode_get_packages), @@ -355,7 +362,7 @@ def scan_resources( with futures.ProcessPoolExecutor(max_workers) as executor: future_to_resource = { - executor.submit(scan_func, resource.location): resource + executor.submit(scan_func, resource.location, **scan_func_kwargs): resource for resource in resource_iterator } @@ -409,6 +416,7 @@ def scan_for_files(project, resource_qs=None, progress_logger=None): def scan_for_application_packages( project, assemble=True, + compiled=False, package_only=False, resource_qs=None, progress_logger=logger.info, @@ -431,6 +439,7 @@ def scan_for_application_packages( scan_func_kwargs = { "package_only": package_only, + "compiled": compiled, } # Collect detected Package data and save it to the CodebaseResource it was diff --git a/scanpipe/tests/data/asgiref/asgiref-3.3.0.spdx.json b/scanpipe/tests/data/asgiref/asgiref-3.3.0.spdx.json index b8382a8891..565e2f4506 100644 --- a/scanpipe/tests/data/asgiref/asgiref-3.3.0.spdx.json +++ b/scanpipe/tests/data/asgiref/asgiref-3.3.0.spdx.json @@ -1,11 +1,11 @@ { "spdxVersion": "SPDX-2.3", "dataLicense": "CC0-1.0", - "SPDXID": "SPDXRef-DOCUMENT-3282ba3d-f525-4b74-9008-919108846d33", + "SPDXID": "SPDXRef-DOCUMENT-92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "name": "scancodeio_asgiref", - "documentNamespace": "https://scancode.io/spdxdocs/3282ba3d-f525-4b74-9008-919108846d33", + "documentNamespace": "https://scancode.io/spdxdocs/92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "documentDescribes": [ - "SPDXRef-scancodeio-project-3282ba3d-f525-4b74-9008-919108846d33" + "SPDXRef-scancodeio-project-92fe63d9-1d53-4b63-b19a-85022fb7a3f3" ], "creationInfo": { "created": "2000-01-01T01:02:03Z", @@ -17,7 +17,7 @@ "packages": [ { "name": "asgiref", - "SPDXID": "SPDXRef-scancodeio-project-3282ba3d-f525-4b74-9008-919108846d33", + "SPDXID": "SPDXRef-scancodeio-project-92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "downloadLocation": "NOASSERTION", "licenseDeclared": "NOASSERTION", "licenseConcluded": "NOASSERTION", @@ -26,7 +26,7 @@ }, { "name": "asgiref", - "SPDXID": "SPDXRef-scancodeio-discoveredpackage-b0467b22-5e83-4b0d-91e2-15121a8cf075", + "SPDXID": "SPDXRef-scancodeio-discoveredpackage-543a3583-3a13-4b5d-a039-c6bc4072de35", "downloadLocation": "NOASSERTION", "licenseDeclared": "BSD-3-Clause", "licenseConcluded": "BSD-3-Clause", @@ -46,7 +46,7 @@ }, { "name": "asgiref", - "SPDXID": "SPDXRef-scancodeio-discoveredpackage-856001ca-aabf-4c6a-8ac2-cc1317f3e05d", + "SPDXID": "SPDXRef-scancodeio-discoveredpackage-b2913908-3a31-4460-b330-a74d14b5ee24", "downloadLocation": "NOASSERTION", "licenseDeclared": "BSD-3-Clause", "licenseConcluded": "BSD-3-Clause", @@ -66,7 +66,7 @@ }, { "name": "pytest", - "SPDXID": "SPDXRef-scancodeio-discovereddependency-ef046d41-ae47-42c3-838c-def8bc956723", + "SPDXID": "SPDXRef-scancodeio-discovereddependency-19138025-32e9-4060-ab38-622b27493b6c", "downloadLocation": "NOASSERTION", "licenseDeclared": "NOASSERTION", "licenseConcluded": "NOASSERTION", @@ -82,7 +82,7 @@ }, { "name": "pytest", - "SPDXID": "SPDXRef-scancodeio-discovereddependency-31fbc19e-c309-4463-8312-390ef734bf78", + "SPDXID": "SPDXRef-scancodeio-discovereddependency-c6282019-1112-43f8-a27a-658cc0c5dcf4", "downloadLocation": "NOASSERTION", "licenseDeclared": "NOASSERTION", "licenseConcluded": "NOASSERTION", @@ -98,7 +98,7 @@ }, { "name": "pytest-asyncio", - "SPDXID": "SPDXRef-scancodeio-discovereddependency-3f928a8b-4505-4a7b-ad3a-a589dec836db", + "SPDXID": "SPDXRef-scancodeio-discovereddependency-52135390-385f-4fc0-8b0a-38f28c1040dd", "downloadLocation": "NOASSERTION", "licenseDeclared": "NOASSERTION", "licenseConcluded": "NOASSERTION", @@ -114,7 +114,7 @@ }, { "name": "pytest-asyncio", - "SPDXID": "SPDXRef-scancodeio-discovereddependency-892b2af3-a9f0-4544-9874-583307d2d387", + "SPDXID": "SPDXRef-scancodeio-discovereddependency-01b33ae9-cb5f-442b-bfac-af8e40a1bf8c", "downloadLocation": "NOASSERTION", "licenseDeclared": "NOASSERTION", "licenseConcluded": "NOASSERTION", @@ -132,33 +132,33 @@ "files": [], "relationships": [ { - "spdxElementId": "SPDXRef-scancodeio-project-3282ba3d-f525-4b74-9008-919108846d33", - "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-b0467b22-5e83-4b0d-91e2-15121a8cf075", + "spdxElementId": "SPDXRef-scancodeio-project-92fe63d9-1d53-4b63-b19a-85022fb7a3f3", + "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-543a3583-3a13-4b5d-a039-c6bc4072de35", "relationshipType": "DEPENDS_ON" }, { - "spdxElementId": "SPDXRef-scancodeio-project-3282ba3d-f525-4b74-9008-919108846d33", - "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-856001ca-aabf-4c6a-8ac2-cc1317f3e05d", + "spdxElementId": "SPDXRef-scancodeio-project-92fe63d9-1d53-4b63-b19a-85022fb7a3f3", + "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-b2913908-3a31-4460-b330-a74d14b5ee24", "relationshipType": "DEPENDS_ON" }, { - "spdxElementId": "SPDXRef-scancodeio-discovereddependency-ef046d41-ae47-42c3-838c-def8bc956723", - "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-b0467b22-5e83-4b0d-91e2-15121a8cf075", + "spdxElementId": "SPDXRef-scancodeio-discovereddependency-19138025-32e9-4060-ab38-622b27493b6c", + "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-543a3583-3a13-4b5d-a039-c6bc4072de35", "relationshipType": "DEPENDENCY_OF" }, { - "spdxElementId": "SPDXRef-scancodeio-discovereddependency-31fbc19e-c309-4463-8312-390ef734bf78", - "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-856001ca-aabf-4c6a-8ac2-cc1317f3e05d", + "spdxElementId": "SPDXRef-scancodeio-discovereddependency-c6282019-1112-43f8-a27a-658cc0c5dcf4", + "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-b2913908-3a31-4460-b330-a74d14b5ee24", "relationshipType": "DEPENDENCY_OF" }, { - "spdxElementId": "SPDXRef-scancodeio-discovereddependency-3f928a8b-4505-4a7b-ad3a-a589dec836db", - "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-b0467b22-5e83-4b0d-91e2-15121a8cf075", + "spdxElementId": "SPDXRef-scancodeio-discovereddependency-52135390-385f-4fc0-8b0a-38f28c1040dd", + "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-543a3583-3a13-4b5d-a039-c6bc4072de35", "relationshipType": "DEPENDENCY_OF" }, { - "spdxElementId": "SPDXRef-scancodeio-discovereddependency-892b2af3-a9f0-4544-9874-583307d2d387", - "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-856001ca-aabf-4c6a-8ac2-cc1317f3e05d", + "spdxElementId": "SPDXRef-scancodeio-discovereddependency-01b33ae9-cb5f-442b-bfac-af8e40a1bf8c", + "relatedSpdxElement": "SPDXRef-scancodeio-discoveredpackage-b2913908-3a31-4460-b330-a74d14b5ee24", "relationshipType": "DEPENDENCY_OF" } ], diff --git a/scanpipe/tests/data/asgiref/asgiref-3.3.0_fixtures.json b/scanpipe/tests/data/asgiref/asgiref-3.3.0_fixtures.json index 86c4a1e609..39213c9281 100644 --- a/scanpipe/tests/data/asgiref/asgiref-3.3.0_fixtures.json +++ b/scanpipe/tests/data/asgiref/asgiref-3.3.0_fixtures.json @@ -1,13 +1,13 @@ [ { "model": "scanpipe.project", - "pk": "3282ba3d-f525-4b74-9008-919108846d33", + "pk": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "fields": { "extra_data": {}, - "created_date": "2025-12-12T08:16:20.590Z", + "created_date": "2026-01-16T12:40:25.319Z", "name": "asgiref", - "slug": "asgiref-3282ba3d", - "work_directory": "/tmp/tmp5qi2cv31/projects/asgiref-3282ba3d", + "slug": "asgiref-92fe63d9", + "work_directory": "/tmp/tmp7o1wn11c/projects/asgiref-92fe63d9", "is_archived": false, "notes": "", "settings": {}, @@ -16,17 +16,17 @@ }, { "model": "scanpipe.run", - "pk": "88cca71e-436d-42ec-804b-ab1af5727413", + "pk": "87e422a3-3ebf-4fbf-99cb-e2f2bbb1ba7a", "fields": { "task_id": null, "task_start_date": null, "task_end_date": null, "task_exitcode": null, "task_output": "", - "log": "2025-12-12 08:16:20.593 Pipeline [scan_codebase] starting\n2025-12-12 08:16:20.594 Step [download_missing_inputs] starting\n2025-12-12 08:16:20.595 Step [download_missing_inputs] completed in 0 seconds\n2025-12-12 08:16:20.596 Step [copy_inputs_to_codebase_directory] starting\n2025-12-12 08:16:20.596 Step [copy_inputs_to_codebase_directory] completed in 0 seconds\n2025-12-12 08:16:20.597 Step [extract_archives] starting\n2025-12-12 08:16:20.661 Step [extract_archives] completed in 0 seconds\n2025-12-12 08:16:20.662 Step [collect_and_create_codebase_resources] starting\n2025-12-12 08:16:20.831 Step [collect_and_create_codebase_resources] completed in 0 seconds\n2025-12-12 08:16:20.832 Step [flag_empty_files] starting\n2025-12-12 08:16:20.834 Step [flag_empty_files] completed in 0 seconds\n2025-12-12 08:16:20.835 Step [flag_ignored_resources] starting\n2025-12-12 08:16:20.838 Step [flag_ignored_resources] completed in 0 seconds\n2025-12-12 08:16:20.839 Step [scan_for_application_packages] starting\n2025-12-12 08:16:20.840 Collecting package data from resources:\n2025-12-12 08:16:20.893 Progress: 11% (2/18)\n2025-12-12 08:16:20.932 Progress: 22% (4/18)\n2025-12-12 08:16:20.936 Progress: 33% (6/18)\n2025-12-12 08:16:20.937 Progress: 44% (8/18)\n2025-12-12 08:16:20.939 Progress: 55% (10/18)\n2025-12-12 08:16:20.940 Progress: 66% (12/18)\n2025-12-12 08:16:20.943 Progress: 77% (14/18)\n2025-12-12 08:16:20.945 Progress: 88% (16/18)\n2025-12-12 08:16:24.518 Progress: 100% (18/18)\n2025-12-12 08:16:24.602 Assembling collected package data:\n2025-12-12 08:16:24.603 Progress: 0%\n2025-12-12 08:16:24.722 Step [scan_for_application_packages] completed in 4 seconds\n2025-12-12 08:16:24.723 Step [scan_for_files] starting\n2025-12-12 08:16:43.753 Progress: 12% (2/16) ETA: 140 seconds (2.3 minutes)\n2025-12-12 08:16:43.947 Progress: 25% (4/16) ETA: 58 seconds\n2025-12-12 08:16:44.080 Progress: 37% (6/16) ETA: 33 seconds\n2025-12-12 08:16:44.379 Progress: 50% (8/16) ETA: 20 seconds\n2025-12-12 08:16:44.551 Progress: 62% (10/16) ETA: 12 seconds\n2025-12-12 08:16:44.666 Progress: 75% (12/16) ETA: 7 seconds\n2025-12-12 08:16:44.708 Progress: 87% (14/16) ETA: 3 seconds\n2025-12-12 08:16:44.998 Progress: 100% (16/16)\n2025-12-12 08:16:45.464 Step [scan_for_files] completed in 21 seconds\n2025-12-12 08:16:45.467 Step [collect_and_create_license_detections] starting\n2025-12-12 08:16:49.430 Step [collect_and_create_license_detections] completed in 4 seconds\n2025-12-12 08:16:49.432 Pipeline completed in 29 seconds\n", - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "log": "2026-01-16 12:40:25.323 Pipeline [scan_codebase] starting\n2026-01-16 12:40:25.324 Step [download_missing_inputs] starting\n2026-01-16 12:40:25.325 Step [download_missing_inputs] completed in 0 seconds\n2026-01-16 12:40:25.325 Step [copy_inputs_to_codebase_directory] starting\n2026-01-16 12:40:25.326 Step [copy_inputs_to_codebase_directory] completed in 0 seconds\n2026-01-16 12:40:25.327 Step [extract_archives] starting\n2026-01-16 12:40:25.378 Step [extract_archives] completed in 0 seconds\n2026-01-16 12:40:25.379 Step [collect_and_create_codebase_resources] starting\n2026-01-16 12:40:25.557 Step [collect_and_create_codebase_resources] completed in 0 seconds\n2026-01-16 12:40:25.559 Step [flag_empty_files] starting\n2026-01-16 12:40:25.560 Step [flag_empty_files] completed in 0 seconds\n2026-01-16 12:40:25.561 Step [flag_ignored_resources] starting\n2026-01-16 12:40:25.563 Step [flag_ignored_resources] completed in 0 seconds\n2026-01-16 12:40:25.564 Step [scan_for_application_packages] starting\n2026-01-16 12:40:25.564 Collecting package data from resources:\n2026-01-16 12:40:28.178 Progress: 11% (2/18) ETA: 21 seconds\n2026-01-16 12:40:28.179 Progress: 22% (4/18) ETA: 9 seconds\n2026-01-16 12:40:28.180 Progress: 33% (6/18) ETA: 5 seconds\n2026-01-16 12:40:28.187 Progress: 44% (8/18) ETA: 3 seconds\n2026-01-16 12:40:28.188 Progress: 55% (10/18) ETA: 2 seconds\n2026-01-16 12:40:28.188 Progress: 66% (12/18) ETA: 1 seconds\n2026-01-16 12:40:28.189 Progress: 77% (14/18) ETA: 1 seconds\n2026-01-16 12:40:28.189 Progress: 88% (16/18)\n2026-01-16 12:40:28.190 Progress: 100% (18/18)\n2026-01-16 12:40:28.191 Assembling collected package data:\n2026-01-16 12:40:28.191 Progress: 0%\n2026-01-16 12:40:28.257 Step [scan_for_application_packages] completed in 3 seconds\n2026-01-16 12:40:28.258 Step [scan_for_files] starting\n2026-01-16 12:40:28.297 Progress: 12% (2/16)\n2026-01-16 12:40:28.300 Progress: 25% (4/16)\n2026-01-16 12:40:28.578 Progress: 37% (6/16) ETA: 1 seconds\n2026-01-16 12:40:28.581 Progress: 50% (8/16)\n2026-01-16 12:40:28.966 Progress: 62% (10/16)\n2026-01-16 12:40:29.237 Progress: 75% (12/16)\n2026-01-16 12:40:30.070 Progress: 87% (14/16)\n2026-01-16 12:40:30.590 Progress: 100% (16/16)\n2026-01-16 12:40:30.855 Step [scan_for_files] completed in 3 seconds\n2026-01-16 12:40:30.857 Step [collect_and_create_license_detections] starting\n2026-01-16 12:40:30.869 Step [collect_and_create_license_detections] completed in 0 seconds\n2026-01-16 12:40:30.870 Pipeline completed in 6 seconds\n", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "pipeline_name": "scan_codebase", - "created_date": "2025-12-12T08:16:20.592Z", + "created_date": "2026-01-16T12:40:25.322Z", "scancodeio_version": "", "description": "Scan a codebase for application packages, licenses, and copyrights.", "current_step": "", @@ -43,7 +43,7 @@ "sha256": "a5098bc870b80e7b872bff60bb363c7f2c2c89078759f6c47b53ff8c525a152e", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -354,7 +354,7 @@ "sha256": "", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -400,7 +400,7 @@ "sha256": "", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -446,7 +446,7 @@ "sha256": "", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -492,7 +492,7 @@ "sha256": "6e89108c2cf0c0446174188f76f60465ae1c1f14f83427807df40d52a27cb2c8", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -538,7 +538,7 @@ "sha256": "b846415d1b514e9c1dff14a22deb906d794bc546ca6129f950a18cd091e2a669", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "bsd-new", "detected_license_expression_spdx": "BSD-3-Clause", "license_detections": [ @@ -619,7 +619,7 @@ "sha256": "70f98f4eb9f6068b192b5464fcdf69e29a8ff09962bfce84bbb052baeee44f33", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -930,7 +930,7 @@ "sha256": "11546323af45e6a5639bf620a9c4d73e74c0bf705f494af4595007b923f75e8a", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -976,7 +976,7 @@ "sha256": "2c1983592aa38f0bfb0afacc73ddc5b46ce10e8e89ceaa9fed1e5fc6361b608d", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1022,7 +1022,7 @@ "sha256": "30f49b9094bff904a42caeec32515715fe625a56dc48bd7c0e3d9988c0ad4bd7", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1068,7 +1068,7 @@ "sha256": "fa4651a3b79201a4dc44a4096cd49ec8f427e912ea0ee05c666357b413a8afe7", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1114,7 +1114,7 @@ "sha256": "ee0fcf4a8e6fa9df8a4643bb48e82892d496afce44b6c8b8aea2721755545e1c", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1160,7 +1160,7 @@ "sha256": "3151f66c476208c3154cb6c4fb557a2a253bab82f0ab33fb3c8b9f7976be9e33", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1206,7 +1206,7 @@ "sha256": "ddd445b778c097fc75c2bf69ad964cbadd3bd6999d1dd2306d39d401855e8e3e", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1252,7 +1252,7 @@ "sha256": "ddbc8d455eceb68fc583c67e7c4ad0277c867fb39095c51ec5b37f70342e8334", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1298,7 +1298,7 @@ "sha256": "126c3e3a8a75a517d2739612304607804cf5f34da63fa25d03a6f11f7edb6f2f", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "apache-2.0", "detected_license_expression_spdx": "Apache-2.0", "license_detections": [ @@ -1383,7 +1383,7 @@ "sha256": "f8bd1ea3fb8afddabb10f8efd66796d41446cad51168ef4d3c44b19c973d0ad0", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1429,7 +1429,7 @@ "sha256": "885267fee0fea687875a02ceb929ca095312d47aaa57e20e4ce382f397caaf4d", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1476,7 +1476,7 @@ "version": "3.3.0", "qualifiers": "", "subpath": "", - "uuid": "b0467b22-5e83-4b0d-91e2-15121a8cf075", + "uuid": "543a3583-3a13-4b5d-a039-c6bc4072de35", "md5": "", "sha1": "", "sha256": "", @@ -1486,7 +1486,7 @@ "Documentation": "https://asgi.readthedocs.io/", "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" }, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "compliance_alert": "", "affected_by_vulnerabilities": [], "filename": "", @@ -1577,7 +1577,7 @@ ], "missing_resources": [], "modified_resources": [], - "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=549d974e-a424-4fe4-9351-edb3b03b391a", + "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=e66230c6-0a0c-4b55-b339-03afd0550002", "keywords": [ "Development Status :: 5 - Production/Stable", "Environment :: Web Environment", @@ -1610,7 +1610,7 @@ "version": "3.3.0", "qualifiers": "", "subpath": "", - "uuid": "856001ca-aabf-4c6a-8ac2-cc1317f3e05d", + "uuid": "b2913908-3a31-4460-b330-a74d14b5ee24", "md5": "", "sha1": "", "sha256": "", @@ -1620,7 +1620,7 @@ "Documentation": "https://asgi.readthedocs.io/", "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" }, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "compliance_alert": "", "affected_by_vulnerabilities": [], "filename": "", @@ -1711,7 +1711,7 @@ ], "missing_resources": [], "modified_resources": [], - "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=5e52877b-c669-414e-bdf9-5d2bcb2443b6", + "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=e522e5ca-fd0c-4566-8632-bb3e82c5f7eb", "keywords": [ "Development Status :: 5 - Production/Stable", "Environment :: Web Environment", @@ -1757,10 +1757,10 @@ "version": "", "qualifiers": "", "subpath": "", - "uuid": "ef046d41-ae47-42c3-838c-def8bc956723", + "uuid": "19138025-32e9-4060-ab38-622b27493b6c", "affected_by_vulnerabilities": [], - "project": "3282ba3d-f525-4b74-9008-919108846d33", - "dependency_uid": "pkg:pypi/pytest?uuid=a1855203-fd9a-441d-98f6-e1e10de689c2", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", + "dependency_uid": "pkg:pypi/pytest?uuid=43b6aa1d-7036-47c8-a40e-7b14bb14171b", "for_package": 1, "resolved_to_package": null, "datafile_resource": 1, @@ -1783,10 +1783,10 @@ "version": "", "qualifiers": "", "subpath": "", - "uuid": "3f928a8b-4505-4a7b-ad3a-a589dec836db", + "uuid": "52135390-385f-4fc0-8b0a-38f28c1040dd", "affected_by_vulnerabilities": [], - "project": "3282ba3d-f525-4b74-9008-919108846d33", - "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=8cfeaa4e-a8db-4004-a416-3230ee0d3aae", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", + "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=b2d0632b-d881-4d8a-ad04-d3d780e7c6e2", "for_package": 1, "resolved_to_package": null, "datafile_resource": 1, @@ -1809,10 +1809,10 @@ "version": "", "qualifiers": "", "subpath": "", - "uuid": "31fbc19e-c309-4463-8312-390ef734bf78", + "uuid": "c6282019-1112-43f8-a27a-658cc0c5dcf4", "affected_by_vulnerabilities": [], - "project": "3282ba3d-f525-4b74-9008-919108846d33", - "dependency_uid": "pkg:pypi/pytest?uuid=3fa1cdcc-c729-4e57-94f3-95212cd8f7d4", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", + "dependency_uid": "pkg:pypi/pytest?uuid=a1481753-660d-4cf0-b0b8-a929ede30720", "for_package": 2, "resolved_to_package": null, "datafile_resource": 7, @@ -1835,10 +1835,10 @@ "version": "", "qualifiers": "", "subpath": "", - "uuid": "892b2af3-a9f0-4544-9874-583307d2d387", + "uuid": "01b33ae9-cb5f-442b-bfac-af8e40a1bf8c", "affected_by_vulnerabilities": [], - "project": "3282ba3d-f525-4b74-9008-919108846d33", - "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=040715c9-6806-402c-b027-141a05b6fd05", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", + "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=66c37d13-079b-412b-a610-1dabf9d7e6aa", "for_package": 2, "resolved_to_package": null, "datafile_resource": 7, diff --git a/scanpipe/tests/data/asgiref/asgiref-3.3.0_scanpipe_output.json b/scanpipe/tests/data/asgiref/asgiref-3.3.0_scanpipe_output.json index a2b303500d..83c5aed912 100644 --- a/scanpipe/tests/data/asgiref/asgiref-3.3.0_scanpipe_output.json +++ b/scanpipe/tests/data/asgiref/asgiref-3.3.0_scanpipe_output.json @@ -2,18 +2,18 @@ "headers": [ { "tool_name": "scanpipe", - "tool_version": "v36.0.0-2-gd030eab3", + "tool_version": "v36.0.1-14-g5999496e", "other_tools": [ - "pkg:pypi/scancode-toolkit@32.4.1" + "pkg:pypi/scancode-toolkit@32.5.0" ], "notice": "Generated with ScanCode.io and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied.\nNo content created from ScanCode.io should be considered or used as legal advice.\nConsult an Attorney for any legal advice.\nScanCode.io is a free software code scanning tool from nexB Inc. and others\nlicensed under the Apache License version 2.0.\nScanCode is a trademark of nexB Inc.\nVisit https://github.com/nexB/scancode.io for support and download.\n", - "uuid": "3282ba3d-f525-4b74-9008-919108846d33", - "created_date": "2025-12-12T08:16:20.590Z", + "uuid": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", + "created_date": "2026-01-16T12:40:25.319Z", "notes": "", "settings": {}, "input_sources": [ { - "uuid": "477b8b4c-4a5d-4481-8ec7-6b499ea069eb", + "uuid": "6619f69c-3ee7-454a-947c-879b289f001f", "filename": "asgiref-3.3.0-py3-none-any.whl", "download_url": "", "is_uploaded": true, @@ -30,15 +30,15 @@ "description": "Scan a codebase for application packages, licenses, and copyrights.", "selected_groups": null, "selected_steps": null, - "uuid": "88cca71e-436d-42ec-804b-ab1af5727413", - "created_date": "2025-12-12T08:16:20.592627Z", + "uuid": "87e422a3-3ebf-4fbf-99cb-e2f2bbb1ba7a", + "created_date": "2026-01-16T12:40:25.322706Z", "scancodeio_version": "", "task_id": null, "task_start_date": null, "task_end_date": null, "task_exitcode": null, "task_output": "", - "log": "2025-12-12 08:16:20.593 Pipeline [scan_codebase] starting\n2025-12-12 08:16:20.594 Step [download_missing_inputs] starting\n2025-12-12 08:16:20.595 Step [download_missing_inputs] completed in 0 seconds\n2025-12-12 08:16:20.596 Step [copy_inputs_to_codebase_directory] starting\n2025-12-12 08:16:20.596 Step [copy_inputs_to_codebase_directory] completed in 0 seconds\n2025-12-12 08:16:20.597 Step [extract_archives] starting\n2025-12-12 08:16:20.661 Step [extract_archives] completed in 0 seconds\n2025-12-12 08:16:20.662 Step [collect_and_create_codebase_resources] starting\n2025-12-12 08:16:20.831 Step [collect_and_create_codebase_resources] completed in 0 seconds\n2025-12-12 08:16:20.832 Step [flag_empty_files] starting\n2025-12-12 08:16:20.834 Step [flag_empty_files] completed in 0 seconds\n2025-12-12 08:16:20.835 Step [flag_ignored_resources] starting\n2025-12-12 08:16:20.838 Step [flag_ignored_resources] completed in 0 seconds\n2025-12-12 08:16:20.839 Step [scan_for_application_packages] starting\n2025-12-12 08:16:20.840 Collecting package data from resources:\n2025-12-12 08:16:20.893 Progress: 11% (2/18)\n2025-12-12 08:16:20.932 Progress: 22% (4/18)\n2025-12-12 08:16:20.936 Progress: 33% (6/18)\n2025-12-12 08:16:20.937 Progress: 44% (8/18)\n2025-12-12 08:16:20.939 Progress: 55% (10/18)\n2025-12-12 08:16:20.940 Progress: 66% (12/18)\n2025-12-12 08:16:20.943 Progress: 77% (14/18)\n2025-12-12 08:16:20.945 Progress: 88% (16/18)\n2025-12-12 08:16:24.518 Progress: 100% (18/18)\n2025-12-12 08:16:24.602 Assembling collected package data:\n2025-12-12 08:16:24.603 Progress: 0%\n2025-12-12 08:16:24.722 Step [scan_for_application_packages] completed in 4 seconds\n2025-12-12 08:16:24.723 Step [scan_for_files] starting\n2025-12-12 08:16:43.753 Progress: 12% (2/16) ETA: 140 seconds (2.3 minutes)\n2025-12-12 08:16:43.947 Progress: 25% (4/16) ETA: 58 seconds\n2025-12-12 08:16:44.080 Progress: 37% (6/16) ETA: 33 seconds\n2025-12-12 08:16:44.379 Progress: 50% (8/16) ETA: 20 seconds\n2025-12-12 08:16:44.551 Progress: 62% (10/16) ETA: 12 seconds\n2025-12-12 08:16:44.666 Progress: 75% (12/16) ETA: 7 seconds\n2025-12-12 08:16:44.708 Progress: 87% (14/16) ETA: 3 seconds\n2025-12-12 08:16:44.998 Progress: 100% (16/16)\n2025-12-12 08:16:45.464 Step [scan_for_files] completed in 21 seconds\n2025-12-12 08:16:45.467 Step [collect_and_create_license_detections] starting\n2025-12-12 08:16:49.430 Step [collect_and_create_license_detections] completed in 4 seconds\n2025-12-12 08:16:49.432 Pipeline completed in 29 seconds\n", + "log": "2026-01-16 12:40:25.323 Pipeline [scan_codebase] starting\n2026-01-16 12:40:25.324 Step [download_missing_inputs] starting\n2026-01-16 12:40:25.325 Step [download_missing_inputs] completed in 0 seconds\n2026-01-16 12:40:25.325 Step [copy_inputs_to_codebase_directory] starting\n2026-01-16 12:40:25.326 Step [copy_inputs_to_codebase_directory] completed in 0 seconds\n2026-01-16 12:40:25.327 Step [extract_archives] starting\n2026-01-16 12:40:25.378 Step [extract_archives] completed in 0 seconds\n2026-01-16 12:40:25.379 Step [collect_and_create_codebase_resources] starting\n2026-01-16 12:40:25.557 Step [collect_and_create_codebase_resources] completed in 0 seconds\n2026-01-16 12:40:25.559 Step [flag_empty_files] starting\n2026-01-16 12:40:25.560 Step [flag_empty_files] completed in 0 seconds\n2026-01-16 12:40:25.561 Step [flag_ignored_resources] starting\n2026-01-16 12:40:25.563 Step [flag_ignored_resources] completed in 0 seconds\n2026-01-16 12:40:25.564 Step [scan_for_application_packages] starting\n2026-01-16 12:40:25.564 Collecting package data from resources:\n2026-01-16 12:40:28.178 Progress: 11% (2/18) ETA: 21 seconds\n2026-01-16 12:40:28.179 Progress: 22% (4/18) ETA: 9 seconds\n2026-01-16 12:40:28.180 Progress: 33% (6/18) ETA: 5 seconds\n2026-01-16 12:40:28.187 Progress: 44% (8/18) ETA: 3 seconds\n2026-01-16 12:40:28.188 Progress: 55% (10/18) ETA: 2 seconds\n2026-01-16 12:40:28.188 Progress: 66% (12/18) ETA: 1 seconds\n2026-01-16 12:40:28.189 Progress: 77% (14/18) ETA: 1 seconds\n2026-01-16 12:40:28.189 Progress: 88% (16/18)\n2026-01-16 12:40:28.190 Progress: 100% (18/18)\n2026-01-16 12:40:28.191 Assembling collected package data:\n2026-01-16 12:40:28.191 Progress: 0%\n2026-01-16 12:40:28.257 Step [scan_for_application_packages] completed in 3 seconds\n2026-01-16 12:40:28.258 Step [scan_for_files] starting\n2026-01-16 12:40:28.297 Progress: 12% (2/16)\n2026-01-16 12:40:28.300 Progress: 25% (4/16)\n2026-01-16 12:40:28.578 Progress: 37% (6/16) ETA: 1 seconds\n2026-01-16 12:40:28.581 Progress: 50% (8/16)\n2026-01-16 12:40:28.966 Progress: 62% (10/16)\n2026-01-16 12:40:29.237 Progress: 75% (12/16)\n2026-01-16 12:40:30.070 Progress: 87% (14/16)\n2026-01-16 12:40:30.590 Progress: 100% (16/16)\n2026-01-16 12:40:30.855 Step [scan_for_files] completed in 3 seconds\n2026-01-16 12:40:30.857 Step [collect_and_create_license_detections] starting\n2026-01-16 12:40:30.869 Step [collect_and_create_license_detections] completed in 0 seconds\n2026-01-16 12:40:30.870 Pipeline completed in 6 seconds\n", "execution_time": null } ], @@ -158,7 +158,7 @@ "Documentation": "https://asgi.readthedocs.io/", "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" }, - "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=549d974e-a424-4fe4-9351-edb3b03b391a", + "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=e66230c6-0a0c-4b55-b339-03afd0550002", "is_private": false, "is_virtual": false, "datasource_ids": [ @@ -283,7 +283,7 @@ "Documentation": "https://asgi.readthedocs.io/", "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" }, - "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=5e52877b-c669-414e-bdf9-5d2bcb2443b6", + "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=e522e5ca-fd0c-4566-8632-bb3e82c5f7eb", "is_private": false, "is_virtual": false, "datasource_ids": [ @@ -307,8 +307,8 @@ "is_optional": true, "is_pinned": false, "is_direct": true, - "dependency_uid": "pkg:pypi/pytest?uuid=a1855203-fd9a-441d-98f6-e1e10de689c2", - "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=549d974e-a424-4fe4-9351-edb3b03b391a", + "dependency_uid": "pkg:pypi/pytest?uuid=43b6aa1d-7036-47c8-a40e-7b14bb14171b", + "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=e66230c6-0a0c-4b55-b339-03afd0550002", "resolved_to_package_uid": null, "datafile_path": "asgiref-3.3.0-py3-none-any.whl", "datasource_id": "pypi_wheel", @@ -323,8 +323,8 @@ "is_optional": true, "is_pinned": false, "is_direct": true, - "dependency_uid": "pkg:pypi/pytest?uuid=3fa1cdcc-c729-4e57-94f3-95212cd8f7d4", - "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=5e52877b-c669-414e-bdf9-5d2bcb2443b6", + "dependency_uid": "pkg:pypi/pytest?uuid=a1481753-660d-4cf0-b0b8-a929ede30720", + "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=e522e5ca-fd0c-4566-8632-bb3e82c5f7eb", "resolved_to_package_uid": null, "datafile_path": "asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info/METADATA", "datasource_id": "pypi_wheel_metadata", @@ -339,8 +339,8 @@ "is_optional": true, "is_pinned": false, "is_direct": true, - "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=8cfeaa4e-a8db-4004-a416-3230ee0d3aae", - "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=549d974e-a424-4fe4-9351-edb3b03b391a", + "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=b2d0632b-d881-4d8a-ad04-d3d780e7c6e2", + "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=e66230c6-0a0c-4b55-b339-03afd0550002", "resolved_to_package_uid": null, "datafile_path": "asgiref-3.3.0-py3-none-any.whl", "datasource_id": "pypi_wheel", @@ -355,8 +355,8 @@ "is_optional": true, "is_pinned": false, "is_direct": true, - "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=040715c9-6806-402c-b027-141a05b6fd05", - "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=5e52877b-c669-414e-bdf9-5d2bcb2443b6", + "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=66c37d13-079b-412b-a610-1dabf9d7e6aa", + "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=e522e5ca-fd0c-4566-8632-bb3e82c5f7eb", "resolved_to_package_uid": null, "datafile_path": "asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info/METADATA", "datasource_id": "pypi_wheel_metadata", @@ -371,7 +371,7 @@ "name": "asgiref-3.3.0-py3-none-any.whl", "status": "application-package", "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=549d974e-a424-4fe4-9351-edb3b03b391a" + "pkg:pypi/asgiref@3.3.0?uuid=e66230c6-0a0c-4b55-b339-03afd0550002" ], "tag": "", "extension": ".whl", @@ -798,7 +798,7 @@ "name": "LICENSE", "status": "scanned", "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=5e52877b-c669-414e-bdf9-5d2bcb2443b6" + "pkg:pypi/asgiref@3.3.0?uuid=e522e5ca-fd0c-4566-8632-bb3e82c5f7eb" ], "tag": "", "extension": "", @@ -875,7 +875,7 @@ "name": "METADATA", "status": "application-package", "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=5e52877b-c669-414e-bdf9-5d2bcb2443b6" + "pkg:pypi/asgiref@3.3.0?uuid=e522e5ca-fd0c-4566-8632-bb3e82c5f7eb" ], "tag": "", "extension": "", @@ -1182,7 +1182,7 @@ "name": "RECORD", "status": "scanned", "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=5e52877b-c669-414e-bdf9-5d2bcb2443b6" + "pkg:pypi/asgiref@3.3.0?uuid=e522e5ca-fd0c-4566-8632-bb3e82c5f7eb" ], "tag": "", "extension": "", @@ -1224,7 +1224,7 @@ "name": "top_level.txt", "status": "scanned", "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=5e52877b-c669-414e-bdf9-5d2bcb2443b6" + "pkg:pypi/asgiref@3.3.0?uuid=e522e5ca-fd0c-4566-8632-bb3e82c5f7eb" ], "tag": "", "extension": ".txt", @@ -1266,7 +1266,7 @@ "name": "WHEEL", "status": "scanned", "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=5e52877b-c669-414e-bdf9-5d2bcb2443b6" + "pkg:pypi/asgiref@3.3.0?uuid=e522e5ca-fd0c-4566-8632-bb3e82c5f7eb" ], "tag": "", "extension": "", @@ -1308,7 +1308,7 @@ "name": "compatibility.py", "status": "scanned", "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=5e52877b-c669-414e-bdf9-5d2bcb2443b6" + "pkg:pypi/asgiref@3.3.0?uuid=e522e5ca-fd0c-4566-8632-bb3e82c5f7eb" ], "tag": "", "extension": ".py", @@ -1350,7 +1350,7 @@ "name": "current_thread_executor.py", "status": "scanned", "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=5e52877b-c669-414e-bdf9-5d2bcb2443b6" + "pkg:pypi/asgiref@3.3.0?uuid=e522e5ca-fd0c-4566-8632-bb3e82c5f7eb" ], "tag": "", "extension": ".py", @@ -1392,7 +1392,7 @@ "name": "__init__.py", "status": "scanned", "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=5e52877b-c669-414e-bdf9-5d2bcb2443b6" + "pkg:pypi/asgiref@3.3.0?uuid=e522e5ca-fd0c-4566-8632-bb3e82c5f7eb" ], "tag": "", "extension": ".py", @@ -1434,7 +1434,7 @@ "name": "local.py", "status": "scanned", "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=5e52877b-c669-414e-bdf9-5d2bcb2443b6" + "pkg:pypi/asgiref@3.3.0?uuid=e522e5ca-fd0c-4566-8632-bb3e82c5f7eb" ], "tag": "", "extension": ".py", @@ -1476,7 +1476,7 @@ "name": "server.py", "status": "scanned", "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=5e52877b-c669-414e-bdf9-5d2bcb2443b6" + "pkg:pypi/asgiref@3.3.0?uuid=e522e5ca-fd0c-4566-8632-bb3e82c5f7eb" ], "tag": "", "extension": ".py", @@ -1518,7 +1518,7 @@ "name": "sync.py", "status": "scanned", "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=5e52877b-c669-414e-bdf9-5d2bcb2443b6" + "pkg:pypi/asgiref@3.3.0?uuid=e522e5ca-fd0c-4566-8632-bb3e82c5f7eb" ], "tag": "", "extension": ".py", @@ -1560,7 +1560,7 @@ "name": "testing.py", "status": "scanned", "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=5e52877b-c669-414e-bdf9-5d2bcb2443b6" + "pkg:pypi/asgiref@3.3.0?uuid=e522e5ca-fd0c-4566-8632-bb3e82c5f7eb" ], "tag": "", "extension": ".py", @@ -1602,7 +1602,7 @@ "name": "timeout.py", "status": "scanned", "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=5e52877b-c669-414e-bdf9-5d2bcb2443b6" + "pkg:pypi/asgiref@3.3.0?uuid=e522e5ca-fd0c-4566-8632-bb3e82c5f7eb" ], "tag": "", "extension": ".py", @@ -1683,7 +1683,7 @@ "name": "wsgi.py", "status": "scanned", "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=5e52877b-c669-414e-bdf9-5d2bcb2443b6" + "pkg:pypi/asgiref@3.3.0?uuid=e522e5ca-fd0c-4566-8632-bb3e82c5f7eb" ], "tag": "", "extension": ".py", diff --git a/scanpipe/tests/data/asgiref/asgiref-3.3.0_toolkit_scan.json b/scanpipe/tests/data/asgiref/asgiref-3.3.0_toolkit_scan.json index 1cacfac87f..319d1ec39e 100644 --- a/scanpipe/tests/data/asgiref/asgiref-3.3.0_toolkit_scan.json +++ b/scanpipe/tests/data/asgiref/asgiref-3.3.0_toolkit_scan.json @@ -2,7 +2,7 @@ "headers": [ { "tool_name": "scancode-toolkit", - "tool_version": "32.4.1", + "tool_version": "32.5.0", "options": { "--copyright": true, "--info": true, @@ -10,10 +10,10 @@ "--package": true }, "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", - "start_timestamp": "2025-12-12T081649.433146", - "end_timestamp": "2025-12-12T081655.140123", + "start_timestamp": "2026-01-16T124030.871108", + "end_timestamp": "2026-01-16T124033.754071", "output_format_version": "4.1.0", - "duration": 5.707000255584717, + "duration": 2.88297438621521, "message": null, "errors": [], "warnings": [], @@ -21,9 +21,9 @@ "system_environment": { "operating_system": "linux", "cpu_architecture": "64", - "platform": "Linux-5.15.0-163-generic-x86_64-with-glibc2.35", - "platform_version": "#173-Ubuntu SMP Tue Oct 14 17:51:00 UTC 2025", - "python_version": "3.10.12 (main, Nov 4 2025, 08:48:33) [GCC 11.4.0]" + "platform": "Linux-5.15.0-164-generic-x86_64-with-glibc2.35", + "platform_version": "#174-Ubuntu SMP Fri Nov 14 20:25:16 UTC 2025", + "python_version": "3.10.12 (main, Jan 8 2026, 06:52:19) [GCC 11.4.0]" }, "spdx_license_list_version": "3.27", "files_count": 15 @@ -140,7 +140,7 @@ "repository_homepage_url": "https://pypi.org/project/asgiref", "repository_download_url": "https://pypi.org/packages/source/a/asgiref/asgiref-3.3.0.tar.gz", "api_data_url": "https://pypi.org/pypi/asgiref/3.3.0/json", - "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=8d961513-130b-422b-8fa7-3cbfdd6646e4", + "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=9d756414-29cf-4cec-871f-bced786b3f82", "datafile_paths": [ "codebase/asgiref-3.3.0-py3-none-any.whl" ], @@ -258,7 +258,7 @@ "repository_homepage_url": "https://pypi.org/project/asgiref", "repository_download_url": "https://pypi.org/packages/source/a/asgiref/asgiref-3.3.0.tar.gz", "api_data_url": "https://pypi.org/pypi/asgiref/3.3.0/json", - "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=c7159857-d80e-40e5-8665-3347b8d35ffb", + "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=4fbdf9c2-d73b-4d3d-a342-4ee38977b41b", "datafile_paths": [ "codebase/asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info/METADATA" ], @@ -279,8 +279,8 @@ "is_direct": true, "resolved_package": {}, "extra_data": {}, - "dependency_uid": "pkg:pypi/pytest?uuid=a5625e52-9223-44ca-aa7a-f0f00c75ef4d", - "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=8d961513-130b-422b-8fa7-3cbfdd6646e4", + "dependency_uid": "pkg:pypi/pytest?uuid=9a208fed-5bd8-42d2-a995-1162e26fb9fe", + "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=9d756414-29cf-4cec-871f-bced786b3f82", "datafile_path": "codebase/asgiref-3.3.0-py3-none-any.whl", "datasource_id": "pypi_wheel" }, @@ -294,8 +294,8 @@ "is_direct": true, "resolved_package": {}, "extra_data": {}, - "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=91a0c4c7-f419-48b5-b22d-00f10b69e69a", - "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=8d961513-130b-422b-8fa7-3cbfdd6646e4", + "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=c14ff398-c25c-4729-9112-265761b1de6c", + "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=9d756414-29cf-4cec-871f-bced786b3f82", "datafile_path": "codebase/asgiref-3.3.0-py3-none-any.whl", "datasource_id": "pypi_wheel" }, @@ -309,8 +309,8 @@ "is_direct": true, "resolved_package": {}, "extra_data": {}, - "dependency_uid": "pkg:pypi/pytest?uuid=25041a97-3bee-4b75-926a-f303ebb81d99", - "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=c7159857-d80e-40e5-8665-3347b8d35ffb", + "dependency_uid": "pkg:pypi/pytest?uuid=e2301b96-0c70-442d-9366-62a8c12a7489", + "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=4fbdf9c2-d73b-4d3d-a342-4ee38977b41b", "datafile_path": "codebase/asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info/METADATA", "datasource_id": "pypi_wheel_metadata" }, @@ -324,8 +324,8 @@ "is_direct": true, "resolved_package": {}, "extra_data": {}, - "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=05572afa-6f15-4b60-94bb-1d9a43d5aba6", - "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=c7159857-d80e-40e5-8665-3347b8d35ffb", + "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=bdc3a89f-9c24-4e36-b41d-daf87efc6ccc", + "for_package_uid": "pkg:pypi/asgiref@3.3.0?uuid=4fbdf9c2-d73b-4d3d-a342-4ee38977b41b", "datafile_path": "codebase/asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info/METADATA", "datasource_id": "pypi_wheel_metadata" } @@ -464,7 +464,7 @@ "base_name": "asgiref-3.3.0-py3-none-any", "extension": ".whl", "size": 19948, - "date": "2025-12-12", + "date": "2026-01-16", "sha1": "c03f67211a311b13d1294ac8af7cb139ee34c4f9", "md5": "5bce1df6dedc53a41a9a6b40d7b1699e", "sha256": "a5098bc870b80e7b872bff60bb363c7f2c2c89078759f6c47b53ff8c525a152e", @@ -745,7 +745,7 @@ } ], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=8d961513-130b-422b-8fa7-3cbfdd6646e4" + "pkg:pypi/asgiref@3.3.0?uuid=9d756414-29cf-4cec-871f-bced786b3f82" ], "detected_license_expression": null, "detected_license_expression_spdx": null, @@ -855,7 +855,7 @@ "is_script": false, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=c7159857-d80e-40e5-8665-3347b8d35ffb" + "pkg:pypi/asgiref@3.3.0?uuid=4fbdf9c2-d73b-4d3d-a342-4ee38977b41b" ], "detected_license_expression": null, "detected_license_expression_spdx": null, @@ -893,7 +893,7 @@ "is_script": true, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=c7159857-d80e-40e5-8665-3347b8d35ffb" + "pkg:pypi/asgiref@3.3.0?uuid=4fbdf9c2-d73b-4d3d-a342-4ee38977b41b" ], "detected_license_expression": null, "detected_license_expression_spdx": null, @@ -931,7 +931,7 @@ "is_script": true, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=c7159857-d80e-40e5-8665-3347b8d35ffb" + "pkg:pypi/asgiref@3.3.0?uuid=4fbdf9c2-d73b-4d3d-a342-4ee38977b41b" ], "detected_license_expression": null, "detected_license_expression_spdx": null, @@ -969,7 +969,7 @@ "is_script": true, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=c7159857-d80e-40e5-8665-3347b8d35ffb" + "pkg:pypi/asgiref@3.3.0?uuid=4fbdf9c2-d73b-4d3d-a342-4ee38977b41b" ], "detected_license_expression": null, "detected_license_expression_spdx": null, @@ -1007,7 +1007,7 @@ "is_script": true, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=c7159857-d80e-40e5-8665-3347b8d35ffb" + "pkg:pypi/asgiref@3.3.0?uuid=4fbdf9c2-d73b-4d3d-a342-4ee38977b41b" ], "detected_license_expression": null, "detected_license_expression_spdx": null, @@ -1045,7 +1045,7 @@ "is_script": true, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=c7159857-d80e-40e5-8665-3347b8d35ffb" + "pkg:pypi/asgiref@3.3.0?uuid=4fbdf9c2-d73b-4d3d-a342-4ee38977b41b" ], "detected_license_expression": null, "detected_license_expression_spdx": null, @@ -1083,7 +1083,7 @@ "is_script": true, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=c7159857-d80e-40e5-8665-3347b8d35ffb" + "pkg:pypi/asgiref@3.3.0?uuid=4fbdf9c2-d73b-4d3d-a342-4ee38977b41b" ], "detected_license_expression": null, "detected_license_expression_spdx": null, @@ -1121,7 +1121,7 @@ "is_script": true, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=c7159857-d80e-40e5-8665-3347b8d35ffb" + "pkg:pypi/asgiref@3.3.0?uuid=4fbdf9c2-d73b-4d3d-a342-4ee38977b41b" ], "detected_license_expression": "apache-2.0", "detected_license_expression_spdx": "Apache-2.0", @@ -1181,7 +1181,7 @@ "is_script": true, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=c7159857-d80e-40e5-8665-3347b8d35ffb" + "pkg:pypi/asgiref@3.3.0?uuid=4fbdf9c2-d73b-4d3d-a342-4ee38977b41b" ], "detected_license_expression": null, "detected_license_expression_spdx": null, @@ -1255,7 +1255,7 @@ "is_script": false, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=c7159857-d80e-40e5-8665-3347b8d35ffb" + "pkg:pypi/asgiref@3.3.0?uuid=4fbdf9c2-d73b-4d3d-a342-4ee38977b41b" ], "detected_license_expression": "bsd-new", "detected_license_expression_spdx": "BSD-3-Clause", @@ -1592,7 +1592,7 @@ } ], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=c7159857-d80e-40e5-8665-3347b8d35ffb" + "pkg:pypi/asgiref@3.3.0?uuid=4fbdf9c2-d73b-4d3d-a342-4ee38977b41b" ], "detected_license_expression": "bsd-new", "detected_license_expression_spdx": "BSD-3-Clause", @@ -1679,7 +1679,7 @@ "is_script": false, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=c7159857-d80e-40e5-8665-3347b8d35ffb" + "pkg:pypi/asgiref@3.3.0?uuid=4fbdf9c2-d73b-4d3d-a342-4ee38977b41b" ], "detected_license_expression": null, "detected_license_expression_spdx": null, @@ -1717,7 +1717,7 @@ "is_script": false, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=c7159857-d80e-40e5-8665-3347b8d35ffb" + "pkg:pypi/asgiref@3.3.0?uuid=4fbdf9c2-d73b-4d3d-a342-4ee38977b41b" ], "detected_license_expression": null, "detected_license_expression_spdx": null, @@ -1755,7 +1755,7 @@ "is_script": false, "package_data": [], "for_packages": [ - "pkg:pypi/asgiref@3.3.0?uuid=c7159857-d80e-40e5-8665-3347b8d35ffb" + "pkg:pypi/asgiref@3.3.0?uuid=4fbdf9c2-d73b-4d3d-a342-4ee38977b41b" ], "detected_license_expression": null, "detected_license_expression_spdx": null, diff --git a/scanpipe/tests/data/asgiref/asgiref-3.3.0_walk_test_fixtures.json b/scanpipe/tests/data/asgiref/asgiref-3.3.0_walk_test_fixtures.json index 055b9c7c29..1f49a9438f 100644 --- a/scanpipe/tests/data/asgiref/asgiref-3.3.0_walk_test_fixtures.json +++ b/scanpipe/tests/data/asgiref/asgiref-3.3.0_walk_test_fixtures.json @@ -1,13 +1,13 @@ [ { "model": "scanpipe.project", - "pk": "3282ba3d-f525-4b74-9008-919108846d33", + "pk": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "fields": { "extra_data": {}, - "created_date": "2025-12-12T08:16:20.590Z", + "created_date": "2026-01-16T12:40:25.319Z", "name": "asgiref", - "slug": "asgiref-3282ba3d", - "work_directory": "/tmp/tmp5qi2cv31/projects/asgiref-3282ba3d", + "slug": "asgiref-92fe63d9", + "work_directory": "/tmp/tmp7o1wn11c/projects/asgiref-92fe63d9", "is_archived": false, "notes": "", "settings": {}, @@ -16,17 +16,17 @@ }, { "model": "scanpipe.run", - "pk": "88cca71e-436d-42ec-804b-ab1af5727413", + "pk": "87e422a3-3ebf-4fbf-99cb-e2f2bbb1ba7a", "fields": { "task_id": null, "task_start_date": null, "task_end_date": null, "task_exitcode": null, "task_output": "", - "log": "2025-12-12 08:16:20.593 Pipeline [scan_codebase] starting\n2025-12-12 08:16:20.594 Step [download_missing_inputs] starting\n2025-12-12 08:16:20.595 Step [download_missing_inputs] completed in 0 seconds\n2025-12-12 08:16:20.596 Step [copy_inputs_to_codebase_directory] starting\n2025-12-12 08:16:20.596 Step [copy_inputs_to_codebase_directory] completed in 0 seconds\n2025-12-12 08:16:20.597 Step [extract_archives] starting\n2025-12-12 08:16:20.661 Step [extract_archives] completed in 0 seconds\n2025-12-12 08:16:20.662 Step [collect_and_create_codebase_resources] starting\n2025-12-12 08:16:20.831 Step [collect_and_create_codebase_resources] completed in 0 seconds\n2025-12-12 08:16:20.832 Step [flag_empty_files] starting\n2025-12-12 08:16:20.834 Step [flag_empty_files] completed in 0 seconds\n2025-12-12 08:16:20.835 Step [flag_ignored_resources] starting\n2025-12-12 08:16:20.838 Step [flag_ignored_resources] completed in 0 seconds\n2025-12-12 08:16:20.839 Step [scan_for_application_packages] starting\n2025-12-12 08:16:20.840 Collecting package data from resources:\n2025-12-12 08:16:20.893 Progress: 11% (2/18)\n2025-12-12 08:16:20.932 Progress: 22% (4/18)\n2025-12-12 08:16:20.936 Progress: 33% (6/18)\n2025-12-12 08:16:20.937 Progress: 44% (8/18)\n2025-12-12 08:16:20.939 Progress: 55% (10/18)\n2025-12-12 08:16:20.940 Progress: 66% (12/18)\n2025-12-12 08:16:20.943 Progress: 77% (14/18)\n2025-12-12 08:16:20.945 Progress: 88% (16/18)\n2025-12-12 08:16:24.518 Progress: 100% (18/18)\n2025-12-12 08:16:24.602 Assembling collected package data:\n2025-12-12 08:16:24.603 Progress: 0%\n2025-12-12 08:16:24.722 Step [scan_for_application_packages] completed in 4 seconds\n2025-12-12 08:16:24.723 Step [scan_for_files] starting\n2025-12-12 08:16:43.753 Progress: 12% (2/16) ETA: 140 seconds (2.3 minutes)\n2025-12-12 08:16:43.947 Progress: 25% (4/16) ETA: 58 seconds\n2025-12-12 08:16:44.080 Progress: 37% (6/16) ETA: 33 seconds\n2025-12-12 08:16:44.379 Progress: 50% (8/16) ETA: 20 seconds\n2025-12-12 08:16:44.551 Progress: 62% (10/16) ETA: 12 seconds\n2025-12-12 08:16:44.666 Progress: 75% (12/16) ETA: 7 seconds\n2025-12-12 08:16:44.708 Progress: 87% (14/16) ETA: 3 seconds\n2025-12-12 08:16:44.998 Progress: 100% (16/16)\n2025-12-12 08:16:45.464 Step [scan_for_files] completed in 21 seconds\n2025-12-12 08:16:45.467 Step [collect_and_create_license_detections] starting\n2025-12-12 08:16:49.430 Step [collect_and_create_license_detections] completed in 4 seconds\n2025-12-12 08:16:49.432 Pipeline completed in 29 seconds\n", - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "log": "2026-01-16 12:40:25.323 Pipeline [scan_codebase] starting\n2026-01-16 12:40:25.324 Step [download_missing_inputs] starting\n2026-01-16 12:40:25.325 Step [download_missing_inputs] completed in 0 seconds\n2026-01-16 12:40:25.325 Step [copy_inputs_to_codebase_directory] starting\n2026-01-16 12:40:25.326 Step [copy_inputs_to_codebase_directory] completed in 0 seconds\n2026-01-16 12:40:25.327 Step [extract_archives] starting\n2026-01-16 12:40:25.378 Step [extract_archives] completed in 0 seconds\n2026-01-16 12:40:25.379 Step [collect_and_create_codebase_resources] starting\n2026-01-16 12:40:25.557 Step [collect_and_create_codebase_resources] completed in 0 seconds\n2026-01-16 12:40:25.559 Step [flag_empty_files] starting\n2026-01-16 12:40:25.560 Step [flag_empty_files] completed in 0 seconds\n2026-01-16 12:40:25.561 Step [flag_ignored_resources] starting\n2026-01-16 12:40:25.563 Step [flag_ignored_resources] completed in 0 seconds\n2026-01-16 12:40:25.564 Step [scan_for_application_packages] starting\n2026-01-16 12:40:25.564 Collecting package data from resources:\n2026-01-16 12:40:28.178 Progress: 11% (2/18) ETA: 21 seconds\n2026-01-16 12:40:28.179 Progress: 22% (4/18) ETA: 9 seconds\n2026-01-16 12:40:28.180 Progress: 33% (6/18) ETA: 5 seconds\n2026-01-16 12:40:28.187 Progress: 44% (8/18) ETA: 3 seconds\n2026-01-16 12:40:28.188 Progress: 55% (10/18) ETA: 2 seconds\n2026-01-16 12:40:28.188 Progress: 66% (12/18) ETA: 1 seconds\n2026-01-16 12:40:28.189 Progress: 77% (14/18) ETA: 1 seconds\n2026-01-16 12:40:28.189 Progress: 88% (16/18)\n2026-01-16 12:40:28.190 Progress: 100% (18/18)\n2026-01-16 12:40:28.191 Assembling collected package data:\n2026-01-16 12:40:28.191 Progress: 0%\n2026-01-16 12:40:28.257 Step [scan_for_application_packages] completed in 3 seconds\n2026-01-16 12:40:28.258 Step [scan_for_files] starting\n2026-01-16 12:40:28.297 Progress: 12% (2/16)\n2026-01-16 12:40:28.300 Progress: 25% (4/16)\n2026-01-16 12:40:28.578 Progress: 37% (6/16) ETA: 1 seconds\n2026-01-16 12:40:28.581 Progress: 50% (8/16)\n2026-01-16 12:40:28.966 Progress: 62% (10/16)\n2026-01-16 12:40:29.237 Progress: 75% (12/16)\n2026-01-16 12:40:30.070 Progress: 87% (14/16)\n2026-01-16 12:40:30.590 Progress: 100% (16/16)\n2026-01-16 12:40:30.855 Step [scan_for_files] completed in 3 seconds\n2026-01-16 12:40:30.857 Step [collect_and_create_license_detections] starting\n2026-01-16 12:40:30.869 Step [collect_and_create_license_detections] completed in 0 seconds\n2026-01-16 12:40:30.870 Pipeline completed in 6 seconds\n", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "pipeline_name": "scan_codebase", - "created_date": "2025-12-12T08:16:20.592Z", + "created_date": "2026-01-16T12:40:25.322Z", "scancodeio_version": "", "description": "Scan a codebase for application packages, licenses, and copyrights.", "current_step": "", @@ -43,7 +43,7 @@ "sha256": "a5098bc870b80e7b872bff60bb363c7f2c2c89078759f6c47b53ff8c525a152e", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -354,7 +354,7 @@ "sha256": "", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -400,7 +400,7 @@ "sha256": "", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -446,7 +446,7 @@ "sha256": "", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -492,7 +492,7 @@ "sha256": "6e89108c2cf0c0446174188f76f60465ae1c1f14f83427807df40d52a27cb2c8", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -538,7 +538,7 @@ "sha256": "b846415d1b514e9c1dff14a22deb906d794bc546ca6129f950a18cd091e2a669", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "bsd-new", "detected_license_expression_spdx": "BSD-3-Clause", "license_detections": [ @@ -619,7 +619,7 @@ "sha256": "70f98f4eb9f6068b192b5464fcdf69e29a8ff09962bfce84bbb052baeee44f33", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -930,7 +930,7 @@ "sha256": "11546323af45e6a5639bf620a9c4d73e74c0bf705f494af4595007b923f75e8a", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -976,7 +976,7 @@ "sha256": "2c1983592aa38f0bfb0afacc73ddc5b46ce10e8e89ceaa9fed1e5fc6361b608d", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1022,7 +1022,7 @@ "sha256": "30f49b9094bff904a42caeec32515715fe625a56dc48bd7c0e3d9988c0ad4bd7", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1068,7 +1068,7 @@ "sha256": "fa4651a3b79201a4dc44a4096cd49ec8f427e912ea0ee05c666357b413a8afe7", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1114,7 +1114,7 @@ "sha256": "ee0fcf4a8e6fa9df8a4643bb48e82892d496afce44b6c8b8aea2721755545e1c", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1160,7 +1160,7 @@ "sha256": "3151f66c476208c3154cb6c4fb557a2a253bab82f0ab33fb3c8b9f7976be9e33", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1206,7 +1206,7 @@ "sha256": "ddd445b778c097fc75c2bf69ad964cbadd3bd6999d1dd2306d39d401855e8e3e", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1252,7 +1252,7 @@ "sha256": "ddbc8d455eceb68fc583c67e7c4ad0277c867fb39095c51ec5b37f70342e8334", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1298,7 +1298,7 @@ "sha256": "126c3e3a8a75a517d2739612304607804cf5f34da63fa25d03a6f11f7edb6f2f", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "apache-2.0", "detected_license_expression_spdx": "Apache-2.0", "license_detections": [ @@ -1383,7 +1383,7 @@ "sha256": "f8bd1ea3fb8afddabb10f8efd66796d41446cad51168ef4d3c44b19c973d0ad0", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1429,7 +1429,7 @@ "sha256": "885267fee0fea687875a02ceb929ca095312d47aaa57e20e4ce382f397caaf4d", "sha512": "", "extra_data": {}, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "detected_license_expression": "", "detected_license_expression_spdx": "", "license_detections": [], @@ -1476,7 +1476,7 @@ "version": "3.3.0", "qualifiers": "", "subpath": "", - "uuid": "b0467b22-5e83-4b0d-91e2-15121a8cf075", + "uuid": "543a3583-3a13-4b5d-a039-c6bc4072de35", "md5": "", "sha1": "", "sha256": "", @@ -1486,7 +1486,7 @@ "Documentation": "https://asgi.readthedocs.io/", "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" }, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "compliance_alert": "", "affected_by_vulnerabilities": [], "filename": "", @@ -1577,7 +1577,7 @@ ], "missing_resources": [], "modified_resources": [], - "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=549d974e-a424-4fe4-9351-edb3b03b391a", + "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=e66230c6-0a0c-4b55-b339-03afd0550002", "keywords": [ "Development Status :: 5 - Production/Stable", "Environment :: Web Environment", @@ -1610,7 +1610,7 @@ "version": "3.3.0", "qualifiers": "", "subpath": "", - "uuid": "856001ca-aabf-4c6a-8ac2-cc1317f3e05d", + "uuid": "b2913908-3a31-4460-b330-a74d14b5ee24", "md5": "", "sha1": "", "sha256": "", @@ -1620,7 +1620,7 @@ "Documentation": "https://asgi.readthedocs.io/", "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" }, - "project": "3282ba3d-f525-4b74-9008-919108846d33", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "compliance_alert": "", "affected_by_vulnerabilities": [], "filename": "", @@ -1711,7 +1711,7 @@ ], "missing_resources": [], "modified_resources": [], - "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=5e52877b-c669-414e-bdf9-5d2bcb2443b6", + "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=e522e5ca-fd0c-4566-8632-bb3e82c5f7eb", "keywords": [ "Development Status :: 5 - Production/Stable", "Environment :: Web Environment", @@ -1757,10 +1757,10 @@ "version": "", "qualifiers": "", "subpath": "", - "uuid": "ef046d41-ae47-42c3-838c-def8bc956723", + "uuid": "19138025-32e9-4060-ab38-622b27493b6c", "affected_by_vulnerabilities": [], - "project": "3282ba3d-f525-4b74-9008-919108846d33", - "dependency_uid": "pkg:pypi/pytest?uuid=a1855203-fd9a-441d-98f6-e1e10de689c2", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", + "dependency_uid": "pkg:pypi/pytest?uuid=43b6aa1d-7036-47c8-a40e-7b14bb14171b", "for_package": 1, "resolved_to_package": null, "datafile_resource": 1, @@ -1783,10 +1783,10 @@ "version": "", "qualifiers": "", "subpath": "", - "uuid": "3f928a8b-4505-4a7b-ad3a-a589dec836db", + "uuid": "52135390-385f-4fc0-8b0a-38f28c1040dd", "affected_by_vulnerabilities": [], - "project": "3282ba3d-f525-4b74-9008-919108846d33", - "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=8cfeaa4e-a8db-4004-a416-3230ee0d3aae", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", + "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=b2d0632b-d881-4d8a-ad04-d3d780e7c6e2", "for_package": 1, "resolved_to_package": null, "datafile_resource": 1, @@ -1809,10 +1809,10 @@ "version": "", "qualifiers": "", "subpath": "", - "uuid": "31fbc19e-c309-4463-8312-390ef734bf78", + "uuid": "c6282019-1112-43f8-a27a-658cc0c5dcf4", "affected_by_vulnerabilities": [], - "project": "3282ba3d-f525-4b74-9008-919108846d33", - "dependency_uid": "pkg:pypi/pytest?uuid=3fa1cdcc-c729-4e57-94f3-95212cd8f7d4", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", + "dependency_uid": "pkg:pypi/pytest?uuid=a1481753-660d-4cf0-b0b8-a929ede30720", "for_package": 2, "resolved_to_package": null, "datafile_resource": 7, @@ -1835,10 +1835,10 @@ "version": "", "qualifiers": "", "subpath": "", - "uuid": "892b2af3-a9f0-4544-9874-583307d2d387", + "uuid": "01b33ae9-cb5f-442b-bfac-af8e40a1bf8c", "affected_by_vulnerabilities": [], - "project": "3282ba3d-f525-4b74-9008-919108846d33", - "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=040715c9-6806-402c-b027-141a05b6fd05", + "project": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", + "dependency_uid": "pkg:pypi/pytest-asyncio?uuid=66c37d13-079b-412b-a610-1dabf9d7e6aa", "for_package": 2, "resolved_to_package": null, "datafile_resource": 7, diff --git a/scanpipe/tests/data/cyclonedx/asgiref-3.3.0.cdx.json b/scanpipe/tests/data/cyclonedx/asgiref-3.3.0.cdx.json index d14b14ce71..e587c0f2e8 100644 --- a/scanpipe/tests/data/cyclonedx/asgiref-3.3.0.cdx.json +++ b/scanpipe/tests/data/cyclonedx/asgiref-3.3.0.cdx.json @@ -6,7 +6,7 @@ "version": 1, "metadata": { "component": { - "bom-ref": "3282ba3d-f525-4b74-9008-919108846d33", + "bom-ref": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3", "name": "asgiref", "type": "library" }, @@ -32,9 +32,16 @@ "name": "Django Software Foundation" } ], - "bom-ref": "pkg:pypi/asgiref@3.3.0?uuid=549d974e-a424-4fe4-9351-edb3b03b391a", + "bom-ref": "pkg:pypi/asgiref@3.3.0?uuid=e522e5ca-fd0c-4566-8632-bb3e82c5f7eb", "copyright": "", "description": "ASGI specs, helper code, and adapters\nasgiref\n=======\n\n.. image:: https://api.travis-ci.org/django/asgiref.svg\n :target: https://travis-ci.org/django/asgiref\n\n.. image:: https://img.shields.io/pypi/v/asgiref.svg\n :target: https://pypi.python.org/pypi/asgiref\n\nASGI is a standard for Python asynchronous web apps and servers to communicate\nwith each other, and positioned as an asynchronous successor to WSGI. You can\nread more at https://asgi.readthedocs.io/en/latest/\n\nThis package includes ASGI base libraries, such as:\n\n* Sync-to-async and async-to-sync function wrappers, ``asgiref.sync``\n* Server base classes, ``asgiref.server``\n* A WSGI-to-ASGI adapter, in ``asgiref.wsgi``\n\n\nFunction wrappers\n-----------------\n\nThese allow you to wrap or decorate async or sync functions to call them from\nthe other style (so you can call async functions from a synchronous thread,\nor vice-versa).\n\nIn particular:\n\n* AsyncToSync lets a synchronous subthread stop and wait while the async\n function is called on the main thread's event loop, and then control is\n returned to the thread when the async function is finished.\n\n* SyncToAsync lets async code call a synchronous function, which is run in\n a threadpool and control returned to the async coroutine when the synchronous\n function completes.\n\nThe idea is to make it easier to call synchronous APIs from async code and\nasynchronous APIs from synchronous code so it's easier to transition code from\none style to the other. In the case of Channels, we wrap the (synchronous)\nDjango view system with SyncToAsync to allow it to run inside the (asynchronous)\nASGI server.\n\nNote that exactly what threads things run in is very specific, and aimed to\nkeep maximum compatibility with old synchronous code. See\n\"Synchronous code & Threads\" below for a full explanation. By default,\n``sync_to_async`` will run all synchronous code in the program in the same\nthread for safety reasons; you can disable this for more performance with\n``@sync_to_async(thread_sensitive=False)``, but make sure that your code does\nnot rely on anything bound to threads (like database connections) when you do.\n\n\nThreadlocal replacement\n-----------------------\n\nThis is a drop-in replacement for ``threading.local`` that works with both\nthreads and asyncio Tasks. Even better, it will proxy values through from a\ntask-local context to a thread-local context when you use ``sync_to_async``\nto run things in a threadpool, and vice-versa for ``async_to_sync``.\n\nIf you instead want true thread- and task-safety, you can set\n``thread_critical`` on the Local object to ensure this instead.\n\n\nServer base classes\n-------------------\n\nIncludes a ``StatelessServer`` class which provides all the hard work of\nwriting a stateless server (as in, does not handle direct incoming sockets\nbut instead consumes external streams or sockets to work out what is happening).\n\nAn example of such a server would be a chatbot server that connects out to\na central chat server and provides a \"connection scope\" per user chatting to\nit. There's only one actual connection, but the server has to separate things\ninto several scopes for easier writing of the code.\n\nYou can see an example of this being used in `frequensgi `_.\n\n\nWSGI-to-ASGI adapter\n--------------------\n\nAllows you to wrap a WSGI application so it appears as a valid ASGI application.\n\nSimply wrap it around your WSGI application like so::\n\n asgi_application = WsgiToAsgi(wsgi_application)\n\nThe WSGI application will be run in a synchronous threadpool, and the wrapped\nASGI application will be one that accepts ``http`` class messages.\n\nPlease note that not all extended features of WSGI may be supported (such as\nfile handles for incoming POST bodies).\n\n\nDependencies\n------------\n\n``asgiref`` requires Python 3.5 or higher.\n\n\nContributing\n------------\n\nPlease refer to the\n`main Channels contributing docs `_.\n\n\nTesting\n'''''''\n\nTo run tests, make sure you have installed the ``tests`` extra with the package::\n\n cd asgiref/\n pip install -e .[tests]\n pytest\n\n\nBuilding the documentation\n''''''''''''''''''''''''''\n\nThe documentation uses `Sphinx `_::\n\n cd asgiref/docs/\n pip install sphinx\n\nTo build the docs, you can use the default tools::\n\n sphinx-build -b html . _build/html # or `make html`, if you've got make set up\n cd _build/html\n python -m http.server\n\n...or you can use ``sphinx-autobuild`` to run a server and rebuild/reload\nyour documentation changes automatically::\n\n pip install sphinx-autobuild\n sphinx-autobuild . _build/html\n\n\nImplementation Details\n----------------------\n\nSynchronous code & threads\n''''''''''''''''''''''''''\n\nThe ``asgiref.sync`` module provides two wrappers that let you go between\nasynchronous and synchronous code at will, while taking care of the rough edges\nfor you.\n\nUnfortunately, the rough edges are numerous, and the code has to work especially\nhard to keep things in the same thread as much as possible. Notably, the\nrestrictions we are working with are:\n\n* All synchronous code called through ``SyncToAsync`` and marked with\n ``thread_sensitive`` should run in the same thread as each other (and if the\n outer layer of the program is synchronous, the main thread)\n\n* If a thread already has a running async loop, ``AsyncToSync`` can't run things\n on that loop if it's blocked on synchronous code that is above you in the\n call stack.\n\nThe first compromise you get to might be that ``thread_sensitive`` code should\njust run in the same thread and not spawn in a sub-thread, fulfilling the first\nrestriction, but that immediately runs you into the second restriction.\n\nThe only real solution is to essentially have a variant of ThreadPoolExecutor\nthat executes any ``thread_sensitive`` code on the outermost synchronous\nthread - either the main thread, or a single spawned subthread.\n\nThis means you now have two basic states:\n\n* If the outermost layer of your program is synchronous, then all async code\n run through ``AsyncToSync`` will run in a per-call event loop in arbitary\n sub-threads, while all ``thread_sensitive`` code will run in the main thread.\n\n* If the outermost layer of your program is asynchronous, then all async code\n runs on the main thread's event loop, and all ``thread_sensitive`` synchronous\n code will run in a single shared sub-thread.\n\nCruicially, this means that in both cases there is a thread which is a shared\nresource that all ``thread_sensitive`` code must run on, and there is a chance\nthat this thread is currently blocked on its own ``AsyncToSync`` call. Thus,\n``AsyncToSync`` needs to act as an executor for thread code while it's blocking.\n\nThe ``CurrentThreadExecutor`` class provides this functionality; rather than\nsimply waiting on a Future, you can call its ``run_until_future`` method and\nit will run submitted code until that Future is done. This means that code\ninside the call can then run code on your thread.\n\n\nMaintenance and Security\n------------------------\n\nTo report security issues, please contact security@djangoproject.com. For GPG\nsignatures and more security process information, see\nhttps://docs.djangoproject.com/en/dev/internals/security/.\n\nTo report bugs or request new features, please open a new GitHub issue.\n\nThis repository is part of the Channels project. For the shepherd and maintenance team, please see the\n`main Channels readme `_.", + "evidence": { + "licenses": [ + { + "expression": "Apache-2.0 AND LicenseRef-test" + } + ] + }, "externalReferences": [ { "type": "bom", @@ -62,7 +69,7 @@ }, { "name": "aboutcode:package_uid", - "value": "pkg:pypi/asgiref@3.3.0?uuid=549d974e-a424-4fe4-9351-edb3b03b391a" + "value": "pkg:pypi/asgiref@3.3.0?uuid=e522e5ca-fd0c-4566-8632-bb3e82c5f7eb" }, { "name": "aboutcode:primary_language", @@ -80,16 +87,9 @@ "name": "Django Software Foundation" } ], - "bom-ref": "pkg:pypi/asgiref@3.3.0?uuid=5e52877b-c669-414e-bdf9-5d2bcb2443b6", + "bom-ref": "pkg:pypi/asgiref@3.3.0?uuid=e66230c6-0a0c-4b55-b339-03afd0550002", "copyright": "", "description": "ASGI specs, helper code, and adapters\nasgiref\n=======\n\n.. image:: https://api.travis-ci.org/django/asgiref.svg\n :target: https://travis-ci.org/django/asgiref\n\n.. image:: https://img.shields.io/pypi/v/asgiref.svg\n :target: https://pypi.python.org/pypi/asgiref\n\nASGI is a standard for Python asynchronous web apps and servers to communicate\nwith each other, and positioned as an asynchronous successor to WSGI. You can\nread more at https://asgi.readthedocs.io/en/latest/\n\nThis package includes ASGI base libraries, such as:\n\n* Sync-to-async and async-to-sync function wrappers, ``asgiref.sync``\n* Server base classes, ``asgiref.server``\n* A WSGI-to-ASGI adapter, in ``asgiref.wsgi``\n\n\nFunction wrappers\n-----------------\n\nThese allow you to wrap or decorate async or sync functions to call them from\nthe other style (so you can call async functions from a synchronous thread,\nor vice-versa).\n\nIn particular:\n\n* AsyncToSync lets a synchronous subthread stop and wait while the async\n function is called on the main thread's event loop, and then control is\n returned to the thread when the async function is finished.\n\n* SyncToAsync lets async code call a synchronous function, which is run in\n a threadpool and control returned to the async coroutine when the synchronous\n function completes.\n\nThe idea is to make it easier to call synchronous APIs from async code and\nasynchronous APIs from synchronous code so it's easier to transition code from\none style to the other. In the case of Channels, we wrap the (synchronous)\nDjango view system with SyncToAsync to allow it to run inside the (asynchronous)\nASGI server.\n\nNote that exactly what threads things run in is very specific, and aimed to\nkeep maximum compatibility with old synchronous code. See\n\"Synchronous code & Threads\" below for a full explanation. By default,\n``sync_to_async`` will run all synchronous code in the program in the same\nthread for safety reasons; you can disable this for more performance with\n``@sync_to_async(thread_sensitive=False)``, but make sure that your code does\nnot rely on anything bound to threads (like database connections) when you do.\n\n\nThreadlocal replacement\n-----------------------\n\nThis is a drop-in replacement for ``threading.local`` that works with both\nthreads and asyncio Tasks. Even better, it will proxy values through from a\ntask-local context to a thread-local context when you use ``sync_to_async``\nto run things in a threadpool, and vice-versa for ``async_to_sync``.\n\nIf you instead want true thread- and task-safety, you can set\n``thread_critical`` on the Local object to ensure this instead.\n\n\nServer base classes\n-------------------\n\nIncludes a ``StatelessServer`` class which provides all the hard work of\nwriting a stateless server (as in, does not handle direct incoming sockets\nbut instead consumes external streams or sockets to work out what is happening).\n\nAn example of such a server would be a chatbot server that connects out to\na central chat server and provides a \"connection scope\" per user chatting to\nit. There's only one actual connection, but the server has to separate things\ninto several scopes for easier writing of the code.\n\nYou can see an example of this being used in `frequensgi `_.\n\n\nWSGI-to-ASGI adapter\n--------------------\n\nAllows you to wrap a WSGI application so it appears as a valid ASGI application.\n\nSimply wrap it around your WSGI application like so::\n\n asgi_application = WsgiToAsgi(wsgi_application)\n\nThe WSGI application will be run in a synchronous threadpool, and the wrapped\nASGI application will be one that accepts ``http`` class messages.\n\nPlease note that not all extended features of WSGI may be supported (such as\nfile handles for incoming POST bodies).\n\n\nDependencies\n------------\n\n``asgiref`` requires Python 3.5 or higher.\n\n\nContributing\n------------\n\nPlease refer to the\n`main Channels contributing docs `_.\n\n\nTesting\n'''''''\n\nTo run tests, make sure you have installed the ``tests`` extra with the package::\n\n cd asgiref/\n pip install -e .[tests]\n pytest\n\n\nBuilding the documentation\n''''''''''''''''''''''''''\n\nThe documentation uses `Sphinx `_::\n\n cd asgiref/docs/\n pip install sphinx\n\nTo build the docs, you can use the default tools::\n\n sphinx-build -b html . _build/html # or `make html`, if you've got make set up\n cd _build/html\n python -m http.server\n\n...or you can use ``sphinx-autobuild`` to run a server and rebuild/reload\nyour documentation changes automatically::\n\n pip install sphinx-autobuild\n sphinx-autobuild . _build/html\n\n\nImplementation Details\n----------------------\n\nSynchronous code & threads\n''''''''''''''''''''''''''\n\nThe ``asgiref.sync`` module provides two wrappers that let you go between\nasynchronous and synchronous code at will, while taking care of the rough edges\nfor you.\n\nUnfortunately, the rough edges are numerous, and the code has to work especially\nhard to keep things in the same thread as much as possible. Notably, the\nrestrictions we are working with are:\n\n* All synchronous code called through ``SyncToAsync`` and marked with\n ``thread_sensitive`` should run in the same thread as each other (and if the\n outer layer of the program is synchronous, the main thread)\n\n* If a thread already has a running async loop, ``AsyncToSync`` can't run things\n on that loop if it's blocked on synchronous code that is above you in the\n call stack.\n\nThe first compromise you get to might be that ``thread_sensitive`` code should\njust run in the same thread and not spawn in a sub-thread, fulfilling the first\nrestriction, but that immediately runs you into the second restriction.\n\nThe only real solution is to essentially have a variant of ThreadPoolExecutor\nthat executes any ``thread_sensitive`` code on the outermost synchronous\nthread - either the main thread, or a single spawned subthread.\n\nThis means you now have two basic states:\n\n* If the outermost layer of your program is synchronous, then all async code\n run through ``AsyncToSync`` will run in a per-call event loop in arbitary\n sub-threads, while all ``thread_sensitive`` code will run in the main thread.\n\n* If the outermost layer of your program is asynchronous, then all async code\n runs on the main thread's event loop, and all ``thread_sensitive`` synchronous\n code will run in a single shared sub-thread.\n\nCruicially, this means that in both cases there is a thread which is a shared\nresource that all ``thread_sensitive`` code must run on, and there is a chance\nthat this thread is currently blocked on its own ``AsyncToSync`` call. Thus,\n``AsyncToSync`` needs to act as an executor for thread code while it's blocking.\n\nThe ``CurrentThreadExecutor`` class provides this functionality; rather than\nsimply waiting on a Future, you can call its ``run_until_future`` method and\nit will run submitted code until that Future is done. This means that code\ninside the call can then run code on your thread.\n\n\nMaintenance and Security\n------------------------\n\nTo report security issues, please contact security@djangoproject.com. For GPG\nsignatures and more security process information, see\nhttps://docs.djangoproject.com/en/dev/internals/security/.\n\nTo report bugs or request new features, please open a new GitHub issue.\n\nThis repository is part of the Channels project. For the shepherd and maintenance team, please see the\n`main Channels readme `_.", - "evidence": { - "licenses": [ - { - "expression": "Apache-2.0 AND LicenseRef-test" - } - ] - }, "externalReferences": [ { "type": "bom", @@ -117,7 +117,7 @@ }, { "name": "aboutcode:package_uid", - "value": "pkg:pypi/asgiref@3.3.0?uuid=5e52877b-c669-414e-bdf9-5d2bcb2443b6" + "value": "pkg:pypi/asgiref@3.3.0?uuid=e66230c6-0a0c-4b55-b339-03afd0550002" }, { "name": "aboutcode:primary_language", @@ -132,23 +132,23 @@ "dependencies": [ { "dependsOn": [ - "pkg:pypi/asgiref@3.3.0?uuid=549d974e-a424-4fe4-9351-edb3b03b391a", - "pkg:pypi/asgiref@3.3.0?uuid=5e52877b-c669-414e-bdf9-5d2bcb2443b6" + "pkg:pypi/asgiref@3.3.0?uuid=e522e5ca-fd0c-4566-8632-bb3e82c5f7eb", + "pkg:pypi/asgiref@3.3.0?uuid=e66230c6-0a0c-4b55-b339-03afd0550002" ], - "ref": "3282ba3d-f525-4b74-9008-919108846d33" + "ref": "92fe63d9-1d53-4b63-b19a-85022fb7a3f3" }, { - "ref": "pkg:pypi/asgiref@3.3.0?uuid=549d974e-a424-4fe4-9351-edb3b03b391a" + "ref": "pkg:pypi/asgiref@3.3.0?uuid=e522e5ca-fd0c-4566-8632-bb3e82c5f7eb" }, { - "ref": "pkg:pypi/asgiref@3.3.0?uuid=5e52877b-c669-414e-bdf9-5d2bcb2443b6" + "ref": "pkg:pypi/asgiref@3.3.0?uuid=e66230c6-0a0c-4b55-b339-03afd0550002" } ], "vulnerabilities": [ { "affects": [ { - "ref": "urn:cdx:pkg:pypi/asgiref@3.3.0?uuid=5e52877b-c669-414e-bdf9-5d2bcb2443b6" + "ref": "urn:cdx:pkg:pypi/asgiref@3.3.0?uuid=e522e5ca-fd0c-4566-8632-bb3e82c5f7eb" } ], "bom-ref": "BomRef", diff --git a/scanpipe/tests/data/dependencies/resolved_dependencies_npm_inspect_packages.json b/scanpipe/tests/data/dependencies/resolved_dependencies_npm_inspect_packages.json index 8ddf31f979..b45ab86b72 100644 --- a/scanpipe/tests/data/dependencies/resolved_dependencies_npm_inspect_packages.json +++ b/scanpipe/tests/data/dependencies/resolved_dependencies_npm_inspect_packages.json @@ -168,7 +168,7 @@ "matcher": "5-undetected", "end_line": 1, "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/package-manifest-unknown-cb9ea49fe36cb2e1ba6d87c68e1195a492f762cf", - "from_file": "resolved_dependencies_npm.zip-extract/package.json", + "from_file": null, "start_line": 1, "matched_text": "license - UNLICENSED", "match_coverage": 100.0, @@ -637,38 +637,17 @@ "source_packages": [], "bug_tracking_url": null, "primary_language": "JavaScript", - "license_detections": [ - { - "matches": [ - { - "score": 100.0, - "matcher": "5-undetected", - "end_line": 1, - "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/package-manifest-unknown-cb9ea49fe36cb2e1ba6d87c68e1195a492f762cf", - "from_file": null, - "start_line": 1, - "matched_text": "license - UNLICENSED", - "match_coverage": 100.0, - "matched_length": 2, - "rule_relevance": 100, - "rule_identifier": "package-manifest-unknown-cb9ea49fe36cb2e1ba6d87c68e1195a492f762cf", - "license_expression": "unknown", - "license_expression_spdx": "LicenseRef-scancode-unknown" - } - ], - "identifier": "unknown-0669ac45-20f6-defd-ec9f-2b6aafc9f944", - "license_expression": "unknown", - "license_expression_spdx": "LicenseRef-scancode-unknown" - } - ], + "license_detections": [], "repository_download_url": null, "repository_homepage_url": null, "other_license_detections": [], "other_license_expression": null, - "declared_license_expression": "unknown", - "extracted_license_statement": "- UNLICENSED\n", + "declared_license_expression": null, + "extracted_license_statement": [ + "UNLICENSED" + ], "other_license_expression_spdx": null, - "declared_license_expression_spdx": "LicenseRef-scancode-unknown" + "declared_license_expression_spdx": null } ], "emails": [], diff --git a/scanpipe/tests/data/dependencies/resolved_dependencies_poetry_inspect_packages.json b/scanpipe/tests/data/dependencies/resolved_dependencies_poetry_inspect_packages.json index 0d1043eb4b..4ac1504d36 100644 --- a/scanpipe/tests/data/dependencies/resolved_dependencies_poetry_inspect_packages.json +++ b/scanpipe/tests/data/dependencies/resolved_dependencies_poetry_inspect_packages.json @@ -322,19 +322,19 @@ "score": 100.0, "matcher": "1-hash", "end_line": 1, - "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/spdx_license_id_apache-2.0_for_apache-2.0.RULE", - "from_file": "resolved_dependencies_poetry.zip-extract/univers/pyproject.toml", + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/apache-2.0_65.RULE", + "from_file": null, "start_line": 1, - "matched_text": "apache-2.0", + "matched_text": "license: apache-2.0", "match_coverage": 100.0, - "matched_length": 3, + "matched_length": 4, "rule_relevance": 100, - "rule_identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", + "rule_identifier": "apache-2.0_65.RULE", "license_expression": "apache-2.0", "license_expression_spdx": "Apache-2.0" } ], - "identifier": "apache_2_0-d66ab77d-a5cc-7104-e702-dc7df61fe9e8", + "identifier": "apache_2_0-ec759ae0-ea5a-f138-793e-388520e080c0", "license_expression": "apache-2.0", "license_expression_spdx": "Apache-2.0" } @@ -1157,38 +1157,15 @@ "source_packages": [], "bug_tracking_url": null, "primary_language": "Python", - "license_detections": [ - { - "matches": [ - { - "score": 100.0, - "matcher": "1-hash", - "end_line": 1, - "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/spdx_license_id_apache-2.0_for_apache-2.0.RULE", - "from_file": null, - "start_line": 1, - "matched_text": "apache-2.0", - "match_coverage": 100.0, - "matched_length": 3, - "rule_relevance": 100, - "rule_identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE", - "license_expression": "apache-2.0", - "license_expression_spdx": "Apache-2.0" - } - ], - "identifier": "apache_2_0-d66ab77d-a5cc-7104-e702-dc7df61fe9e8", - "license_expression": "apache-2.0", - "license_expression_spdx": "Apache-2.0" - } - ], + "license_detections": [], "repository_download_url": "https://pypi.org/packages/source/u/univers/univers-0.1.0.tar.gz", "repository_homepage_url": "https://pypi.org/project/univers", "other_license_detections": [], "other_license_expression": null, - "declared_license_expression": "apache-2.0", + "declared_license_expression": null, "extracted_license_statement": "license: apache-2.0\n", "other_license_expression_spdx": null, - "declared_license_expression_spdx": "Apache-2.0" + "declared_license_expression_spdx": null } ], "emails": [], diff --git a/scanpipe/tests/data/scancode/package_assembly_codebase.json b/scanpipe/tests/data/scancode/package_assembly_codebase.json index 39d0483821..b8dda71390 100644 --- a/scanpipe/tests/data/scancode/package_assembly_codebase.json +++ b/scanpipe/tests/data/scancode/package_assembly_codebase.json @@ -2,16 +2,16 @@ "headers": [ { "tool_name": "scancode-toolkit", - "tool_version": "32.4.1", + "tool_version": "32.5.0", "options": { "--info": true, "--package": true }, "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", - "start_timestamp": "2025-12-12T081655.345103", - "end_timestamp": "2025-12-12T081655.456071", + "start_timestamp": "2026-01-16T124033.902159", + "end_timestamp": "2026-01-16T124034.007409", "output_format_version": "4.1.0", - "duration": 0.11098170280456543, + "duration": 0.10526299476623535, "message": null, "errors": [], "warnings": [], @@ -19,9 +19,9 @@ "system_environment": { "operating_system": "linux", "cpu_architecture": "64", - "platform": "Linux-5.15.0-163-generic-x86_64-with-glibc2.35", - "platform_version": "#173-Ubuntu SMP Tue Oct 14 17:51:00 UTC 2025", - "python_version": "3.10.12 (main, Nov 4 2025, 08:48:33) [GCC 11.4.0]" + "platform": "Linux-5.15.0-164-generic-x86_64-with-glibc2.35", + "platform_version": "#174-Ubuntu SMP Fri Nov 14 20:25:16 UTC 2025", + "python_version": "3.10.12 (main, Jan 8 2026, 06:52:19) [GCC 11.4.0]" }, "spdx_license_list_version": "3.27", "files_count": 3 @@ -91,7 +91,7 @@ "repository_homepage_url": "https://www.npmjs.com/package/test", "repository_download_url": "https://registry.npmjs.org/test/-/test-0.1.0.tgz", "api_data_url": "https://registry.npmjs.org/test/0.1.0", - "package_uid": "pkg:npm/test@0.1.0?uuid=966ed823-6627-4192-9608-1d1170ea4528", + "package_uid": "pkg:npm/test@0.1.0?uuid=bc515e36-4d09-4c13-b689-533065ae7d09", "datafile_paths": [ "package_assembly_codebase.tar.gz-extract/test/get_package_resources/package.json" ], @@ -278,7 +278,7 @@ } ], "for_packages": [ - "pkg:npm/test@0.1.0?uuid=966ed823-6627-4192-9608-1d1170ea4528" + "pkg:npm/test@0.1.0?uuid=bc515e36-4d09-4c13-b689-533065ae7d09" ], "files_count": 0, "dirs_count": 0, @@ -308,7 +308,7 @@ "is_script": false, "package_data": [], "for_packages": [ - "pkg:npm/test@0.1.0?uuid=966ed823-6627-4192-9608-1d1170ea4528" + "pkg:npm/test@0.1.0?uuid=bc515e36-4d09-4c13-b689-533065ae7d09" ], "files_count": 0, "dirs_count": 0, diff --git a/scanpipe/tests/pipes/test_scancode.py b/scanpipe/tests/pipes/test_scancode.py index 147086bc4a..e82eb81e01 100644 --- a/scanpipe/tests/pipes/test_scancode.py +++ b/scanpipe/tests/pipes/test_scancode.py @@ -502,7 +502,7 @@ def test_scanpipe_pipes_scancode_assemble_package_function(self): resource = project.codebaseresources.get(name="package.json") # This assembly should not trigger that many queries. - with self.assertNumQueries(17): + with self.assertNumQueries(16): scancode.assemble_package(resource, project, processed_paths) self.assertEqual(1, project.discoveredpackages.count()) From 1602d6a014215db3b62abf978c657b47ff87fdbb Mon Sep 17 00:00:00 2001 From: Aryan-SINGH-GIT Date: Thu, 18 Dec 2025 05:37:05 +0530 Subject: [PATCH 030/110] exclude github folder from deploy resource Signed-off-by: Aryan-SINGH-GIT --- how e323d10 --stat | 78 ++++++++++++++++++++++++++++++++++++ scanpipe/pipes/d2d_config.py | 2 +- 2 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 how e323d10 --stat diff --git a/how e323d10 --stat b/how e323d10 --stat new file mode 100644 index 0000000000..a91648e3db --- /dev/null +++ b/how e323d10 --stat @@ -0,0 +1,78 @@ +commit e323d1044f283e654de745864db531236051cc80 +Author: Aryan-SINGH-GIT +Date: Thu Dec 18 05:09:18 2025 +0530 + + Add .github/ exclusion to D2D mapping + + Signed-off-by: Aryan-SINGH-GIT + +diff --git a/scanpipe/pipes/d2d_config.py b/scanpipe/pipes/d2d_config.py +index d965fea..232ed68 100644 +--- a/scanpipe/pipes/d2d_config.py ++++ b/scanpipe/pipes/d2d_config.py +@@ -80,7 +80,7 @@ ECOSYSTEM_CONFIGS = { + ".odt", + ".odp", + ], +- deployed_resource_path_exclusions=["*.properties", "*.html"], ++ deployed_resource_path_exclusions=["*.properties", "*.html", "*/.github/*"], + ), + "Java": EcosystemConfig( + ecosystem_option="Java", +diff --git a/test_from/package.json b/test_from/package.json +new file mode 100644 +index 0000000..e078130 +--- /dev/null ++++ b/test_from/package.json +@@ -0,0 +1 @@ ++{"name": "test-app", "version": "1.0.0"}  +diff --git a/test_from/src/main/app.js b/test_from/src/main/app.js +new file mode 100644 +index 0000000..7946d15 +--- /dev/null ++++ b/test_from/src/main/app.js +@@ -0,0 +1 @@ ++console.log('Hello from development');  +diff --git a/test_from/src/main/utils.js b/test_from/src/main/utils.js +new file mode 100644 +index 0000000..bc11d31 +--- /dev/null ++++ b/test_from/src/main/utils.js +@@ -0,0 +1 @@ ++export function helper() { return true; }  +diff --git a/test_to/.github/README.md b/test_to/.github/README.md +new file mode 100644 +index 0000000..2063509 +--- /dev/null ++++ b/test_to/.github/README.md +@@ -0,0 +1 @@ ++GitHub workflows and configs  +diff --git a/test_to/.github/workflows/ci.yml b/test_to/.github/workflows/ci.yml +new file mode 100644 +index 0000000..b954009 +--- /dev/null ++++ b/test_to/.github/workflows/ci.yml +@@ -0,0 +1,2 @@ ++name: CI ++run: npm test  +diff --git a/test_to/dist/app.js b/test_to/dist/app.js +new file mode 100644 +index 0000000..a981b69 +--- /dev/null ++++ b/test_to/dist/app.js +@@ -0,0 +1 @@ ++console.log('Hello from deployment');  +diff --git a/test_to/dist/utils.js b/test_to/dist/utils.js +new file mode 100644 +index 0000000..bc11d31 +--- /dev/null ++++ b/test_to/dist/utils.js +@@ -0,0 +1 @@ ++export function helper() { return true; }  +diff --git a/test_to/package.json b/test_to/package.json +new file mode 100644 +index 0000000..e078130 +--- /dev/null ++++ b/test_to/package.json +@@ -0,0 +1 @@ ++{"name": "test-app", "version": "1.0.0"}  diff --git a/scanpipe/pipes/d2d_config.py b/scanpipe/pipes/d2d_config.py index 05ea09ca44..c7fd95338b 100644 --- a/scanpipe/pipes/d2d_config.py +++ b/scanpipe/pipes/d2d_config.py @@ -80,7 +80,7 @@ class EcosystemConfig: ".odt", ".odp", ], - deployed_resource_path_exclusions=["*.properties", "*.html"], + deployed_resource_path_exclusions=["*.properties", "*.html","*/.github/*"], ), "Java": EcosystemConfig( ecosystem_option="Java", From 5406ece256b464f940740f70f49c1010159891c1 Mon Sep 17 00:00:00 2001 From: Aryan-SINGH-GIT Date: Thu, 18 Dec 2025 05:41:17 +0530 Subject: [PATCH 031/110] removing unnecessary git stat Signed-off-by: Aryan-SINGH-GIT --- how e323d10 --stat | 78 ---------------------------------------------- 1 file changed, 78 deletions(-) delete mode 100644 how e323d10 --stat diff --git a/how e323d10 --stat b/how e323d10 --stat deleted file mode 100644 index a91648e3db..0000000000 --- a/how e323d10 --stat +++ /dev/null @@ -1,78 +0,0 @@ -commit e323d1044f283e654de745864db531236051cc80 -Author: Aryan-SINGH-GIT -Date: Thu Dec 18 05:09:18 2025 +0530 - - Add .github/ exclusion to D2D mapping - - Signed-off-by: Aryan-SINGH-GIT - -diff --git a/scanpipe/pipes/d2d_config.py b/scanpipe/pipes/d2d_config.py -index d965fea..232ed68 100644 ---- a/scanpipe/pipes/d2d_config.py -+++ b/scanpipe/pipes/d2d_config.py -@@ -80,7 +80,7 @@ ECOSYSTEM_CONFIGS = { - ".odt", - ".odp", - ], -- deployed_resource_path_exclusions=["*.properties", "*.html"], -+ deployed_resource_path_exclusions=["*.properties", "*.html", "*/.github/*"], - ), - "Java": EcosystemConfig( - ecosystem_option="Java", -diff --git a/test_from/package.json b/test_from/package.json -new file mode 100644 -index 0000000..e078130 ---- /dev/null -+++ b/test_from/package.json -@@ -0,0 +1 @@ -+{"name": "test-app", "version": "1.0.0"}  -diff --git a/test_from/src/main/app.js b/test_from/src/main/app.js -new file mode 100644 -index 0000000..7946d15 ---- /dev/null -+++ b/test_from/src/main/app.js -@@ -0,0 +1 @@ -+console.log('Hello from development');  -diff --git a/test_from/src/main/utils.js b/test_from/src/main/utils.js -new file mode 100644 -index 0000000..bc11d31 ---- /dev/null -+++ b/test_from/src/main/utils.js -@@ -0,0 +1 @@ -+export function helper() { return true; }  -diff --git a/test_to/.github/README.md b/test_to/.github/README.md -new file mode 100644 -index 0000000..2063509 ---- /dev/null -+++ b/test_to/.github/README.md -@@ -0,0 +1 @@ -+GitHub workflows and configs  -diff --git a/test_to/.github/workflows/ci.yml b/test_to/.github/workflows/ci.yml -new file mode 100644 -index 0000000..b954009 ---- /dev/null -+++ b/test_to/.github/workflows/ci.yml -@@ -0,0 +1,2 @@ -+name: CI -+run: npm test  -diff --git a/test_to/dist/app.js b/test_to/dist/app.js -new file mode 100644 -index 0000000..a981b69 ---- /dev/null -+++ b/test_to/dist/app.js -@@ -0,0 +1 @@ -+console.log('Hello from deployment');  -diff --git a/test_to/dist/utils.js b/test_to/dist/utils.js -new file mode 100644 -index 0000000..bc11d31 ---- /dev/null -+++ b/test_to/dist/utils.js -@@ -0,0 +1 @@ -+export function helper() { return true; }  -diff --git a/test_to/package.json b/test_to/package.json -new file mode 100644 -index 0000000..e078130 ---- /dev/null -+++ b/test_to/package.json -@@ -0,0 +1 @@ -+{"name": "test-app", "version": "1.0.0"}  From 96535de868f3c78ca98b311a67bf8070e17bd88b Mon Sep 17 00:00:00 2001 From: Aryan-SINGH-GIT Date: Thu, 18 Dec 2025 05:52:01 +0530 Subject: [PATCH 032/110] reformat file Signed-off-by: Aryan-SINGH-GIT --- scanpipe/pipes/d2d_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanpipe/pipes/d2d_config.py b/scanpipe/pipes/d2d_config.py index c7fd95338b..917e004bf7 100644 --- a/scanpipe/pipes/d2d_config.py +++ b/scanpipe/pipes/d2d_config.py @@ -80,7 +80,7 @@ class EcosystemConfig: ".odt", ".odp", ], - deployed_resource_path_exclusions=["*.properties", "*.html","*/.github/*"], + deployed_resource_path_exclusions=["*.properties", "*.html", "*/.github/*"], ), "Java": EcosystemConfig( ecosystem_option="Java", From d09aa27929e74255c96cbc6f0eb446d94966ba09 Mon Sep 17 00:00:00 2001 From: Aryan-SINGH-GIT Date: Thu, 18 Dec 2025 06:01:23 +0530 Subject: [PATCH 033/110] adding github in deployed_resource_path_exclusions Signed-off-by: Aryan-SINGH-GIT --- scanpipe/tests/data/d2d/config/ecosystem_config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanpipe/tests/data/d2d/config/ecosystem_config.json b/scanpipe/tests/data/d2d/config/ecosystem_config.json index c152ea85b6..e920b6672d 100644 --- a/scanpipe/tests/data/d2d/config/ecosystem_config.json +++ b/scanpipe/tests/data/d2d/config/ecosystem_config.json @@ -3,7 +3,7 @@ "matchable_package_extensions": [".jar", ".war", ".gem", ".zip", ".tar.gz", ".tar.xz"], "matchable_resource_extensions": [".map", ".js", ".mjs", ".ts", ".d.ts", ".jsx", ".tsx", ".css", ".scss", ".less", ".sass", ".soy",".class", ".rb"], "doc_extensions": [".pdf", ".doc", ".docx", ".ppt", ".pptx", ".tex", ".odt", ".odp"], - "deployed_resource_path_exclusions": ["*META-INF/*", "*/module-info.class", "*/OSGI-INF/*.xml", "*/OSGI-INF/*.json", "*spring-configuration-metadata.json", "*checksums.yaml.gz*", "*metadata.gz*", "*data.tar.gz.sig", "*.properties", "*.html"], + "deployed_resource_path_exclusions": ["*META-INF/*", "*/module-info.class", "*/OSGI-INF/*.xml", "*/OSGI-INF/*.json", "*spring-configuration-metadata.json", "*checksums.yaml.gz*", "*metadata.gz*", "*data.tar.gz.sig", "*.properties", "*.html", "*/.github/*"], "devel_resource_path_exclusions": ["*/tests/*"], "standard_symbols_to_exclude": [], "source_symbol_extensions": [] From e0d7391f25b42f6cf4465912a3da6b4e7b6e7f6f Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Fri, 13 Feb 2026 09:06:47 +1300 Subject: [PATCH 034/110] chore: upgrade django to version 6.x (#2052) Signed-off-by: tdruez --- .github/workflows/run-unit-tests-macos.yml | 2 +- .github/workflows/run-unit-tests.yml | 2 +- CHANGELOG.rst | 7 +++++ etc/thirdparty/virtualenv.pyz | Bin 8414106 -> 8428467 bytes etc/thirdparty/virtualenv.pyz.ABOUT | 6 ++-- pyproject.toml | 32 ++++++++++----------- scancodeio/__init__.py | 1 + scanpipe/filters.py | 2 +- 8 files changed, 29 insertions(+), 23 deletions(-) diff --git a/.github/workflows/run-unit-tests-macos.yml b/.github/workflows/run-unit-tests-macos.yml index 59e2ccbf27..df128bfa6a 100644 --- a/.github/workflows/run-unit-tests-macos.yml +++ b/.github/workflows/run-unit-tests-macos.yml @@ -21,7 +21,7 @@ jobs: strategy: matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.12", "3.13", "3.14"] steps: - name: Checkout code diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 96b191eaa9..2d8c286ca0 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -39,7 +39,7 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.12", "3.13", "3.14"] steps: - name: Checkout code diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 891b6398fd..0966a61a34 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,13 @@ Docker Compose users with existing data: run `./migrate-pg13-to-17.sh` before st the stack. Fresh installations require no action. +v37.0.0 (unreleased) +-------------------- + +- Upgrade Django to release 6.x + +- Drop support for Python3.10 and Python3.11 + v36.1.0 (2026-01-22) -------------------- diff --git a/etc/thirdparty/virtualenv.pyz b/etc/thirdparty/virtualenv.pyz index 06a228d36dd64d84d8b1d1982cdcbdacd748ebcc..914dcfb62a9620f33414478ef0c111327a243855 100644 GIT binary patch delta 26385 zcmdsfd0H_PvUI-rRx+;gdyT8SQUv>9;a|HC;@B8zc-IAB?>gww1 z>h7xQdhfOuxA{Hl`k7xu+e*1#jD-B2?#NoVE8^6#O49}A-oj~)%57tEeMjvMoz zO)8(?8+hb-nM6V$RszRE_;`5Yd&fN2xt6h&GQa#hWdWpKax4ta&&%ctinBQtv`a^o zjEXZQRK}imq`v6So>phPB=nYE_>wGw^zp`pzlZ6=%X1WDFIHy z0JoM)<#PHtwYsOVqpQ2RNnO{{qgHe47jyPOy*Y%#pI$d;;S@avtKFe7a(eTK!D=y^ z^kzHPZ?Nihc8hiVviAKJy^S+l?3{kgV6&$xxFrknDY4q_j*hyPE_GXVSA(;1ex_Bg zF%D_$gPdaimb@uNmJiah#tuZmUd$Sko?v*l7LmOeuT=OF6w#5+RIXB)tH@G>=#4f# z?WCu1u6x9lGsYh>UoJn_Xy*!XT&2N569z~@4<@SfJNSWSyPdqFxLM1TozEW53=Czq(qQ2;=p#i0QSr zWx?}LIMS!joeYEn`(y!+K69Hx%DOJOeU-rf^gXizSlRyjr~TMd<$*o1?CI3=8yP#( zjW3nZAwBz8cK_c(B$B^{OdotL0d6nRhCF)krLRH+8m}n?_FsAPc~%Bh%jL<_2hMPG zFe{WN9-Kd0A|a46z3)uoqyIirGMhHWF6K6S-x2`pFX+nNRVy|qkUJM`q%yr zTnimx(@VZdfU0tBsH-hqx{j?G*FlwZHJe}vFO$}@*1cuYY&ICz_7>?EQLJ>dPkM?? zn`^aBdVd5hg+Qych1IXNN>{V>_tLf5;bpNBNm(pt%A^^vcZoL0bs&1Da3ZXc!y!6Ika-u|%oXBumllWjboWzGre;mHZWxQ8< zF@ZJNbXfXX4t>10_pFo+5Y}Flc2X(oj@&5!ZGc2Vpk%uGn^@SJfP2rgb9jH(`HRvQ z*^IiLlFA~4Nq99>R!%wC@ZIe<8Wu<-h6Qk_Lze8y3zO~f56>OZTWto5Imb4n*TG9s zGJTJdQ_!VGb`kPZ^e_~Tpcn=W_!pO{S8h~hb4w}_Q&gK{zvRLhUsH&33O@DW3zH4B%>{JRJ&AZQ@Ii5L3A|bE<&NRysT&8?kSj@6C zuHItR>ACK%`kYeEWa-y)b_+MCG4~twoW)@ua@e^6tHs1M8+2BS%`#x;I&nwSb8X}H zL5rEQ*&SLqGOCS(Kh{Wd(@Irr8zZB^op7z9ZFN^eYYS2@RjlVKIfY8$8mgC>q;n~T z)oQWYDivCdZnMp(u?<@FLq?5GuK>-!WCHLt6MnAGm&-n3+l=e=X4!E+D&gwLSsxgz zC}FD$C0tb%oXxd0b>x+)@?95hlD+O1k2^P!p;C~!S`a9xx+mvA#9DbEgkCLEz{UBd z2>F0w3taj^8Vem)%c2)3w}@>F<^hY+M20=(J55L&S|67s1*PJE1`K8mXu9Pwu!8xKM>wZ>I%XR*S`P78|ykC2qqwmN5xCTC*( zqO5hcg;`5UIq7dn=F;-Y%F0Rn|JKUM)XK?<%E=}8-^nFe%0*d^KQFsZ3VfyAFGZYP zHm&Huq4_8pU5g430{a5zEZv;5>k}ekB9yEr#tS`!{0^t2hW02CSHfij-K_uoLI63{h>N;AKv?kML zw;IeFsSYbalV^;@6>jvS>X0JJR3L&%=4{;>R2SqdCH zr44|?D^SMCr{qzv*E;0~2M29oL87FfZ`amXTI5H@PS)T&`057KSjy>t_!K%|)$7%E z+?6!_aN)NTvCvmK8Ri*mvS+ZB-t$a$ZrHX?n+$DlPecI!2nuKCTCE>E{;DhuQclUk zog$+sAm4>g*!3+jX*OA`Gl~r6%ZqTI^2O3vb%VYo#E|qY!M)u*M16lH8>_xQ8>DOH zF>`)md{$_Smc_c1f08ZqOHk_u^}5aKnKH0~=FhSuIQ--m|61HLg=&u)at0e`)7T9* zlvlq}q44aoIty+bW;CH~<(v@>ZA#71klx&%Ii#`LnYm0hQ|zCZ+aAJ@x(&DdzFyqv_Npg>OWe%ciRLPl z3gsq?!A#a3qr?qZtQ?`q8O+{fhMROraNysLV3nZFJ?sq6D)-sYh0;u~ZnEIid&Z;~ zRDqetlpa!Dy4YE@M5^2n`VarL>%%~NsNVxOKdQ>kmbRR)o% zs=)c>cw7;|!`{PN(uslY2a##=z(g_@LQ|BgQaBs4;N(A@^I+AVWpPm)mxlT7saU}giKU0`OV zII&~{r$#AMSIJ{xN3Syw^4^s7N2YPLxJBBD8EQa-#0Ghx6LXy6u5QK6pPU6`XY;WE zLWDKpt`zIVx^nqW$Dcu) z&r~B99Iv0($R8J~(ZgKK_;fCp-yg#sbJoc(vTk9KUf#e8NA+@TJbi@sCy_jf0l12$ zlG3Sfh;)k zj6BKJ_kz5v5DwfW4~MoD(jdT%$P0U;3~ z2_YF_4#Hf7c?c=4#}m1i5@!*yEXX}j^GsT@1Ock1a82!O(Jp`=68%E9|G9ueNJUT} zq#?{lNJm(Jkb#hikcF@iVG%+$LJoowAr~PJL4}ZyP=HW~P=rv7P=Zj3P=-+M`g6hJ zXVU28Y`?j8F+v4GCBhPfr3h6B)d)2RwFt`)>JaJ?mLoJEG$O1(Sc%Yt(CoUox8*IS z2-_`|7KB!WHiUMB4unpGE`)A`9)wHnwN(Fp<7$Y9k+3J=~MB-T=5)z_Aui577OEJ3xJ7+sCClJdq z8aC_QHW-fEh9>$?Z~@ghj(9lzo;)CQgjqCdw1r^jeGWAD- zu8s5|*5N3L6NAPRh8H}Rh8KD#3tx~EAKe}sN@KGgU1JjNP-L@zm3f{g4JnPM)yh0= zzNf;fQz@YROIcF5n>ac5K!0Aw#iEy(shQMHT6I~O>-raEWi6erV8(738J?1GIh2_N z((7c&AUoP04bJVhfO!Mrfguj3oO6c>Y;w-iKP!var@loT3B+=F?lA}y@U*`Ig@Vf|6l8f($R`qD zb|M;%YWw&Q2^)BKSeQjjB26}@&E~8+Gzp<9T^>`4$YzJ>3OyEG-eZx_Be5xW|Hct3 z<0>KixGWtWsW4@LGkPiz+8&yk9mw>u2c9`5p96b!=nP)WkcRcS!x@#F1!L=0(QM<0 zPfJJdVO>OVm-ygNj=fQ^03!?+3MS*A>N@lm+I}Gm1a@5?bx^hgVknt&mplaiLEXoVf@HzsBq|)^7!e4r~G5Z`U;=w zGiT(9MXVcQ{tHN%ltNYr?WA8nz^TdVLRP$R;N)xC_!YP%h+Jax9I+E&@!EVGur#i5 zKpcyWh8WSa&=|AXmIR0EO!2xj^d5+%?xuwP#71feTJds3W5$Ua_pZN84DQ(z9T75% zOdvuPZ2L?W4<}Ae#Lg0BH?tM9AM>?0I*8L!{X=krsvh( ziz@BKO)rhBL3&h3Ug$PKR;UwnxsmH44z)?Ex4K)Bu~@8lrGN`;3&sk|IaM~-uF&EG zhI&a? zxS3)6xq@RVs3aY@i@o$wZPcsPnb4>-r6yw3T9_PcOeamyadQRHSUxy}w;DV6NTwxi~TQwMp_bwwn zl5@eeLD zE)dzL+q*!Y9NY7j{zw=X!@Zdp%Ej}VZK7fiGh;j;J=z~bkL|ghW1ApT-xUvTq2ZNI z?G&m5Zq|H{NL5w{oGdoQwxWP%oSykv?s0kzv;p@syhRaZEZPptv=J`}&>?q;GJk1!IX%IIN)orD_V$3rV8o5tMsKh*7=2_$DGlve^>}aM775&#Y75VyTu`Z! zlhqySA|uQer;|)vIiUnYi_pOM{w7)IBC%hAz8K`R;bh>UNccL?( zg!9?`F_E6-hH=-*O1!bc$gI;%M6{OHqL9e|L~SEx{{d;!(@(bq z=`_q0Yu(ep>_mgjVz#lFh!Q&|^2C{7QxO-7A#gXXEGla(f#E&+q!=6-eueAQ+O3@pjF$B?` zcPl{V)=G>kcwM_7Z|;faD!{U8EE%|~wWOy2IK9&32hYAH3!LXZ6=vvp7G^1M`Ec@& zvXm-x=Ji%J?z`y8<2izNI6m98XNBgd)_YEHB4pMQa&NaKFQ(6zZHc~@Gg(xFhRC&y zSEpI9Z-X>;w%2f2Z3wjYnqq)ErVoa&U*Y|w+b^9pBSKB1_{=|D_zzupvfH*1oi@<` zxuR?74|m2p{XI6)v$-av=dT~JMLR_+MYKl6E`G`s#qj!1TBGjM2KOfW6(c^CLg48~ zEK!AlW4a<@sT6C56s##Q^aGC6Fp>(&1FWr81uMCoOhCy7Mpm`IMTNkaDu zqQz0cw$Uvd4a3BsAgyd`@XRM1s_+iD%$^}2)c6^TGs7)Y(!1NqL@QM=`)#EBCl+di zFD8)XINbMaOQoJ zYr#J?a^Wr|=UdGV;R5B^?=)`;nRmBd(rgiKSQ3J?t6Acp%Mq$gkS; z-kqq8m*r)+F2-nM!)OrYHoUVUkaML5 z=Z^Hvbpp{*{edbNg^kSLgDL zy=9H{Dnn0yN5N1}Pl>s^qp4=4rC8THmQ$TwlAl*9(D-349}1PV+P?hJhE0Q=1v=H} z=4GRWL#9F7=wMBEZHGp++_2m{T&o_}3_FUAy&Z+4-P!qN`K4uy!tUo}A#k=<8>A{J z?9z-hjI8KT_11K?uNf)mv|00uT6?;S##ffDXkMWjw~TIXuvT|t{tV_EvQcEZT9;{G z3(9MsC|g}!J3KI2*QT#m*SB>~m{lv~&DpA=q7pdY zqD^#dZq^=>JIyQmH(Q3Z#zI5az=*a3I~rFF7?-sdtXVcOp=;>1Tk2Xy^ri-*-I86D zmsb+ZERDGuEYUtR;&BfjRA!? zoPttUQ@eJq40gth%&Q$VZEh&iZPc}`Sy?u^xxca7q#w79IC{rcH z>^;S+v|V?x(j6DI1uwD3V@Gwjw6n(zG5rs{$R20C*Z*h!`<4&+wcF}j9_uy|M^htO_ZS@&fle!>_CQgP=>Y#Zyk08Z?#74p!8hx z!feYQVvmL=ZOStC_^YpMe~qE{mFF((jW*+sWQM#)(2^MX%vlVV`kT=74z!O^WLOtu zcd%w}1lhL>nb`ZDwl5UkgjmWeTn8iVU$d)i_}g=0O_b4|0DI3k16+mA+uto^)Xcwe zu!gSc@9mXxI)-N=~K60ERj4BJd>9tk2wQfZwESt{L(!%ew+;j9QsTa6ONXn2b-`m z4|tbjRekqt~zx`U0ZcWXPvsO zwXvlO<&Za)M}CEQ>;kc|rKh^7ahckyl@ptFQp|ecCoSEA3 z>?o86g0=HfS@iNUJP+FL5N+~&$_tHoU?58nkeUhz)Aa0vo+(E)RqXyxn8sfi{TQ&Hnjce_s2~FkXs!b4zXRy5-ja3SI zpdmpT;o^i|tqq_Ab1)i|GvG%Y8fgEQBhHBeCC>Iwh_IIm$>VPm$*@+H+!b_$sEBVG zBfc(xJ#*ysC3A-G_7XRaeq|14uNloJm0PJ-C(EWLbGXHrnt(b3w$G`XdVSuP}6S1BENW|z= z=t8ZW7P(^~fkA-~ zzFMH!fk|tj?@PmBozoHQ`Xb2@%eKsrzeFwkFi#@+Fb}F$VRwCW%Y>*0oC;TWsv}qG zsvWUEoy(4^i0_5t3RL^;S#-+{Kfh<+Q9P56%6!@%so+!Lc!y(yE4OtlR2ER6q%R6; z?i|kuD#+&OyMpvR!60I^{QIMwzk+S|*|VetuaADRktIC{^=U(cI3$6 z_$tS5KpCw~-<>-e0S69D2D`Y2?Q%a#hj==$=#d$8POf(>7Tg;)j9FnL!f{PHljY}u zs$+2+^o?%_bTvO}f1VNBu)l5$s-s30Fm6QD)UontIH7Y~%&V(kv#FqN^FU7bu-@pf zY%1KODY09|C&tI?^NVvv$7^*fHnkO-##gSgWUERFO5j&d+LQ8%H#OKht%ZXd4PAN7 z!%YqS8&|h3?_FVXpczxqpRLh)821u-<+LaSXv6<(IXpGIT}+@;b2#P^QN)Q z1LH+4YwBz5>XHWB3r6~D?Io6)36*yBa`kYtsW7{sq%aqrXyG7@yE|?gYsNS@qsjSzeX{)z2*_N#?L?5acUoAB1iRR=8leVk7 z2`P1IgBGo5YF@cfTc|0}4wmQ*oz-^78huM~VQr_adChX|>gKMZ>Y}^>eRrp>5Dq+kP3!y5?F$V-g>Pg`hvR?)|ejZG<+1K1iWpvEFd+e)Iw8Duc57XI{PUw{jBI9tdqMM*A#s;3>vaNsw{%2!X@zY3rRar&pf z=3Xt8NM3~t&)F3WMYLi6U%2ZRQ;rXqPvAnaZ}yh+&JUfJ$@$Yq@HOEp$hi`R?so>v z6mi9&hu_?WBHkv5IIn?U0DT`hLw`)fgZ{AeDaTsZ{qr2X?7hYY(+=GMl=cBZ+DIY~ zx>6k_u4`X${LCZmUM6V6_l|!`Y3k}r7dr}he5aK_3#bwt^T^l};l>sGY*5~(4k8hm z7}pz@93$+VR%i_!d74onyNeJKs*D=Bmd(PVJ3gub?(wmO@*?yGPyWT3;i?=Mjb$Cf z4dLTTv0^SLwWGnVhkif$0iyytpBtsMt|QNl_Omom2#1k8sS2bI;Wpg|BP-@JEqM4T zT8xOvaN}w|2tA&$r`g+x92%hw|A%q{qd60qa58#)K1GXa<LAHoO^g zQ}88e8)yc{FqE4xIxCRf{7kt<=Z!uuGqB64hLv@z)lIFn_@Z}nb!|gqOC7bqY~wZz z6X%Q?@byUc5-=Xb6jywOy1TQkgQDUgZl$kyV*F1hBGa>faI=0K?H(|mktL=Rqf}=z z^cyK#BueVTCcEQ>w$#xCXdfO+_9Km5gQ=rIEV=oSFHXGamoJeJ=z?vwku;o+GwViE zKyjZsh6U^*C&m(pEL^h|j=rVxoKR=ij%JJ4(?|NEhtu^+8r2v<=xiP}ZZ;N`j2Oq(n7c|XJ)7DL z-L?6>YFq!9x>P+_M$Q1J>>r(9vudPvV6$;~cXQ9!3hgR`y-2lbXmxW-jTv-I}uYO~suXvsL+}d4)6Sz^)^G(QwBsdr83Rc6^l*>f+)?;=C3itKmBMP_G5(!UcGXXWc0oQp!eE zfJ+S{D?Lk3sJX(z+rYL4Txum*>~TxW7{~TY*PZQSUuLk-AcRlmJU(U;_A}g8iiIQY#pv-H*$`Z7 zO2#J`&#u3$9Ut`C9U$)$d_Bj#Gr0EK$N!nh7~8vN{97ToednFyZgyWBf0=#h0XF<| z+?(CJb-dEW|7(1$z|{xej*qa%YgSKOvwF*8t0yCEjI_x%`BgC+>Drr}h4AcpUIBd% zOa{W?`zMVIC+~sDA|bwIcw#b*orc?Y{%#Us%C+-%TTkA_C?K&yxDm;?US03J-b3~e zj85h)&g?L%4EHX~-VUZGCi58f;eF1hq_pD2XD`&auDaV9?T^+=Du#35@Egt|uP6+j zf0crPddAt~Re@GT6}F+F8VrxPpD|KV%gPn-{bSC>+ga$K6M^L{qs0@@1Jwl zP$gwY9JGCjHsi|=JJ-YMuWXU<%X7{=kMG5vuGnPk@6NsdJ>jHd{}aMH&N^>$8NYOz zgiZSW^Ujxqt@6HLzKZQOU%H=EA$$cZ8xG-@P}|x4;T(86gfC7Zes1-Fb>~X>=^_UH%S+SX zU?^Y6k?!(MNAJ8AeRBf%H4OKUm!2z_zR;Egxw+a1ct4mAqv)Qm@XSPis+gkRK+yRx zUiebS&M-bn_HZp$&6|#_4W)!AbBPtt#~i_+EdjI&-9KLX>F#j8gvn`fIJQ39Ok1}# zN5G~j<_?ieAG~=Tp0{OV3Q>;&+qS(_IyN ze9Mi1Z^w^spywK0r$! zo-K<01S#EijF!rx`6Md=hDIYd@M*L zh5z!>kq@zLD%|l%N(e3_vY>%7`_?z0N|(mKpMynivtyCy?0>N0b+LRLrF{1#O;I!M z4+N5YC|4)Ka2y{4hZFFv&4*(}VLn61ypYc7$cKmE-&jX!9lmK7(5AAS5d(Kc>ovV=Yk_GC)}3gZ07ntmE-SK9%)(M^&Q}S0 zD&^nj%S~=_M5VjG^=@<#H-*7Dd{`>cZQI0O+n&LzMgqk?jW)<+aO5JsdU?EB8w9Ul z1IEUCbND3rP?!u(zA1}?S#$Y#O33r^C;a-B521%eT>KtXNY2dD!{-Z4KJB^+gg!`) z20S%aWZ;i;k%7alLM0|&Fa|u|Mw-dMbeDkJG>?y?%zD1(GZVM$MTSdKGA`qF=6miU zyS!BODQ%Vo4?md_g4s1hv~w5!(-0OXkr41b&Hi$hV$XwRDLi}S9M(?NDN`CXZE*p?uV2aXdrr^OtWpJC1X9B)9QbW0bdnrpEdYMRkUbYwIf&cQ-Lk)6hFO{dkyQzJm zNc7=P6eMA~5&_N!yT3)~rEj@5Byy&b!1YT>~zS0~Yt`u|RhQ^Is(r0ysbJA1^&Cw0KLwCsM(`bK}j&e;*`~5b#C4-N=Z~ zYSf0u&g7Aj1l14C(SE%2HV45JY}0*d$m(BdNjkjOsGW7$KIdi!AAGl-rW**Mbcme7 znu!D2t8Qz$ zaF&hT^zhg#$P)ntg81X5$De|I3(zWjHJy*AEd4rcS|5zrbOd}A=-sCo>f8ka)uJ=i z+U0nn5#U6U_%B+~#~JE@Chg@Lrq8Dke472{P0~!TaNk>i3jSQQ9}LU=QXnBi*yw({ z@zbtVc%T#TJ+#Kt{9soBHxI5=h>K@y2GWRMNXygVa0buOF?xNsV20Q~s@W$AkgL6(bLdO-sRPHX0Q)aL7vMSQJPA|XINODX<%Y54^>m&Grj zJ-I(0fS(YE7g6wEUi#=)_{p~qH3`tNkPmAid)&!BsGc!=dUU})4OS~oLosT+nZ=cwFW_GEk`R(mj zXnv-*WscDC%rsL^%Yu$8w8ZSUi~`2;%!E@<%Y;>zwWR5%a4D=sPwk=+*efMQGW-i+;?86{BplD40ird?)GXT^0B%ud4P4=bT5)N=8J77 zNo8+8ty}_qpSVSJd(ebb@-pf;f-nD%e7KR2Dus#yVI8j1%M&I0r4j;_J`)l3&?bia zK!NCM94Wvajh#lu&YvE^JhP>L^^7#T5UZ-;%3>^k_%j>fvY5ovqToQWsID)Q%F_zFyD|~TiWba8cNQcHYNd~`UM}%8iQ27i z!p~FoEOt-hvn5D%U#S-gC|g?MMFpxV#psY5!npeyc@{PEJ_>vlr|mQEW^&90f|J>z3tPDYS&sa>H-6@{Rw;Y&ys ziQpOiP=}!ZY>rE{*oXT zlf01yj0cLXUP&YRSy$SR@~Ts8<|SGZv-1q>z>ugQ-Gf{9RnE z@|(nRGA~qNeX>t{mftUND{tf*SPBZ+vN1wL43nR~bIA^SYWa9+4MAAV1VAULw3an# zjU9=kR**OeN64}pXkfM8v(w&=6<;+$brY{(7n2KM?@Ccc53a;o-928lEMo<}58>QL z!Qu`?+H=2%gqM}+aA$`|;sB{F{*zdXH=3z%I;6$jI;9iq-}*|d7q2>n+1}EOLi;>) z6i&WAk}BSW;-z&6e9$Zg^u8oG&0%gFyc|_PON(G+lk}FuXgCn4_mS_lMeZgi_l&2* z*)>8FPln6PCJ&ano8VomiYkXsa)?g{*K6F^o@5cPv)6kIMHA5dw9pxFZd#$WTG^D23?XgV??^1kCSh^)t3sUj?59Mka?GXyJr-3b@7!XJ;H7KkvW$e z{~^?1dV~-qC(|t7|E*Ad*ftU}bC#E;z|Jmy38btRtt&dYr)ya;rb>9ts_dI}toRDE zDeGC!>W1DJ38Yg=m&&VmEy}y{RPOpY!S7-#;fFP-QIB26DBKyjI7S@vx^6OpLt}90 z&{#86VfG2Ngv9Doq`Kr|R(+YlL&g5RP)n0D=mHbgB+bsCLVw~jN}!XRQFJ~`A2*(Y zr7Lkh^3RSz)H=Qb4)=Mry*mW`Yy!)Cu+C0H`)a;}Nr|j1*_sDg&+a*(6y#67xBJ#i zbYu}~eJDIF;JfFA(Yl^U{P8y!t=zB{x5U|0yu~WWR4CXMw)l7Te_yARNC038w=P5mi==SiO z+lL+P|B6&BAc-vs%C{ilALo0^EikO**8t|sT!Hc7S6JlY z^7%A{hf*D%!gOw}j!#PeI+K^E@%!oqvs#VcfNWdokBeC%`2_!WdNsyUt9ktE{{vI| BF>(L^ delta 15744 zcmd^GcUV-{wm)YE5Qh#@ls2G%(V=%MMWhd1KtY(HD=?IqVIT-cjWH$~At);`YBa^< zTB63}o#e%qZi-RA-1Nj~Vq%&#-ZYKAwa-3hU@-T)zQ6B$-%rl0w%6W!?X~xuy*$9L zu-f$2ax2IDY`N70hI}UNs8}@Uc;1Tq-T%iM_O9fi`pI4FRa=H3aAlx9g`05ToWl)U zLF&Lc50kY65wo+Zbo^PyFh3t?@>wIB5E~W~R$FI`jGEq%7!gww7aNrr4U6K8A9c5fx$JVFlxNi-%duK>I2@grcaR}$L#6TU(v^8j-=_gvVJPq zYsWBq?S`r)lUni>I+j?DW-bcUE&V~3Kcjoio;^hoOpc;x?IR(@i=w`Kv}Txz))yV7MPx z7ua^F+1|EZ*IZlIahC$xcgft~OeO0uH0jaFu)|m83eL~yy8>Fw>ea=)Tiof>`ZpTHvHJ5^yfC&$UQpG3uV9$(6+@bTdcmaz ztzzh$!|VfF-+n6#b+YY^K&aKJU|?5>1}@ys?w|$$HnR(; z=#|aveud!Kd&fR(^q$Bt-V@CC z*2|yA5G9C*vq7x;a53itQCrv;_);c|9bTaI1WqpV;uCMM;dF-hW|_=U9MC>{*$lyh zKEKDmV}fyk2SdSIGH-alL#yC_vX>26!+@)-nhk{HTKNQczfnGg=iFqc{ROS#XA)gA zycs6L8!lfYS|ywd|8TVIx*L32$vW~2GGq%_2v_wwLPeI$mcNiG`;vw^|9+nAgq2_< zBL{kZ(=p6%I?}Ws|8}t~FbTNddpvl>M%iI2sJ?Z#73}*)=FF#Umff5tC=CV0EZgbE zFgx9d($f&JRkp;x$#(F5$7Q)T zu&syf^1YE4DGuL>6lWpP!L=3_Z8ioA*5{n>0*rczk(HlDI$am5-`D@X5AQU9BZgBWpnFi|aXI*^ryY-E_)*|#f zP^G9;SY~N{?s?f0G^qLSF3NH|XgtMP%kL9I>s3GbkK+X@Sy(eE|I$?X?{rP`KSjzL z#K#3QV#V}vNI3b)7E(lkI8)5>& z9pQoSM0g=4A|@fc5k3fCgdf7651OVtMyGl8l~g4n01=3ojF^H5LIfj15K|F%A*Law zBUFe`L>M9*5rK$AL?NOPF^E`193mc(fJj8l;IE|4betm0Msr4uq0iHeA(%^N|75?` z3tIXyBehJkQmn3MoQar)n2nf&n2Sh4BqLG~sfc-qG{k&FIwAv+iO51^BXSVAd_|+W zkH*95)BS2h9wHx6fG9*1A&LZzSu@JEcQGuvLEJjrEr~9j)E)ha(^_mZ> z`86MER(R8h!&r}{VVH46V_=0CIsK26l*ce1J;uLtRdbz%t&cX3gEMc|I$L#M3V`Zq zvUnJHuG8MOO+yH{&+gA>C-K4m(#Y*$##1$RbSvJiG&=AaR;v`FbWN~UEZiJ+(b*;wG zsEDxWzd3`!_N)2{5HVL9^S387zG1HRl(jr6EQ%jY)4JQolU7o8?p2neb%N+SY9$&@ zr5I;I>PADH;HT>6wP0azwyi1(Qw_l#TG+0g=?-n* zn#S=pW!hO*LRVgze(k9br!ovd%pExr&A(BhtzjeTjHZIr?$+X(uA+sFtqm#h23=IQ zvAfHpj!da)OiwFlFxJOMN0cNlNKnPcBq)ZGLp=D_#oGJW$he%PO%e6_;)e9v#E$re zyu!??qPhjK>Bf3pd$F-0t+p`5piiyIDK^%qq7oA$1zq7l3K#Jd%d4xkMb;7XHT9Wg znnHbTc30EF*tjJvd08zj+LoUA9rNZFv^Mp0l%;7Fq=sgfmei;sW8&iZQ*~M=hlsfB z{FscC0!>X%Zu!D=b$oYvX>7V7qb)VVu%N!NEUTlZCch}HxFw>sK$RFDl>h;Ht%tDH z$E!LUHBqr~vr9F(F$F0R37Yuay2j+rc`0r69qA34(w3Uqw2Z9C?xH1)`lUsoEw!~e zRa{~MXgV+ntm@P*jVx-f)fH)58=G63a>_C~qST?Ci3tV9rqHI2xYFc??4?C5H7(gO zEw#-hs;GpxxMvJCf3$gWot@{QcRsq$tmeYhK_7Qp02pNtUwhL7Zb1e&#O5a&d$|)LDvOS zc;J7Q7Scg!kXGv~&mk+Edib_Y}39q9++X zudHQ!CFNAS(85=|&{|qgzw+L?)x38b&wIBodWe!do^9tnp6w`oQ25HX&+1IyLmv5CUA1?;#zoQ+5PvZ3-2t?SMpf^zV)I|Mi@h8hCzv!n~VK$H8&nD^JMWX+8kqAii z;*&D;XA`K6tQZ3or<{QpLlE6I;H7wpbLHoLf8Hj+@RE(NyQRaD55mVYOgOyqfPM-a z!C#qUC=-v4xfzm@i#O-bZ^|$nqGujhlqYcn7q4r*v+duTBG%DcwE`2B) zL0s|VlXmNWji63etQn>23H@K{T?J2Oh5hdSD!xN_6^^bk1PHwrbYY(}r+`Z_Mn7H= z*Oe`^iVlm40L>cHUDnYme4zjjt}#Ul7MAV{UN#3`RLlXtwWBO}z`!NF4VR6+!GUTQ&~50NE^KkM zPkuj2T@{5iS9*Q;-Koact!&9I+VIE2havxx{w_KmqEJVo-W5(}80)~{vVJ4HlWEL? z?9GO6;KUXEhS|xbvAK0=OR{5%H3g-mSsm%xs^sRBu7v#NmifBOn%4MqU2S?*@`9qg zTvdEToT9#|xvp7Pvour~pEDI|Tp95_wzC^D8yeedYqaTV+_1W0quSIFQB|76Jk$L4 zp2j7~3C7UG^v-#udbO@W6%`p7VPObP9jJB#-33#;gQflIE4@LmHODwhhBMHfsCR=) zIYtK>z!z#cw1b77>5ec z9tPf*;-P3(nZYO098#sUX?~4ip_r81*BYLrqJYzQ&^%GoNGp@z(k6o|tncNVM@>_O z!}4r0O~HZ)k0SHLpEKB{>2UHZgE#MYP5&2N1$^HQ!(`eyz|KwQk)JfO-bDFZ@rI+K zyx)(8X(LNZoZ(bZCn)s%Q_l#Tb7T4~v`PjNjV{@pbQK zxAgGJ215w(aUz7T;5=!FI)?Yejr9EKwp63FpwL?TMmO#0z7(Sh!s|I39wSjjyE~j~ zjDV<0lMnvV=Ir^hT%$W3Uub_);^fJdcKop?jbG72EbQ8eoQLk`f_cS_uDO$Gqq+X3 zFg6wj@8|qreOIJ4oYZx{#OIwbJ|qfUOcewoD^0#I*Gg&2=Wpl^u!b}FCKrC`@5cMx zsGUFTx&(*9=+{N|^s5iQuhR6>RG~+9KVRGcn_|0EqT?TRPbK*xPX^jGrs?PeNfW{b zvFG7|E9*iSW?cwui|q;)8&Nwh=~2S4>kQ<`jgr?IX!KCU^swNsx~v}9psYiD_SPku*tVN6Ya zrlvl+Afq_TP&}`^Oq~mw&0SOA;LoOHG-l2Z(_7}77B1IlBpIMW}#fbk$05w{eK2WpeX0~LSlUXzl( zI<=heDS}^g!i8o7d0#NV_#U0FXUF5Xsq+^?C$<` zst~BnJMS9*C>~@UWl5k;0YR|MjkANFW4ru#hgf3<3*5)ubN?o+CFvnNTRA6oJ>kP2 z++n&VzJq!GR`)w$BZ(vPoM{>{zJWSLVv>9f8&2qP=0{@jc-;0uAr=KDa2q)rz#vre zt73Yt(Y&{MSwauuZhl!puX86Iu)$ruSH&FE|6YF#v;h|bm%i@RPv3 zWSYEJKYz6sv5bHIYTv2Hsat3#2DsyBVX?2;nIFHqS4wSY_HAk`=Xk%$Ug+4<(^RWr z@N#bizwgujORPZKx#0Om{^`&A-K=pNE||!bnhzOXfTt)MsQ9^m2Hl~(01r>Lu>Nd+ zA`HG5AB%1%ZSoqOwuycmh%kJ97zUk|*Gi#}4{0N)|oIIVOcJV_&i=a-?Un z4xsN`e~OSk!SQnc9P;E#GiK}M{v?{xNPdJQKW?_9h{?|u9KP#M7Wl*dT$aSaY-Y!Q z`CY&De??CLd+G{X0=Wvfxk_u#hkw!kfYpD+6o>xxUoypfi7mI;Q&`1*u_Y(r4}b0r zx>Cin^pzma9WE<5;g}%q+MPkTh%SEDUkTenxFqw^{7)L21G#N?W_!+G{}0>ygSjDo zAc$+Bsgh^!;*N>i!0}{skNi#mmlWx#B!!zrX&Y0xIl@L|zGa0oDV&&6C#7oyvJCiJ6d9Z#q7JuiFR$@lBWX;aDy}kld2Wc?pt5u{WPygBNE6L6R8$<=E^H zLGnT>C%zlCpNB1Z#t77;d7O`+KXUtu_e&P~-^2-$(tRv-tYdQ7Z`bfQ9s!=f%pZ>Z zC`Aw_H?ic!7Zo45rezp>X=na$tke7v@zgX<9LK6ObkQSsg!Kp#1V1mln)JOF>JwP@ z^TT{e+zI|n3i@{yW= zL%FbZ4qj9fSWZIbx}jGF{oz)I)T>FE*y`~Cq=w&S#(YJMG3js7c;qk)W^$o|i?h6* zP*296@C11KX#Q~QU7LxN2VBeK#2*EAS=idDCnZ*rf6KvK(I&6DZC4WTYP6 zNk!MzYwckmOB$+e#7y{J%KNrn>w25(;peu_osPXH2(&PH@^z{foXt)0Co69hm+>9- zg=1$zRW?U|!@+tKXN`p54o9*%55e)+S8Q5@8VU>{9}CroKY;3NZUT(Y5k*t$8V){$ zlSn`!TKaJ8;g1B-p~pV9f$|)wi_T@(#e2_ED@jdSE7+1F23A(s$=y@%27$nGTHY9u zcL0ai{1pMTMnr0i{I%?2Fer1SF6QTAlbUPLex02FTrNlNC}9&J4|u`$JnT=>60PlR zgFSMUKi0f^5LI(kOYs(}Mr)5pQgWqQT#KU}RE_$=v9}V4=w!H*$U1<3DW?zy-mS0j zAKzfS6NFpHXC_kl#2Kt3DD$M@jL5@gyVaDZ%i~-HGjBhjeDP77I|9ok+Eh(M`&15z?hHd<+0ke8qm+^gI9DN|;y0&?#neKpKY4VN(UY55cVWosD5wf?6rwhd5;&vNj)&etZh~NSq-=s=EUlj3VX5Q>7Yn6{ zxk;>6933Tq&Duh45$7&+$h!UEmv>?YC9t%b3CoMPZ0g&|BD9kGr`gIB!6S=$40A6@ zUU_mv)B#o&bF?r7HWzb~fp3S_0WKGdo(`8F?}(iY+M{J25M9D~29WGA zQmQd_G5#gVlIjh3poFs*6h^8v#%?<^q6Fh1@D<9^eqADszL_gF8n{4gbfidQY@?49 zin47BI1iz{k=l&0vd3Q)We-WRuT^>_YQ`}Pf#tsO{70hf$hD25OsTM;`2|8t(a%Mv zMVabS&X@YXoX~e&lfl+fE<|uA;Zuh*Jifdpz#1F#hhyU%<#5VRK54j2>OIMU@*poc zELY1tVP2V(L#hb9R-)$^ahkT7mG%*4b%3aJu1wl`t`qvZcZu}ya>-a$InrYiM0#Jj zbTHXO=;vn(^Z@vzTypAJ4btbP$l=pMxgRLGYFlX#A{Sy1no=NlAs0fsFC_7(bU3?+ z^P)u+eq6O8kPcdK&Dqix%wGhP&Er6#vt87sJ_U1c??leV%~!N-mu$ zAFRYdFWe=Eq!V&HH1}e83irbdbNJaW<|bJ!-YsWfuvZ%Z*^8ygT7(Swj>Cdp06c

@LDYHax4(wRcT5sfpajF`Fi$-riHi7P`eSF(YBIaZ@nOY(N$ui+*O&0fqI%nI{n z83M~0wSS{pkT2u3zOYpzby^7iht6og?}`S4e~Q|63hnSO#_3!tzV{-qJl?!}M#Bs* z$6Aw6tvLGT>I|&Vz<@JBjI9wU z%{EA1{jX~vG8FH$D_3CR^(~d0*VLem{Wk>Wl+dwFEWG~Nrg^x&2rSP?58M*%Ij_`? zg9|m1y}yV(g{@X-EV`?yHLR_Btkt16Kk)wwH+%xi9~ljHS~x#b8v-Pc&Zna|wmwAa z7RTQ2sJ-KAk<0CBMLIP&wiIN0zOZb;#%Su2~ z&*cdzu`=`9@ax+dhQM;cO)Vo*32@>*?fBcnIiLZPA@?U0IsHXHrn3eqtOk%L=U=UX zcKf<`IO&dO$e+#38-$rZp_Vf98>L-rX(Q6vF9doDywfNd`;5?EbgLusi4f7mO%N8( zSF-P|;1Q^#MwddD@?JfAJmA3bnx6Td*e)BhPQR@i%g=OuOeHPV(0bBc5ljEh` z3`1afUU#4_+RF{McIeP3j(!;)Vixg zDg*Go#1=@xkcP8LhkkaA)5G#sE**4+9DD_05EFO7#O>jj9T|cm3v+5)QDMtswWqiT ze%Q0~PylZ01UVMcZ%NYr%{eyEwq6R>n?_s}h0d_e$obQ^d4UjqV`L1I+mZi>=P14x z9PQ$0i72pr$bCKEf-BqzRQ5>?v=EW#Vo8M5RlpXL#NA^;?(9WoF1b4RvP0rt=s@mU zwGx+^V_Q@rlc{uOqOQ6=9h^{zpGb zO}Lm)fVxNWWpNMs@~@{XM6f7B0RwvR2eP>`LpIiO{tgQ@p;F_$d#e@rUVU!$21 z#^9V-28mqH7)IGX(kJ~wBrziI8#kI+eW8E+m^hK!x{Q_>_{A_@h!tVU|NB_+@EWBx z%t`&Zg0&Cw-;|?xz?B*7#>9i%vv9P;z8=n9Xl~YT9gQqT0fA+Vbyz4|){8$bUwpm$ z+ANG?0?VXvvUsG0Xd!Eii3Lwt<%q;|@M~8)jS2#J!s5|fEadQ^C)&Ne*unam(M;uI zhOwDJezif6hy&kicuhwVgP3W8&OfC*j){=K@*P2FZ-~YR1g{ ziCrS|XrH)UvJYtwe}Kb5V7U)``W%%V?v?_KgQAI~PKo|(KhoV_6Y1OXZ~{pl zpd;YJA@kb1lG1Km%9g8L&io}Z&Ffs4-SCgbAKZc)6oKXJE}{xKv;oa>Y@pI#Rp8k~i7OHF^a0@cD@MhRePikOp^ ztU=z(Sk}9e3lxhyT<|XZ#2TsNUl9RS%P4`o0qIOL=N{zMcTyhSu0{!an)dFs*OKv9 z2SJKOs=YOA78NgI^Sss4hQEKKPvCQ23_~!_LT%g2qguxcTUTi-JUWQ&KmSiNHy9k( za^ip0@mY&Jj}Of}AE>^Ua}yGoP}MTH786;)%~5>kkGd6*8((eSY1wr+B?h(&+E;P` zba0LsJQTyeYALQsS&J3^9w#Wa@VWl35w($e7mL*FvVrrEf3fH`IiVDUZ0U6Ns_#QD z9dr%>L7RZ@x$p4^bAsZKC3E~AVoXVEPa-b$+mBsu{SM$Vz|sFs0ZRr@NCR}5uyW!B77V;^v;i{Q((+J~ak#@0#>G|Q>tXpz zUkpDom0<`{ErR)BQIxD8VeUtYXc%U`h3G?DsMWt+OtQhpC|6oWlBW+0p{Fa}BFx!P zJ(Y9b=+rab;?(P~0VVQ(9g%>~Q)1p7wOb=|^)te5xALi(PQ-UK8?QOjp$%b7oO-Pe6rwIc zrG<@>op5BBTS_A)3{X38m!Mq^7b?`|Svg2Ji(V8teTi5*L(Wry+%i#6wwWCA2@nEhL3KlLG0{@!z|cyMYLcE}nS=)hBJF>U|zL zmYY!PCgf&qr(AW1+F3kekU}81xry_b-Tp*B)1s?2G}m>8E$P&?j(z1qiVeQNjhevK zVx(R>-wy?ka{j{3d16_~!tJ;j5a8W{`GapWFZaXXqud0?bIdvBr*aJ0@f@=M4 Date: Fri, 13 Feb 2026 09:42:56 +1300 Subject: [PATCH 035/110] fix: upgrade python in the rtd config file (#2053) Signed-off-by: tdruez --- .readthedocs.yaml | 4 ++-- docs/installation.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index e3df240c0b..00da93c89c 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -6,9 +6,9 @@ version: 2 # Set the version of Python and other tools you might need build: - os: ubuntu-22.04 + os: ubuntu-24.04 tools: - python: "3.11" + python: "3.14" # Optionally declare the Python requirements required to build your docs python: diff --git a/docs/installation.rst b/docs/installation.rst index b29de683a2..0dbafd4516 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -283,7 +283,7 @@ Pre-installation Checklist Before you install ScanCode.io, make sure you have the following prerequisites: - * **Python: versions 3.10 to 3.13** found at https://www.python.org/downloads/ + * **Python: versions 3.12 to 3.14** found at https://www.python.org/downloads/ * **Git**: most recent release available at https://git-scm.com/ * **PostgreSQL**: release 17 or later found at https://www.postgresql.org/ or https://postgresapp.com/ on macOS From 7d23536b7daab34affd2e05eb01333e850efcef8 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 19 Feb 2026 21:45:51 +0530 Subject: [PATCH 036/110] Match MacOS binary symbols to Go TypeScript symbols (#2056) Signed-off-by: Ayan Sinha Mahapatra --- scanpipe/pipes/d2d_config.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scanpipe/pipes/d2d_config.py b/scanpipe/pipes/d2d_config.py index 917e004bf7..fbe8cc4219 100644 --- a/scanpipe/pipes/d2d_config.py +++ b/scanpipe/pipes/d2d_config.py @@ -164,7 +164,16 @@ class EcosystemConfig: ), "MacOS": EcosystemConfig( ecosystem_option="MacOS", - source_symbol_extensions=[".c", ".cpp", ".h", ".m", ".swift"], + source_symbol_extensions=[ + ".c", + ".cpp", + ".h", + ".m", + ".swift", + ".go", + ".ts", + ".tsx", + ], ), "Windows": EcosystemConfig( ecosystem_option="Windows", From 706dbe599ba60cc1e99038fac38f18454b2dd742 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Fri, 27 Feb 2026 02:28:17 +0530 Subject: [PATCH 037/110] Bump android_inspector to v0.2.0 (#2069) Signed-off-by: Keshav Priyadarshi --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8eedf0a4d9..fdd676995d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -114,7 +114,7 @@ dev = [ "sphinxcontrib-django==2.5", ] android_analysis = [ - "android_inspector==0.0.1" + "android_inspector==0.2.0" ] mining = [ "minecode_pipelines==0.1.1" From 0adfb1e9c716c4d6b0613a3580ea3fcf85686e98 Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Mon, 2 Mar 2026 15:03:10 +1300 Subject: [PATCH 038/110] chore: automatically closes low-quality and AI slop PRs (#2072) Signed-off-by: tdruez --- .github/pull_request_template.md | 16 ++++++++++++ .github/workflows/pr-quality.yml | 32 +++++++++++++++++++++++ CONTRIBUTING.md | 45 ++++++++++++++++++++++++++++++++ README.rst | 13 ++++++--- 4 files changed, 102 insertions(+), 4 deletions(-) create mode 100644 .github/pull_request_template.md create mode 100644 .github/workflows/pr-quality.yml create mode 100644 CONTRIBUTING.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000000..9ce4f39e8a --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,16 @@ + + +## Issues + +- Closes: # + +## Changes + + + +## Checklist + +- [ ] I have read the [contributing guidelines](https://github.com/aboutcode-org/scancode.io/blob/main/CONTRIBUTING.md) +- [ ] I have linked an existing issue above +- [ ] I have added unit tests covering the new code +- [ ] I have reviewed and understood every line of this PR diff --git a/.github/workflows/pr-quality.yml b/.github/workflows/pr-quality.yml new file mode 100644 index 0000000000..0eaa5b048b --- /dev/null +++ b/.github/workflows/pr-quality.yml @@ -0,0 +1,32 @@ +name: PR Quality + +permissions: + contents: read + issues: read + pull-requests: write + +on: + pull_request_target: + types: [opened, reopened] + +jobs: + anti-slop: + runs-on: ubuntu-24.04 + name: Detects and automatically closes low-quality and AI slop PRs + steps: + - uses: peakoss/anti-slop@v0 + with: + # Number of check failures needed before failure actions are triggered + max-failures: 3 + # List of commit author usernames to block + blocked-commit-authors: "claude,copilot" + # Require the PR to reference at least one issue in the PR description. + require-linked-issue: true + # List of terms blocked from appearing in the PR description + blocked-terms: "MANGO" + # Require all changed files to end with a newline character + require-final-newline: false + # PR does not allow maintainers to push to the source + require-maintainer-can-modify: false + # Minimum number of profile signals the user must have to pass. Disabled. + min-profile-completeness: 0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000000..7538eebdca --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,45 @@ +# Contributing to ScanCode.io + +Thank you for your interest in contributing to AboutCode projects. +Please **read the following guidelines carefully before submitting a pull request**. + +## Pull request rules + +- **Attach your PR to an existing issue.** Discuss the approach and implementation + details in the issue first, before writing any code. Please note that if the issue + has already been assigned to someone else, you are welcome to comment on it, but it + would be best to contribute code in such a case only if you are invited to do so. + +- **You must fully understand your code changes** and be able to explain them + clearly. If you cannot walk a reviewer through your changes, the PR will not + be accepted. + +- **Write your own descriptions, comments, and documentation.** All written + content in a PR must be authored by a human. + +- **Disclose any AI usage.** If any part of your contribution was generated or + assisted by AI, you must disclose this and specify the tools used. + +- **Include tests.** Any code change or addition must be accompanied by proper + unit tests. Untested code will not be merged. + +**Any PR that violates these rules will be closed.** + +## Pull request tips + +To maximize your chances of a successful contribution: + +- **Keep changes focused.** Address only what the issue requires. Do not bundle + unrelated changes together. + +- **Be patient.** Do not solicit maintainers for a review. + +- **Remember:** project maintainers have no interest in reviewing code + generated by machines. + +- **Do not make structural changes** to the codebase such as adding type hints. + +## Ways to Contribute + +See https://scancodeio.readthedocs.io/en/latest/contributing.html for more ways to +contribute. diff --git a/README.rst b/README.rst index 78afe33409..707224414b 100644 --- a/README.rst +++ b/README.rst @@ -27,8 +27,14 @@ The ScanCode.io documentation also provides: If you have questions that are not covered by our documentation, please ask them in `Discussions `_. +Contributing +============ + +Thank you for your interest in contributing to AboutCode projects. +Please `read the following guidelines carefully `_ before getting started. + Build and tests status ----------------------- +====================== +------------+-------------------+ | **Tests** | **Documentation** | @@ -37,7 +43,7 @@ Build and tests status +------------+-------------------+ License -------- +======= SPDX-License-Identifier: Apache-2.0 @@ -58,7 +64,6 @@ ScanCode.io should be considered or used as legal advice. Consult an Attorney for any legal advice. - .. |ci-tests| image:: https://github.com/aboutcode-org/scancode.io/actions/workflows/run-unit-tests.yml/badge.svg?branch=main :target: https://github.com/aboutcode-org/scancode.io/actions/workflows/run-unit-tests.yml :alt: CI Tests Status @@ -69,7 +74,7 @@ for any legal advice. Acknowledgements, Funding, Support and Sponsoring --------------------------------------------------------- +================================================= This project is funded, supported and sponsored by: From c9be5889b7e804ea2bb72566f1c9493fb219219a Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Mon, 2 Mar 2026 19:44:32 +1300 Subject: [PATCH 039/110] chore: upgrade RQ, Redis, and more to their latest version (#2073) Signed-off-by: tdruez --- pyproject.toml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fdd676995d..04d94d3f56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,23 +37,23 @@ dependencies = [ "setuptools==82.0.0", # Django related "Django==6.0.2", - "django-environ==0.12.0", - "django-crispy-forms==2.5", + "django-environ==0.13.0", + "django-crispy-forms==2.6", "crispy-bootstrap3==2024.1", "django-filter==25.2", "djangorestframework==3.16.1", "django-taggit==6.1.0", "django-htmx==1.27.0", # Database - "psycopg[binary]==3.3.2", + "psycopg[binary]==3.3.3", # wait_for_database Django management command "django-probes==1.8.0", # Task queue - "rq==2.6.1", + "rq==2.7.0", "django-rq==3.2.2", - "redis==7.1.1", + "redis==7.2.1", # WSGI server - "gunicorn==25.0.3", + "gunicorn==25.1.0", # Docker "container-inspector==33.1.0", # ScanCode-toolkit @@ -103,7 +103,7 @@ dependencies = [ [project.optional-dependencies] dev = [ # Validation - "ruff==0.15.0", + "ruff==0.15.4", "doc8==2.0.0", # Debug "django-debug-toolbar==6.2.0", From 3387a47149b10017d4b62d6e9a3e4af83b7c8f71 Mon Sep 17 00:00:00 2001 From: tdruez Date: Mon, 9 Mar 2026 11:05:15 +1300 Subject: [PATCH 040/110] chore: upgrade Django and Redis to their latest version Signed-off-by: tdruez --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 04d94d3f56..84d1da5608 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ dependencies = [ # Task queue "rq==2.7.0", "django-rq==3.2.2", - "redis==7.2.1", + "redis==7.3.0", # WSGI server "gunicorn==25.1.0", # Docker @@ -103,7 +103,7 @@ dependencies = [ [project.optional-dependencies] dev = [ # Validation - "ruff==0.15.4", + "ruff==0.15.5", "doc8==2.0.0", # Debug "django-debug-toolbar==6.2.0", From ac1331a81a1741d02fa72fca3e71dc63fba65b91 Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Mon, 9 Mar 2026 17:35:19 +1300 Subject: [PATCH 041/110] fix: failing test by updating the expectation fixture (#2084) Signed-off-by: tdruez --- pyproject.toml | 4 ++-- scanpipe/tests/data/docker/centos_scan_codebase.json | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 84d1da5608..04d94d3f56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ dependencies = [ # Task queue "rq==2.7.0", "django-rq==3.2.2", - "redis==7.3.0", + "redis==7.2.1", # WSGI server "gunicorn==25.1.0", # Docker @@ -103,7 +103,7 @@ dependencies = [ [project.optional-dependencies] dev = [ # Validation - "ruff==0.15.5", + "ruff==0.15.4", "doc8==2.0.0", # Debug "django-debug-toolbar==6.2.0", diff --git a/scanpipe/tests/data/docker/centos_scan_codebase.json b/scanpipe/tests/data/docker/centos_scan_codebase.json index cd0f9d9ad3..10a86f80f9 100644 --- a/scanpipe/tests/data/docker/centos_scan_codebase.json +++ b/scanpipe/tests/data/docker/centos_scan_codebase.json @@ -192441,8 +192441,8 @@ "sha256": "512a4d0b117712025a3bfe2e276188e7478dba7f66173d9ebb046879b87e13cf", "sha512": "", "sha1_git": "17504e69bbb3d2f2353494848c1182220e83a286", - "is_binary": true, - "is_text": false, + "is_binary": false, + "is_text": true, "is_archive": false, "is_media": false, "is_legal": false, @@ -192479,8 +192479,8 @@ "sha256": "2823235dd7bf108bb7385d1341d7a25a6ffecd19b83406babd77c2da50612b5a", "sha512": "", "sha1_git": "cf308590ba310e562d3247811e942cc12c434848", - "is_binary": true, - "is_text": false, + "is_binary": false, + "is_text": true, "is_archive": false, "is_media": false, "is_legal": false, @@ -192517,8 +192517,8 @@ "sha256": "9caa0fb74ee48688ccc44656c1a75270adda701fcfdd6ba63f00c0f390ee1ea4", "sha512": "", "sha1_git": "1e40bdfb6510cb30a53289e9271a994320eaaa9d", - "is_binary": true, - "is_text": false, + "is_binary": false, + "is_text": true, "is_archive": false, "is_media": false, "is_legal": false, From 621d890f6aa28a35f1f9c05aa97a7888222072c3 Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Mon, 9 Mar 2026 17:48:37 +1300 Subject: [PATCH 042/110] chore: upgrade Django to latest version (#2085) Signed-off-by: tdruez --- pyproject.toml | 6 +++--- ...lter_discoveredpackage_children_packages.py | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 scanpipe/migrations/0077_alter_discoveredpackage_children_packages.py diff --git a/pyproject.toml b/pyproject.toml index 04d94d3f56..916bd237b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ dependencies = [ "importlib-metadata==8.7.1", "setuptools==82.0.0", # Django related - "Django==6.0.2", + "Django==6.0.3", "django-environ==0.13.0", "django-crispy-forms==2.6", "crispy-bootstrap3==2024.1", @@ -51,7 +51,7 @@ dependencies = [ # Task queue "rq==2.7.0", "django-rq==3.2.2", - "redis==7.2.1", + "redis==7.3.0", # WSGI server "gunicorn==25.1.0", # Docker @@ -103,7 +103,7 @@ dependencies = [ [project.optional-dependencies] dev = [ # Validation - "ruff==0.15.4", + "ruff==0.15.5", "doc8==2.0.0", # Debug "django-debug-toolbar==6.2.0", diff --git a/scanpipe/migrations/0077_alter_discoveredpackage_children_packages.py b/scanpipe/migrations/0077_alter_discoveredpackage_children_packages.py new file mode 100644 index 0000000000..19b3c93b93 --- /dev/null +++ b/scanpipe/migrations/0077_alter_discoveredpackage_children_packages.py @@ -0,0 +1,18 @@ +# Generated by Django 6.0.3 on 2026-03-09 04:36 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('scanpipe', '0076_discoveredpackagescore_scorecardcheck_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='discoveredpackage', + name='children_packages', + field=models.ManyToManyField(related_name='parent_packages', through='scanpipe.DiscoveredDependency', through_fields=('for_package', 'resolved_to_package'), to='scanpipe.discoveredpackage'), + ), + ] From 2216bf224d42f2ad804c6db13b158bf51f2582a7 Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Mon, 9 Mar 2026 18:08:26 +1300 Subject: [PATCH 043/110] feat: remove the chown service in compose file (#2086) Signed-off-by: tdruez --- docker-compose.yml | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index f078e88cf4..426ae80bfe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -71,38 +71,6 @@ services: - redis_data:/data restart: always - # This service is responsible for ensuring the correct ownership of files - # in the shared volumes used by the application (static and workspace). - # It ensures that all files inside the `/var/scancodeio/` directory are owned - # by the user and group with the UID and GID defined in the environment variables - # APP_UID and APP_GID, which default to 1000 if not set. - # - # The service runs only once (due to "restart: no") and performs a `chown` operation - # to change the ownership of the static and workspace directories, ensuring proper - # file access rights for the running application containers. - # - # Volumes mounted: - # - static: Ensures the ownership of static files in the /var/scancodeio/static directory - # - media: Ensures the ownership of media files in the /var/scancodeio/workspace directory - # - # Notes: This service can be removed in future ScanCode.io release. - chown: - image: docker.io/library/alpine:latest - restart: "no" - command: sh -c " - if [ ! -f /var/scancodeio/workspace/.chown_done ]; then - chown -R ${APP_UID:-1000}:${APP_GID:-1000} /var/scancodeio/ && - touch /var/scancodeio/workspace/.chown_done; - echo 'Chown applied!'; - else - echo 'Chown already applied, skipping...'; - fi" - env_file: - - docker.env - volumes: - - static:/var/scancodeio/static/ - - workspace:/var/scancodeio/workspace/ - web: build: . command: sh -c " @@ -124,8 +92,6 @@ services: condition: service_healthy redis: condition: service_started - chown: - condition: service_completed_successfully worker: build: . @@ -144,7 +110,6 @@ services: - redis - db - web - - chown nginx: image: docker.io/library/nginx:alpine From e541b86b99327d3cab915f65f2b3f478d7a6dc19 Mon Sep 17 00:00:00 2001 From: Uttam Raj Date: Mon, 9 Mar 2026 11:48:00 +0530 Subject: [PATCH 044/110] fix: URL-encode programming language filter values in resource list (#2079) Signed-off-by: uttam282005 --- .../scanpipe/panels/scan_summary_panel.html | 4 ++-- .../templates/scanpipe/project_charts.html | 2 +- .../templates/scanpipe/resource_list.html | 2 +- scanpipe/tests/test_api.py | 12 ++++++++++ scanpipe/tests/test_views.py | 23 +++++++++++++++++++ 5 files changed, 39 insertions(+), 4 deletions(-) diff --git a/scanpipe/templates/scanpipe/panels/scan_summary_panel.html b/scanpipe/templates/scanpipe/panels/scan_summary_panel.html index 39d3e71470..1407efcfe1 100644 --- a/scanpipe/templates/scanpipe/panels/scan_summary_panel.html +++ b/scanpipe/templates/scanpipe/panels/scan_summary_panel.html @@ -57,7 +57,7 @@ {% for entry in scan_summary.primary_language %} {% if entry.value %}

  • - + {{ entry.value }} {% if entry.count %} @@ -123,7 +123,7 @@
    		• - {{ resource.programming_language }} + {{ resource.programming_language }} {{ resource.mime_type }} diff --git a/scanpipe/tests/test_api.py b/scanpipe/tests/test_api.py index 03d68cfaca..8bb5f63ef3 100644 --- a/scanpipe/tests/test_api.py +++ b/scanpipe/tests/test_api.py @@ -788,6 +788,18 @@ def test_scanpipe_api_project_action_resources_filterset(self): response = self.csrf_client.get(url + "?slug=aaa") self.assertEqual(2, response.data["count"]) + def test_scanpipe_api_project_action_resources_filterset_special_chars(self): + make_resource_file( + self.project1, + path="csharp_file.cs", + programming_language="C#", + ) + url = reverse("project-resources", args=[self.project1.uuid]) + response = self.csrf_client.get(url + "?programming_language=C%23") + self.assertEqual(1, response.data["count"]) + self.assertEqual("csharp_file.cs", response.data["results"][0]["path"]) + self.assertEqual("C#", response.data["results"][0]["programming_language"]) + def test_scanpipe_api_project_action_packages(self): url = reverse("project-packages", args=[self.project1.uuid]) response = self.csrf_client.get(url) diff --git a/scanpipe/tests/test_views.py b/scanpipe/tests/test_views.py index 63e9b9f295..09603c5316 100644 --- a/scanpipe/tests/test_views.py +++ b/scanpipe/tests/test_views.py @@ -590,6 +590,18 @@ def test_scanpipe_views_project_details_scan_summary_panels(self): self.assertContains(response, expected1) self.assertContains(response, expected2) + def test_scanpipe_views_project_details_scan_summary_language_url_encoding(self): + summary_file = self.project1.get_output_file_path("summary", "json") + scan_summary_json = { + "primary_language": [{"value": "C#", "count": 1}], + "other_languages": [{"value": "C#", "count": 1}], + } + summary_file.write_text(json.dumps(scan_summary_json)) + url = self.project1.get_absolute_url() + response = self.client.get(url) + self.assertContains(response, "?programming_language=C%23") + self.assertNotContains(response, "?programming_language=C#") + def test_scanpipe_views_project_details_get_license_clarity_data(self): get_license_clarity_data = ProjectDetailView.get_license_clarity_data @@ -1039,6 +1051,17 @@ def test_scanpipe_views_codebase_resource_list_view_bad_search_query(self): expected_error = "The provided search value is invalid: No closing quotation" self.assertContains(response, expected_error) + def test_scanpipe_views_codebase_resource_list_programming_language_url_encoding( + self, + ): + make_resource_file( + self.project1, path="csharp_file.cs", programming_language="C#" + ) + url = reverse("project_resources", args=[self.project1.slug]) + response = self.client.get(url) + self.assertContains(response, "?programming_language=C%23") + self.assertNotContains(response, "?programming_language=C#") + def test_scanpipe_views_codebase_resource_details_view_tab_image(self): resource1 = make_resource_file(self.project1, "file1.ext") response = self.client.get(resource1.get_absolute_url()) From 492b6937779e6116cf4feeb6fdcfb6d1bc51cdcd Mon Sep 17 00:00:00 2001 From: loll <146756227+Monal-Reddy@users.noreply.github.com> Date: Tue, 10 Mar 2026 00:37:31 +0530 Subject: [PATCH 045/110] fix: rename "Grammar" optional step group to "Antlr" in d2d pipeline (#2059) Signed-off-by: Monal-Reddy --- scanpipe/pipelines/deploy_to_develop.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scanpipe/pipelines/deploy_to_develop.py b/scanpipe/pipelines/deploy_to_develop.py index fa481fa95f..e8b915bfdb 100644 --- a/scanpipe/pipelines/deploy_to_develop.py +++ b/scanpipe/pipelines/deploy_to_develop.py @@ -250,21 +250,21 @@ def map_jar_to_kotlin_source(self): project=self.project, jvm_lang=jvm.KotlinLanguage, logger=self.log ) - @optional_step("Grammar") + @optional_step("Antlr") def find_grammar_packages(self): """Find the java package of the .g/.g4 source files.""" d2d.find_jvm_packages( project=self.project, jvm_lang=jvm.GrammarLanguage, logger=self.log ) - @optional_step("Grammar") + @optional_step("Antlr") def map_grammar_to_class(self): """Map a .class compiled file to its .g/.g4 source.""" d2d.map_jvm_to_class( project=self.project, jvm_lang=jvm.GrammarLanguage, logger=self.log ) - @optional_step("Grammar") + @optional_step("Antlr") def map_jar_to_grammar_source(self): """Map .jar files to their related source directory.""" d2d.map_jar_to_jvm_source( From 1f1e74c0ad78a9497821f491aa00548e779cf6a4 Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Wed, 11 Mar 2026 13:38:28 +1300 Subject: [PATCH 046/110] feat!: replace plain-text DRF token with PBKDF2-hashed API token (#2087) Signed-off-by: tdruez --- docs/command-line-interface.rst | 24 ++++++-- pyproject.toml | 1 + scancodeio/settings.py | 9 ++- scancodeio/urls.py | 12 ++++ scanpipe/management/commands/create-user.py | 35 +++++++++-- scanpipe/migrations/0078_apitoken.py | 29 +++++++++ scanpipe/migrations/0079_apitoken_data.py | 60 +++++++++++++++++++ scanpipe/models.py | 15 ++--- scanpipe/templates/account/profile.html | 53 +++++++++++----- .../scanpipe/includes/navbar_header.html | 19 ++++-- .../profile_generate_api_key_modal.html | 27 +++++++++ .../modals/profile_revoke_api_key_modal.html | 27 +++++++++ scanpipe/tests/test_api.py | 4 +- scanpipe/tests/test_auth.py | 24 ++++++-- scanpipe/tests/test_commands.py | 20 ++++++- scanpipe/tests/test_models.py | 5 -- scanpipe/views.py | 17 ++++++ 17 files changed, 320 insertions(+), 61 deletions(-) create mode 100644 scanpipe/migrations/0078_apitoken.py create mode 100644 scanpipe/migrations/0079_apitoken_data.py create mode 100644 scanpipe/templates/scanpipe/modals/profile_generate_api_key_modal.html create mode 100644 scanpipe/templates/scanpipe/modals/profile_revoke_api_key_modal.html diff --git a/docs/command-line-interface.rst b/docs/command-line-interface.rst index ef2925127c..da6b5fb9e4 100644 --- a/docs/command-line-interface.rst +++ b/docs/command-line-interface.rst @@ -717,21 +717,32 @@ Optional arguments: .. note:: This command is to be used when ScanCode.io's authentication system :ref:`scancodeio_settings_require_authentication` is enabled. -Creates a user and generates an API key for authentication. +Creates a new user and optionally generates an API key for authentication. You will be prompted for a password. After you enter one, the user will be created immediately. -The API key for the new user account will be displayed on the terminal output. - .. code-block:: console - User created with API key: abcdef123456 + $ scanpipe create-user + User created. + +Use the ``--generate-api-key`` option to generate an API key for this user and print it +to the console. + +.. code-block:: console -The API key can also be retrieved from the :guilabel:`Profile settings` menu in the UI. + $ scanpipe create-user --generate-api-key + User created. + API key: 1234567890abcdef .. warning:: - Your API key is like a password and should be treated with the same care. + Treat your API key like a password and keep it secure. + For security reasons, the key is only shown once at generation time. + If you lose it, you will need to regenerate a new one. + +.. tip:: + The API key can be regenerated from the :guilabel:`Profile settings` menu in the UI. By default, this command will prompt for a password for the new user account. When run non-interactively with the ``--no-input`` option, no password will be set, @@ -741,6 +752,7 @@ API key. Optional arguments: - ``--no-input`` Does not prompt the user for input of any kind. +- ``--generate-api-key`` Generate an API key for this user and print it to the console. - ``--admin`` Specifies that the user should be created as an admin user. - ``--super`` Specifies that the user should be created as a superuser. diff --git a/pyproject.toml b/pyproject.toml index 916bd237b6..8b2862b3a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,6 +96,7 @@ dependencies = [ "aboutcode.hashid==0.2.0", # AboutCode pipeline "aboutcode.pipeline==0.2.1", + "aboutcode.api-auth==0.2.0", # ScoreCode "scorecode==0.0.4" ] diff --git a/scancodeio/settings.py b/scancodeio/settings.py index 4bb3673ec6..9a94a13383 100644 --- a/scancodeio/settings.py +++ b/scancodeio/settings.py @@ -194,7 +194,6 @@ "crispy_bootstrap3", # required for the djangorestframework browsable API "django_filters", "rest_framework", - "rest_framework.authtoken", "django_rq", "django_probes", "taggit", @@ -401,12 +400,12 @@ CLAMD_USE_TCP = env.bool("CLAMD_USE_TCP", default=True) CLAMD_TCP_ADDR = env.str("CLAMD_TCP_ADDR", default="clamav") -# Django restframework +# REST API + +API_TOKEN_MODEL = "scanpipe.APIToken" # noqa: S105 REST_FRAMEWORK = { - "DEFAULT_AUTHENTICATION_CLASSES": ( - "rest_framework.authentication.TokenAuthentication", - ), + "DEFAULT_AUTHENTICATION_CLASSES": ("aboutcode.api_auth.APITokenAuthentication",), "DEFAULT_PERMISSION_CLASSES": ("rest_framework.permissions.IsAuthenticated",), "DEFAULT_RENDERER_CLASSES": ( "rest_framework.renderers.JSONRenderer", diff --git a/scancodeio/urls.py b/scancodeio/urls.py index f0e475e173..ecac3d6da8 100644 --- a/scancodeio/urls.py +++ b/scancodeio/urls.py @@ -32,6 +32,8 @@ from scanpipe.api.views import ProjectViewSet from scanpipe.api.views import RunViewSet from scanpipe.views import AccountProfileView +from scanpipe.views import GenerateAPIKeyView +from scanpipe.views import RevokeAPIKeyView api_router = DefaultRouter() api_router.register(r"projects", ProjectViewSet) @@ -45,6 +47,16 @@ name="logout", ), path("accounts/profile/", AccountProfileView.as_view(), name="account_profile"), + path( + "accounts/profile/api_key/generate/", + GenerateAPIKeyView.as_view(), + name="generate_api_key", + ), + path( + "accounts/profile/api_key/revoke/", + RevokeAPIKeyView.as_view(), + name="revoke_api_key", + ), ] diff --git a/scanpipe/management/commands/create-user.py b/scanpipe/management/commands/create-user.py index 8e537209a3..787e6c0ec1 100644 --- a/scanpipe/management/commands/create-user.py +++ b/scanpipe/management/commands/create-user.py @@ -28,7 +28,7 @@ from django.core.management.base import BaseCommand from django.core.management.base import CommandError -from rest_framework.authtoken.models import Token +from scanpipe.models import APIToken class Command(BaseCommand): @@ -43,13 +43,21 @@ def __init__(self, *args, **kwargs): ) def add_arguments(self, parser): - parser.add_argument("username", help="Specifies the username for the user.") + parser.add_argument( + "username", + help=f"Specifies the {self.UserModel.USERNAME_FIELD} for the user.", + ) parser.add_argument( "--no-input", action="store_false", dest="interactive", help="Do not prompt the user for input of any kind.", ) + parser.add_argument( + "--generate-api-key", + action="store_true", + help="Generate an API key for this user and print it to the console.", + ) parser.add_argument( "--admin", action="store_true", @@ -63,9 +71,16 @@ def add_arguments(self, parser): def handle(self, *args, **options): username = options["username"] + generate_api_key = options["generate_api_key"] is_admin = options["admin"] is_superuser = options["super"] + if options["verbosity"] <= 0 and generate_api_key: + raise CommandError( + "Cannot display the API key with verbosity disabled. " + "The key is only shown once at generation time." + ) + error_msg = self._validate_username(username) if error_msg: raise CommandError(error_msg) @@ -75,19 +90,27 @@ def handle(self, *args, **options): password = self.get_password_from_stdin(username) user_kwargs = { - "username": username, + self.UserModel.USERNAME_FIELD: username, "password": password, "is_staff": is_admin or is_superuser, "is_superuser": is_superuser, } - user = self.UserModel._default_manager.create_user(**user_kwargs) - token, _ = Token._default_manager.get_or_create(user=user) if options["verbosity"] > 0: - msg = f"User {username} created with API key: {token.key}" + msg = f"User {username} created." self.stdout.write(msg, self.style.SUCCESS) + if generate_api_key: + plain_api_key = APIToken.create_token(user=user) + self.stdout.write(f"API key: {plain_api_key}", self.style.SUCCESS) + warning_msg = ( + "Treat your API key like a password and keep it secure. " + "For security reasons, the key is only shown once at generation time. " + "If you lose it, you will need to regenerate a new one." + ) + self.stdout.write(warning_msg, self.style.WARNING) + def get_password_from_stdin(self, username): # Validators, such as UserAttributeSimilarityValidator, depends on other user's # fields data for password validation. diff --git a/scanpipe/migrations/0078_apitoken.py b/scanpipe/migrations/0078_apitoken.py new file mode 100644 index 0000000000..aa22986c15 --- /dev/null +++ b/scanpipe/migrations/0078_apitoken.py @@ -0,0 +1,29 @@ +# Generated by Django 6.0.3 on 2026-03-09 05:23 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('scanpipe', '0077_alter_discoveredpackage_children_packages'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='APIToken', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('key_hash', models.CharField(max_length=128)), + ('prefix', models.CharField(db_index=True, max_length=8, unique=True)), + ('created', models.DateTimeField(auto_now_add=True, db_index=True)), + ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='api_token', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'verbose_name': 'API Token', + }, + ), + ] diff --git a/scanpipe/migrations/0079_apitoken_data.py b/scanpipe/migrations/0079_apitoken_data.py new file mode 100644 index 0000000000..22443817ef --- /dev/null +++ b/scanpipe/migrations/0079_apitoken_data.py @@ -0,0 +1,60 @@ +# Generated by Django 6.0.3 on 2026-03-10 22:09 + +from django.db import migrations +from django.contrib.auth.hashers import make_password + + +def migrate_api_tokens(apps, schema_editor): + """Migrate existing plain-text DRF tokens to the new hashed APIToken model.""" + APIToken = apps.get_model("scanpipe", "APIToken") + PREFIX_LENGTH = 8 + + with schema_editor.connection.cursor() as cursor: + cursor.execute( + "SELECT EXISTS (SELECT 1 FROM information_schema.tables " + "WHERE table_name = 'authtoken_token')" + ) + table_exists = cursor.fetchone()[0] + + if not table_exists: + return + + cursor.execute("SELECT user_id, key, created FROM authtoken_token") + rows = cursor.fetchall() + if not rows: + return + + tokens_to_create = [ + APIToken( + user_id=user_id, + prefix=key[:PREFIX_LENGTH], + key_hash=make_password(key), + created=created, + ) + for user_id, key, created in rows + ] + migrated_tokens = APIToken.objects.bulk_create(tokens_to_create, ignore_conflicts=True) + if migrated_tokens: + print(f" -> {len(migrated_tokens)} tokens migrated.") + + +def reverse_migrate_api_tokens(apps, schema_editor): + """Reverse migration: remove all migrated tokens.""" + APIToken = apps.get_model("scanpipe", "APIToken") + APIToken.objects.all().delete() + + +class Migration(migrations.Migration): + + dependencies = [ + ('scanpipe', '0078_apitoken'), + ] + + operations = [ + migrations.RunPython(migrate_api_tokens, reverse_migrate_api_tokens), + migrations.RunSQL( + sql="DROP TABLE IF EXISTS authtoken_token", + reverse_sql=migrations.RunSQL.noop, + ), + ] + diff --git a/scanpipe/models.py b/scanpipe/models.py index ac8e2da155..f065fb4bb5 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -57,7 +57,6 @@ from django.db.models import When from django.db.models.functions import Cast from django.db.models.functions import Lower -from django.dispatch import receiver from django.forms import model_to_dict from django.urls import NoReverseMatch from django.urls import reverse @@ -70,6 +69,7 @@ import redis import requests import saneyaml +from aboutcode.api_auth import AbstractAPIToken from commoncode.fileutils import parent_directory from cyclonedx import model as cyclonedx_model from cyclonedx.model import component as cyclonedx_component @@ -86,7 +86,6 @@ from packageurl.contrib.django.models import PACKAGE_URL_FIELDS from packageurl.contrib.django.models import PackageURLMixin from packageurl.contrib.django.models import PackageURLQuerySetMixin -from rest_framework.authtoken.models import Token from rq.command import send_stop_job_command from rq.exceptions import NoSuchJobError from rq.job import Job @@ -146,6 +145,11 @@ class Meta: abstract = True +class APIToken(AbstractAPIToken): + class Meta: + verbose_name = "API Token" + + class HashFieldsMixin(models.Model): """ The hash fields are not indexed by default, use the `indexes` in Meta as needed: @@ -4963,13 +4967,6 @@ def success(self): return self.response_status_code in (200, 201, 202) -@receiver(models.signals.post_save, sender=settings.AUTH_USER_MODEL) -def create_auth_token(sender, instance=None, created=False, **kwargs): - """Create an API key token on user creation, using the signal system.""" - if created: - Token.objects.create(user_id=instance.pk) - - class DiscoveredPackageScore(UUIDPKModel, PackageScoreMixin): """Represents a security or quality score for a DiscoveredPackage.""" diff --git a/scanpipe/templates/account/profile.html b/scanpipe/templates/account/profile.html index ec96eac82d..f71a732cd1 100644 --- a/scanpipe/templates/account/profile.html +++ b/scanpipe/templates/account/profile.html @@ -13,23 +13,46 @@ - -
    -
    - An API key is like a password and should be treated with the same care. -
    -
    - -
    - -
    - - - - +
    +
    +
    +
    + Your personal API key provides access to the + REST API +
    + Treat it like a password and keep it secure. +
    +
    + {% if request.user.api_token %} +
    + Your API key {{ request.user.api_token.prefix }}... + was generated on {{ request.user.api_token.created }}
    + For security reasons, the full key is only shown once at generation time.
    + If you lose it, you will need to regenerate a new one. +
    + {% else %} +
    + No API key created.
    + Generate one using the button below to access the REST API. +
    + {% endif %} +
    +
    + + {% if request.user.api_token %} + + {% endif %} +
    - + {% include 'scanpipe/modals/profile_generate_api_key_modal.html' %} + {% if request.user.api_token %} + {% include 'scanpipe/modals/profile_revoke_api_key_modal.html' %} + {% endif %}
    {% endblock %} \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/includes/navbar_header.html b/scanpipe/templates/scanpipe/includes/navbar_header.html index caa3bc74e7..cb12597fa8 100644 --- a/scanpipe/templates/scanpipe/includes/navbar_header.html +++ b/scanpipe/templates/scanpipe/includes/navbar_header.html @@ -35,14 +35,11 @@
    + + + Profile settings -
    - {% csrf_token %} - -
    ScanCode.io {{ SCANCODEIO_VERSION }} @@ -50,6 +47,16 @@
    ScanCode-toolkit {{ SCANCODE_TOOLKIT_VERSION }}
    +
    +
    + {% csrf_token %} + +
    {% else %} diff --git a/scanpipe/templates/scanpipe/modals/profile_generate_api_key_modal.html b/scanpipe/templates/scanpipe/modals/profile_generate_api_key_modal.html new file mode 100644 index 0000000000..b302f25ea2 --- /dev/null +++ b/scanpipe/templates/scanpipe/modals/profile_generate_api_key_modal.html @@ -0,0 +1,27 @@ + \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/modals/profile_revoke_api_key_modal.html b/scanpipe/templates/scanpipe/modals/profile_revoke_api_key_modal.html new file mode 100644 index 0000000000..6bfa65d118 --- /dev/null +++ b/scanpipe/templates/scanpipe/modals/profile_revoke_api_key_modal.html @@ -0,0 +1,27 @@ + \ No newline at end of file diff --git a/scanpipe/tests/test_api.py b/scanpipe/tests/test_api.py index 8bb5f63ef3..7bb66da2f5 100644 --- a/scanpipe/tests/test_api.py +++ b/scanpipe/tests/test_api.py @@ -45,6 +45,7 @@ from scanpipe.api.serializers import ProjectSerializer from scanpipe.api.serializers import get_model_serializer from scanpipe.api.serializers import get_serializer_fields +from scanpipe.models import APIToken from scanpipe.models import CodebaseRelation from scanpipe.models import CodebaseResource from scanpipe.models import DiscoveredDependency @@ -91,7 +92,8 @@ def setUp(self): self.project1_detail_url = reverse("project-detail", args=[self.project1.uuid]) self.user = User.objects.create_user("username", "e@mail.com", "secret") - self.auth = f"Token {self.user.auth_token.key}" + self.user_api_key = APIToken.create_token(user=self.user) + self.auth = f"Token {self.user_api_key}" self.csrf_client = APIClient(enforce_csrf_checks=True) self.csrf_client.credentials(HTTP_AUTHORIZATION=self.auth) diff --git a/scanpipe/tests/test_auth.py b/scanpipe/tests/test_auth.py index 32e178a7d4..2f9213c6a4 100644 --- a/scanpipe/tests/test_auth.py +++ b/scanpipe/tests/test_auth.py @@ -22,6 +22,7 @@ import uuid +from django.apps import apps from django.conf import settings from django.contrib.auth import get_user_model from django.contrib.auth.models import AnonymousUser @@ -35,6 +36,7 @@ TEST_PASSWORD = str(uuid.uuid4()) +APIToken = apps.get_model("scanpipe", "APIToken") login_url = reverse("login") project_list_url = reverse("project_list") logout_url = reverse("logout") @@ -93,10 +95,10 @@ def test_scancodeio_auth_logged_in_navbar_header(self): response = self.client.get(project_list_url) expected = 'basic_user' self.assertContains(response, expected, html=True) - expected = f'Profile settings' - self.assertContains(response, expected, html=True) expected = f'
    ' self.assertContains(response, expected) + self.assertContains(response, profile_url) + self.assertContains(response, "Profile settings") def test_scancodeio_auth_logout_view(self): response = self.client.get(logout_url) @@ -111,10 +113,22 @@ def test_scancodeio_auth_logout_view(self): def test_scancodeio_account_profile_view(self): self.client.login(username=self.basic_user.username, password=TEST_PASSWORD) + + expected1 = "No API key created." + expected2 = "Generate API key" + expected3 = "Revoke API key" + response = self.client.get(profile_url) - expected = '' - self.assertContains(response, expected, html=True) - self.assertContains(response, self.basic_user.auth_token.key) + self.assertContains(response, expected1) + self.assertContains(response, expected2) + self.assertNotContains(response, expected3) + + APIToken.create_token(user=self.basic_user) + response = self.client.get(profile_url) + self.assertNotContains(response, expected1) + self.assertContains(response, expected2) + self.assertContains(response, expected3) + self.assertContains(response, self.basic_user.api_token.prefix) def test_scancodeio_auth_views_are_protected(self): a_uuid = uuid.uuid4() diff --git a/scanpipe/tests/test_commands.py b/scanpipe/tests/test_commands.py index 7e93bc1c64..749fc0269b 100644 --- a/scanpipe/tests/test_commands.py +++ b/scanpipe/tests/test_commands.py @@ -31,6 +31,7 @@ from django.apps import apps from django.contrib.auth import get_user_model +from django.core.exceptions import ObjectDoesNotExist from django.core.management import CommandError from django.core.management import call_command from django.core.management.base import BaseCommand @@ -928,15 +929,28 @@ def test_scanpipe_management_command_create_user(self): username = "my_username" call_command("create-user", "--no-input", username, stdout=out) - self.assertIn(f"User {username} created with API key:", out.getvalue()) + self.assertIn(f"User {username} created.", out.getvalue()) + self.assertNotIn("API key", out.getvalue()) user = get_user_model().objects.get(username=username) - self.assertTrue(user.auth_token) self.assertFalse(user.is_staff) self.assertFalse(user.is_superuser) + message = "User has no api_token" + with self.assertRaisesMessage(ObjectDoesNotExist, message): + user.api_token + + username = "verbosity_issue" + options = ["--no-input", "--generate-api-key", "--verbosity=0"] + expected = ( + "Cannot display the API key with verbosity disabled. " + "The key is only shown once at generation time." + ) + with self.assertRaisesMessage(CommandError, expected): + call_command("create-user", username, *options) + expected = "Error: That username is already taken." with self.assertRaisesMessage(CommandError, expected): - call_command("create-user", "--no-input", username) + call_command("create-user", "--no-input", user.username) username = "^&*" expected = ( diff --git a/scanpipe/tests/test_models.py b/scanpipe/tests/test_models.py index e4a5b4cb7d..0251e7b729 100644 --- a/scanpipe/tests/test_models.py +++ b/scanpipe/tests/test_models.py @@ -2635,11 +2635,6 @@ def test_scanpipe_discovered_package_model_get_author_names(self): expected = ["Debian X Strike Force", "JBoss.org Community"] self.assertEqual(expected, package1.get_author_names(roles)) - def test_scanpipe_model_create_user_creates_auth_token(self): - basic_user = User.objects.create_user(username="basic_user") - self.assertTrue(basic_user.auth_token.key) - self.assertEqual(40, len(basic_user.auth_token.key)) - def test_scanpipe_discovered_dependency_model_update_from_data(self): DiscoveredPackage.create_from_data(self.project1, package_data1) CodebaseResource.objects.create( diff --git a/scanpipe/views.py b/scanpipe/views.py index 5e657b874b..038a3e8722 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -60,6 +60,8 @@ import saneyaml import xlsxwriter +from aboutcode.api_auth.views import BaseGenerateAPIKeyView +from aboutcode.api_auth.views import BaseRevokeAPIKeyView from django_filters.views import FilterView from django_htmx.http import HttpResponseClientRedirect from licensedcode.spans import Span @@ -2835,3 +2837,18 @@ def get_context_data(self, **kwargs): context["parent_path"] = "/".join(parent_segments) return context + + +class GenerateAPIKeyView(ConditionalLoginRequired, BaseGenerateAPIKeyView): + success_url = reverse_lazy("account_profile") + success_message = ( + "Copy your API key now, it will not be shown again:" + '
    '
+        '{plain_key}'
+        "
    " + ) + + +class RevokeAPIKeyView(ConditionalLoginRequired, BaseRevokeAPIKeyView): + success_url = reverse_lazy("account_profile") + success_message = "API key revoked." From beb18078dac443e5a171b3fbacb25f99b4ff3f3d Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Wed, 11 Mar 2026 15:29:55 +1300 Subject: [PATCH 047/110] chore: set explicit workflow permissions and pin down actions (#2090) Signed-off-by: tdruez --- .github/workflows/generate-sboms.yml | 10 ++- .github/workflows/pr-quality.yml | 5 +- .github/workflows/publish-docker-image.yml | 25 ++++-- ...ublish-pypi-release-aboutcode-pipeline.yml | 48 ++++++++--- .github/workflows/publish-pypi-release.yml | 79 ++++++++++++++----- .github/workflows/run-unit-tests-docker.yml | 6 +- .github/workflows/run-unit-tests-macos.yml | 10 ++- .github/workflows/run-unit-tests.yml | 10 ++- .github/workflows/sca-integration-anchore.yml | 6 +- .github/workflows/sca-integration-cdxgen.yml | 6 +- .../sca-integration-cyclonedx-gomod.yml | 9 ++- .github/workflows/sca-integration-depscan.yml | 8 +- .../sca-integration-ort-package-file.yml | 22 +++--- .github/workflows/sca-integration-ort.yml | 40 +++++----- .../workflows/sca-integration-osv-scanner.yml | 7 +- .../workflows/sca-integration-sbom-tool.yml | 7 +- .github/workflows/sca-integration-trivy.yml | 8 +- 17 files changed, 202 insertions(+), 104 deletions(-) diff --git a/.github/workflows/generate-sboms.yml b/.github/workflows/generate-sboms.yml index 0cb0fc5ee6..6440dc0875 100644 --- a/.github/workflows/generate-sboms.yml +++ b/.github/workflows/generate-sboms.yml @@ -12,10 +12,14 @@ env: jobs: generate-sboms: runs-on: ubuntu-24.04 + permissions: + contents: read steps: - - name: Checkout code - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false # do not keep the token around - name: Ensure INPUTS_PATH directory exists run: mkdir -p "${{ env.INPUTS_PATH }}" @@ -32,7 +36,7 @@ jobs: find scancodeio/ -type f -name "*.ABOUT" -exec cp {} "${{ env.INPUTS_PATH }}/about-files/" \; - name: Resolve the dependencies using ScanCode-action - uses: aboutcode-org/scancode-action@main + uses: aboutcode-org/scancode-action@8adbf888f487c3cdf6c15386035769cd03a94c66 with: pipelines: "resolve_dependencies:DynamicResolver" inputs-path: ${{ env.INPUTS_PATH }} diff --git a/.github/workflows/pr-quality.yml b/.github/workflows/pr-quality.yml index 0eaa5b048b..1e938fe95e 100644 --- a/.github/workflows/pr-quality.yml +++ b/.github/workflows/pr-quality.yml @@ -6,6 +6,9 @@ permissions: pull-requests: write on: + # pull_request_target is required so the action can close/comment on fork PRs. + # This is safe because: no untrusted code is checked out, and no attacker-controlled + # values are interpolated into shell commands. All action inputs are hardcoded. pull_request_target: types: [opened, reopened] @@ -14,7 +17,7 @@ jobs: runs-on: ubuntu-24.04 name: Detects and automatically closes low-quality and AI slop PRs steps: - - uses: peakoss/anti-slop@v0 + - uses: peakoss/anti-slop@e158eeefe5c43e1d3ba8533b84e0e35d9d6761de with: # Number of check failures needed before failure actions are triggered max-failures: 3 diff --git a/.github/workflows/publish-docker-image.yml b/.github/workflows/publish-docker-image.yml index be24c863d8..9bc575214e 100644 --- a/.github/workflows/publish-docker-image.yml +++ b/.github/workflows/publish-docker-image.yml @@ -22,15 +22,19 @@ jobs: permissions: contents: read packages: write + attestations: write + id-token: write steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false # do not keep the token around # Uses the `docker/login-action` action to log in to the Container registry using # the account and password that will publish the packages. - name: Log in to the Container registry - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -42,7 +46,7 @@ jobs: # The `images` value provides the base name for the tags and labels. - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} @@ -53,11 +57,22 @@ jobs: # It uses the `tags` and `labels` parameters to tag and label the image with # the output from the "meta" step. - name: Build and push Docker image - uses: docker/build-push-action@v5 + id: push + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: context: . push: true tags: | ${{ steps.meta.outputs.tags }} - ${{ env.REGISTRY }}/aboutcode-org/scancode.io:latest + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest labels: ${{ steps.meta.outputs.labels }} + + # This step generates an artifact attestation for the image, which is an + # unforgeable statement about where and how it was built. + # It increases supply chain security for people who consume the image. + - name: Generate artifact attestation + uses: actions/attest-build-provenance@a2bbfa25375fe432b6a289bc6b6cd05ecd0c4c32 # v4.1.0 + with: + subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}} + subject-digest: ${{ steps.push.outputs.digest }} + push-to-registry: true diff --git a/.github/workflows/publish-pypi-release-aboutcode-pipeline.yml b/.github/workflows/publish-pypi-release-aboutcode-pipeline.yml index cef72ed191..62ff6c388c 100644 --- a/.github/workflows/publish-pypi-release-aboutcode-pipeline.yml +++ b/.github/workflows/publish-pypi-release-aboutcode-pipeline.yml @@ -7,32 +7,56 @@ on: - "aboutcode.pipeline/*" jobs: - build-and-publish: + build: name: Build and publish library to PyPI runs-on: ubuntu-24.04 + permissions: + contents: read steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false # do not keep the token around - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: 3.14 - name: Install flot - run: python -m pip install flot --user + run: python -m pip install flot==0.7.2 --user - name: Build a binary wheel and a source tarball run: python -m flot --pyproject pipeline-pyproject.toml --sdist --wheel --output-dir dist/ - - name: Publish to PyPI - if: startsWith(github.ref, 'refs/tags') - uses: pypa/gh-action-pypi-publish@release/v1 + - name: Upload package distributions as GitHub workflow artifacts + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: - password: ${{ secrets.PYPI_API_TOKEN_ABOUTCODE_PIPELINE }} + name: python-package-distributions + path: dist/ + + # Only set the id-token: write permission in the job that does publishing, not globally. + # Also, separate building from publishing — this makes sure that any scripts + # maliciously injected into the build or test environment won't be able to elevate + # privileges while flying under the radar. + pypi-publish: + name: Upload package distributions to PyPI + if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes + needs: + - build + runs-on: ubuntu-24.04 + environment: + name: pypi + url: https://pypi.org/p/aboutcode.pipeline + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing - - name: Upload built archives - uses: actions/upload-artifact@v4 + steps: + - name: Download all the dists + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: - name: pypi_archives - path: dist/* + name: python-package-distributions + path: dist/ + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 diff --git a/.github/workflows/publish-pypi-release.yml b/.github/workflows/publish-pypi-release.yml index 7d13564a9c..2e5163a147 100644 --- a/.github/workflows/publish-pypi-release.yml +++ b/.github/workflows/publish-pypi-release.yml @@ -1,4 +1,4 @@ -name: Build Python distributions and publish on PyPI +name: Build Python distributions, publish on PyPI, and create a GH release on: workflow_dispatch: @@ -6,16 +6,24 @@ on: tags: - "v*.*.*" +env: + PYPI_PROJECT_URL: "https://pypi.org/p/scancodeio" + jobs: - build-and-publish: - name: Build and publish library to PyPI + build-python-dist: + name: Build Python distributions runs-on: ubuntu-24.04 + permissions: + contents: read steps: - - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false # do not keep the token around - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: 3.14 @@ -23,23 +31,56 @@ jobs: run: python -m pip install build --user - name: Build a binary wheel and a source tarball - run: python -m build --sdist --wheel --outdir dist/ . + run: python -m build --sdist --wheel --outdir dist/ - - name: Publish to PyPI - if: startsWith(github.ref, 'refs/tags') - uses: pypa/gh-action-pypi-publish@release/v1 + - name: Upload package distributions as GitHub workflow artifacts + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: - password: ${{ secrets.PYPI_API_TOKEN }} + name: python-package-distributions + path: dist/ - - name: Upload built archives - uses: actions/upload-artifact@v4 + # Only set the id-token: write permission in the job that does publishing, not globally. + # Also, separate building from publishing — this makes sure that any scripts + # maliciously injected into the build or test environment won't be able to elevate + # privileges while flying under the radar. + pypi-publish: + name: Upload package distributions to PyPI + if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes + needs: + - build-python-dist + runs-on: ubuntu-24.04 + environment: + name: pypi + url: ${{ env.PYPI_PROJECT_URL }} + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + + steps: + - name: Download package distributions + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: - name: pypi_archives - path: dist/* + name: python-package-distributions + path: dist/ - - name: Create a GitHub release - uses: softprops/action-gh-release@v2 + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 + + create-gh-release: + name: Create GitHub release + needs: + - build-python-dist + runs-on: ubuntu-24.04 + permissions: + contents: write + + steps: + - name: Download package distributions + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: - generate_release_notes: true - draft: false - files: dist/* + name: python-package-distributions + path: dist/ + + - name: Create GitHub release + run: gh release create "$GITHUB_REF_NAME" dist/* --generate-notes + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/run-unit-tests-docker.yml b/.github/workflows/run-unit-tests-docker.yml index 609fdfab09..73a20835c3 100644 --- a/.github/workflows/run-unit-tests-docker.yml +++ b/.github/workflows/run-unit-tests-docker.yml @@ -15,8 +15,10 @@ jobs: runs-on: ubuntu-24.04 steps: - - name: Checkout code - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false # do not keep the token around - name: Generate the .env file and the SECRET_KEY run: make envfile diff --git a/.github/workflows/run-unit-tests-macos.yml b/.github/workflows/run-unit-tests-macos.yml index df128bfa6a..b48dc764fd 100644 --- a/.github/workflows/run-unit-tests-macos.yml +++ b/.github/workflows/run-unit-tests-macos.yml @@ -24,16 +24,18 @@ jobs: python-version: ["3.12", "3.13", "3.14"] steps: - - name: Checkout code - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false # do not keep the token around - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: ${{ matrix.python-version }} - name: Set up PostgreSQL - uses: ikalnytskyi/action-setup-postgres@v8 + uses: ikalnytskyi/action-setup-postgres@c4dda34aae1c821e3a771b68b73b13af3198a7ee # v8 with: postgres-version: "17" database: ${{ env.POSTGRES_DB }} diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 2d8c286ca0..59c9f6657f 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -22,7 +22,7 @@ jobs: services: postgres: - image: postgres:17 + image: postgres:17.9 env: POSTGRES_DB: ${{ env.POSTGRES_DB }} POSTGRES_USER: ${{ env.POSTGRES_USER }} @@ -42,11 +42,13 @@ jobs: python-version: ["3.12", "3.13", "3.14"] steps: - - name: Checkout code - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false # do not keep the token around - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/sca-integration-anchore.yml b/.github/workflows/sca-integration-anchore.yml index d8ac014829..f57339fd02 100644 --- a/.github/workflows/sca-integration-anchore.yml +++ b/.github/workflows/sca-integration-anchore.yml @@ -28,7 +28,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Generate CycloneDX SBOM with Anchore Grype scanner - uses: anchore/scan-action@v6 + uses: anchore/scan-action@7037fa011853d5a11690026fb85feee79f4c946c # v7.3.2 with: image: ${{ env.IMAGE_REFERENCE }} output-format: cyclonedx-json @@ -36,14 +36,14 @@ jobs: fail-build: false - name: Upload SBOM as GitHub Artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: anchore-sbom-report path: "anchore-grype-sbom.cdx.json" retention-days: 20 - name: Import SBOM into ScanCode.io - uses: aboutcode-org/scancode-action@main + uses: aboutcode-org/scancode-action@8adbf888f487c3cdf6c15386035769cd03a94c66 with: pipelines: "load_sbom" inputs-path: "anchore-grype-sbom.cdx.json" diff --git a/.github/workflows/sca-integration-cdxgen.yml b/.github/workflows/sca-integration-cdxgen.yml index ad7f050fac..ded93df560 100644 --- a/.github/workflows/sca-integration-cdxgen.yml +++ b/.github/workflows/sca-integration-cdxgen.yml @@ -28,7 +28,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Install CycloneDX cdxgen - run: npm install @cyclonedx/cdxgen + run: npm install @cyclonedx/cdxgen@12.1.2 - name: Generate SBOM with CycloneDX cdxgen run: | @@ -39,14 +39,14 @@ jobs: --json-pretty - name: Upload SBOM as GitHub Artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: cdxgen-sbom path: "cdxgen-sbom.cdx.json" retention-days: 20 - name: Import SBOM into ScanCode.io - uses: aboutcode-org/scancode-action@main + uses: aboutcode-org/scancode-action@8adbf888f487c3cdf6c15386035769cd03a94c66 with: pipelines: "load_sbom" inputs-path: "cdxgen-sbom.cdx.json" diff --git a/.github/workflows/sca-integration-cyclonedx-gomod.yml b/.github/workflows/sca-integration-cyclonedx-gomod.yml index bbbf724f7c..ea39bd659f 100644 --- a/.github/workflows/sca-integration-cyclonedx-gomod.yml +++ b/.github/workflows/sca-integration-cyclonedx-gomod.yml @@ -27,25 +27,26 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Checkout minimal Go repo - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: repository: opencontainers/runc + persist-credentials: false # do not keep the token around - name: Generate SBOM with cyclonedx-gomod - uses: CycloneDX/gh-gomod-generate-sbom@v2 + uses: CycloneDX/gh-gomod-generate-sbom@efc74245d6802c8cefd925620515442756c70d8f # v2.0.0 with: version: v1 args: mod -licenses -json -output gomod-sbom.cdx.json - name: Upload SBOM as GitHub Artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: sbom-report path: "gomod-sbom.cdx.json" retention-days: 20 - name: Import SBOM into ScanCode.io - uses: aboutcode-org/scancode-action@main + uses: aboutcode-org/scancode-action@8adbf888f487c3cdf6c15386035769cd03a94c66 with: pipelines: "load_sbom" inputs-path: "gomod-sbom.cdx.json" diff --git a/.github/workflows/sca-integration-depscan.yml b/.github/workflows/sca-integration-depscan.yml index 6b824a6a60..76cb16aace 100644 --- a/.github/workflows/sca-integration-depscan.yml +++ b/.github/workflows/sca-integration-depscan.yml @@ -29,8 +29,8 @@ jobs: steps: - name: Install OWASP dep-scan run: | - sudo npm install -g @cyclonedx/cdxgen - pip install owasp-depscan + sudo npm install -g @cyclonedx/cdxgen@12.1.2 + pip install owasp-depscan==6.1.0 - name: Generate SBOM with OWASP dep-scan run: | @@ -41,7 +41,7 @@ jobs: --explain - name: Upload SBOM as GitHub Artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: depscan-sbom path: reports/ @@ -51,7 +51,7 @@ jobs: run: pip uninstall --yes owasp-depscan - name: Import SBOM into ScanCode.io - uses: aboutcode-org/scancode-action@main + uses: aboutcode-org/scancode-action@8adbf888f487c3cdf6c15386035769cd03a94c66 with: pipelines: "load_sbom" inputs-path: "reports/sbom-docker.vdr.json" diff --git a/.github/workflows/sca-integration-ort-package-file.yml b/.github/workflows/sca-integration-ort-package-file.yml index 50cc4780dc..14ee50419d 100644 --- a/.github/workflows/sca-integration-ort-package-file.yml +++ b/.github/workflows/sca-integration-ort-package-file.yml @@ -17,49 +17,49 @@ permissions: env: SCIO_IMAGE_INPUT: "docker://osadl/alpine-docker-base-image:v3.22-latest" - ORT_VERSION: "68.1.0" + ORT_VERSION: "82.0.0" jobs: generate-and-load-sbom: runs-on: ubuntu-24.04 steps: - name: Analyze Docker image with ScanCode.io - uses: aboutcode-org/scancode-action@main + uses: aboutcode-org/scancode-action@8adbf888f487c3cdf6c15386035769cd03a94c66 with: pipelines: "analyze_docker_image" input-urls: "${{ env.SCIO_IMAGE_INPUT }}" scancodeio-repo-branch: "main" - output-formats: "ort-package-list spdx:2.2 cyclonedx json xlsx" + output-formats: "ort-package-list" - name: Copy package-list.yml to workspace root run: | - FILE=$(ls ${{ env.PROJECT_WORK_DIRECTORY }}/output/*.package-list.yml | head -n 1) - sudo mkdir -p ${GITHUB_WORKSPACE}/ort-data/ + FILE=$(ls "${PROJECT_WORK_DIRECTORY}/output/"*.package-list.yml | head -n 1) + sudo mkdir -p "${GITHUB_WORKSPACE}/ort-data/" sudo cp "$FILE" "${GITHUB_WORKSPACE}/ort-data/package-list.yml" - sudo chmod -R 777 ${GITHUB_WORKSPACE}/ort-data/ + sudo chmod -R 777 "${GITHUB_WORKSPACE}/ort-data/" ls -lh "${GITHUB_WORKSPACE}/ort-data/" - name: Generates an ORT analyzer-result.yml file run: | - docker run --rm -v ${GITHUB_WORKSPACE}/ort-data:/data \ + docker run --rm -v "${GITHUB_WORKSPACE}/ort-data:/data" \ --entrypoint /opt/ort/bin/orth \ - ghcr.io/oss-review-toolkit/ort:${{ env.ORT_VERSION }} \ + "ghcr.io/oss-review-toolkit/ort:${ORT_VERSION}" \ create-analyzer-result-from-package-list \ --package-list-file /data/package-list.yml \ --ort-file /data/analyzer-result.yml - name: Report as CycloneDX and SPDX using the analyzer-result.yml file run: | - docker run --rm -v ${GITHUB_WORKSPACE}/ort-data:/data \ - ghcr.io/oss-review-toolkit/ort:${{ env.ORT_VERSION }} \ + docker run --rm -v "${GITHUB_WORKSPACE}/ort-data:/data" \ + "ghcr.io/oss-review-toolkit/ort:${ORT_VERSION}" \ report \ --ort-file /data/analyzer-result.yml \ --output-dir /data/results/ \ --report-formats CycloneDX,SpdxDocument - name: Upload SBOMs as GitHub Artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: ort-report path: "${GITHUB_WORKSPACE}/ort-data/results" diff --git a/.github/workflows/sca-integration-ort.yml b/.github/workflows/sca-integration-ort.yml index 2c777845ff..25f5d82fe4 100644 --- a/.github/workflows/sca-integration-ort.yml +++ b/.github/workflows/sca-integration-ort.yml @@ -21,11 +21,13 @@ jobs: checkout-ort-test-assets-from-scancode-io-repo: runs-on: ubuntu-24.04 steps: - - name: Checkout ScanCode.io repository - uses: actions/checkout@v5 + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false # do not keep the token around - name: Upload orthw mime types example - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: npm-mime-types-2.1.26-scan-result.json path: scanpipe/tests/data/integrations-ort/orthw-example-scan-result/npm-mime-types-2.1.26-scan-result.json @@ -44,7 +46,7 @@ jobs: EOF - name: Run GitHub Action for ORT - uses: oss-review-toolkit/ort-ci-github-action@v1 + uses: oss-review-toolkit/ort-ci-github-action@1805edcf1f4f55f35ae6e4d2d9795ccfb29b6021 # v1.1.0 with: ort-cli-report-args: "-O CycloneDX=output.file.formats=json -O CycloneDX=schema.version=1.5" report-formats: "CycloneDx" @@ -55,7 +57,7 @@ jobs: reporter - name: Import SBOM into ScanCode.io - uses: aboutcode-org/scancode-action@main + uses: aboutcode-org/scancode-action@8adbf888f487c3cdf6c15386035769cd03a94c66 with: pipelines: "load_sbom" inputs-path: "${{ env.ORT_RESULTS_PATH }}/bom.cyclonedx.json" @@ -83,7 +85,7 @@ jobs: EOF - name: Run GitHub Action for ORT - uses: oss-review-toolkit/ort-ci-github-action@v1 + uses: oss-review-toolkit/ort-ci-github-action@1805edcf1f4f55f35ae6e4d2d9795ccfb29b6021 # v1.1.0 with: ort-cli-report-args: "-O CycloneDX=output.file.formats=json -O CycloneDX=schema.version=1.6" report-formats: "CycloneDx" @@ -94,7 +96,7 @@ jobs: reporter - name: Import SBOM into ScanCode.io - uses: aboutcode-org/scancode-action@main + uses: aboutcode-org/scancode-action@8adbf888f487c3cdf6c15386035769cd03a94c66 with: pipelines: "load_sbom" inputs-path: "${{ env.ORT_RESULTS_PATH }}/bom.cyclonedx.json" @@ -114,7 +116,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Download mime-type-2.1.26-scan-result file - uses: actions/download-artifact@v5 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: npm-mime-types-2.1.26-scan-result.json @@ -126,16 +128,16 @@ jobs: cat $HOME/.ort/ort-results/current-result.json - name: Run GitHub Action for ORT - uses: oss-review-toolkit/ort-ci-github-action@v1 + uses: oss-review-toolkit/ort-ci-github-action@1805edcf1f4f55f35ae6e4d2d9795ccfb29b6021 # v1.1.0 with: report-formats: "CycloneDx,SpdxDocument" run: > evaluator, advisor, reporter - - name: Upload orthw mime type example - - uses: actions/upload-artifact@v4 + - name: Upload orthw mime type example + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: npm-mime-types-2.1.26-ort-sboms path: | @@ -151,12 +153,12 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Download ORT CycloneDX JSON SBOM for mime-types 2.1.26 - uses: actions/download-artifact@v5 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: npm-mime-types-2.1.26-ort-sboms - name: Import SBOM into ScanCode.io - uses: aboutcode-org/scancode-action@main + uses: aboutcode-org/scancode-action@8adbf888f487c3cdf6c15386035769cd03a94c66 with: pipelines: "load_sbom" inputs-path: "bom.cyclonedx.json" @@ -177,12 +179,12 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Download ORT CycloneDX JSON SBOM for mime-types 2.1.26 - uses: actions/download-artifact@v5 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: npm-mime-types-2.1.26-ort-sboms - name: Import SBOM into ScanCode.io - uses: aboutcode-org/scancode-action@main + uses: aboutcode-org/scancode-action@8adbf888f487c3cdf6c15386035769cd03a94c66 with: pipelines: "load_sbom" inputs-path: "bom.cyclonedx.xml" @@ -203,12 +205,12 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Download ORT SPDX JSON SBOM for mime-types 2.1.26 - uses: actions/download-artifact@v5 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: npm-mime-types-2.1.26-ort-sboms - name: Import SBOM into ScanCode.io - uses: aboutcode-org/scancode-action@main + uses: aboutcode-org/scancode-action@8adbf888f487c3cdf6c15386035769cd03a94c66 with: pipelines: "load_sbom" inputs-path: "bom.spdx.json" @@ -229,12 +231,12 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Download ORT SPDX YAML SBOM for mime-types 2.1.26 - uses: actions/download-artifact@v5 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: npm-mime-types-2.1.26-ort-sboms - name: Import SBOM into ScanCode.io - uses: aboutcode-org/scancode-action@main + uses: aboutcode-org/scancode-action@8adbf888f487c3cdf6c15386035769cd03a94c66 with: pipelines: "load_sbom" inputs-path: "bom.spdx.yml" diff --git a/.github/workflows/sca-integration-osv-scanner.yml b/.github/workflows/sca-integration-osv-scanner.yml index 0edaa49d9c..2bc594026f 100644 --- a/.github/workflows/sca-integration-osv-scanner.yml +++ b/.github/workflows/sca-integration-osv-scanner.yml @@ -19,6 +19,7 @@ permissions: env: IMAGE_REFERENCE: "python:3.13.0-slim" + OSV_SCANNER_URL: "https://github.com/google/osv-scanner/releases/download/v2.3.3/osv-scanner_linux_amd64" EXPECTED_PACKAGE: 100 EXPECTED_VULNERABLE_PACKAGE: 0 EXPECTED_DEPENDENCY: 90 @@ -29,7 +30,7 @@ jobs: steps: - name: Install OSV-Scanner run: | - curl -sLO https://github.com/google/osv-scanner/releases/latest/download/osv-scanner_linux_amd64 + curl -sLO "$OSV_SCANNER_URL" chmod +x osv-scanner_linux_amd64 sudo mv osv-scanner_linux_amd64 /usr/local/bin/osv-scanner @@ -43,14 +44,14 @@ jobs: || true - name: Upload SBOM as GitHub Artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: osv-scanner-sbom-report path: osv-sbom.spdx.json retention-days: 20 - name: Import SBOM into ScanCode.io - uses: aboutcode-org/scancode-action@main + uses: aboutcode-org/scancode-action@8adbf888f487c3cdf6c15386035769cd03a94c66 with: pipelines: "load_sbom" inputs-path: "osv-sbom.spdx.json" diff --git a/.github/workflows/sca-integration-sbom-tool.yml b/.github/workflows/sca-integration-sbom-tool.yml index 01bc2fe96d..14ef30c99b 100644 --- a/.github/workflows/sca-integration-sbom-tool.yml +++ b/.github/workflows/sca-integration-sbom-tool.yml @@ -19,6 +19,7 @@ permissions: env: IMAGE_REFERENCE: "python:3.13.0-slim" + SBOM_TOOL_URL: "https://github.com/microsoft/sbom-tool/releases/download/v4.1.5/sbom-tool-linux-x64" EXPECTED_PACKAGE: 90 EXPECTED_VULNERABLE_PACKAGE: 0 EXPECTED_DEPENDENCY: 90 @@ -29,7 +30,7 @@ jobs: steps: - name: Download SBOM tool run: | - curl -Lo $RUNNER_TEMP/sbom-tool https://github.com/microsoft/sbom-tool/releases/latest/download/sbom-tool-linux-x64 + curl -Lo $RUNNER_TEMP/sbom-tool "$SBOM_TOOL_URL" chmod +x $RUNNER_TEMP/sbom-tool - name: Generate SBOM with SBOM tool @@ -45,13 +46,13 @@ jobs: -V Verbose - name: Upload SBOM artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: sbom-output path: sbom-output - name: Import SBOM into ScanCode.io - uses: aboutcode-org/scancode-action@main + uses: aboutcode-org/scancode-action@8adbf888f487c3cdf6c15386035769cd03a94c66 with: pipelines: "load_sbom" inputs-path: "sbom-output/_manifest/spdx_2.2/manifest.spdx.json" diff --git a/.github/workflows/sca-integration-trivy.yml b/.github/workflows/sca-integration-trivy.yml index d135e00322..c05e538cce 100644 --- a/.github/workflows/sca-integration-trivy.yml +++ b/.github/workflows/sca-integration-trivy.yml @@ -28,24 +28,24 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Generate CycloneDX SBOM with Trivy - uses: aquasecurity/trivy-action@0.32.0 + uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0 with: scan-type: "image" image-ref: ${{ env.IMAGE_REFERENCE }} format: "cyclonedx" output: "trivy-report.sbom.json" scanners: "vuln,license" - version: "latest" + version: "v0.69.3" - name: Upload SBOM as GitHub Artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: trivy-sbom-report path: "trivy-report.sbom.json" retention-days: 20 - name: Import SBOM into ScanCode.io - uses: aboutcode-org/scancode-action@main + uses: aboutcode-org/scancode-action@8adbf888f487c3cdf6c15386035769cd03a94c66 with: pipelines: "load_sbom" inputs-path: "trivy-report.sbom.json" From 13ab091484a26de8ca2c93c5d33c61beb89b4438 Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Wed, 11 Mar 2026 18:48:53 +1300 Subject: [PATCH 048/110] chore: bump version to v37.0.0 for release (#2091) Signed-off-by: tdruez --- CHANGELOG.rst | 26 ++++++++++++++++++++------ pyproject.toml | 2 +- scancodeio/__init__.py | 2 +- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0966a61a34..6a779ef3db 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,18 +1,32 @@ Changelog ========= -**v36 Breaking Change:** PostgreSQL 17 is now required (previously 13). +**v37 Breaking Changes:** -Docker Compose users with existing data: run `./migrate-pg13-to-17.sh` before starting -the stack. -Fresh installations require no action. +- Drop support for Python3.10 and Python3.11 -v37.0.0 (unreleased) +v37.0.0 (2026-03-11) -------------------- - Upgrade Django to release 6.x -- Drop support for Python3.10 and Python3.11 +- Remove the chown service in compose file + https://github.com/aboutcode-org/scancode.io/issues/2086 + +- Replace plain-text DRF token with PBKDF2-hashed API token + https://github.com/aboutcode-org/scancode.io/issues/2087 + +- Rename "Grammar" optional step group to "Antlr" in d2d pipeline + https://github.com/aboutcode-org/scancode.io/issues/2059 + +- Fix URL-encode programming language filter values in resource list + https://github.com/aboutcode-org/scancode.io/issues/2079 + +**v36 Breaking Change:** PostgreSQL 17 is now required (previously 13). + +Docker Compose users with existing data: run `./migrate-pg13-to-17.sh` before starting +the stack. +Fresh installations require no action. v36.1.0 (2026-01-22) -------------------- diff --git a/pyproject.toml b/pyproject.toml index 8b2862b3a1..8d8b714411 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "scancodeio" -version = "36.1.0" +version = "37.0.0" description = "Automate software composition analysis pipelines" readme = "README.rst" requires-python = ">=3.12,<3.15" diff --git a/scancodeio/__init__.py b/scancodeio/__init__.py index a2272e7bb6..1093a8a8a6 100644 --- a/scancodeio/__init__.py +++ b/scancodeio/__init__.py @@ -28,7 +28,7 @@ import git -VERSION = "36.1.0" +VERSION = "37.0.0" PROJECT_DIR = Path(__file__).resolve().parent ROOT_DIR = PROJECT_DIR.parent From edf44da1b4cc5b19168a759bb299c0f8e0a8a929 Mon Sep 17 00:00:00 2001 From: tdruez Date: Wed, 11 Mar 2026 18:59:30 +1300 Subject: [PATCH 049/110] fix: add the checkout step to pypi release workflow Signed-off-by: tdruez --- .github/workflows/publish-pypi-release.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish-pypi-release.yml b/.github/workflows/publish-pypi-release.yml index 2e5163a147..fb5b0f2b6f 100644 --- a/.github/workflows/publish-pypi-release.yml +++ b/.github/workflows/publish-pypi-release.yml @@ -1,4 +1,4 @@ -name: Build Python distributions, publish on PyPI, and create a GH release +name: Build Python distributions, publish on PyPI, and create a GitHub release on: workflow_dispatch: @@ -74,6 +74,11 @@ jobs: contents: write steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Download package distributions uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: From 44371f6ca034b482bcb7917f9db5fb6b04e3ca03 Mon Sep 17 00:00:00 2001 From: tdruez Date: Wed, 11 Mar 2026 21:08:26 +1300 Subject: [PATCH 050/110] chore: refine gh workflows for security and consistency Signed-off-by: tdruez --- .../publish-pypi-release-aboutcode-pipeline.yml | 13 +++++++++---- .github/workflows/publish-pypi-release.yml | 4 ++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/publish-pypi-release-aboutcode-pipeline.yml b/.github/workflows/publish-pypi-release-aboutcode-pipeline.yml index 62ff6c388c..51235ed27a 100644 --- a/.github/workflows/publish-pypi-release-aboutcode-pipeline.yml +++ b/.github/workflows/publish-pypi-release-aboutcode-pipeline.yml @@ -6,6 +6,11 @@ on: tags: - "aboutcode.pipeline/*" +env: + PYPI_PROJECT_URL: "https://pypi.org/p/aboutcode.pipeline" + PYPROJECT_TOML: "pipeline-pyproject.toml" + FLOT_VERSION: "0.7.2" + jobs: build: name: Build and publish library to PyPI @@ -24,10 +29,10 @@ jobs: python-version: 3.14 - name: Install flot - run: python -m pip install flot==0.7.2 --user + run: python -m pip install "flot==${FLOT_VERSION}" --user - name: Build a binary wheel and a source tarball - run: python -m flot --pyproject pipeline-pyproject.toml --sdist --wheel --output-dir dist/ + run: python -m flot --pyproject "$PYPROJECT_TOML" --sdist --wheel --output-dir dist/ - name: Upload package distributions as GitHub workflow artifacts uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 @@ -36,7 +41,7 @@ jobs: path: dist/ # Only set the id-token: write permission in the job that does publishing, not globally. - # Also, separate building from publishing — this makes sure that any scripts + # Also, separate building from publishing, this makes sure that any scripts # maliciously injected into the build or test environment won't be able to elevate # privileges while flying under the radar. pypi-publish: @@ -47,7 +52,7 @@ jobs: runs-on: ubuntu-24.04 environment: name: pypi - url: https://pypi.org/p/aboutcode.pipeline + url: ${{ env.PYPI_PROJECT_URL }} permissions: id-token: write # IMPORTANT: this permission is mandatory for trusted publishing diff --git a/.github/workflows/publish-pypi-release.yml b/.github/workflows/publish-pypi-release.yml index fb5b0f2b6f..6ab7203dc3 100644 --- a/.github/workflows/publish-pypi-release.yml +++ b/.github/workflows/publish-pypi-release.yml @@ -28,7 +28,7 @@ jobs: python-version: 3.14 - name: Install pypa/build - run: python -m pip install build --user + run: python -m pip install build==1.4.0 --user - name: Build a binary wheel and a source tarball run: python -m build --sdist --wheel --outdir dist/ @@ -40,7 +40,7 @@ jobs: path: dist/ # Only set the id-token: write permission in the job that does publishing, not globally. - # Also, separate building from publishing — this makes sure that any scripts + # Also, separate building from publishing, this makes sure that any scripts # maliciously injected into the build or test environment won't be able to elevate # privileges while flying under the radar. pypi-publish: From 40e45a59436b386a84107f4835edd3edae19a330 Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Mon, 16 Mar 2026 17:59:37 +0400 Subject: [PATCH 051/110] feat: display scio and toolkit versions in place of django version (#2101) Signed-off-by: tdruez --- scancodeio/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scancodeio/__init__.py b/scancodeio/__init__.py index 1093a8a8a6..af281b87fa 100644 --- a/scancodeio/__init__.py +++ b/scancodeio/__init__.py @@ -95,6 +95,14 @@ def command_line(): """Command line entry point.""" from django.core.management import execute_from_command_line + # Display ScanCode.io and ScanCode-toolkit versions in place of Django version. + if "--version" in sys.argv: + from scancode_config import __version__ as scancode_toolkit_version + + print(f"ScanCode.io version: {__version__}") + print(f"ScanCode-toolkit version: v{scancode_toolkit_version}") + sys.exit(0) + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "scancodeio.settings") execute_from_command_line(sys.argv) From cf257043a9712362a6ecfce3b43878811794689c Mon Sep 17 00:00:00 2001 From: Rishabh Rohil <130820750+rishabh23rohil@users.noreply.github.com> Date: Mon, 16 Mar 2026 09:03:59 -0500 Subject: [PATCH 052/110] fix missing space in scan_max_file_size help text (#2097) Signed-off-by: Rishabh Rohil --- scanpipe/forms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanpipe/forms.py b/scanpipe/forms.py index e3f93e3af4..8329e36cf9 100644 --- a/scanpipe/forms.py +++ b/scanpipe/forms.py @@ -553,7 +553,7 @@ class ProjectSettingsForm(forms.ModelForm): label="Max file size to scan", required=False, help_text=( - "Maximum file size in bytes which should be skipped from scanning." + "Maximum file size in bytes which should be skipped from scanning. " "File size is in bytes. Example: 5 MB is 5242880 bytes." ), widget=forms.NumberInput(attrs={"class": "input"}), From e13b78f6167fb070373806f88cc8c944f3065d52 Mon Sep 17 00:00:00 2001 From: Rishabh Rohil <130820750+rishabh23rohil@users.noreply.github.com> Date: Mon, 16 Mar 2026 09:08:49 -0500 Subject: [PATCH 053/110] feat: add tests for chunked and get_purls utilities (#2100) Signed-off-by: Rishabh Rohil --- scanpipe/tests/pipes/test_vulnerablecode.py | 28 +++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/scanpipe/tests/pipes/test_vulnerablecode.py b/scanpipe/tests/pipes/test_vulnerablecode.py index e39dbf69ed..f33a68cdca 100644 --- a/scanpipe/tests/pipes/test_vulnerablecode.py +++ b/scanpipe/tests/pipes/test_vulnerablecode.py @@ -28,8 +28,10 @@ from django.test import TestCase from scanpipe.models import Project +from scanpipe.pipes.vulnerablecode import chunked from scanpipe.pipes.vulnerablecode import fetch_vulnerabilities from scanpipe.pipes.vulnerablecode import filter_vulnerabilities +from scanpipe.pipes.vulnerablecode import get_purls from scanpipe.tests import make_package @@ -81,3 +83,29 @@ def test_scanpipe_pipes_vulnerablecode_filter_vulnerabilities(self): vulnerability2 = vulnerability_data[1] ignore_set.add(vulnerability2.get("aliases")[1]) self.assertEqual([], filter_vulnerabilities(vulnerability_data, ignore_set)) + + def test_scanpipe_pipes_vulnerablecode_chunked(self): + result = list(chunked([1, 2, 3, 4, 5], 2)) + self.assertEqual([[1, 2], [3, 4], [5]], result) + + result = list(chunked([1, 2, 3, 4, 5], 3)) + self.assertEqual([[1, 2, 3], [4, 5]], result) + + result = list(chunked([], 10)) + self.assertEqual([], result) + + result = list(chunked([1], 5)) + self.assertEqual([[1]], result) + + result = list(chunked([1, 2, 3], 3)) + self.assertEqual([[1, 2, 3]], result) + + def test_scanpipe_pipes_vulnerablecode_get_purls(self): + pkg1 = make_package(self.project1, "pkg:pypi/django@5.0") + pkg2 = make_package(self.project1, "pkg:npm/express@4.18.2") + purls = get_purls([pkg1, pkg2]) + self.assertEqual(["pkg:pypi/django@5.0", "pkg:npm/express@4.18.2"], purls) + + def test_scanpipe_pipes_vulnerablecode_get_purls_empty(self): + purls = get_purls([]) + self.assertEqual([], purls) From 85ea093c6d6a7a2eca308fd74c77974a161ab9c0 Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Mon, 16 Mar 2026 18:16:20 +0400 Subject: [PATCH 054/110] feat: display layers information (created_by, comment) in tree view (#2102) Signed-off-by: tdruez --- .../templates/scanpipe/resource_tree.html | 27 ++- .../scanpipe/tree/resource_left_pane.html | 4 + .../tree/resource_left_pane_header.html | 4 +- .../tree/resource_path_breadcrumb.html | 2 +- .../scanpipe/tree/resource_table.html | 181 ++++++------------ .../tree/resource_table_resource_row.html | 78 ++++++++ scanpipe/views.py | 9 +- 7 files changed, 162 insertions(+), 143 deletions(-) create mode 100644 scanpipe/templates/scanpipe/tree/resource_left_pane.html create mode 100644 scanpipe/templates/scanpipe/tree/resource_table_resource_row.html diff --git a/scanpipe/templates/scanpipe/resource_tree.html b/scanpipe/templates/scanpipe/resource_tree.html index 73cfc75816..24d2bea087 100644 --- a/scanpipe/templates/scanpipe/resource_tree.html +++ b/scanpipe/templates/scanpipe/resource_tree.html @@ -20,24 +20,21 @@
    - {% include "scanpipe/tree/resource_left_pane_header.html" only %} -
    - {% include "scanpipe/tree/resource_left_pane_tree.html" with children=children path=path %} -
    + {% include 'scanpipe/tree/resource_left_pane.html' %}
    -
    -
    +
    +
    diff --git a/scanpipe/templates/scanpipe/tree/resource_left_pane.html b/scanpipe/templates/scanpipe/tree/resource_left_pane.html new file mode 100644 index 0000000000..62d1bc70b5 --- /dev/null +++ b/scanpipe/templates/scanpipe/tree/resource_left_pane.html @@ -0,0 +1,4 @@ +{% include "scanpipe/tree/resource_left_pane_header.html" only %} +
    + {% include "scanpipe/tree/resource_left_pane_tree.html" with children=children path=path %} +
    \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/tree/resource_left_pane_header.html b/scanpipe/templates/scanpipe/tree/resource_left_pane_header.html index 1dfbc8482c..3250d5f1e1 100644 --- a/scanpipe/templates/scanpipe/tree/resource_left_pane_header.html +++ b/scanpipe/templates/scanpipe/tree/resource_left_pane_header.html @@ -1,12 +1,12 @@
    -
    +
    Resources
    \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/tree/resource_path_breadcrumb.html b/scanpipe/templates/scanpipe/tree/resource_path_breadcrumb.html index 0a1bd943e7..766af14741 100644 --- a/scanpipe/templates/scanpipe/tree/resource_path_breadcrumb.html +++ b/scanpipe/templates/scanpipe/tree/resource_path_breadcrumb.html @@ -1,7 +1,7 @@
    NameStatusLanguageLicenseAlert
    +
    + + + + .. +
    +
    + +{% if is_paginated %} + {% endif %} \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/tree/resource_table_resource_row.html b/scanpipe/templates/scanpipe/tree/resource_table_resource_row.html new file mode 100644 index 0000000000..4f47fb414d --- /dev/null +++ b/scanpipe/templates/scanpipe/tree/resource_table_resource_row.html @@ -0,0 +1,78 @@ + + +
    + + {% if resource.is_dir %} + + {% else %} + + {% endif %} + + {% if resource.is_dir %} + + {{ resource.name }} + + {% else %} + + {{ resource.name }} + + {% endif %} + {% if resource.tag %} + {{ resource.tag }} + {% endif %} + {% if resource.extra_data.layer %} +
    + + +
    + {% endif %} +
    + + + {{ resource.status }} + + + {{ resource.programming_language }} + + + {{ resource.detected_license_expression }} + + + {{ resource.compliance_alert }} + + \ No newline at end of file diff --git a/scanpipe/views.py b/scanpipe/views.py index 038a3e8722..2e1ac5f362 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -2759,13 +2759,14 @@ def get_node(self, package): class ProjectResourceTreeView(ConditionalLoginRequired, generic.DetailView): + model = Project template_name = "scanpipe/resource_tree.html" def get(self, request, *args, **kwargs): - slug = self.kwargs.get("slug") - project = get_object_or_404(Project, slug=slug) + project = self.get_object() path = self.kwargs.get("path", "") - parent_path = path if request.GET.get("tree_panel") == "true" else "" + is_tree_panel = request.GET.get("tree_panel") == "true" + parent_path = path if is_tree_panel else "" children = ( project.codebaseresources.filter(parent_path=parent_path) @@ -2786,7 +2787,7 @@ def get(self, request, *args, **kwargs): "resource": resource, } - if request.GET.get("tree_panel") == "true": + if is_tree_panel: return render( request, "scanpipe/tree/resource_left_pane_tree.html", context ) From 0cd684fcea590c7507cd500b4cc5c99f5b05d73c Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Tue, 17 Mar 2026 14:23:40 +0400 Subject: [PATCH 055/110] feat: enhance the global HTML structure (#2103) Signed-off-by: tdruez --- scancodeio/context_processors.py | 2 - .../{add-inputs.js => js/add_inputs.js} | 0 scancodeio/static/js/dependency_tree.js | 95 ++++++++++ scancodeio/static/{ => js}/main.js | 50 ++++++ scancodeio/static/js/project_form.js | 170 ++++++++++++++++++ scancodeio/static/js/resource_tree.js | 146 +++++++++++++++ scancodeio/static/js/resource_viewer.js | 160 +++++++++++++++++ scanpipe/forms.py | 2 +- scanpipe/templates/scanpipe/base.html | 18 +- scanpipe/templates/scanpipe/base_htmx.html | 3 - .../templates/scanpipe/dependency_tree.html | 77 +------- .../scanpipe/includes/admin_edit_link.html | 2 +- .../scanpipe/includes/search_field.html | 1 + .../scanpipe/modals/add_inputs_modal.html | 2 +- .../scanpipe/modals/add_labels_modal.html | 2 +- .../scanpipe/modals/add_pipeline_modal.html | 4 +- .../scanpipe/modals/clone_modal.html | 4 +- .../scanpipe/modals/edit_input_tag_modal.html | 2 +- .../scanpipe/modals/project_delete_modal.html | 2 +- .../modals/project_inputs_delete_modal.html | 15 +- .../scanpipe/modals/project_reset_modal.html | 2 +- .../modals/project_webhook_add_modal.html | 4 +- .../modals/project_webhook_delete_modal.html | 11 +- .../modals/projects_archive_modal.html | 2 +- .../modals/projects_delete_modal.html | 2 +- scanpipe/templates/scanpipe/package_list.html | 17 -- .../scanpipe/panels/project_inputs.html | 4 +- .../templates/scanpipe/project_charts.html | 2 + .../templates/scanpipe/project_detail.html | 23 +-- scanpipe/templates/scanpipe/project_form.html | 143 +-------------- scanpipe/templates/scanpipe/project_list.html | 2 + .../templates/scanpipe/project_settings.html | 16 +- .../templates/scanpipe/resource_detail.html | 133 +------------- .../templates/scanpipe/resource_tree.html | 130 +------------- .../scanpipe/tabset/tab_vulnerabilities.html | 2 +- scanpipe/views.py | 6 + 36 files changed, 692 insertions(+), 564 deletions(-) rename scancodeio/static/{add-inputs.js => js/add_inputs.js} (100%) create mode 100644 scancodeio/static/js/dependency_tree.js rename scancodeio/static/{ => js}/main.js (89%) create mode 100644 scancodeio/static/js/project_form.js create mode 100644 scancodeio/static/js/resource_tree.js create mode 100644 scancodeio/static/js/resource_viewer.js diff --git a/scancodeio/context_processors.py b/scancodeio/context_processors.py index 41bdc4e5a0..25bb148fae 100644 --- a/scancodeio/context_processors.py +++ b/scancodeio/context_processors.py @@ -20,7 +20,6 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. -from django.conf import settings from scancode_config import __version__ as scancode_toolkit_version @@ -31,5 +30,4 @@ def versions(request): return { "SCANCODEIO_VERSION": scancodeio_version.lstrip("v"), "SCANCODE_TOOLKIT_VERSION": scancode_toolkit_version, - "VULNERABLECODE_URL": settings.VULNERABLECODE_URL, } diff --git a/scancodeio/static/add-inputs.js b/scancodeio/static/js/add_inputs.js similarity index 100% rename from scancodeio/static/add-inputs.js rename to scancodeio/static/js/add_inputs.js diff --git a/scancodeio/static/js/dependency_tree.js b/scancodeio/static/js/dependency_tree.js new file mode 100644 index 0000000000..343b953c50 --- /dev/null +++ b/scancodeio/static/js/dependency_tree.js @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// http://nexb.com and https://github.com/aboutcode-org/scancode.io +// The ScanCode.io software is licensed under the Apache License version 2.0. +// Data generated with ScanCode.io is provided as-is without warranties. +// ScanCode is a trademark of nexB Inc. +// +// You may not use this software except in compliance with the License. +// You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +// +// Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +// OR CONDITIONS OF ANY KIND, either express or implied. No content created from +// ScanCode.io should be considered or used as legal advice. Consult an Attorney +// for any legal advice. +// +// ScanCode.io is a free software code scanning tool from nexB Inc. and others. +// Visit https://github.com/aboutcode-org/scancode.io for support and download. + +'use strict'; + +(function() { + const treeContainer = document.getElementById('tree'); + const collapseAllButton = document.getElementById('collapseAll'); + const expendAllButton = document.getElementById('expendAll'); + + if (!collapseAllButton) return; + + function collapseAllDetails() { + document.querySelectorAll('details').forEach(details => { + details.removeAttribute('open'); + }); + showAllListItems(); + } + + function expendAllDetails() { + document.querySelectorAll('details').forEach(details => { + details.setAttribute('open', ''); // Adding 'open' attribute to open the details + }); + showAllListItems(); + } + + collapseAllButton.addEventListener('click', collapseAllDetails); + expendAllButton.addEventListener('click', expendAllDetails); + + // Following function are use to limit the display to specific elements. + + function expandAncestors(detailsElement) { + let parent = detailsElement.parentElement.closest('details'); + while (parent) { + parent.setAttribute('open', ''); + parent.parentElement.style.display = ''; + parent = parent.parentElement.closest('details'); + } + } + + function showAllListItems() { + const listItems = treeContainer.querySelectorAll('li'); + listItems.forEach(item => { + item.style.display = ''; + }); + } + + function hideAllListItems() { + const listItems = treeContainer.querySelectorAll('li'); + listItems.forEach(item => { + item.style.display = 'none'; + }); + } + + function handleItems(attribute, value) { + collapseAllDetails(); + hideAllListItems(); + + const items = document.querySelectorAll(`li[${attribute}="${value}"]`); + items.forEach(item => { + item.style.display = 'block'; + expandAncestors(item); + }); + } + + function handleVulnerableItems() { + handleItems('data-is-vulnerable', 'true'); + } + + function handleComplianceAlertItems() { + handleItems('data-compliance-alert', 'true'); + } + + showVulnerableOnlyButton.addEventListener('click', handleVulnerableItems); + showComplianceAlertOnlyButton.addEventListener('click', handleComplianceAlertItems); +})(); diff --git a/scancodeio/static/main.js b/scancodeio/static/js/main.js similarity index 89% rename from scancodeio/static/main.js rename to scancodeio/static/js/main.js index def511d7fe..a738951049 100644 --- a/scancodeio/static/main.js +++ b/scancodeio/static/js/main.js @@ -1,3 +1,25 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// http://nexb.com and https://github.com/aboutcode-org/scancode.io +// The ScanCode.io software is licensed under the Apache License version 2.0. +// Data generated with ScanCode.io is provided as-is without warranties. +// ScanCode is a trademark of nexB Inc. +// +// You may not use this software except in compliance with the License. +// You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +// +// Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +// OR CONDITIONS OF ANY KIND, either express or implied. No content created from +// ScanCode.io should be considered or used as legal advice. Consult an Attorney +// for any legal advice. +// +// ScanCode.io is a free software code scanning tool from nexB Inc. and others. +// Visit https://github.com/aboutcode-org/scancode.io for support and download. + // // Selected parts from https://bulma.io/lib/main.js?v=202104191409 // @@ -58,6 +80,26 @@ function setupCloseModalButtons() { } } +// Populate dynamic modals on open. +// - Sets form action from the button's data-url +// - Fills [data-label] elements from matching button data attributes +// Usage: diff --git a/scanpipe/templates/scanpipe/modals/project_inputs_delete_modal.html b/scanpipe/templates/scanpipe/modals/project_inputs_delete_modal.html index 61b5f6eff4..87a27c25db 100644 --- a/scanpipe/templates/scanpipe/modals/project_inputs_delete_modal.html +++ b/scanpipe/templates/scanpipe/modals/project_inputs_delete_modal.html @@ -10,7 +10,7 @@ This action cannot be undone.

    - This will delete the file from the project inputs. + This will delete the file from the project inputs.

    Are you sure you want to do this? @@ -25,15 +25,4 @@ - - \ No newline at end of file + \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/modals/project_reset_modal.html b/scanpipe/templates/scanpipe/modals/project_reset_modal.html index ade682ca12..4b4244e985 100644 --- a/scanpipe/templates/scanpipe/modals/project_reset_modal.html +++ b/scanpipe/templates/scanpipe/modals/project_reset_modal.html @@ -5,7 +5,7 @@

    Reset this project, are you sure?

    -
    {% csrf_token %} + {% csrf_token %}
    This action cannot be undone. diff --git a/scanpipe/templates/scanpipe/modals/project_webhook_add_modal.html b/scanpipe/templates/scanpipe/modals/project_webhook_add_modal.html index e091c13480..2e3dcfb4f5 100644 --- a/scanpipe/templates/scanpipe/modals/project_webhook_add_modal.html +++ b/scanpipe/templates/scanpipe/modals/project_webhook_add_modal.html @@ -8,7 +8,9 @@ + hx-on::after-settle="removeOverlay()" + data-submit-overlay + >
    {% include 'scanpipe/forms/project_webhook_form.html' with form=webhook_form %}
    diff --git a/scanpipe/templates/scanpipe/modals/project_webhook_delete_modal.html b/scanpipe/templates/scanpipe/modals/project_webhook_delete_modal.html index d081311fc8..1ec3c8066d 100644 --- a/scanpipe/templates/scanpipe/modals/project_webhook_delete_modal.html +++ b/scanpipe/templates/scanpipe/modals/project_webhook_delete_modal.html @@ -25,13 +25,4 @@
    - - \ No newline at end of file + \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/modals/projects_archive_modal.html b/scanpipe/templates/scanpipe/modals/projects_archive_modal.html index 4efade13b0..464c584a60 100644 --- a/scanpipe/templates/scanpipe/modals/projects_archive_modal.html +++ b/scanpipe/templates/scanpipe/modals/projects_archive_modal.html @@ -6,7 +6,7 @@

    Archive selected projects, are you sure?

    -
    {% csrf_token %} + {% csrf_token %}
    The selected projects will be marked as archived and will become read-only. diff --git a/scanpipe/templates/scanpipe/modals/projects_delete_modal.html b/scanpipe/templates/scanpipe/modals/projects_delete_modal.html index de1f22f85e..7407cf056a 100644 --- a/scanpipe/templates/scanpipe/modals/projects_delete_modal.html +++ b/scanpipe/templates/scanpipe/modals/projects_delete_modal.html @@ -21,7 +21,7 @@ related database entries and all data on disk except for the input/ directories.

    - {% csrf_token %} + {% csrf_token %}
    {% endif %} +{% endblock %} +{% block modals %} {% include 'scanpipe/modals/run_modal.html' %} {% include 'scanpipe/modals/clone_modal.html' %} {% include "scanpipe/modals/add_labels_modal.html" %} @@ -208,28 +211,14 @@ {% endblock %} {% block scripts %} - {% endblock %} \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/project_form.html b/scanpipe/templates/scanpipe/project_form.html index deb8236428..062d00d86a 100644 --- a/scanpipe/templates/scanpipe/project_form.html +++ b/scanpipe/templates/scanpipe/project_form.html @@ -1,4 +1,9 @@ {% extends "scanpipe/base.html" %} +{% load static %} + +{% block extrahead %} + +{% endblock %} {% block content %}
    @@ -97,144 +102,12 @@

    Pipelines:

    {% endfor %} - {% include "scanpipe/modals/pipeline_help_modal.html" %} + {{ pipelines_available_groups|json_script:"pipelines_available_groups" }} {% endblock %} -{% block scripts %} - - - {{ pipelines_available_groups|json_script:"pipelines_available_groups" }} - - - +{% block modals %} + {% include "scanpipe/modals/pipeline_help_modal.html" %} {% endblock %} \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/project_list.html b/scanpipe/templates/scanpipe/project_list.html index 6871206137..9575f16bf4 100644 --- a/scanpipe/templates/scanpipe/project_list.html +++ b/scanpipe/templates/scanpipe/project_list.html @@ -56,7 +56,9 @@ {% endif %} +{% endblock %} +{% block modals %} {% include 'scanpipe/modals/run_modal.html' %} {% include "scanpipe/modals/projects_download_modal.html" %} {% include "scanpipe/modals/projects_report_modal.html" %} diff --git a/scanpipe/templates/scanpipe/project_settings.html b/scanpipe/templates/scanpipe/project_settings.html index 1651f7978a..e83792596f 100644 --- a/scanpipe/templates/scanpipe/project_settings.html +++ b/scanpipe/templates/scanpipe/project_settings.html @@ -21,7 +21,9 @@ +{% endblock %} +{% block modals %} {% include "scanpipe/modals/project_webhook_add_modal.html" %} {% include "scanpipe/modals/project_webhook_delete_modal.html" %} {% if not project.is_archived %} @@ -42,19 +44,5 @@ window.location.reload(); } }); - - onSubmitOverlay = function (selector) { - let element = document.querySelector(selector); - if (element) { - element.addEventListener("submit", function() { - displayOverlay(); - }); - } - }; - - onSubmitOverlay("#modal-archive form"); - onSubmitOverlay("#modal-reset form"); - onSubmitOverlay("#modal-delete form"); - onSubmitOverlay("#modal-webhook-add form"); {% endblock %} \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/resource_detail.html b/scanpipe/templates/scanpipe/resource_detail.html index 58c23e0f55..8aebda40b5 100644 --- a/scanpipe/templates/scanpipe/resource_detail.html +++ b/scanpipe/templates/scanpipe/resource_detail.html @@ -4,8 +4,9 @@ {% block title %}ScanCode.io: {{ project.name }} - {{ object.name }}{% endblock %} {% block extrahead %} - - + + + {% endblock %} {% block content %} @@ -19,134 +20,8 @@ {% include 'scanpipe/tree/resource_path_breadcrumb.html' with path=object.path %} {% endif %} {% include 'scanpipe/tabset/tabset.html' %} + {{ detected_values|json_script:"detected_values" }} {% endblock %} -{% endblock %} - -{% block scripts %} - {{ detected_values|json_script:"detected_values" }} - {% endblock %} \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/resource_tree.html b/scanpipe/templates/scanpipe/resource_tree.html index 24d2bea087..8c7cad2e3c 100644 --- a/scanpipe/templates/scanpipe/resource_tree.html +++ b/scanpipe/templates/scanpipe/resource_tree.html @@ -3,8 +3,10 @@ {% block title %}ScanCode.io: {{ project.name }} - Resources tree{% endblock %} {% block extrahead %} - - + + + + {% endblock %} {% block content %} @@ -17,7 +19,7 @@ -
    +
    {% include 'scanpipe/tree/resource_left_pane.html' %} @@ -38,126 +40,4 @@
    -{% endblock %} - -{% block scripts %} - {% endblock %} \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/tabset/tab_vulnerabilities.html b/scanpipe/templates/scanpipe/tabset/tab_vulnerabilities.html index 0d7a5e4053..e9d296b77d 100644 --- a/scanpipe/templates/scanpipe/tabset/tab_vulnerabilities.html +++ b/scanpipe/templates/scanpipe/tabset/tab_vulnerabilities.html @@ -11,7 +11,7 @@ {% for vulnerability in tab_data.fields.affected_by_vulnerabilities.value %} - {% include 'scanpipe/includes/vulnerability_id.html' with vulnerability=vulnerability %} + {% include 'scanpipe/includes/vulnerability_id.html' with vulnerability=vulnerability VULNERABLECODE_URL=tab_data.VULNERABLECODE_URL %} {% include 'scanpipe/includes/vulnerability_summary.html' with vulnerability=vulnerability only %} diff --git a/scanpipe/views.py b/scanpipe/views.py index 2e1ac5f362..e5fd2bca2b 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -260,6 +260,7 @@ class TabSetMixin: "icon_class": "", "display_condition": , "disable_condition": , + "tab_context": {}, } } """ @@ -291,6 +292,8 @@ def get_tab_data(self, tab_definition): elif fields_have_no_values(fields_data): is_disabled = True + tab_context = tab_definition.get("tab_context") or {} + tab_data = { "verbose_name": tab_definition.get("verbose_name"), "icon_class": tab_definition.get("icon_class"), @@ -298,6 +301,7 @@ def get_tab_data(self, tab_definition): "fields": fields_data, "disabled": is_disabled, "label_count": self.get_label_count(fields_data), + **tab_context, } return tab_data @@ -1972,6 +1976,7 @@ def get_queryset(self): def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context["object_list"] = self.project.vulnerabilities + context["VULNERABLECODE_URL"] = settings.VULNERABLECODE_URL return context @@ -2336,6 +2341,7 @@ class DiscoveredPackageDetailsView( ], "icon_class": "fa-solid fa-bug", "template": "scanpipe/tabset/tab_vulnerabilities.html", + "tab_context": {"VULNERABLECODE_URL": settings.VULNERABLECODE_URL}, }, "extra_data": { "fields": ["extra_data"], From e1a47cd574c967790f55da044a927dd01282f5ac Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Wed, 18 Mar 2026 18:42:28 +0400 Subject: [PATCH 056/110] feat: remove the need for context processor and improve base template (#2106) Signed-off-by: tdruez --- scancodeio/settings.py | 1 - scanpipe/templates/account/profile.html | 3 --- scanpipe/templates/registration/login.html | 3 --- .../templates/scanpipe/app_monitoring.html | 23 ------------------- scanpipe/templates/scanpipe/base.html | 6 +++++ .../templates/scanpipe/dependency_detail.html | 2 -- .../templates/scanpipe/dependency_list.html | 1 - .../templates/scanpipe/dependency_tree.html | 1 - .../scanpipe/includes/navbar_header.html | 3 +++ .../templates/scanpipe/license_detail.html | 2 -- .../scanpipe/license_detection_detail.html | 2 -- .../scanpipe/license_detection_list.html | 1 - scanpipe/templates/scanpipe/license_list.html | 1 - scanpipe/templates/scanpipe/message_list.html | 1 - .../templates/scanpipe/package_detail.html | 4 +--- scanpipe/templates/scanpipe/package_list.html | 1 - .../templates/scanpipe/project_detail.html | 3 --- scanpipe/templates/scanpipe/project_form.html | 2 -- scanpipe/templates/scanpipe/project_list.html | 3 --- .../templates/scanpipe/project_settings.html | 2 -- .../templates/scanpipe/relation_list.html | 1 - .../templates/scanpipe/resource_detail.html | 2 -- .../templates/scanpipe/resource_list.html | 1 - .../templates/scanpipe/resource_tree.html | 1 - .../scanpipe/vulnerability_list.html | 1 - scanpipe/templatetags/__init__.py | 0 .../templatetags/scanpipe_tags.py | 18 +++++++++++---- scanpipe/urls.py | 2 -- 28 files changed, 23 insertions(+), 68 deletions(-) delete mode 100644 scanpipe/templates/scanpipe/app_monitoring.html create mode 100644 scanpipe/templatetags/__init__.py rename scancodeio/context_processors.py => scanpipe/templatetags/scanpipe_tags.py (79%) diff --git a/scancodeio/settings.py b/scancodeio/settings.py index 9a94a13383..b82fff243c 100644 --- a/scancodeio/settings.py +++ b/scancodeio/settings.py @@ -252,7 +252,6 @@ "django.contrib.auth.context_processors.auth", "django.contrib.messages.context_processors.messages", "django.template.context_processors.request", - "scancodeio.context_processors.versions", ], }, }, diff --git a/scanpipe/templates/account/profile.html b/scanpipe/templates/account/profile.html index f71a732cd1..a2f7eeb8f2 100644 --- a/scanpipe/templates/account/profile.html +++ b/scanpipe/templates/account/profile.html @@ -2,9 +2,6 @@ {% block content %}
    - {% include 'scanpipe/includes/navbar_header.html' %} -
    {% include 'scanpipe/includes/messages.html' %}
    -