From 83c10dba5d45eba3f52e16a366f74a12b0f5043c Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 22 Apr 2026 03:41:41 +0200 Subject: [PATCH 01/25] feat: create pipeline for symbol reachability and add a test Signed-off-by: ziad hany --- pyproject.toml | 1 + .../pipelines/collect_symbols_reachability.py | 35 + scanpipe/pipes/reachability.py | 730 ++++++++++++++++++ scanpipe/pipes/symbols.py | 155 ++++ scanpipe/tests/data/reachability/app.py | 35 + .../tests/data/reachability/diff-app.patch | 39 + scanpipe/tests/data/reachability/fixed-app.py | 41 + scanpipe/tests/data/reachability/vuln-app.py | 35 + .../tests/pipes/test_symbols_reachability.py | 293 +++++++ 9 files changed, 1364 insertions(+) create mode 100644 scanpipe/pipelines/collect_symbols_reachability.py create mode 100644 scanpipe/pipes/reachability.py create mode 100644 scanpipe/tests/data/reachability/app.py create mode 100644 scanpipe/tests/data/reachability/diff-app.patch create mode 100644 scanpipe/tests/data/reachability/fixed-app.py create mode 100644 scanpipe/tests/data/reachability/vuln-app.py create mode 100644 scanpipe/tests/pipes/test_symbols_reachability.py diff --git a/pyproject.toml b/pyproject.toml index 350c59a0b7..4b3f331a1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -151,6 +151,7 @@ run = "scancodeio:combined_run" analyze_docker_image = "scanpipe.pipelines.analyze_docker:Docker" analyze_root_filesystem_or_vm_image = "scanpipe.pipelines.analyze_root_filesystem:RootFS" analyze_windows_docker_image = "scanpipe.pipelines.analyze_docker_windows:DockerWindows" +analyze_symbols_reachability = "scanpipe.pipelines.collect_symbols_reachability:SymbolReachability" benchmark_purls = "scanpipe.pipelines.benchmark_purls:BenchmarkPurls" collect_strings_gettext = "scanpipe.pipelines.collect_strings_gettext:CollectStringsGettext" collect_symbols_ctags = "scanpipe.pipelines.collect_symbols_ctags:CollectSymbolsCtags" diff --git a/scanpipe/pipelines/collect_symbols_reachability.py b/scanpipe/pipelines/collect_symbols_reachability.py new file mode 100644 index 0000000000..15519fc661 --- /dev/null +++ b/scanpipe/pipelines/collect_symbols_reachability.py @@ -0,0 +1,35 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from scanpipe.pipelines import Pipeline +from scanpipe.pipes import reachability + + +class SymbolReachability(Pipeline): + """ + Patch reachability analysis, for given a vulnerability patches + """ + + download_inputs = False + is_addon = True + results_url = "/project/{slug}/resources/?extra_data=symbol_reachability" + + @classmethod + def steps(cls): + return (cls.analyze_and_store_symbol_reachability,) + + def analyze_and_store_symbol_reachability(self): + """ + Perform symbol-level reachability analysis for each patch. + This step compares the AST of patched/vulnerable files against the codebase resources. + Results are stored directly in the 'extra_data' of each CodebaseResource. + """ + reachability.collect_and_store_symbol_reachability_results( + project=self.project, logger=self.log + ) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py new file mode 100644 index 0000000000..5531a1f495 --- /dev/null +++ b/scanpipe/pipes/reachability.py @@ -0,0 +1,730 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + +import os +import shutil +import tempfile +from enum import Enum +from pathlib import Path + +from git import Repo +from git.diff import NULL_TREE +from git.exc import BadName +from matchcode_toolkit.fingerprinting import create_file_fingerprints +from scancode.api import get_file_info +from unidiff import PatchSet + +from scanpipe.pipes.symbols import TS_QUERIES +from scanpipe.pipes.symbols import _root_of +from scanpipe.pipes.symbols import collect_definitions +from scanpipe.pipes.symbols import extract_calls_in_node +from scanpipe.pipes.symbols import extract_definitions +from scanpipe.pipes.symbols import extract_symbols +from scanpipe.pipes.symbols import parse_code_to_ast +from scanpipe.pipes.symbols import qualified_name_from_index + +EMPTY_TREE_SHA = "4b825dc642cb6eb9a060e54bf8b8e6f9b79b4d2b" + + +class ReachabilityStatus(str, Enum): + REACHABLE = "REACHABLE" + POTENTIALLY_REACHABLE = "POTENTIALLY_REACHABLE" + NOT_REACHABLE = "NOT_REACHABLE" + + +def api_mocker(): + """ + TODO: Remove this once the API patch url is done + """ + return [ + { + "vcs_url": "https://github.com/pallets/flask", + "commit_hash": "089cb86dd22bff589a4eafb7ab8e42dc357623b4", + }, + ] + + +def clone_repo(vcs_url, commit_hash=None): + repo_path = tempfile.mkdtemp(prefix="symbol-reachability-") + + try: + repo = Repo.clone_from(vcs_url, repo_path) + + if commit_hash: + repo.git.checkout(commit_hash) + + return repo_path + + except BadName as exc: + cleanup_repo(repo_path) + raise ValueError(f"Commit {commit_hash} not found") from exc + + except Exception: + cleanup_repo(repo_path) + raise + + +def cleanup_repo(repo_path): + if repo_path and os.path.exists(repo_path): + shutil.rmtree(repo_path, ignore_errors=True) + + +def normalize_text(content): + if content is None: + return "" + + if isinstance(content, bytes): + return content.decode("utf-8", errors="replace") + + return str(content) + + +def is_supported_language(language): + """A language is supported if we have tree-sitter queries for it.""" + return bool(language) and language in TS_QUERIES + + +def detect_language_with_scancode(file_path, content): + """ + Write `content` to a temp file preserving `file_path`'s basename + so the extension is meaningful, then ask ScanCode's `get_file_info` + to return the programming language. + """ + content = normalize_text(content) + + if not content: + return None + + tmp_dir = tempfile.mkdtemp(prefix="patch-lang-") + + try: + target = Path(tmp_dir) / Path(file_path).name + target.write_text(content, encoding="utf-8", errors="replace") + + info = get_file_info(location=str(target)) or {} + return info.get("programming_language") or None + + finally: + shutil.rmtree(tmp_dir, ignore_errors=True) + + +def get_commit_and_parent(repo, commit_hash): + commit = repo.commit(commit_hash) + parent = commit.parents[0] if commit.parents else None + return commit, parent + + +def get_commit_diff_text(repo, parent_commit, commit): + """Whole-commit unified diff (used to extract changed line numbers).""" + base = parent_commit.hexsha if parent_commit else EMPTY_TREE_SHA + return repo.git.diff(base, commit.hexsha, unified=0) + + +def get_changed_files(parent_commit, commit): + """ + Return: + { + file_path: { + "vulnerable_text": "...", + "fixed_text": "...", + } + } + + """ + diffs = ( + parent_commit.diff(commit, create_patch=False) + if parent_commit + else commit.diff(NULL_TREE, create_patch=False) + ) + + files = {} + for diff in diffs: + change_type = diff.change_type + old_path = diff.a_path if change_type in ("D", "M", "R") else None + new_path = diff.b_path if change_type in ("A", "M", "R") else None + path_key = new_path or old_path + + if not path_key: + continue + + entry = files.setdefault( + path_key, + { + "vulnerable_text": "", + "fixed_text": "", + }, + ) + + if old_path and parent_commit: + entry["vulnerable_text"] = ( + (parent_commit.tree / old_path) + .data_stream.read() + .decode("utf-8", errors="replace") + ) + + if new_path: + entry["fixed_text"] = ( + (commit.tree / new_path) + .data_stream.read() + .decode("utf-8", errors="replace") + ) + + return files + + +def get_changed_lines(diff_text, file_path): + """Return `(removed_lines, added_lines)` for one file from a unified diff.""" + removed = [] + added = [] + + if not diff_text: + return removed, added + + for patched_file in PatchSet.from_string(diff_text): + candidates = { + patched_file.path, + (patched_file.source_file or "").removeprefix("a/"), + (patched_file.target_file or "").removeprefix("b/"), + } + + if file_path not in candidates: + continue + + for hunk in patched_file: + for line in hunk: + if line.is_removed and line.source_line_no: + removed.append(line.source_line_no) + elif line.is_added and line.target_line_no: + added.append(line.target_line_no) + + return removed, added + + +def query_captures(language, kind, node): + """ + Re-run a definition query on the root of `node`'s tree so ancestors can + be compared. Query caching is handled by `scanpipe.pipes.symbols`. + """ + from scanpipe.pipes.symbols import get_query + from scanpipe.pipes.symbols import run_query + + root = node + + while root.parent is not None: + root = root.parent + + query = get_query(language, kind) + return list(run_query(query, root)) + + +def is_nested_function(node, language): + function_nodes = { + captured_node + for captured_node, _ in query_captures(language, "functions", node) + } + class_nodes = { + captured_node for captured_node, _ in query_captures(language, "classes", node) + } + + if node not in function_nodes: + return False + + function_types = {captured_node.type for captured_node in function_nodes} + class_types = {captured_node.type for captured_node in class_nodes} + + parent = node.parent + + while parent is not None: + if parent.type in function_types: + return True + + if parent.type in class_types: + return False + + parent = parent.parent + + return False + + +def diff_changed_symbols(vuln_meta, fixed_meta): + """ + Keep only symbols whose body actually differs between vulnerable and fixed + versions. Pair by qualified name first. + """ + fixed_by_qn = { + metadata["qualified_name"]: metadata for metadata in fixed_meta.values() + } + + vuln_by_qn = { + metadata["qualified_name"]: metadata for metadata in vuln_meta.values() + } + + vuln_only = { + key: metadata + for key, metadata in vuln_meta.items() + if fixed_by_qn.get(metadata["qualified_name"], {}).get("text") + != metadata["text"] + } + + fixed_only = { + key: metadata + for key, metadata in fixed_meta.items() + if vuln_by_qn.get(metadata["qualified_name"], {}).get("text") + != metadata["text"] + } + + return vuln_only, fixed_only + + +def analyze_patched_file(vulnerable_text, fixed_text, diff_text, file_path): + """ + Return `(vuln_metadata, fixed_metadata, language)` for one changed file, + restricted to symbols actually touched by the patch. + """ + vulnerable_text = normalize_text(vulnerable_text) + fixed_text = normalize_text(fixed_text) + + language = detect_language_with_scancode( + file_path, fixed_text + ) or detect_language_with_scancode(file_path, vulnerable_text) + + if not is_supported_language(language): + return {}, {}, language + + vuln_tree, _ = ( + parse_code_to_ast(vulnerable_text, language) + if vulnerable_text + else (None, None) + ) + + fixed_tree, _ = ( + parse_code_to_ast(fixed_text, language) if fixed_text else (None, None) + ) + + if vuln_tree is None and fixed_tree is None: + return {}, {}, language + + removed_lines, added_lines = get_changed_lines(diff_text, file_path) + + vuln_nodes = ( + extract_symbols(vuln_tree, removed_lines, language) if vuln_tree else [] + ) + + fixed_nodes = ( + extract_symbols(fixed_tree, added_lines, language) if fixed_tree else [] + ) + + vuln_meta, fixed_meta = diff_changed_symbols( + build_symbol_metadata(vuln_nodes, language), + build_symbol_metadata(fixed_nodes, language), + ) + + return vuln_meta, fixed_meta, language + + +def collect_patch_symbols(repo, commit_hash): + """ + Return: + { + language: { + "vulnerable": { + "file_path::symbol_key": metadata, + ... + }, + "fixed": { + "file_path::symbol_key": metadata, + ... + }, + }, + ... + } + + Symbols are bucketed by language so resources are only matched against + patch symbols extracted from the same language. + + """ + commit, parent = get_commit_and_parent(repo, commit_hash) + diff_text = get_commit_diff_text(repo, parent, commit) + changed = get_changed_files(parent, commit) + + by_language = {} + for file_path, texts in changed.items(): + vulnerable_text = texts["vulnerable_text"] + fixed_text = texts["fixed_text"] + vuln_meta, fixed_meta, language = analyze_patched_file( + vulnerable_text=vulnerable_text, + fixed_text=fixed_text, + diff_text=diff_text, + file_path=file_path, + ) + + if not language or not (vuln_meta or fixed_meta): + continue + + language_bucket = by_language.setdefault( + language, + { + "vulnerable": {}, + "fixed": {}, + }, + ) + + language_bucket["vulnerable"].update( + {f"{file_path}::{key}": metadata for key, metadata in vuln_meta.items()} + ) + + language_bucket["fixed"].update( + {f"{file_path}::{key}": metadata for key, metadata in fixed_meta.items()} + ) + + return by_language + + +def append_symbol_reachability_result(resource, result): + """ + Append one symbol reachability result to the resource extra_data without + overwriting previous results. + """ + extra_data = resource.extra_data or {} + existing_results = extra_data.get("symbols_reachability", []) + + if not isinstance(existing_results, list): + existing_results = [existing_results] + + existing_results.append(result) + + resource.update_extra_data( + { + "symbols_reachability": existing_results, + } + ) + + +def collect_and_store_symbol_reachability_results(project, logger=None): + """ + For each known patch commit, determine whether each project codebase + resource is reachable to the vulnerable code by comparing tree-sitter ASTs + of the patch versus the resource. + + Result classification: + - REACHABLE + - POTENTIALLY_REACHABLE + - NOT_REACHABLE + """ + candidate_resources = project.codebaseresources.files().filter( + is_binary=False, + is_archive=False, + is_media=False, + ) + + for patch in api_mocker(): + vcs_url = patch["vcs_url"] + commit_hash = patch["commit_hash"] + repo_path = None + try: + repo_path = clone_repo(vcs_url, commit_hash) + repo = Repo(repo_path) + + patch_symbols_by_language = collect_patch_symbols(repo, commit_hash) + + if not patch_symbols_by_language: + continue + + for resource in candidate_resources: + resource_language = resource.programming_language + + if resource_language not in patch_symbols_by_language: + continue + + resource_text = normalize_text(resource.file_content) + + if not resource_text: + continue + + patch_symbols = patch_symbols_by_language[resource_language] + vuln_metadata = patch_symbols["vulnerable"] + fixed_metadata = patch_symbols["fixed"] + + resource_index = build_resource_index( + resource_text, + resource_language, + ) + + if not resource_index: + continue + + vuln_match_symbols = match_symbols_against_resource( + vuln_metadata, + resource_index, + ) + + fixed_match_symbols = match_symbols_against_resource( + fixed_metadata, + resource_index, + ) + + if not vuln_match_symbols and not fixed_match_symbols: + continue + + result = { + "reachability_status": classify_reachability(vuln_match_symbols), + "summary": { + "vulnerable_symbols": sorted(vuln_match_symbols), + "fixed_symbols": sorted(fixed_match_symbols), + "call_paths": { + qn: ev.get("reachable_from", []) + for qn, ev in vuln_match_symbols.items() + if ev.get("called") + }, + }, + "evidence": vuln_match_symbols, + "patch": { + "vcs_url": vcs_url, + "commit_hash": commit_hash, + }, + } + + append_symbol_reachability_result(resource, result) + + except Exception as e: + logger.exception( + "Failed to collect symbol reachability for " + f"{vcs_url}@{commit_hash}: {e}" + ) + finally: + cleanup_repo(repo_path) + + +def compute_reachable_symbols(call_graph, target_simple_names): + if not call_graph or not target_simple_names: + return set(), False + + edges = call_graph["edges"] + targets = set(target_simple_names) + + callers_of = {} + for caller_qn, callees in edges.items(): + for callee_simple in callees: + callers_of.setdefault(callee_simple, set()).add(caller_qn) + + direct_callers = set() + for target in targets: + direct_callers |= callers_of.get(target, set()) + + has_direct_call = bool(direct_callers) + + by_simple = call_graph["by_simple_name"] + qn_to_simple = {qn: meta["simple_name"] for qn, meta in call_graph["nodes"].items()} + + reachable = set(direct_callers) + frontier = list(direct_callers) + + while frontier: + current_qn = frontier.pop() + current_simple = qn_to_simple.get(current_qn) + if not current_simple: + continue + for parent_qn in callers_of.get(current_simple, ()): + if parent_qn not in reachable: + reachable.add(parent_qn) + frontier.append(parent_qn) + + return reachable, has_direct_call + + +def build_resource_index(resource_text, language): + resource_text = normalize_text(resource_text) + + if not is_supported_language(language) or not resource_text: + return None + + tree, _ = parse_code_to_ast(resource_text, language) + + if tree is None: + return None + + call_graph = build_call_graph(tree, language) + + meta = ( + call_graph["nodes"] + if call_graph + else build_symbol_metadata( + extract_definitions(tree, language), + language, + ) + ) + + return { + "definitions": {metadata["qualified_name"] for metadata in meta.values()}, + "fingerprints": { + metadata["fingerprint"] + for metadata in meta.values() + if metadata["fingerprint"] + }, + "call_graph": call_graph, + } + + +def match_symbols_against_resource(symbols, resource_index): + if not symbols or not resource_index: + return {} + + call_graph = resource_index.get("call_graph") + target_simple_names = {metadata["simple_name"] for metadata in symbols.values()} + + reachable_callers, _has_direct = compute_reachable_symbols( + call_graph, + target_simple_names, + ) + + called_simple_names = set() + if call_graph: + for callees in call_graph["edges"].values(): + called_simple_names |= callees + + matched = {} + for metadata in symbols.values(): + qualified_name = metadata["qualified_name"] + simple_name = metadata["simple_name"] + fingerprint = metadata["fingerprint"] + + defined = qualified_name in resource_index["definitions"] + fingerprint_hit = bool( + fingerprint and fingerprint in resource_index["fingerprints"] + ) + called = simple_name in called_simple_names + + if not (defined or fingerprint_hit or called): + continue + + entry = matched.setdefault( + qualified_name, + { + "defined": False, + "called": False, + "reachable_from": [], + }, + ) + + entry["defined"] = entry["defined"] or defined + entry["called"] = entry["called"] or called + + if fingerprint_hit: + entry["exact_match_fingerprint"] = fingerprint + + if called: + entry["reachable_from"] = sorted(reachable_callers) + + return matched + + +def classify_reachability(evidence): + if not evidence: + return ReachabilityStatus.NOT_REACHABLE + + SEVERITY_RANK = { + ReachabilityStatus.NOT_REACHABLE: 0, + ReachabilityStatus.POTENTIALLY_REACHABLE: 1, + ReachabilityStatus.REACHABLE: 2, + } + + highest_status = ReachabilityStatus.NOT_REACHABLE + + for item in evidence.values(): + is_called = bool(item.get("called")) + has_path = bool(item.get("reachable_from")) + is_exact = "exact_match_fingerprint" in item + is_defined = bool(item.get("defined")) + + if is_called or (has_path and is_exact): + return ReachabilityStatus.REACHABLE + + elif has_path or is_exact or is_defined: + current_item_status = ReachabilityStatus.POTENTIALLY_REACHABLE + + else: + current_item_status = ReachabilityStatus.NOT_REACHABLE + + if SEVERITY_RANK[current_item_status] > SEVERITY_RANK[highest_status]: + highest_status = current_item_status + return highest_status + + +def build_symbol_metadata(nodes, language, index=None): + if index is None and nodes: + index = collect_definitions(_root_of(nodes[0]), language) + + metadata = {} + for node in nodes: + if is_nested_function(node, language): + continue + + qualified_name = qualified_name_from_index(node, index) + if not qualified_name: + continue + + body_text = node.text.decode("utf-8", errors="replace") + fingerprints = create_file_fingerprints(content=body_text) or {} + + key = qualified_name + suffix = 1 + while key in metadata: + suffix += 1 + key = f"{qualified_name}#{suffix}" + + metadata[key] = { + "qualified_name": qualified_name, + "simple_name": qualified_name.rsplit(".", 1)[-1], + "text": body_text, + "fingerprint": fingerprints.get("halo1"), + "start_line": node.start_point[0] + 1, + "end_line": node.end_point[0] + 1, + "node_type": node.type, + } + return metadata + + +def build_call_graph(tree, language): + if tree is None or not is_supported_language(language): + return None + + index = collect_definitions(tree.root_node, language) + definition_nodes = [d["node"] for d in index.values()] + metadata = build_symbol_metadata(definition_nodes, language, index=index) + + qualified_name_to_node = {} + for node in definition_nodes: + qualified_name = qualified_name_from_index(node, index) + if qualified_name: + qualified_name_to_node.setdefault(qualified_name, node) + + edges = {} + by_simple_name = {} + for qualified_name, meta in metadata.items(): + canonical = meta["qualified_name"] + node = qualified_name_to_node.get(canonical) + if node is None: + continue + edges.setdefault(canonical, set()).update(extract_calls_in_node(node, language)) + by_simple_name.setdefault(meta["simple_name"], set()).add(canonical) + + return {"nodes": metadata, "edges": edges, "by_simple_name": by_simple_name} diff --git a/scanpipe/pipes/symbols.py b/scanpipe/pipes/symbols.py index 76493d8dac..c6247dbc44 100644 --- a/scanpipe/pipes/symbols.py +++ b/scanpipe/pipes/symbols.py @@ -20,8 +20,20 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. +import importlib +from functools import cache + from django.db.models import Q +from source_inspector import symbols_ctags +from source_inspector import symbols_pygments +from source_inspector import symbols_tree_sitter +from source_inspector.symbols_tree_sitter import TS_LANGUAGE_WHEELS +from source_inspector.symbols_tree_sitter import TreeSitterWheelNotInstalled +from tree_sitter import Language +from tree_sitter import Parser +from tree_sitter import Query + from aboutcode.pipeline import LoopProgress @@ -171,3 +183,146 @@ def _collect_and_store_tree_sitter_symbols_and_strings(resource): "source_strings": result.get("source_strings"), } ) + + +SYMBOLS_TYPE_SUPPORTED = { + "ctags": symbols_ctags.get_symbols, + "tree_sitter": symbols_tree_sitter.get_treesitter_symbols, + "pygments": symbols_pygments.get_pygments_symbols, +} + +# https://github.com/Aider-AI/aider/tree/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/queries/tree-sitter-language-pack +TS_QUERIES = { + "Python": { + "functions": """ + (function_definition name: (identifier) @name) @function + """, + "classes": """ + (class_definition name: (identifier) @name) @class + """, + "calls": """ + (call function: (identifier) @callee) + (call function: (attribute attribute: (identifier) @callee)) + """, + }, +} + +@cache +def load_language(language: str) -> Language: + if language not in TS_LANGUAGE_WHEELS: + raise ValueError(f"Unsupported language: {language}") + + wheel = TS_LANGUAGE_WHEELS[language]["wheel"] + try: + grammar = importlib.import_module(wheel) + except ModuleNotFoundError as exc: + raise TreeSitterWheelNotInstalled( + f"Grammar wheel '{wheel}' is not installed." + ) from exc + return Language(grammar.language()) + + +@cache +def get_query(language: str, kind: str) -> Query | None: + source = TS_QUERIES.get(language, {}).get(kind, "").strip() + if not source: + return None + return Query(load_language(language), source) + + +def parse_code_to_ast(code_text: str, language: str): + if not code_text or not language or language not in TS_LANGUAGE_WHEELS: + return None, None + + ts_language = load_language(language) + parser = Parser(language=ts_language) + return parser.parse(code_text.encode("utf-8")), TS_LANGUAGE_WHEELS[language] + + +def run_query(query: Query, root_node): + """Yield ``(definition_node, name)`` pairs for function/class queries.""" + if query is None: + return + + for _pattern_index, captures in query.matches(root_node): + def_nodes = captures.get("function") or captures.get("class") or [] + if not def_nodes: + continue + + name_nodes = captures.get("name") or [] + name = ( + name_nodes[0].text.decode("utf-8", errors="replace") if name_nodes else None + ) + yield def_nodes[0], name + + +def extract_calls_in_node(node, language: str) -> set[str]: + query = get_query(language, "calls") + if query is None or node is None: + return set() + + names = set() + for _pattern_index, captures in query.matches(node): + for callee_node in captures.get("callee", []): + name = callee_node.text.decode("utf-8", errors="replace") + if name: + names.add(name) + return names + + +def collect_definitions(root_node, language: str) -> dict[int, dict]: + index: dict[int, dict] = {} + for kind in ("functions", "classes"): + query = get_query(language, kind) + for node, name in run_query(query, root_node): + index[node.id] = {"node": node, "name": name, "kind": kind} + return index + + +def extract_definitions(tree, language: str, kinds=("functions", "classes")): + if tree is None: + return [] + index = collect_definitions(tree.root_node, language) + return [d["node"] for d in index.values() if d["kind"] in kinds] + + +def extract_symbols(tree, changed_lines: list[int], language: str): + if tree is None or not changed_lines: + return [] + + definition_ids = set(collect_definitions(tree.root_node, language).keys()) + if not definition_ids: + return [] + + seen = set() + enclosing = [] + + for line in changed_lines: + row = max(0, line - 1) + node = tree.root_node.descendant_for_point_range((row, 0), (row, 0)) + + while node is not None: + if node.id in definition_ids and node.id not in seen: + seen.add(node.id) + enclosing.append(node) + break + node = node.parent + + return enclosing + + +def _root_of(node): + while node.parent is not None: + node = node.parent + return node + + +def qualified_name_from_index(node, index: dict[int, dict]) -> str: + parts = [] + curr = node + while curr is not None: + definition = index.get(curr.id) + if definition is not None and definition["name"]: + parts.append(definition["name"]) + curr = curr.parent + return ".".join(reversed(parts)) diff --git a/scanpipe/tests/data/reachability/app.py b/scanpipe/tests/data/reachability/app.py new file mode 100644 index 0000000000..c64ae7d9d1 --- /dev/null +++ b/scanpipe/tests/data/reachability/app.py @@ -0,0 +1,35 @@ +import os + + +class ReportGenerator: + """A dummy class to test AST class method parsing.""" + + def __init__(self, base_dir): + self.base_dir = base_dir + + +def serve_report(request_payload): + """Top-level function handling a request.""" + generator = ReportGenerator("/var/reports") + requested_file = request_payload.get("file") + + # Helper function nested inside serve_report + def build_file_path(filename): + # VULNERABLE: Direct concatenation allows Path Traversal + # An attacker passing "../../etc/passwd" could read system files. + return os.path.join(generator.base_dir, filename) + + if not requested_file: + return "Error: No file specified" + + target_path = build_file_path(requested_file) + + if os.path.exists(target_path): + return f"Serving content of {target_path}" + + return "Error: File not found" + + +def unrelated_top_level_function(): + """An extra function to test AST node boundaries.""" + return "I am just here to add AST complexity." diff --git a/scanpipe/tests/data/reachability/diff-app.patch b/scanpipe/tests/data/reachability/diff-app.patch new file mode 100644 index 0000000000..ccb86953a8 --- /dev/null +++ b/scanpipe/tests/data/reachability/diff-app.patch @@ -0,0 +1,39 @@ +From 8f7b1c3d9a4e2b6f5d8c1a2e3f4b5c6d7e8f9a0b Mon Sep 17 00:00:00 2001 +From: Security Team +Date: Tue, 2 Jun 2026 10:00:00 +0000 +Subject: [PATCH] Fix path traversal vulnerability in report generator + +- Validates that target paths stay within the designated base_dir. +- Catches ValueError on invalid path resolution. +--- + app.py | 12 +++++++++--- + 1 file changed, 9 insertions(+), 3 deletions(-) + +diff --git a/app.py b/app.py +index a1b2c3d..e4f5g6h 100644 +--- a/app.py ++++ b/app.py +@@ -15,13 +15,19 @@ def serve_report(request_payload): + # Helper function nested inside serve_report + def build_file_path(filename): +- # VULNERABLE: Direct concatenation allows Path Traversal +- # An attacker passing "../../etc/passwd" could read system files. +- return os.path.join(generator.base_dir, filename) ++ # FIXED: Validate that the resolved path stays within the base_dir ++ base = os.path.abspath(generator.base_dir) ++ target = os.path.abspath(os.path.join(base, filename)) ++ if not target.startswith(base): ++ raise ValueError("Path Traversal Detected") ++ return target + + if not requested_file: + return "Error: No file specified" + +- target_path = build_file_path(requested_file) ++ try: ++ target_path = build_file_path(requested_file) ++ except ValueError: ++ return "Error: Invalid path" + + if os.path.exists(target_path): + return f"Serving content of {target_path}" \ No newline at end of file diff --git a/scanpipe/tests/data/reachability/fixed-app.py b/scanpipe/tests/data/reachability/fixed-app.py new file mode 100644 index 0000000000..3296bb843e --- /dev/null +++ b/scanpipe/tests/data/reachability/fixed-app.py @@ -0,0 +1,41 @@ +import os + + +class ReportGenerator: + """A dummy class to test AST class method parsing.""" + + def __init__(self, base_dir): + self.base_dir = base_dir + + +def serve_report(request_payload): + """Top-level function handling a request.""" + generator = ReportGenerator("/var/reports") + requested_file = request_payload.get("file") + + # Helper function nested inside serve_report + def build_file_path(filename): + # FIXED: Validate that the resolved path stays within the base_dir + base = os.path.abspath(generator.base_dir) + target = os.path.abspath(os.path.join(base, filename)) + if not target.startswith(base): + raise ValueError("Path Traversal Detected") + return target + + if not requested_file: + return "Error: No file specified" + + try: + target_path = build_file_path(requested_file) + except ValueError: + return "Error: Invalid path" + + if os.path.exists(target_path): + return f"Serving content of {target_path}" + + return "Error: File not found" + + +def unrelated_top_level_function(): + """An extra function to test AST node boundaries.""" + return "I am just here to add AST complexity." diff --git a/scanpipe/tests/data/reachability/vuln-app.py b/scanpipe/tests/data/reachability/vuln-app.py new file mode 100644 index 0000000000..c64ae7d9d1 --- /dev/null +++ b/scanpipe/tests/data/reachability/vuln-app.py @@ -0,0 +1,35 @@ +import os + + +class ReportGenerator: + """A dummy class to test AST class method parsing.""" + + def __init__(self, base_dir): + self.base_dir = base_dir + + +def serve_report(request_payload): + """Top-level function handling a request.""" + generator = ReportGenerator("/var/reports") + requested_file = request_payload.get("file") + + # Helper function nested inside serve_report + def build_file_path(filename): + # VULNERABLE: Direct concatenation allows Path Traversal + # An attacker passing "../../etc/passwd" could read system files. + return os.path.join(generator.base_dir, filename) + + if not requested_file: + return "Error: No file specified" + + target_path = build_file_path(requested_file) + + if os.path.exists(target_path): + return f"Serving content of {target_path}" + + return "Error: File not found" + + +def unrelated_top_level_function(): + """An extra function to test AST node boundaries.""" + return "I am just here to add AST complexity." diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py new file mode 100644 index 0000000000..30d4c924dd --- /dev/null +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -0,0 +1,293 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/nexB/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode.io for support and download. + +from pathlib import Path +from unittest.mock import patch + +from django.test import TestCase + +from scanpipe.models import Project +from scanpipe.pipes import collect_and_create_codebase_resources +from scanpipe.pipes.reachability import ReachabilityStatus +from scanpipe.pipes.reachability import analyze_patched_file +from scanpipe.pipes.reachability import build_call_graph +from scanpipe.pipes.reachability import classify_reachability +from scanpipe.pipes.reachability import collect_and_store_symbol_reachability_results +from scanpipe.pipes.symbols import collect_definitions +from scanpipe.pipes.symbols import extract_definitions +from scanpipe.pipes.symbols import parse_code_to_ast +from scanpipe.pipes.symbols import qualified_name_from_index + + +class SymbolReachabilityPipesTest(TestCase): + data = Path(__file__).parent.parent / "data" / "reachability" + + def setUp(self): + self.project1 = Project.objects.create(name="Analysis") + self.project1.codebase_path.mkdir(parents=True, exist_ok=True) + + @patch("scanpipe.pipes.reachability.Repo") + @patch("scanpipe.pipes.reachability.clone_repo") + @patch("scanpipe.pipes.reachability.api_mocker") + @patch("scanpipe.pipes.reachability.collect_patch_symbols") + def test_collect_and_store_symbol_reachability_results( + self, mock_collect_symbols, mock_api, mock_clone_repo, mock_repo + ): + app_text = (self.data / "app.py").read_text() + vuln_text = (self.data / "vuln-app.py").read_text() + fixed_text = (self.data / "fixed-app.py").read_text() + diff_text = (self.data / "diff-app.patch").read_text() + + vuln_meta, fixed_meta, lang = analyze_patched_file( + vulnerable_text=vuln_text, + fixed_text=fixed_text, + diff_text=diff_text, + file_path="app.py", + ) + + self.assertTrue(lang) + self.assertTrue(vuln_meta or fixed_meta) + mock_api.return_value = [ + { + "vcs_url": "https://github.com/aboutcode-org/test", + "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", + } + ] + + mock_clone_repo.return_value = str(self.project1.codebase_path) + mock_collect_symbols.return_value = { + lang: { + "vulnerable": { + f"app.py::{key}": metadata for key, metadata in vuln_meta.items() + }, + "fixed": { + f"app.py::{key}": metadata for key, metadata in fixed_meta.items() + }, + } + } + + resource_file = self.project1.codebase_path / "app.py" + resource_file.write_text(app_text) + collect_and_create_codebase_resources(self.project1) + + resource = self.project1.codebaseresources.get(path="app.py") + resource.programming_language = lang + resource.save() + + collect_and_store_symbol_reachability_results(self.project1) + + resource.refresh_from_db() + results = resource.extra_data.get("symbols_reachability") + + assert results == [ + { + "patch": { + "vcs_url": "https://github.com/aboutcode-org/test", + "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", + }, + "summary": { + "call_paths": {}, + "fixed_symbols": ["serve_report"], + "vulnerable_symbols": ["serve_report"], + }, + "evidence": { + "serve_report": { + "called": False, + "defined": True, + "reachable_from": [], + "exact_match_fingerprint": "000000556d322a47595af353274b000aa324e014", + } + }, + "reachability_status": "POTENTIALLY_REACHABLE", + } + ] + + def test_build_call_graph(self): + source_code = """ +def calculate_total(price, tax): + return price + get_tax_amount(price, tax) + +def get_tax_amount(price, tax): + return price * tax + +def process_order(): + total = calculate_total(100, 0.05) + print("Done") +""" + tree, _ = parse_code_to_ast(source_code, "Python") + result = build_call_graph(tree, "Python") + + assert result == { + "nodes": { + "calculate_total": { + "qualified_name": "calculate_total", + "simple_name": "calculate_total", + "text": "def calculate_total(price, tax):\n return price + get_tax_amount(price, tax)", + "fingerprint": "00000008060105fd3624134884412006ce880936", + "start_line": 2, + "end_line": 3, + "node_type": "function_definition", + }, + "get_tax_amount": { + "qualified_name": "get_tax_amount", + "simple_name": "get_tax_amount", + "text": "def get_tax_amount(price, tax):\n return price * tax", + "fingerprint": "000000058f0ee87d9669f20b1f473137b665bb20", + "start_line": 5, + "end_line": 6, + "node_type": "function_definition", + }, + "process_order": { + "qualified_name": "process_order", + "simple_name": "process_order", + "text": 'def process_order():\n total = calculate_total(100, 0.05)\n print("Done")', + "fingerprint": "000000071c3e6902da5c2b322386eff29068e3e2", + "start_line": 8, + "end_line": 10, + "node_type": "function_definition", + }, + }, + "edges": { + "calculate_total": {"get_tax_amount"}, + "get_tax_amount": set(), + "process_order": {"print", "calculate_total"}, + }, + "by_simple_name": { + "calculate_total": {"calculate_total"}, + "get_tax_amount": {"get_tax_amount"}, + "process_order": {"process_order"}, + }, + } + + def test_extract_definitions(self): + source_code = """ +class OrderManager: + def __init__(self, order_id): + self.order_id = order_id + + def process_payment(self): + print("Processing...") + +def calculate_discount(price): + return price * 0.10 + +class InventoryItem: + pass +""" + tree, _ = parse_code_to_ast(source_code, "Python") + functions = extract_definitions(tree, "Python", kinds=("functions",)) + assert ( + len(functions) == 3 + ) # '__init__', 'process_payment', and 'calculate_discount' + + assert functions[0].type == "function_definition" + first_func_text = functions[0].text.decode("utf-8") + assert "def __init__" in first_func_text + + classes = extract_definitions(tree, "Python", kinds=("classes",)) + assert len(classes) == 2 # OrderManager, InventoryItem + second_class_text = classes[1].text.decode("utf-8") + assert "class InventoryItem" in second_class_text + + def test_extract_definitions_empty(self): + tree, _ = parse_code_to_ast("", "Python") + assert extract_definitions(tree, "Python", kinds=("functions",)) == [] + assert extract_definitions(tree, "Python", kinds=("functions",)) == [] + assert extract_definitions(None, "Python", kinds=("classes",)) == [] + assert extract_definitions(None, "Python", kinds=("classes",)) == [] + + def test_get_qualified_name_functions(self): + source_code = """ +class CoreService: + class Validator: + def validate_payload(self, data): + return True + +def global_utility(): + pass + """ + + tree, _ = parse_code_to_ast(source_code, "Python") + index = collect_definitions(tree.root_node, "Python") + + functions = extract_definitions(tree, "Python", kinds=("functions",)) + assert len(functions) == 2 + + outer_function_name = qualified_name_from_index(functions[0], index) + inner_function_name = qualified_name_from_index(functions[1], index) + + assert outer_function_name == "CoreService.Validator.validate_payload" + assert inner_function_name == "global_utility" + + def test_get_qualified_classes(self): + source_code = """ +class FleetManagement: + class DroneController: + pass + """ + tree, _ = parse_code_to_ast(source_code, "Python") + index = collect_definitions(tree.root_node, "Python") + + classes = extract_definitions(tree, "Python", kinds=("classes",)) + assert len(classes) == 2 + + outer_class_name = qualified_name_from_index(classes[0], index) + inner_class_name = qualified_name_from_index(classes[1], index) + + assert outer_class_name == "FleetManagement" + assert inner_class_name == "FleetManagement.DroneController" + + def test_classify_reachability(self): + assert classify_reachability(None) == ReachabilityStatus.NOT_REACHABLE + assert classify_reachability({}) == ReachabilityStatus.NOT_REACHABLE + assert ( + classify_reachability( + {"sym1": {"exact_match_fingerprint": "hash123", "called": True}} + ) + == ReachabilityStatus.REACHABLE + ) + + assert ( + classify_reachability( + { + "sym1": { + "called": True, + "reachable_from": ["main_function", "api_handler"], + } + } + ) + == ReachabilityStatus.REACHABLE + ) + assert ( + classify_reachability({"sym1": {"defined": True, "called": False}}) + == ReachabilityStatus.POTENTIALLY_REACHABLE + ) + assert ( + classify_reachability( + {"sym1": {"exact_match_fingerprint": "hash123", "called": False}} + ) + == ReachabilityStatus.POTENTIALLY_REACHABLE + ) + assert ( + classify_reachability({"sym1": {"file_path": "src/vulnerable.py"}}) + == ReachabilityStatus.NOT_REACHABLE + ) From 3799817fd19aad3b9e6f1e4ef0d76ae443e81e68 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Fri, 5 Jun 2026 11:03:43 +0300 Subject: [PATCH 02/25] Add more test for reachability and remove redundant code Signed-off-by: ziad hany --- scanpipe/pipes/reachability.py | 67 ++----- scanpipe/pipes/symbols.py | 54 +++++- .../tests/pipes/test_symbols_reachability.py | 175 +++++++++++++++++- 3 files changed, 236 insertions(+), 60 deletions(-) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index 5531a1f495..5169eba94d 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -39,6 +39,7 @@ from scanpipe.pipes.symbols import extract_calls_in_node from scanpipe.pipes.symbols import extract_definitions from scanpipe.pipes.symbols import extract_symbols +from scanpipe.pipes.symbols import is_nested_function from scanpipe.pipes.symbols import parse_code_to_ast from scanpipe.pipes.symbols import qualified_name_from_index @@ -219,52 +220,6 @@ def get_changed_lines(diff_text, file_path): return removed, added -def query_captures(language, kind, node): - """ - Re-run a definition query on the root of `node`'s tree so ancestors can - be compared. Query caching is handled by `scanpipe.pipes.symbols`. - """ - from scanpipe.pipes.symbols import get_query - from scanpipe.pipes.symbols import run_query - - root = node - - while root.parent is not None: - root = root.parent - - query = get_query(language, kind) - return list(run_query(query, root)) - - -def is_nested_function(node, language): - function_nodes = { - captured_node - for captured_node, _ in query_captures(language, "functions", node) - } - class_nodes = { - captured_node for captured_node, _ in query_captures(language, "classes", node) - } - - if node not in function_nodes: - return False - - function_types = {captured_node.type for captured_node in function_nodes} - class_types = {captured_node.type for captured_node in class_nodes} - - parent = node.parent - - while parent is not None: - if parent.type in function_types: - return True - - if parent.type in class_types: - return False - - parent = parent.parent - - return False - - def diff_changed_symbols(vuln_meta, fixed_meta): """ Keep only symbols whose body actually differs between vulnerable and fixed @@ -506,7 +461,7 @@ def collect_and_store_symbol_reachability_results(project, logger=None): append_symbol_reachability_result(resource, result) except Exception as e: - logger.exception( + logger( "Failed to collect symbol reachability for " f"{vcs_url}@{commit_hash}: {e}" ) @@ -515,24 +470,36 @@ def collect_and_store_symbol_reachability_results(project, logger=None): def compute_reachable_symbols(call_graph, target_simple_names): + """ + Find all symbols that can transitively reach any of ``target_simple_names``. + + Reachability is matched on *simple* names (the call graph records callee + tokens, not fully-qualified names), so distinct symbols sharing a name are + treated as equivalent. This can over-approximate reachability. + + Returns: + (reachable_callers, has_direct_call) + reachable_callers: qualified names of all transitive callers + has_direct_call: whether any symbol calls a target directly + + """ if not call_graph or not target_simple_names: return set(), False edges = call_graph["edges"] targets = set(target_simple_names) - callers_of = {} + callers_of: dict[str, set[str]] = {} for caller_qn, callees in edges.items(): for callee_simple in callees: callers_of.setdefault(callee_simple, set()).add(caller_qn) - direct_callers = set() + direct_callers: set[str] = set() for target in targets: direct_callers |= callers_of.get(target, set()) has_direct_call = bool(direct_callers) - by_simple = call_graph["by_simple_name"] qn_to_simple = {qn: meta["simple_name"] for qn, meta in call_graph["nodes"].items()} reachable = set(direct_callers) diff --git a/scanpipe/pipes/symbols.py b/scanpipe/pipes/symbols.py index c6247dbc44..cac76562e3 100644 --- a/scanpipe/pipes/symbols.py +++ b/scanpipe/pipes/symbols.py @@ -207,6 +207,7 @@ def _collect_and_store_tree_sitter_symbols_and_strings(resource): }, } + @cache def load_language(language: str) -> Language: if language not in TS_LANGUAGE_WHEELS: @@ -256,7 +257,48 @@ def run_query(query: Query, root_node): yield def_nodes[0], name -def extract_calls_in_node(node, language: str) -> set[str]: +def query_captures(language, kind, node): + """Re-run a definition query on the root of node's tree.""" + query = get_query(language, kind) + return list(run_query(query, _root_of(node))) + + +def _root_of(node): + while node.parent is not None: + node = node.parent + return node + + +def is_nested_function(node, language): + function_nodes = { + captured_node + for captured_node, _ in query_captures(language, "functions", node) + } + class_nodes = { + captured_node for captured_node, _ in query_captures(language, "classes", node) + } + + if node not in function_nodes: + return False + + function_types = {captured_node.type for captured_node in function_nodes} + class_types = {captured_node.type for captured_node in class_nodes} + + parent = node.parent + + while parent is not None: + if parent.type in function_types: + return True + + if parent.type in class_types: + return False + + parent = parent.parent + + return False + + +def extract_calls_in_node(node, language: str): query = get_query(language, "calls") if query is None or node is None: return set() @@ -270,7 +312,7 @@ def extract_calls_in_node(node, language: str) -> set[str]: return names -def collect_definitions(root_node, language: str) -> dict[int, dict]: +def collect_definitions(root_node, language: str): index: dict[int, dict] = {} for kind in ("functions", "classes"): query = get_query(language, kind) @@ -311,13 +353,7 @@ def extract_symbols(tree, changed_lines: list[int], language: str): return enclosing -def _root_of(node): - while node.parent is not None: - node = node.parent - return node - - -def qualified_name_from_index(node, index: dict[int, dict]) -> str: +def qualified_name_from_index(node, index): parts = [] curr = node while curr is not None: diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index 30d4c924dd..1a5fd7b3ad 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -30,9 +30,12 @@ from scanpipe.pipes.reachability import ReachabilityStatus from scanpipe.pipes.reachability import analyze_patched_file from scanpipe.pipes.reachability import build_call_graph +from scanpipe.pipes.reachability import build_symbol_metadata from scanpipe.pipes.reachability import classify_reachability from scanpipe.pipes.reachability import collect_and_store_symbol_reachability_results -from scanpipe.pipes.symbols import collect_definitions +from scanpipe.pipes.reachability import diff_changed_symbols +from scanpipe.pipes.reachability import get_changed_lines +from scanpipe.pipes.symbols import collect_definitions, extract_symbols from scanpipe.pipes.symbols import extract_definitions from scanpipe.pipes.symbols import parse_code_to_ast from scanpipe.pipes.symbols import qualified_name_from_index @@ -291,3 +294,173 @@ def test_classify_reachability(self): classify_reachability({"sym1": {"file_path": "src/vulnerable.py"}}) == ReachabilityStatus.NOT_REACHABLE ) + + def test_get_changed_lines(self): + data = Path(__file__).parent.parent / "data" / "reachability" + diff_text = (data / "diff-app.patch").read_text(encoding="utf-8") + + removed, added = get_changed_lines(diff_text, "app.py") + assert removed == [17, 18, 19, 24] + assert added == [17, 18, 19, 20, 21, 22, 27, 28, 29, 30] + + def test_build_symbol_metadata_processing(self): + source_code = """ +class Controller: + def process_data(payload): + def inner_helper(): + return True + return payload.strip() + +if True: + def process_data(payload): + return payload +""" + tree, _ = parse_code_to_ast(source_code, "Python") + nodes = extract_definitions(tree, "Python", kinds=("functions",)) + + metadata = build_symbol_metadata(nodes, "Python") + assert metadata == { + "Controller.process_data": { + "qualified_name": "Controller.process_data", + "simple_name": "process_data", + "text": "def process_data(payload):\n def inner_helper():\n return True\n return payload.strip()", + "fingerprint": "0000000888014a04b037189a42b238a2c50f218c", + "start_line": 3, + "end_line": 6, + "node_type": "function_definition", + }, + "process_data": { + "qualified_name": "process_data", + "simple_name": "process_data", + "text": "def process_data(payload):\n return payload", + "fingerprint": "000000022020300e882a900807880d0300010000", + "start_line": 9, + "end_line": 10, + "node_type": "function_definition", + }, + } + + def test_diff_changed_symbols(self): + vuln_meta = { + "serve_report": { + "qualified_name": "app.serve_report", + "text": "def serve_report():\n return os.path.join(base, filename)", + }, + "sanitize_input": { + "qualified_name": "app.sanitize_input", + "text": "def sanitize_input(x):\n return x.strip()", + }, + "deprecated_logger": { + "qualified_name": "app.deprecated_logger", + "text": "def deprecated_logger():\n print('legacy')", + }, + } + + fixed_meta = { + "serve_report": { + "qualified_name": "app.serve_report", + "text": "def serve_report():\n if not target.startswith(base): raise ValueError\n return target", + }, + "sanitize_input": { + "qualified_name": "app.sanitize_input", + "text": "def sanitize_input(x):\n return x.strip()", + }, + "audit_trail": { + "qualified_name": "app.audit_trail", + "text": "def audit_trail():\n log.info('action')", + }, + } + + vuln_only, fixed_only = diff_changed_symbols(vuln_meta, fixed_meta) + + assert vuln_only == { + "serve_report": { + "qualified_name": "app.serve_report", + "text": "def serve_report():\n return os.path.join(base, filename)", + }, + "deprecated_logger": { + "qualified_name": "app.deprecated_logger", + "text": "def deprecated_logger():\n print('legacy')", + }, + } + assert fixed_only == { + "serve_report": { + "qualified_name": "app.serve_report", + "text": "def serve_report():\n if not target.startswith(base): raise ValueError\n return target", + }, + "audit_trail": { + "qualified_name": "app.audit_trail", + "text": "def audit_trail():\n log.info('action')", + }, + } + + def test_analyze_patched_file(self): + vuln_text = (self.data / "vuln-app.py").read_text(encoding="utf-8") + fixed_text = (self.data / "fixed-app.py").read_text(encoding="utf-8") + diff_text = (self.data / "diff-app.patch").read_text(encoding="utf-8") + + vuln_meta, fixed_meta, lang = analyze_patched_file( + vulnerable_text=vuln_text, + fixed_text=fixed_text, + diff_text=diff_text, + file_path="app.py", + ) + + assert vuln_meta == { + "serve_report": { + "qualified_name": "serve_report", + "simple_name": "serve_report", + "text": 'def serve_report(request_payload):\n """Top-level function handling a request."""\n generator = ReportGenerator("/var/reports")\n requested_file = request_payload.get("file")\n\n # Helper function nested inside serve_report\n def build_file_path(filename):\n # VULNERABLE: Direct concatenation allows Path Traversal\n # An attacker passing "../../etc/passwd" could read system files.\n return os.path.join(generator.base_dir, filename)\n\n if not requested_file:\n return "Error: No file specified"\n\n target_path = build_file_path(requested_file)\n\n if os.path.exists(target_path):\n return f"Serving content of {target_path}"\n\n return "Error: File not found"', + "fingerprint": "000000556d322a47595af353274b000aa324e014", + "start_line": 11, + "end_line": 30, + "node_type": "function_definition", + } + } + assert fixed_meta == { + "serve_report": { + "qualified_name": "serve_report", + "simple_name": "serve_report", + "text": 'def serve_report(request_payload):\n """Top-level function handling a request."""\n generator = ReportGenerator("/var/reports")\n requested_file = request_payload.get("file")\n\n # Helper function nested inside serve_report\n def build_file_path(filename):\n # FIXED: Validate that the resolved path stays within the base_dir\n base = os.path.abspath(generator.base_dir)\n target = os.path.abspath(os.path.join(base, filename))\n if not target.startswith(base):\n raise ValueError("Path Traversal Detected")\n return target\n\n if not requested_file:\n return "Error: No file specified"\n\n try:\n target_path = build_file_path(requested_file)\n except ValueError:\n return "Error: Invalid path"\n\n if os.path.exists(target_path):\n return f"Serving content of {target_path}"\n\n return "Error: File not found"', + "fingerprint": "0000006cceea8aedf1da91830f67b64927086d24", + "start_line": 11, + "end_line": 36, + "node_type": "function_definition", + } + } + + def test_extract_symbols(self): + source_code = ( + "def serve_report(request):\n" # Line 1 (Row 0) + " # Some processing here\n" # Line 2 (Row 1) + " def build_path(filename):\n" # Line 3 (Row 2) + " return filename.strip()\n" # Line 4 (Row 3) <- Targeted Change + " return build_path(request)\n" # Line 5 (Row 4) + ) + + tree, _ = parse_code_to_ast(source_code, "Python") + + changed_lines = [4] + enclosing_symbols = extract_symbols(tree, changed_lines, "Python") + + assert len(enclosing_symbols) == 1 + target_node = enclosing_symbols[0] + assert target_node.type == "function_definition" + + node_text = target_node.text.decode("utf-8") + assert "def build_path" in node_text + assert "def serve_report" not in node_text + + def test_extract_symbols_deduplication(self): + source_code = ( + "def calculate_total(price, tax):\n" + " amount = price * tax\n" # Line 2 -> Changed + " return price + amount\n" # Line 3 -> Changed + ) + + tree, _ = parse_code_to_ast(source_code, "Python") + changed_lines = [2, 3] + + enclosing_symbols = extract_symbols(tree, changed_lines, "Python") + assert len(enclosing_symbols) == 1 + assert enclosing_symbols[0].type == "function_definition" \ No newline at end of file From a3d1172c01d433b2186f9b1d8a5e4b9913de687b Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 10 Jun 2026 15:08:45 +0300 Subject: [PATCH 03/25] Fix the format bugs and refactor the code Signed-off-by: ziad hany --- .../pipelines/collect_symbols_reachability.py | 8 +- scanpipe/pipes/reachability.py | 394 +++++++++++------- scanpipe/pipes/symbols.py | 14 +- scanpipe/tests/data/reachability/app.py | 2 +- scanpipe/tests/data/reachability/fixed-app.py | 2 +- scanpipe/tests/data/reachability/vuln-app.py | 2 +- .../tests/pipes/test_symbols_reachability.py | 381 +++++++++-------- 7 files changed, 473 insertions(+), 330 deletions(-) diff --git a/scanpipe/pipelines/collect_symbols_reachability.py b/scanpipe/pipelines/collect_symbols_reachability.py index 15519fc661..c1d5fb11c4 100644 --- a/scanpipe/pipelines/collect_symbols_reachability.py +++ b/scanpipe/pipelines/collect_symbols_reachability.py @@ -12,9 +12,7 @@ class SymbolReachability(Pipeline): - """ - Patch reachability analysis, for given a vulnerability patches - """ + """Patch reachability analysis for given vulnerability patches.""" download_inputs = False is_addon = True @@ -26,8 +24,8 @@ def steps(cls): def analyze_and_store_symbol_reachability(self): """ - Perform symbol-level reachability analysis for each patch. - This step compares the AST of patched/vulnerable files against the codebase resources. + Perform symbol-level reachability analysis for each patch. This step compares + the AST of patched/vulnerable files against the codebase resources. Results are stored directly in the 'extra_data' of each CodebaseResource. """ reachability.collect_and_store_symbol_reachability_results( diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index 5169eba94d..e9ebd97378 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -29,16 +29,16 @@ from git import Repo from git.diff import NULL_TREE from git.exc import BadName -from matchcode_toolkit.fingerprinting import create_file_fingerprints from scancode.api import get_file_info from unidiff import PatchSet from scanpipe.pipes.symbols import TS_QUERIES from scanpipe.pipes.symbols import _root_of from scanpipe.pipes.symbols import collect_definitions -from scanpipe.pipes.symbols import extract_calls_in_node +from scanpipe.pipes.symbols import create_exact_symbol_fingerprint from scanpipe.pipes.symbols import extract_definitions from scanpipe.pipes.symbols import extract_symbols +from scanpipe.pipes.symbols import get_query from scanpipe.pipes.symbols import is_nested_function from scanpipe.pipes.symbols import parse_code_to_ast from scanpipe.pipes.symbols import qualified_name_from_index @@ -53,14 +53,16 @@ class ReachabilityStatus(str, Enum): def api_mocker(): - """ - TODO: Remove this once the API patch url is done - """ + """TODO: Remove this once the API patch url is done""" return [ { "vcs_url": "https://github.com/pallets/flask", "commit_hash": "089cb86dd22bff589a4eafb7ab8e42dc357623b4", }, + # { + # "vcs_url": "https://github.com/aio-libs/aiohttp", + # "commit_hash": "0c2e9da51126238a421568eb7c5b53e5b5d17b36", + # } ] @@ -100,7 +102,7 @@ def normalize_text(content): def is_supported_language(language): - """A language is supported if we have tree-sitter queries for it.""" + """Return True if the language is supported by tree-sitter queries.""" return bool(language) and language in TS_QUERIES @@ -223,28 +225,18 @@ def get_changed_lines(diff_text, file_path): def diff_changed_symbols(vuln_meta, fixed_meta): """ Keep only symbols whose body actually differs between vulnerable and fixed - versions. Pair by qualified name first. + versions. Utilizes the unique suffix keys generated by build_symbol_metadata. """ - fixed_by_qn = { - metadata["qualified_name"]: metadata for metadata in fixed_meta.values() - } - - vuln_by_qn = { - metadata["qualified_name"]: metadata for metadata in vuln_meta.values() - } - vuln_only = { key: metadata for key, metadata in vuln_meta.items() - if fixed_by_qn.get(metadata["qualified_name"], {}).get("text") - != metadata["text"] + if fixed_meta.get(key, {}).get("text") != metadata["text"] } fixed_only = { key: metadata for key, metadata in fixed_meta.items() - if vuln_by_qn.get(metadata["qualified_name"], {}).get("text") - != metadata["text"] + if vuln_meta.get(key, {}).get("text") != metadata["text"] } return vuln_only, fixed_only @@ -313,9 +305,6 @@ def collect_patch_symbols(repo, commit_hash): ... } - Symbols are bucketed by language so resources are only matched against - patch symbols extracted from the same language. - """ commit, parent = get_commit_and_parent(repo, commit_hash) diff_text = get_commit_diff_text(repo, parent, commit) @@ -379,11 +368,6 @@ def collect_and_store_symbol_reachability_results(project, logger=None): For each known patch commit, determine whether each project codebase resource is reachable to the vulnerable code by comparing tree-sitter ASTs of the patch versus the resource. - - Result classification: - - REACHABLE - - POTENTIALLY_REACHABLE - - NOT_REACHABLE """ candidate_resources = project.codebaseresources.files().filter( is_binary=False, @@ -394,10 +378,10 @@ def collect_and_store_symbol_reachability_results(project, logger=None): for patch in api_mocker(): vcs_url = patch["vcs_url"] commit_hash = patch["commit_hash"] - repo_path = None try: - repo_path = clone_repo(vcs_url, commit_hash) - repo = Repo(repo_path) + # repo_path = clone_repo(vcs_url, commit_hash) + # repo = Repo("/home/ziad-hany/PycharmProjects/flask/") + repo = Repo("/home/ziad-hany/PycharmProjects/aiohttp") patch_symbols_by_language = collect_patch_symbols(repo, commit_hash) @@ -406,18 +390,16 @@ def collect_and_store_symbol_reachability_results(project, logger=None): for resource in candidate_resources: resource_language = resource.programming_language - if resource_language not in patch_symbols_by_language: continue - resource_text = normalize_text(resource.file_content) - + resource_text = resource.file_content if not resource_text: continue patch_symbols = patch_symbols_by_language[resource_language] - vuln_metadata = patch_symbols["vulnerable"] - fixed_metadata = patch_symbols["fixed"] + vuln_patch_metadata = patch_symbols["vulnerable"] + fixed_patch_metadata = patch_symbols["fixed"] resource_index = build_resource_index( resource_text, @@ -428,12 +410,12 @@ def collect_and_store_symbol_reachability_results(project, logger=None): continue vuln_match_symbols = match_symbols_against_resource( - vuln_metadata, + vuln_patch_metadata, resource_index, ) fixed_match_symbols = match_symbols_against_resource( - fixed_metadata, + fixed_patch_metadata, resource_index, ) @@ -443,84 +425,36 @@ def collect_and_store_symbol_reachability_results(project, logger=None): result = { "reachability_status": classify_reachability(vuln_match_symbols), "summary": { - "vulnerable_symbols": sorted(vuln_match_symbols), - "fixed_symbols": sorted(fixed_match_symbols), "call_paths": { - qn: ev.get("reachable_from", []) - for qn, ev in vuln_match_symbols.items() + qualified_name: ev.get("reachable_from", []) + for qualified_name, ev in vuln_match_symbols.items() if ev.get("called") }, }, "evidence": vuln_match_symbols, + "vulnerable_symbols": sorted(vuln_match_symbols), + "fixed_symbols": sorted(fixed_match_symbols), "patch": { "vcs_url": vcs_url, "commit_hash": commit_hash, }, } - + print(result) append_symbol_reachability_result(resource, result) except Exception as e: logger( - "Failed to collect symbol reachability for " + f"Failed to collect symbol reachability for " f"{vcs_url}@{commit_hash}: {e}" ) finally: - cleanup_repo(repo_path) - - -def compute_reachable_symbols(call_graph, target_simple_names): - """ - Find all symbols that can transitively reach any of ``target_simple_names``. - - Reachability is matched on *simple* names (the call graph records callee - tokens, not fully-qualified names), so distinct symbols sharing a name are - treated as equivalent. This can over-approximate reachability. - - Returns: - (reachable_callers, has_direct_call) - reachable_callers: qualified names of all transitive callers - has_direct_call: whether any symbol calls a target directly - - """ - if not call_graph or not target_simple_names: - return set(), False - - edges = call_graph["edges"] - targets = set(target_simple_names) - - callers_of: dict[str, set[str]] = {} - for caller_qn, callees in edges.items(): - for callee_simple in callees: - callers_of.setdefault(callee_simple, set()).add(caller_qn) - - direct_callers: set[str] = set() - for target in targets: - direct_callers |= callers_of.get(target, set()) - - has_direct_call = bool(direct_callers) - - qn_to_simple = {qn: meta["simple_name"] for qn, meta in call_graph["nodes"].items()} - - reachable = set(direct_callers) - frontier = list(direct_callers) - - while frontier: - current_qn = frontier.pop() - current_simple = qn_to_simple.get(current_qn) - if not current_simple: - continue - for parent_qn in callers_of.get(current_simple, ()): - if parent_qn not in reachable: - reachable.add(parent_qn) - frontier.append(parent_qn) + if repo: + repo.close() - return reachable, has_direct_call + # cleanup_repo(repo_path) def build_resource_index(resource_text, language): - resource_text = normalize_text(resource_text) - if not is_supported_language(language) or not resource_text: return None @@ -551,34 +485,38 @@ def build_resource_index(resource_text, language): } -def match_symbols_against_resource(symbols, resource_index): - if not symbols or not resource_index: +def match_symbols_against_resource(patch_symbols_metadata, resource_index): + if not patch_symbols_metadata or not resource_index: return {} call_graph = resource_index.get("call_graph") - target_simple_names = {metadata["simple_name"] for metadata in symbols.values()} - reachable_callers, _has_direct = compute_reachable_symbols( + target_qualified_names = { + metadata["qualified_name"] for metadata in patch_symbols_metadata.values() + } + + reachable_callers, _ = compute_reachable_symbols( call_graph, - target_simple_names, + target_qualified_names, ) - called_simple_names = set() + called_qualified_names = set() + if call_graph: - for callees in call_graph["edges"].values(): - called_simple_names |= callees + for callees in call_graph.get("edges_qualified", {}).values(): + called_qualified_names |= set(callees) matched = {} - for metadata in symbols.values(): + + for metadata in patch_symbols_metadata.values(): qualified_name = metadata["qualified_name"] - simple_name = metadata["simple_name"] fingerprint = metadata["fingerprint"] - defined = qualified_name in resource_index["definitions"] + defined = qualified_name in resource_index.get("definitions", {}) fingerprint_hit = bool( - fingerprint and fingerprint in resource_index["fingerprints"] + fingerprint and fingerprint in resource_index.get("fingerprints", {}) ) - called = simple_name in called_simple_names + called = qualified_name in called_qualified_names if not (defined or fingerprint_hit or called): continue @@ -608,12 +546,6 @@ def classify_reachability(evidence): if not evidence: return ReachabilityStatus.NOT_REACHABLE - SEVERITY_RANK = { - ReachabilityStatus.NOT_REACHABLE: 0, - ReachabilityStatus.POTENTIALLY_REACHABLE: 1, - ReachabilityStatus.REACHABLE: 2, - } - highest_status = ReachabilityStatus.NOT_REACHABLE for item in evidence.values(): @@ -622,17 +554,12 @@ def classify_reachability(evidence): is_exact = "exact_match_fingerprint" in item is_defined = bool(item.get("defined")) - if is_called or (has_path and is_exact): + if is_called or has_path: return ReachabilityStatus.REACHABLE - elif has_path or is_exact or is_defined: - current_item_status = ReachabilityStatus.POTENTIALLY_REACHABLE - - else: - current_item_status = ReachabilityStatus.NOT_REACHABLE + if is_exact or is_defined: + highest_status = ReachabilityStatus.POTENTIALLY_REACHABLE - if SEVERITY_RANK[current_item_status] > SEVERITY_RANK[highest_status]: - highest_status = current_item_status return highest_status @@ -650,7 +577,7 @@ def build_symbol_metadata(nodes, language, index=None): continue body_text = node.text.decode("utf-8", errors="replace") - fingerprints = create_file_fingerprints(content=body_text) or {} + fingerprints = create_exact_symbol_fingerprint(body_text) key = qualified_name suffix = 1 @@ -660,9 +587,8 @@ def build_symbol_metadata(nodes, language, index=None): metadata[key] = { "qualified_name": qualified_name, - "simple_name": qualified_name.rsplit(".", 1)[-1], "text": body_text, - "fingerprint": fingerprints.get("halo1"), + "fingerprint": fingerprints, "start_line": node.start_point[0] + 1, "end_line": node.end_point[0] + 1, "node_type": node.type, @@ -675,23 +601,211 @@ def build_call_graph(tree, language): return None index = collect_definitions(tree.root_node, language) - definition_nodes = [d["node"] for d in index.values()] - metadata = build_symbol_metadata(definition_nodes, language, index=index) - qualified_name_to_node = {} - for node in definition_nodes: + graph_meta = {} + for definition in index.values(): + node = definition["node"] qualified_name = qualified_name_from_index(node, index) - if qualified_name: - qualified_name_to_node.setdefault(qualified_name, node) - - edges = {} - by_simple_name = {} - for qualified_name, meta in metadata.items(): - canonical = meta["qualified_name"] - node = qualified_name_to_node.get(canonical) - if node is None: + + if not qualified_name: continue - edges.setdefault(canonical, set()).update(extract_calls_in_node(node, language)) - by_simple_name.setdefault(meta["simple_name"], set()).add(canonical) - return {"nodes": metadata, "edges": edges, "by_simple_name": by_simple_name} + body_text = node.text.decode("utf-8", errors="replace") + fingerprints = create_exact_symbol_fingerprint(body_text) or {} + + graph_meta[qualified_name] = { + "qualified_name": qualified_name, + "node": node, + "node_type": node.type, + "fingerprint": fingerprints, + } + + definitions_by_name = {} + class_methods = set() + + for qualified_name, metadata in graph_meta.items(): + name = qualified_name.rsplit(".", 1)[-1] + definitions_by_name.setdefault(name, set()).add(qualified_name) + + if metadata["node_type"] == "function_definition" and "." in qualified_name: + class_methods.add(qualified_name) + + edges_qualified = {} + for qualified_name, metadata in graph_meta.items(): + direct_calls = extract_direct_calls(metadata["node"], language, index) + + resolved_callees = set() + + for receiver_name, callee_name in direct_calls: + resolved_callees |= resolve_callee( + receiver_name=receiver_name, + callee_name=callee_name, + owner_qn=qualified_name, + definitions_by_name=definitions_by_name, + class_methods=class_methods, + ) + + edges_qualified[qualified_name] = resolved_callees + + return { + "nodes": graph_meta, + "edges_qualified": edges_qualified, + } + + +def extract_direct_calls(node, language, definition_index): + """ + Return direct calls inside `node`, excluding calls inside nested definitions. + + Returns: + list of (receiver_name, callee_name) + + Examples: + foo() -> (None, "foo") + self.foo() -> ("self", "foo") + obj.foo() -> ("obj", "foo") + + """ + query = get_query(language, "calls") + if query is None or node is None: + return [] + + definition_ids = set(definition_index) + calls = [] + + for _, captures in query.matches(node): + for callee_node in captures.get("callee", []): + if is_inside_nested_definition( + node=callee_node, + owner_node=node, + definition_ids=definition_ids, + ): + continue + + receiver_name = get_call_receiver(callee_node) + callee_name = node_text(callee_node) + + if callee_name: + calls.append((receiver_name, callee_name)) + + return calls + + +def is_inside_nested_definition(node, owner_node, definition_ids): + """ + Return True if node is inside a nested function/class within `owner_node`. + + Example: + def outer(): + foo() # belongs to outer + + def inner(): + bar() # nested; should not count as outer's call + + """ + current = node.parent + + while current is not None and current is not owner_node: + if current.id in definition_ids: + return True + + current = current.parent + + return False + + +def node_text(node): + return node.text.decode("utf-8", errors="replace") + + +def get_call_receiver(callee_node): + """ + Return receiver name for attribute calls. + + Examples: + foo() -> None + self.foo() -> "self" + obj.foo() -> "obj" + + """ + parent = callee_node.parent + + if parent is None or parent.type != "attribute": + return None + + object_node = parent.child_by_field_name("object") + if object_node is None: + return None + + return node_text(object_node) + + +def resolve_callee( + receiver_name, callee_name, owner_qn, definitions_by_name, class_methods +): + """ + Resolve a call to candidate qualified names. + + Examples: + self.foo() from class A -> {"A.foo"} if A.foo exists + foo() -> definitions named "foo" + + """ + if receiver_name == "self": + owner_class = get_owner_class_name(owner_qn) + + if owner_class: + method_qn = f"{owner_class}.{callee_name}" + + if method_qn in class_methods: + return {method_qn} + + candidates = definitions_by_name.get(callee_name, set()) + return set(candidates) + + +def get_owner_class_name(owner_qn): + """ + Return enclosing class name from a qualified name. + + Examples: + "User.save" -> "User" + "User.Inner.save" -> "User.Inner" + "save" -> None + + """ + if "." not in owner_qn: + return None + + return owner_qn.rsplit(".", 1)[0] + + +def compute_reachable_symbols(call_graph, target_qualified_names): + """Transitive callers using resolved qualified-name edges.""" + if not call_graph or not target_qualified_names: + return set(), False + + edges = call_graph.get("edges_qualified") + if not edges: + return set(), False + + callers_of = {} + for caller, callees in edges.items(): + for callee in callees: + callers_of.setdefault(callee, set()).add(caller) + + targets = set(target_qualified_names) + direct = set() + for target in targets: + direct |= callers_of.get(target, set()) + + reachable = set(direct) + frontier = list(direct) + while frontier: + cur = frontier.pop() + for parent in callers_of.get(cur, ()): + if parent not in reachable: + reachable.add(parent) + frontier.append(parent) + + return reachable, bool(direct) diff --git a/scanpipe/pipes/symbols.py b/scanpipe/pipes/symbols.py index cac76562e3..da6f359f65 100644 --- a/scanpipe/pipes/symbols.py +++ b/scanpipe/pipes/symbols.py @@ -20,6 +20,7 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. +import hashlib import importlib from functools import cache @@ -191,7 +192,6 @@ def _collect_and_store_tree_sitter_symbols_and_strings(resource): "pygments": symbols_pygments.get_pygments_symbols, } -# https://github.com/Aider-AI/aider/tree/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/queries/tree-sitter-language-pack TS_QUERIES = { "Python": { "functions": """ @@ -202,7 +202,9 @@ def _collect_and_store_tree_sitter_symbols_and_strings(resource): """, "calls": """ (call function: (identifier) @callee) - (call function: (attribute attribute: (identifier) @callee)) + (call function: (attribute + object: (_) @receiver + attribute: (identifier) @callee)) """, }, } @@ -362,3 +364,11 @@ def qualified_name_from_index(node, index): parts.append(definition["name"]) curr = curr.parent return ".".join(reversed(parts)) + + +def create_exact_symbol_fingerprint(text): + if text is None: + return None + + text = text.encode("utf-8", errors="replace") + return hashlib.sha256(text).hexdigest() diff --git a/scanpipe/tests/data/reachability/app.py b/scanpipe/tests/data/reachability/app.py index c64ae7d9d1..b8c9eff5e0 100644 --- a/scanpipe/tests/data/reachability/app.py +++ b/scanpipe/tests/data/reachability/app.py @@ -31,5 +31,5 @@ def build_file_path(filename): def unrelated_top_level_function(): - """An extra function to test AST node boundaries.""" + """Test AST node boundaries.""" return "I am just here to add AST complexity." diff --git a/scanpipe/tests/data/reachability/fixed-app.py b/scanpipe/tests/data/reachability/fixed-app.py index 3296bb843e..ca5a6f4c8b 100644 --- a/scanpipe/tests/data/reachability/fixed-app.py +++ b/scanpipe/tests/data/reachability/fixed-app.py @@ -37,5 +37,5 @@ def build_file_path(filename): def unrelated_top_level_function(): - """An extra function to test AST node boundaries.""" + """Test AST node boundaries.""" return "I am just here to add AST complexity." diff --git a/scanpipe/tests/data/reachability/vuln-app.py b/scanpipe/tests/data/reachability/vuln-app.py index c64ae7d9d1..b8c9eff5e0 100644 --- a/scanpipe/tests/data/reachability/vuln-app.py +++ b/scanpipe/tests/data/reachability/vuln-app.py @@ -31,5 +31,5 @@ def build_file_path(filename): def unrelated_top_level_function(): - """An extra function to test AST node boundaries.""" + """Test AST node boundaries.""" return "I am just here to add AST complexity." diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index 1a5fd7b3ad..68bbe9b964 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -29,14 +29,15 @@ from scanpipe.pipes import collect_and_create_codebase_resources from scanpipe.pipes.reachability import ReachabilityStatus from scanpipe.pipes.reachability import analyze_patched_file -from scanpipe.pipes.reachability import build_call_graph from scanpipe.pipes.reachability import build_symbol_metadata from scanpipe.pipes.reachability import classify_reachability from scanpipe.pipes.reachability import collect_and_store_symbol_reachability_results +from scanpipe.pipes.reachability import compute_reachable_symbols from scanpipe.pipes.reachability import diff_changed_symbols from scanpipe.pipes.reachability import get_changed_lines -from scanpipe.pipes.symbols import collect_definitions, extract_symbols +from scanpipe.pipes.symbols import collect_definitions from scanpipe.pipes.symbols import extract_definitions +from scanpipe.pipes.symbols import extract_symbols from scanpipe.pipes.symbols import parse_code_to_ast from scanpipe.pipes.symbols import qualified_name_from_index @@ -101,85 +102,31 @@ def test_collect_and_store_symbol_reachability_results( resource.refresh_from_db() results = resource.extra_data.get("symbols_reachability") - assert results == [ - { - "patch": { - "vcs_url": "https://github.com/aboutcode-org/test", - "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", - }, - "summary": { - "call_paths": {}, + self.assertEqual( + results, + [ + { + "patch": { + "vcs_url": "https://github.com/aboutcode-org/test", + "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", + }, + "summary": {"call_paths": {}}, + "evidence": { + "serve_report": { + "called": False, + "defined": True, + "reachable_from": [], + "exact_match_fingerprint": ( + "e341b914f9823915e0685396a730d421ec9e3635" + ), + } + }, "fixed_symbols": ["serve_report"], "vulnerable_symbols": ["serve_report"], - }, - "evidence": { - "serve_report": { - "called": False, - "defined": True, - "reachable_from": [], - "exact_match_fingerprint": "000000556d322a47595af353274b000aa324e014", - } - }, - "reachability_status": "POTENTIALLY_REACHABLE", - } - ] - - def test_build_call_graph(self): - source_code = """ -def calculate_total(price, tax): - return price + get_tax_amount(price, tax) - -def get_tax_amount(price, tax): - return price * tax - -def process_order(): - total = calculate_total(100, 0.05) - print("Done") -""" - tree, _ = parse_code_to_ast(source_code, "Python") - result = build_call_graph(tree, "Python") - - assert result == { - "nodes": { - "calculate_total": { - "qualified_name": "calculate_total", - "simple_name": "calculate_total", - "text": "def calculate_total(price, tax):\n return price + get_tax_amount(price, tax)", - "fingerprint": "00000008060105fd3624134884412006ce880936", - "start_line": 2, - "end_line": 3, - "node_type": "function_definition", - }, - "get_tax_amount": { - "qualified_name": "get_tax_amount", - "simple_name": "get_tax_amount", - "text": "def get_tax_amount(price, tax):\n return price * tax", - "fingerprint": "000000058f0ee87d9669f20b1f473137b665bb20", - "start_line": 5, - "end_line": 6, - "node_type": "function_definition", - }, - "process_order": { - "qualified_name": "process_order", - "simple_name": "process_order", - "text": 'def process_order():\n total = calculate_total(100, 0.05)\n print("Done")', - "fingerprint": "000000071c3e6902da5c2b322386eff29068e3e2", - "start_line": 8, - "end_line": 10, - "node_type": "function_definition", - }, - }, - "edges": { - "calculate_total": {"get_tax_amount"}, - "get_tax_amount": set(), - "process_order": {"print", "calculate_total"}, - }, - "by_simple_name": { - "calculate_total": {"calculate_total"}, - "get_tax_amount": {"get_tax_amount"}, - "process_order": {"process_order"}, - }, - } + "reachability_status": "POTENTIALLY_REACHABLE", + } + ], + ) def test_extract_definitions(self): source_code = """ @@ -198,25 +145,25 @@ class InventoryItem: """ tree, _ = parse_code_to_ast(source_code, "Python") functions = extract_definitions(tree, "Python", kinds=("functions",)) - assert ( - len(functions) == 3 + self.assertEqual( + len(functions), 3 ) # '__init__', 'process_payment', and 'calculate_discount' - assert functions[0].type == "function_definition" + self.assertEqual(functions[0].type, "function_definition") first_func_text = functions[0].text.decode("utf-8") - assert "def __init__" in first_func_text + self.assertIn("def __init__", first_func_text) classes = extract_definitions(tree, "Python", kinds=("classes",)) - assert len(classes) == 2 # OrderManager, InventoryItem + self.assertEqual(len(classes), 2) second_class_text = classes[1].text.decode("utf-8") - assert "class InventoryItem" in second_class_text + self.assertIn("class InventoryItem", second_class_text) def test_extract_definitions_empty(self): tree, _ = parse_code_to_ast("", "Python") - assert extract_definitions(tree, "Python", kinds=("functions",)) == [] - assert extract_definitions(tree, "Python", kinds=("functions",)) == [] - assert extract_definitions(None, "Python", kinds=("classes",)) == [] - assert extract_definitions(None, "Python", kinds=("classes",)) == [] + self.assertEqual(extract_definitions(tree, "Python", kinds=("functions",)), []) + self.assertEqual(extract_definitions(tree, "Python", kinds=("functions",)), []) + self.assertEqual(extract_definitions(None, "Python", kinds=("classes",)), []) + self.assertEqual(extract_definitions(None, "Python", kinds=("classes",)), []) def test_get_qualified_name_functions(self): source_code = """ @@ -233,13 +180,13 @@ def global_utility(): index = collect_definitions(tree.root_node, "Python") functions = extract_definitions(tree, "Python", kinds=("functions",)) - assert len(functions) == 2 + self.assertEqual(len(functions), 2) outer_function_name = qualified_name_from_index(functions[0], index) inner_function_name = qualified_name_from_index(functions[1], index) - assert outer_function_name == "CoreService.Validator.validate_payload" - assert inner_function_name == "global_utility" + self.assertEqual(outer_function_name, "CoreService.Validator.validate_payload") + self.assertEqual(inner_function_name, "global_utility") def test_get_qualified_classes(self): source_code = """ @@ -251,25 +198,25 @@ class DroneController: index = collect_definitions(tree.root_node, "Python") classes = extract_definitions(tree, "Python", kinds=("classes",)) - assert len(classes) == 2 + self.assertEqual(len(classes), 2) outer_class_name = qualified_name_from_index(classes[0], index) inner_class_name = qualified_name_from_index(classes[1], index) - assert outer_class_name == "FleetManagement" - assert inner_class_name == "FleetManagement.DroneController" + self.assertEqual(outer_class_name, "FleetManagement") + self.assertEqual(inner_class_name, "FleetManagement.DroneController") def test_classify_reachability(self): - assert classify_reachability(None) == ReachabilityStatus.NOT_REACHABLE - assert classify_reachability({}) == ReachabilityStatus.NOT_REACHABLE - assert ( + self.assertEqual(classify_reachability(None), ReachabilityStatus.NOT_REACHABLE) + self.assertEqual(classify_reachability({}), ReachabilityStatus.NOT_REACHABLE) + self.assertEqual( classify_reachability( {"sym1": {"exact_match_fingerprint": "hash123", "called": True}} - ) - == ReachabilityStatus.REACHABLE + ), + ReachabilityStatus.REACHABLE, ) - assert ( + self.assertEqual( classify_reachability( { "sym1": { @@ -277,22 +224,22 @@ def test_classify_reachability(self): "reachable_from": ["main_function", "api_handler"], } } - ) - == ReachabilityStatus.REACHABLE + ), + ReachabilityStatus.REACHABLE, ) - assert ( - classify_reachability({"sym1": {"defined": True, "called": False}}) - == ReachabilityStatus.POTENTIALLY_REACHABLE + self.assertEqual( + classify_reachability({"sym1": {"defined": True, "called": False}}), + ReachabilityStatus.POTENTIALLY_REACHABLE, ) - assert ( + self.assertEqual( classify_reachability( {"sym1": {"exact_match_fingerprint": "hash123", "called": False}} - ) - == ReachabilityStatus.POTENTIALLY_REACHABLE + ), + ReachabilityStatus.POTENTIALLY_REACHABLE, ) - assert ( - classify_reachability({"sym1": {"file_path": "src/vulnerable.py"}}) - == ReachabilityStatus.NOT_REACHABLE + self.assertEqual( + classify_reachability({"sym1": {"file_path": "src/vulnerable.py"}}), + ReachabilityStatus.NOT_REACHABLE, ) def test_get_changed_lines(self): @@ -300,8 +247,8 @@ def test_get_changed_lines(self): diff_text = (data / "diff-app.patch").read_text(encoding="utf-8") removed, added = get_changed_lines(diff_text, "app.py") - assert removed == [17, 18, 19, 24] - assert added == [17, 18, 19, 20, 21, 22, 27, 28, 29, 30] + self.assertEqual(removed, [17, 18, 19, 24]) + self.assertEqual(added, [17, 18, 19, 20, 21, 22, 27, 28, 29, 30]) def test_build_symbol_metadata_processing(self): source_code = """ @@ -319,26 +266,30 @@ def process_data(payload): nodes = extract_definitions(tree, "Python", kinds=("functions",)) metadata = build_symbol_metadata(nodes, "Python") - assert metadata == { - "Controller.process_data": { - "qualified_name": "Controller.process_data", - "simple_name": "process_data", - "text": "def process_data(payload):\n def inner_helper():\n return True\n return payload.strip()", - "fingerprint": "0000000888014a04b037189a42b238a2c50f218c", - "start_line": 3, - "end_line": 6, - "node_type": "function_definition", - }, - "process_data": { - "qualified_name": "process_data", - "simple_name": "process_data", - "text": "def process_data(payload):\n return payload", - "fingerprint": "000000022020300e882a900807880d0300010000", - "start_line": 9, - "end_line": 10, - "node_type": "function_definition", + self.assertEqual( + metadata, + { + "Controller.process_data": { + "qualified_name": "Controller.process_data", + "text": "def process_data(payload):\n" + " def inner_helper():\n" + " return True\n" + " return payload.strip()", + "fingerprint": "0000000888014a04b037189a42b238a2c50f218c", + "start_line": 3, + "end_line": 6, + "node_type": "function_definition", + }, + "process_data": { + "qualified_name": "process_data", + "text": "def process_data(payload):\n return payload", + "fingerprint": "000000022020300e882a900807880d0300010000", + "start_line": 9, + "end_line": 10, + "node_type": "function_definition", + }, }, - } + ) def test_diff_changed_symbols(self): vuln_meta = { @@ -359,7 +310,10 @@ def test_diff_changed_symbols(self): fixed_meta = { "serve_report": { "qualified_name": "app.serve_report", - "text": "def serve_report():\n if not target.startswith(base): raise ValueError\n return target", + "text": "def serve_report():\n " + " if not target.startswith(base): " + "raise ValueError\n " + " return target", }, "sanitize_input": { "qualified_name": "app.sanitize_input", @@ -373,26 +327,34 @@ def test_diff_changed_symbols(self): vuln_only, fixed_only = diff_changed_symbols(vuln_meta, fixed_meta) - assert vuln_only == { - "serve_report": { - "qualified_name": "app.serve_report", - "text": "def serve_report():\n return os.path.join(base, filename)", - }, - "deprecated_logger": { - "qualified_name": "app.deprecated_logger", - "text": "def deprecated_logger():\n print('legacy')", - }, - } - assert fixed_only == { - "serve_report": { - "qualified_name": "app.serve_report", - "text": "def serve_report():\n if not target.startswith(base): raise ValueError\n return target", + self.assertEqual( + vuln_only, + { + "serve_report": { + "qualified_name": "app.serve_report", + "text": "def serve_report():\n " + " return os.path.join(base, filename)", + }, + "deprecated_logger": { + "qualified_name": "app.deprecated_logger", + "text": "def deprecated_logger():\n print('legacy')", + }, }, - "audit_trail": { - "qualified_name": "app.audit_trail", - "text": "def audit_trail():\n log.info('action')", + ) + self.assertEqual( + fixed_only, + { + "serve_report": { + "qualified_name": "app.serve_report", + "text": "def serve_report():\n if not target.startswith(base): " + "raise ValueError\n return target", + }, + "audit_trail": { + "qualified_name": "app.audit_trail", + "text": "def audit_trail():\n log.info('action')", + }, }, - } + ) def test_analyze_patched_file(self): vuln_text = (self.data / "vuln-app.py").read_text(encoding="utf-8") @@ -406,28 +368,70 @@ def test_analyze_patched_file(self): file_path="app.py", ) - assert vuln_meta == { - "serve_report": { - "qualified_name": "serve_report", - "simple_name": "serve_report", - "text": 'def serve_report(request_payload):\n """Top-level function handling a request."""\n generator = ReportGenerator("/var/reports")\n requested_file = request_payload.get("file")\n\n # Helper function nested inside serve_report\n def build_file_path(filename):\n # VULNERABLE: Direct concatenation allows Path Traversal\n # An attacker passing "../../etc/passwd" could read system files.\n return os.path.join(generator.base_dir, filename)\n\n if not requested_file:\n return "Error: No file specified"\n\n target_path = build_file_path(requested_file)\n\n if os.path.exists(target_path):\n return f"Serving content of {target_path}"\n\n return "Error: File not found"', - "fingerprint": "000000556d322a47595af353274b000aa324e014", - "start_line": 11, - "end_line": 30, - "node_type": "function_definition", - } - } - assert fixed_meta == { - "serve_report": { - "qualified_name": "serve_report", - "simple_name": "serve_report", - "text": 'def serve_report(request_payload):\n """Top-level function handling a request."""\n generator = ReportGenerator("/var/reports")\n requested_file = request_payload.get("file")\n\n # Helper function nested inside serve_report\n def build_file_path(filename):\n # FIXED: Validate that the resolved path stays within the base_dir\n base = os.path.abspath(generator.base_dir)\n target = os.path.abspath(os.path.join(base, filename))\n if not target.startswith(base):\n raise ValueError("Path Traversal Detected")\n return target\n\n if not requested_file:\n return "Error: No file specified"\n\n try:\n target_path = build_file_path(requested_file)\n except ValueError:\n return "Error: Invalid path"\n\n if os.path.exists(target_path):\n return f"Serving content of {target_path}"\n\n return "Error: File not found"', - "fingerprint": "0000006cceea8aedf1da91830f67b64927086d24", - "start_line": 11, - "end_line": 36, - "node_type": "function_definition", - } - } + self.assertEqual( + vuln_meta, + { + "serve_report": { + "qualified_name": "serve_report", + "text": "def serve_report(request_payload):\n " + ' """Top-level function handling a request."""\n ' + ' generator = ReportGenerator("/var/reports")\n ' + ' requested_file = request_payload.get("file")\n\n ' + "# Helper function nested inside serve_report\n " + "def build_file_path(filename):\n " + " # VULNERABLE: Direct concatenation allows Path Traversal\n " + ' # An attacker passing "../../etc/passwd" ' + "could read system files.\n " + " return os.path.join(generator.base_dir, filename)\n\n " + " if not requested_file:\n " + ' return "Error: No file specified"\n\n ' + " target_path = build_file_path(requested_file)\n\n" + " " + " " + "if os.path.exists(target_path):\n" + " " + ' return f"Serving content of {target_path}"\n\n ' + ' return "Error: File not found"', + "fingerprint": "000000556d322a47595af353274b000aa324e014", + "start_line": 11, + "end_line": 30, + "node_type": "function_definition", + } + }, + ) + + self.assertEqual( + fixed_meta, + { + "serve_report": { + "qualified_name": "serve_report", + "text": "def serve_report(request_payload):\n " + ' """Top-level function handling a request."""\n ' + ' generator = ReportGenerator("/var/reports")\n ' + ' requested_file = request_payload.get("file")\n\n ' + " # Helper function nested inside serve_report\n " + " def build_file_path(filename):\n " + " # FIXED: Validate that the resolved " + "path stays within the base_dir\n " + " base = os.path.abspath(generator.base_dir)\n " + " target = os.path.abspath(os.path.join(base, filename))\n " + " if not target.startswith(base):\n " + ' raise ValueError("Path Traversal Detected")\n ' + " return target\n\n if not requested_file:\n " + ' return "Error: No file specified"\n\n try:\n ' + " target_path = build_file_path(requested_file)\n " + " except ValueError:\n " + ' return "Error: Invalid path"\n\n' + " if os.path.exists(target_path):\n " + ' return f"Serving content of {target_path}"\n\n ' + ' return "Error: File not found"', + "fingerprint": "0000006cceea8aedf1da91830f67b64927086d24", + "start_line": 11, + "end_line": 36, + "node_type": "function_definition", + } + }, + ) def test_extract_symbols(self): source_code = ( @@ -443,13 +447,13 @@ def test_extract_symbols(self): changed_lines = [4] enclosing_symbols = extract_symbols(tree, changed_lines, "Python") - assert len(enclosing_symbols) == 1 + self.assertEqual(len(enclosing_symbols), 1) target_node = enclosing_symbols[0] - assert target_node.type == "function_definition" + self.assertEqual(target_node.type, "function_definition") node_text = target_node.text.decode("utf-8") - assert "def build_path" in node_text - assert "def serve_report" not in node_text + self.assertIn("def build_path", node_text) + self.assertNotIn("def serve_report", node_text) def test_extract_symbols_deduplication(self): source_code = ( @@ -462,5 +466,22 @@ def test_extract_symbols_deduplication(self): changed_lines = [2, 3] enclosing_symbols = extract_symbols(tree, changed_lines, "Python") - assert len(enclosing_symbols) == 1 - assert enclosing_symbols[0].type == "function_definition" \ No newline at end of file + self.assertEqual(len(enclosing_symbols), 1) + self.assertEqual(enclosing_symbols[0].type, "function_definition") + + def test_compute_reachable_symbols(self): + call_graph = { + "edges_qualified": { + "app.main": {"app.helper", "app.safe_func"}, + "app.helper": {"app.vuln_func"}, + "app.direct_caller": {"app.vuln_func"}, + "app.unrelated": {"app.safe_func"}, + } + } + + target_qns = ["app.vuln_func"] + reachable, has_direct = compute_reachable_symbols(call_graph, target_qns) + self.assertTrue(has_direct) + + expected_reachable = {"app.main", "app.helper", "app.direct_caller"} + self.assertEqual(reachable, expected_reachable) From 2f0dd8496d0a38b933ae8d5827ab21c69613441c Mon Sep 17 00:00:00 2001 From: ziad hany Date: Thu, 11 Jun 2026 18:47:37 +0300 Subject: [PATCH 04/25] Fix a bug in the import-catching logic and add a test. Signed-off-by: ziad hany --- scanpipe/pipes/reachability.py | 255 +++++++++++------- scanpipe/pipes/symbols.py | 15 ++ .../tests/pipes/test_symbols_reachability.py | 39 ++- 3 files changed, 215 insertions(+), 94 deletions(-) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index e9ebd97378..ad8a347e8f 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -55,14 +55,14 @@ class ReachabilityStatus(str, Enum): def api_mocker(): """TODO: Remove this once the API patch url is done""" return [ - { - "vcs_url": "https://github.com/pallets/flask", - "commit_hash": "089cb86dd22bff589a4eafb7ab8e42dc357623b4", - }, # { - # "vcs_url": "https://github.com/aio-libs/aiohttp", - # "commit_hash": "0c2e9da51126238a421568eb7c5b53e5b5d17b36", - # } + # "vcs_url": "https://github.com/pallets/flask", + # "commit_hash": "089cb86dd22bff589a4eafb7ab8e42dc357623b4", + # }, + { + "vcs_url": "https://github.com/aio-libs/aiohttp", + "commit_hash": "0c2e9da51126238a421568eb7c5b53e5b5d17b36", + } ] @@ -139,7 +139,7 @@ def get_commit_and_parent(repo, commit_hash): def get_commit_diff_text(repo, parent_commit, commit): """Whole-commit unified diff (used to extract changed line numbers).""" base = parent_commit.hexsha if parent_commit else EMPTY_TREE_SHA - return repo.git.diff(base, commit.hexsha, unified=0) + return repo.git.diff(base, commit.hexsha, unified=3) def get_changed_files(parent_commit, commit): @@ -195,7 +195,13 @@ def get_changed_files(parent_commit, commit): def get_changed_lines(diff_text, file_path): - """Return `(removed_lines, added_lines)` for one file from a unified diff.""" + """ + Return `(removed_lines, added_lines)` for one file. + + For pure-insertion hunks (no removed lines) we anchor the vulnerable side + to the hunk's source location so the enclosing old symbol is still found. + For pure-deletion hunks we do the mirror image on the added side. + """ removed = [] added = [] @@ -208,20 +214,37 @@ def get_changed_lines(diff_text, file_path): (patched_file.source_file or "").removeprefix("a/"), (patched_file.target_file or "").removeprefix("b/"), } - if file_path not in candidates: continue for hunk in patched_file: - for line in hunk: - if line.is_removed and line.source_line_no: - removed.append(line.source_line_no) - elif line.is_added and line.target_line_no: - added.append(line.target_line_no) + hunk_removed = [ + line.source_line_no + for line in hunk + if line.is_removed and line.source_line_no + ] + hunk_added = [ + line.target_line_no + for line in hunk + if line.is_added and line.target_line_no + ] + + # Pure insertion: nothing removed -> anchor old side to the + # line just before the insertion point in the source file. + if hunk_added and not hunk_removed: + anchor = max(hunk.source_start, 1) + hunk_removed = [anchor] + + # Pure deletion: nothing added -> anchor new side similarly. + if hunk_removed and not hunk_added: + anchor = max(hunk.target_start, 1) + hunk_added = [anchor] + + removed.extend(hunk_removed) + added.extend(hunk_added) return removed, added - def diff_changed_symbols(vuln_meta, fixed_meta): """ Keep only symbols whose body actually differs between vulnerable and fixed @@ -343,26 +366,6 @@ def collect_patch_symbols(repo, commit_hash): return by_language -def append_symbol_reachability_result(resource, result): - """ - Append one symbol reachability result to the resource extra_data without - overwriting previous results. - """ - extra_data = resource.extra_data or {} - existing_results = extra_data.get("symbols_reachability", []) - - if not isinstance(existing_results, list): - existing_results = [existing_results] - - existing_results.append(result) - - resource.update_extra_data( - { - "symbols_reachability": existing_results, - } - ) - - def collect_and_store_symbol_reachability_results(project, logger=None): """ For each known patch commit, determine whether each project codebase @@ -398,9 +401,6 @@ def collect_and_store_symbol_reachability_results(project, logger=None): continue patch_symbols = patch_symbols_by_language[resource_language] - vuln_patch_metadata = patch_symbols["vulnerable"] - fixed_patch_metadata = patch_symbols["fixed"] - resource_index = build_resource_index( resource_text, resource_language, @@ -409,38 +409,34 @@ def collect_and_store_symbol_reachability_results(project, logger=None): if not resource_index: continue - vuln_match_symbols = match_symbols_against_resource( - vuln_patch_metadata, + vuln_evidence = match_symbols_against_resource( + patch_symbols["vulnerable"], resource_index, ) - - fixed_match_symbols = match_symbols_against_resource( - fixed_patch_metadata, + fixed_evidence = match_symbols_against_resource( + patch_symbols["fixed"], resource_index, ) - if not vuln_match_symbols and not fixed_match_symbols: + if not vuln_evidence and not fixed_evidence: continue result = { - "reachability_status": classify_reachability(vuln_match_symbols), - "summary": { - "call_paths": { - qualified_name: ev.get("reachable_from", []) - for qualified_name, ev in vuln_match_symbols.items() - if ev.get("called") - }, - }, - "evidence": vuln_match_symbols, - "vulnerable_symbols": sorted(vuln_match_symbols), - "fixed_symbols": sorted(fixed_match_symbols), + "reachability_status": classify_reachability(vuln_evidence).value, + "vulnerable_symbols": sorted(vuln_evidence), + "fixed_symbols": sorted(fixed_evidence), + "evidence": vuln_evidence, "patch": { "vcs_url": vcs_url, "commit_hash": commit_hash, }, } - print(result) - append_symbol_reachability_result(resource, result) + + resource.update_extra_data( + { + "symbols_reachability": result, + } + ) except Exception as e: logger( @@ -448,11 +444,8 @@ def collect_and_store_symbol_reachability_results(project, logger=None): f"{vcs_url}@{commit_hash}: {e}" ) finally: - if repo: - repo.close() - # cleanup_repo(repo_path) - + pass def build_resource_index(resource_text, language): if not is_supported_language(language) or not resource_text: @@ -489,7 +482,11 @@ def match_symbols_against_resource(patch_symbols_metadata, resource_index): if not patch_symbols_metadata or not resource_index: return {} - call_graph = resource_index.get("call_graph") + call_graph = resource_index.get("call_graph") or {} + imports = call_graph.get("imports", {}) + + # Set of fully-qualified names the resource imports, e.g. "aiohttp.ClientSession" + imported_fq_names = set(imports.values()) target_qualified_names = { metadata["qualified_name"] for metadata in patch_symbols_metadata.values() @@ -501,24 +498,37 @@ def match_symbols_against_resource(patch_symbols_metadata, resource_index): ) called_qualified_names = set() - - if call_graph: - for callees in call_graph.get("edges_qualified", {}).values(): - called_qualified_names |= set(callees) + for callees in call_graph.get("edges_qualified", {}).values(): + called_qualified_names |= set(callees) matched = {} - for metadata in patch_symbols_metadata.values(): qualified_name = metadata["qualified_name"] fingerprint = metadata["fingerprint"] - defined = qualified_name in resource_index.get("definitions", {}) + defined = qualified_name in resource_index.get("definitions", set()) fingerprint_hit = bool( - fingerprint and fingerprint in resource_index.get("fingerprints", {}) + fingerprint and fingerprint in resource_index.get("fingerprints", set()) ) - called = qualified_name in called_qualified_names - if not (defined or fingerprint_hit or called): + # Does the resource *import* this symbol? + # Match either the bare name (import key) or any fq import target + # that ends with ".". + imported = ( + qualified_name in imports + or qualified_name in imported_fq_names + or any( + fq == qualified_name or fq.endswith("." + qualified_name) + for fq in imported_fq_names + ) + ) + + called = any( + fq_name == qualified_name or fq_name.endswith("." + qualified_name) + for fq_name in called_qualified_names + ) + + if not (defined or fingerprint_hit or called or imported): continue entry = matched.setdefault( @@ -526,18 +536,29 @@ def match_symbols_against_resource(patch_symbols_metadata, resource_index): { "defined": False, "called": False, + "imported": False, + "fingerprint": None, "reachable_from": [], + "external": False, }, ) - entry["defined"] = entry["defined"] or defined - entry["called"] = entry["called"] or called + if defined: + entry["defined"] = True - if fingerprint_hit: - entry["exact_match_fingerprint"] = fingerprint + if imported: + entry["imported"] = True + if not defined: + entry["external"] = True if called: + entry["called"] = True entry["reachable_from"] = sorted(reachable_callers) + if not defined: + entry["external"] = True + + if fingerprint_hit: + entry["fingerprint"] = fingerprint return matched @@ -553,8 +574,9 @@ def classify_reachability(evidence): has_path = bool(item.get("reachable_from")) is_exact = "exact_match_fingerprint" in item is_defined = bool(item.get("defined")) + is_imported = bool(item.get("imported")) - if is_called or has_path: + if is_called or has_path or is_imported: return ReachabilityStatus.REACHABLE if is_exact or is_defined: @@ -601,6 +623,7 @@ def build_call_graph(tree, language): return None index = collect_definitions(tree.root_node, language) + import_map = collect_imports(tree.root_node, language) graph_meta = {} for definition in index.values(): @@ -611,13 +634,12 @@ def build_call_graph(tree, language): continue body_text = node.text.decode("utf-8", errors="replace") - fingerprints = create_exact_symbol_fingerprint(body_text) or {} - + fingerprint = create_exact_symbol_fingerprint(body_text) graph_meta[qualified_name] = { "qualified_name": qualified_name, "node": node, "node_type": node.type, - "fingerprint": fingerprints, + "fingerprint": fingerprint, } definitions_by_name = {} @@ -635,7 +657,6 @@ def build_call_graph(tree, language): direct_calls = extract_direct_calls(metadata["node"], language, index) resolved_callees = set() - for receiver_name, callee_name in direct_calls: resolved_callees |= resolve_callee( receiver_name=receiver_name, @@ -643,6 +664,7 @@ def build_call_graph(tree, language): owner_qn=qualified_name, definitions_by_name=definitions_by_name, class_methods=class_methods, + import_map=import_map, ) edges_qualified[qualified_name] = resolved_callees @@ -650,6 +672,7 @@ def build_call_graph(tree, language): return { "nodes": graph_meta, "edges_qualified": edges_qualified, + "imports": import_map, } @@ -741,27 +764,30 @@ def get_call_receiver(callee_node): def resolve_callee( - receiver_name, callee_name, owner_qn, definitions_by_name, class_methods + receiver_name, + callee_name, + owner_qn, + definitions_by_name, + class_methods, + import_map=None, ): - """ - Resolve a call to candidate qualified names. + import_map = import_map or {} - Examples: - self.foo() from class A -> {"A.foo"} if A.foo exists - foo() -> definitions named "foo" - - """ if receiver_name == "self": owner_class = get_owner_class_name(owner_qn) - if owner_class: method_qn = f"{owner_class}.{callee_name}" - if method_qn in class_methods: return {method_qn} - candidates = definitions_by_name.get(callee_name, set()) - return set(candidates) + if callee_name in import_map: + return {import_map[callee_name]} + + if receiver_name is not None and receiver_name in import_map: + base = import_map[receiver_name] + return {f"{base}.{callee_name}"} + + return set(definitions_by_name.get(callee_name, set())) def get_owner_class_name(owner_qn): @@ -809,3 +835,46 @@ def compute_reachable_symbols(call_graph, target_qualified_names): frontier.append(parent) return reachable, bool(direct) + + +def collect_imports(root_node, language: str): + """ + Returns a dict mapping local names/aliases to their absolute import path. + Examples: + 'from django.db import models' -> {'models': 'django.db.models'} + 'import os.path' -> {'os.path': 'os.path'} + 'import numpy as np' -> {'np': 'numpy'} + 'from a.b import c as d' -> {'d': 'a.b.c'} + """ + import_map = {} + query = get_query(language, "imports") + if not query or not root_node: + return import_map + + for _pattern_index, captures in query.matches(root_node): + module_name = None + import_name = None + alias = None + + for node_name, nodes in captures.items(): + if not nodes: + continue + + text = nodes[0].text.decode("utf-8", errors="replace") + if node_name == "module_name": + module_name = text + elif node_name == "import_name": + import_name = text + elif node_name == "alias": + alias = text + + if not import_name: + continue + + local_name = alias or import_name + if module_name: + import_map[local_name] = f"{module_name}.{import_name}" + else: + import_map[local_name] = import_name + + return import_map diff --git a/scanpipe/pipes/symbols.py b/scanpipe/pipes/symbols.py index da6f359f65..6637bbf1c2 100644 --- a/scanpipe/pipes/symbols.py +++ b/scanpipe/pipes/symbols.py @@ -206,6 +206,21 @@ def _collect_and_store_tree_sitter_symbols_and_strings(resource): object: (_) @receiver attribute: (identifier) @callee)) """, + "imports": """ + (import_statement name: (dotted_name) @import_name) + (import_statement + name: (aliased_import + name: (dotted_name) @import_name + alias: (identifier) @alias)) + (import_from_statement + module_name: (dotted_name) @module_name + name: (dotted_name) @import_name) + (import_from_statement + module_name: (dotted_name) @module_name + name: (aliased_import + name: (dotted_name) @import_name + alias: (identifier) @alias)) + """, }, } diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index 68bbe9b964..3d5f27ab28 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -27,7 +27,7 @@ from scanpipe.models import Project from scanpipe.pipes import collect_and_create_codebase_resources -from scanpipe.pipes.reachability import ReachabilityStatus +from scanpipe.pipes.reachability import ReachabilityStatus, collect_imports, extract_direct_calls from scanpipe.pipes.reachability import analyze_patched_file from scanpipe.pipes.reachability import build_symbol_metadata from scanpipe.pipes.reachability import classify_reachability @@ -485,3 +485,40 @@ def test_compute_reachable_symbols(self): expected_reachable = {"app.main", "app.helper", "app.direct_caller"} self.assertEqual(reachable, expected_reachable) + + def test_collect_imports(self): + source_code = """ +from django.db import models +import os.path +import numpy as np +from a.b import c as d + """.strip() + + tree, _ = parse_code_to_ast(source_code, "Python") + real_root_node = tree.root_node + result = collect_imports(real_root_node, language="Python") + + expected_map = { + "models": "django.db.models", + "os.path": "os.path", + "np": "numpy", + "d": "a.b.c", + } + self.assertEqual(result, expected_map) + + def test_extract_direct(self): + source_code = """ +def hello(): + return 10 + +def clean_function(): + x = 10 + y = 20 + return hello() + x + y + """.strip() + + tree, _ = parse_code_to_ast(source_code, "Python") + functions = extract_definitions(tree, "Python", kinds=("functions",)) + + result = extract_direct_calls(functions[1], "Python", []) + self.assertEqual(result, [(None, 'hello')]) \ No newline at end of file From 00fd333a6ca3ba7e52961fd2aeda68e991cd6828 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Tue, 16 Jun 2026 17:56:41 +0300 Subject: [PATCH 05/25] Add an unidiff dependency to pyproject.toml file Fix the test Signed-off-by: ziad hany --- scanpipe/pipes/reachability.py | 43 ++--- .../tests/pipes/test_symbols_reachability.py | 160 ++++++++---------- 2 files changed, 91 insertions(+), 112 deletions(-) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index ad8a347e8f..26a1da1d8d 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -55,14 +55,14 @@ class ReachabilityStatus(str, Enum): def api_mocker(): """TODO: Remove this once the API patch url is done""" return [ - # { - # "vcs_url": "https://github.com/pallets/flask", - # "commit_hash": "089cb86dd22bff589a4eafb7ab8e42dc357623b4", - # }, { - "vcs_url": "https://github.com/aio-libs/aiohttp", - "commit_hash": "0c2e9da51126238a421568eb7c5b53e5b5d17b36", - } + "vcs_url": "https://github.com/pallets/flask", + "commit_hash": "089cb86dd22bff589a4eafb7ab8e42dc357623b4", + }, + # { + # "vcs_url": "https://github.com/aio-libs/aiohttp", + # "commit_hash": "0c2e9da51126238a421568eb7c5b53e5b5d17b36", + # } ] @@ -245,6 +245,7 @@ def get_changed_lines(diff_text, file_path): return removed, added + def diff_changed_symbols(vuln_meta, fixed_meta): """ Keep only symbols whose body actually differs between vulnerable and fixed @@ -382,10 +383,8 @@ def collect_and_store_symbol_reachability_results(project, logger=None): vcs_url = patch["vcs_url"] commit_hash = patch["commit_hash"] try: - # repo_path = clone_repo(vcs_url, commit_hash) - # repo = Repo("/home/ziad-hany/PycharmProjects/flask/") - repo = Repo("/home/ziad-hany/PycharmProjects/aiohttp") - + repo_path = clone_repo(vcs_url, commit_hash) + repo = Repo(repo_path) patch_symbols_by_language = collect_patch_symbols(repo, commit_hash) if not patch_symbols_by_language: @@ -413,6 +412,7 @@ def collect_and_store_symbol_reachability_results(project, logger=None): patch_symbols["vulnerable"], resource_index, ) + fixed_evidence = match_symbols_against_resource( patch_symbols["fixed"], resource_index, @@ -447,6 +447,7 @@ def collect_and_store_symbol_reachability_results(project, logger=None): # cleanup_repo(repo_path) pass + def build_resource_index(resource_text, language): if not is_supported_language(language) or not resource_text: return None @@ -484,8 +485,6 @@ def match_symbols_against_resource(patch_symbols_metadata, resource_index): call_graph = resource_index.get("call_graph") or {} imports = call_graph.get("imports", {}) - - # Set of fully-qualified names the resource imports, e.g. "aiohttp.ClientSession" imported_fq_names = set(imports.values()) target_qualified_names = { @@ -511,9 +510,6 @@ def match_symbols_against_resource(patch_symbols_metadata, resource_index): fingerprint and fingerprint in resource_index.get("fingerprints", set()) ) - # Does the resource *import* this symbol? - # Match either the bare name (import key) or any fq import target - # that ends with ".". imported = ( qualified_name in imports or qualified_name in imported_fq_names @@ -539,7 +535,6 @@ def match_symbols_against_resource(patch_symbols_metadata, resource_index): "imported": False, "fingerprint": None, "reachable_from": [], - "external": False, }, ) @@ -548,14 +543,10 @@ def match_symbols_against_resource(patch_symbols_metadata, resource_index): if imported: entry["imported"] = True - if not defined: - entry["external"] = True if called: entry["called"] = True entry["reachable_from"] = sorted(reachable_callers) - if not defined: - entry["external"] = True if fingerprint_hit: entry["fingerprint"] = fingerprint @@ -572,14 +563,14 @@ def classify_reachability(evidence): for item in evidence.values(): is_called = bool(item.get("called")) has_path = bool(item.get("reachable_from")) - is_exact = "exact_match_fingerprint" in item is_defined = bool(item.get("defined")) is_imported = bool(item.get("imported")) + is_exact = bool(item.get("fingerprint")) - if is_called or has_path or is_imported: + if is_exact or (is_imported and (is_called or has_path)): return ReachabilityStatus.REACHABLE - if is_exact or is_defined: + if (is_imported or is_defined) and not is_exact: highest_status = ReachabilityStatus.POTENTIALLY_REACHABLE return highest_status @@ -839,12 +830,14 @@ def compute_reachable_symbols(call_graph, target_qualified_names): def collect_imports(root_node, language: str): """ - Returns a dict mapping local names/aliases to their absolute import path. + Return a dict mapping local names/aliases to their absolute import path. + Examples: 'from django.db import models' -> {'models': 'django.db.models'} 'import os.path' -> {'os.path': 'os.path'} 'import numpy as np' -> {'np': 'numpy'} 'from a.b import c as d' -> {'d': 'a.b.c'} + """ import_map = {} query = get_query(language, "imports") diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index 3d5f27ab28..1c633c3cc0 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -27,13 +27,15 @@ from scanpipe.models import Project from scanpipe.pipes import collect_and_create_codebase_resources -from scanpipe.pipes.reachability import ReachabilityStatus, collect_imports, extract_direct_calls +from scanpipe.pipes.reachability import ReachabilityStatus from scanpipe.pipes.reachability import analyze_patched_file from scanpipe.pipes.reachability import build_symbol_metadata from scanpipe.pipes.reachability import classify_reachability from scanpipe.pipes.reachability import collect_and_store_symbol_reachability_results +from scanpipe.pipes.reachability import collect_imports from scanpipe.pipes.reachability import compute_reachable_symbols from scanpipe.pipes.reachability import diff_changed_symbols +from scanpipe.pipes.reachability import extract_direct_calls from scanpipe.pipes.reachability import get_changed_lines from scanpipe.pipes.symbols import collect_definitions from scanpipe.pipes.symbols import extract_definitions @@ -104,28 +106,25 @@ def test_collect_and_store_symbol_reachability_results( self.assertEqual( results, - [ - { - "patch": { - "vcs_url": "https://github.com/aboutcode-org/test", - "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", - }, - "summary": {"call_paths": {}}, - "evidence": { - "serve_report": { - "called": False, - "defined": True, - "reachable_from": [], - "exact_match_fingerprint": ( - "e341b914f9823915e0685396a730d421ec9e3635" - ), - } - }, - "fixed_symbols": ["serve_report"], - "vulnerable_symbols": ["serve_report"], - "reachability_status": "POTENTIALLY_REACHABLE", - } - ], + { + "patch": { + "vcs_url": "https://github.com/aboutcode-org/test", + "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", + }, + "evidence": { + "serve_report": { + "called": False, + "defined": True, + "imported": False, + "fingerprint": "d7675efb263896da2a3c0067951183" + "3553907e7e6ea619115a6dfc8625c3457e", + "reachable_from": [], + } + }, + "fixed_symbols": ["serve_report"], + "vulnerable_symbols": ["serve_report"], + "reachability_status": "REACHABLE", + }, ) def test_extract_definitions(self): @@ -210,35 +209,20 @@ def test_classify_reachability(self): self.assertEqual(classify_reachability(None), ReachabilityStatus.NOT_REACHABLE) self.assertEqual(classify_reachability({}), ReachabilityStatus.NOT_REACHABLE) self.assertEqual( - classify_reachability( - {"sym1": {"exact_match_fingerprint": "hash123", "called": True}} - ), + classify_reachability({"evidence": {"fingerprint": "hash123"}}), ReachabilityStatus.REACHABLE, ) self.assertEqual( - classify_reachability( - { - "sym1": { - "called": True, - "reachable_from": ["main_function", "api_handler"], - } - } - ), + classify_reachability({"evidence": {"imported": True, "called": True}}), ReachabilityStatus.REACHABLE, ) self.assertEqual( - classify_reachability({"sym1": {"defined": True, "called": False}}), - ReachabilityStatus.POTENTIALLY_REACHABLE, - ) - self.assertEqual( - classify_reachability( - {"sym1": {"exact_match_fingerprint": "hash123", "called": False}} - ), + classify_reachability({"evidence": {"imported": True, "called": False}}), ReachabilityStatus.POTENTIALLY_REACHABLE, ) self.assertEqual( - classify_reachability({"sym1": {"file_path": "src/vulnerable.py"}}), + classify_reachability({"evidence": {"imported": False, "called": False}}), ReachabilityStatus.NOT_REACHABLE, ) @@ -275,7 +259,8 @@ def process_data(payload): " def inner_helper():\n" " return True\n" " return payload.strip()", - "fingerprint": "0000000888014a04b037189a42b238a2c50f218c", + "fingerprint": "b0d0ad9a92209a6d79b84e932ce302" + "a8bc9054a405131adf7dc21e06e2e7c0c1", "start_line": 3, "end_line": 6, "node_type": "function_definition", @@ -283,7 +268,8 @@ def process_data(payload): "process_data": { "qualified_name": "process_data", "text": "def process_data(payload):\n return payload", - "fingerprint": "000000022020300e882a900807880d0300010000", + "fingerprint": "9b2797712c9ab60ea8452a441396" + "5c94d1b2f63739cab7de695e7b1dc0cf439a", "start_line": 9, "end_line": 10, "node_type": "function_definition", @@ -373,26 +359,24 @@ def test_analyze_patched_file(self): { "serve_report": { "qualified_name": "serve_report", - "text": "def serve_report(request_payload):\n " - ' """Top-level function handling a request."""\n ' - ' generator = ReportGenerator("/var/reports")\n ' - ' requested_file = request_payload.get("file")\n\n ' - "# Helper function nested inside serve_report\n " - "def build_file_path(filename):\n " - " # VULNERABLE: Direct concatenation allows Path Traversal\n " - ' # An attacker passing "../../etc/passwd" ' - "could read system files.\n " - " return os.path.join(generator.base_dir, filename)\n\n " - " if not requested_file:\n " - ' return "Error: No file specified"\n\n ' - " target_path = build_file_path(requested_file)\n\n" - " " - " " - "if os.path.exists(target_path):\n" - " " - ' return f"Serving content of {target_path}"\n\n ' - ' return "Error: File not found"', - "fingerprint": "000000556d322a47595af353274b000aa324e014", + "text": "def serve_report(request_payload):\n" + ' """Top-level function handling a request."""\n' + ' generator = ReportGenerator("/var/reports")\n' + ' requested_file = request_payload.get("file")\n\n' + " # Helper function nested inside serve_report\n" + " def build_file_path(filename):\n" + " # VULNERABLE: Direct concatenation allows Path Traversal\n" + ' # An attacker passing "../../etc/passwd"' + " could read system files.\n" + " return os.path.join(generator.base_dir, filename)\n\n" + " if not requested_file:\n" + ' return "Error: No file specified"\n\n' + " target_path = build_file_path(requested_file)\n\n" + " if os.path.exists(target_path):\n" + ' return f"Serving content of {target_path}"\n\n' + ' return "Error: File not found"', + "fingerprint": "d7675efb263896da2a3c0067951183" + "3553907e7e6ea619115a6dfc8625c3457e", "start_line": 11, "end_line": 30, "node_type": "function_definition", @@ -405,27 +389,30 @@ def test_analyze_patched_file(self): { "serve_report": { "qualified_name": "serve_report", - "text": "def serve_report(request_payload):\n " - ' """Top-level function handling a request."""\n ' - ' generator = ReportGenerator("/var/reports")\n ' - ' requested_file = request_payload.get("file")\n\n ' - " # Helper function nested inside serve_report\n " - " def build_file_path(filename):\n " - " # FIXED: Validate that the resolved " - "path stays within the base_dir\n " - " base = os.path.abspath(generator.base_dir)\n " - " target = os.path.abspath(os.path.join(base, filename))\n " - " if not target.startswith(base):\n " - ' raise ValueError("Path Traversal Detected")\n ' - " return target\n\n if not requested_file:\n " - ' return "Error: No file specified"\n\n try:\n ' - " target_path = build_file_path(requested_file)\n " - " except ValueError:\n " - ' return "Error: Invalid path"\n\n' - " if os.path.exists(target_path):\n " - ' return f"Serving content of {target_path}"\n\n ' - ' return "Error: File not found"', - "fingerprint": "0000006cceea8aedf1da91830f67b64927086d24", + "text": "def serve_report(request_payload):\n" + ' """Top-level function handling a request."""\n' + ' generator = ReportGenerator("/var/reports")\n' + ' requested_file = request_payload.get("file")\n\n' + " # Helper function nested inside serve_report\n" + " def build_file_path(filename):\n" + " # FIXED: Validate that the resolved" + " path stays within the base_dir\n" + " base = os.path.abspath(generator.base_dir)\n" + " target = os.path.abspath(os.path.join(base, filename))\n" + " if not target.startswith(base):\n" + ' raise ValueError("Path Traversal Detected")\n' + " return target\n\n" + " if not requested_file:\n " + ' return "Error: No file specified"\n\n' + " try:\n" + " target_path = build_file_path(requested_file)\n" + " except ValueError:\n" + ' return "Error: Invalid path"\n\n ' + " if os.path.exists(target_path):\n" + ' return f"Serving content of {target_path}"\n\n' + ' return "Error: File not found"', + "fingerprint": "2deedb21d5f9b1409c59f0b1e5512d7" + "3d9afdfc3f469ccf86e8835915d240e76", "start_line": 11, "end_line": 36, "node_type": "function_definition", @@ -510,7 +497,6 @@ def test_extract_direct(self): source_code = """ def hello(): return 10 - def clean_function(): x = 10 y = 20 @@ -521,4 +507,4 @@ def clean_function(): functions = extract_definitions(tree, "Python", kinds=("functions",)) result = extract_direct_calls(functions[1], "Python", []) - self.assertEqual(result, [(None, 'hello')]) \ No newline at end of file + self.assertEqual(result, [(None, "hello")]) From 5da2504d7265b67a0bc475ca0c66a12e03a32bb3 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Tue, 16 Jun 2026 18:53:55 +0300 Subject: [PATCH 06/25] Add unidiff to package dependencies to parse diff text Signed-off-by: ziad hany --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 4b3f331a1e..50a4286150 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,6 +84,7 @@ dependencies = [ "urllib3==2.7.0", "idna==3.18", "GitPython==3.1.50", + "unidiff==0.7.5", "lxml==6.1.1", "certifi==2026.6.17", # Profiling From 1e7f76e921605a600d8c269b2bcb47eaf894057f Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 24 Jun 2026 02:03:15 +0300 Subject: [PATCH 07/25] Simplify the pipeline logic for imports and direct calls Add missing logic for imports Signed-off-by: ziad hany --- scanpipe/pipes/reachability.py | 511 ++++++++++-------- scanpipe/pipes/symbols.py | 77 +-- .../tests/pipes/test_symbols_reachability.py | 63 ++- 3 files changed, 357 insertions(+), 294 deletions(-) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index 26a1da1d8d..1e52eadeff 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -23,8 +23,11 @@ import os import shutil import tempfile +from dataclasses import dataclass +from dataclasses import field from enum import Enum from pathlib import Path +from typing import Any from git import Repo from git.diff import NULL_TREE @@ -35,11 +38,10 @@ from scanpipe.pipes.symbols import TS_QUERIES from scanpipe.pipes.symbols import _root_of from scanpipe.pipes.symbols import collect_definitions -from scanpipe.pipes.symbols import create_exact_symbol_fingerprint +from scanpipe.pipes.symbols import create_sha256_fingerprint from scanpipe.pipes.symbols import extract_definitions from scanpipe.pipes.symbols import extract_symbols from scanpipe.pipes.symbols import get_query -from scanpipe.pipes.symbols import is_nested_function from scanpipe.pipes.symbols import parse_code_to_ast from scanpipe.pipes.symbols import qualified_name_from_index @@ -330,7 +332,7 @@ def collect_patch_symbols(repo, commit_hash): } """ - commit, parent = get_commit_and_parent(repo, commit_hash) + commit, parent = get_commit_and_parent(repo=repo, commit_hash=commit_hash) diff_text = get_commit_diff_text(repo, parent, commit) changed = get_changed_files(parent, commit) @@ -383,8 +385,11 @@ def collect_and_store_symbol_reachability_results(project, logger=None): vcs_url = patch["vcs_url"] commit_hash = patch["commit_hash"] try: - repo_path = clone_repo(vcs_url, commit_hash) - repo = Repo(repo_path) + # repo_path = clone_repo(vcs_url, commit_hash) + # repo = Repo(repo_path) + + repo = Repo("/home/ziad-hany/PycharmProjects/flask") + # repo = Repo("/home/ziad-hany/PycharmProjects/vulnerablecode") patch_symbols_by_language = collect_patch_symbols(repo, commit_hash) if not patch_symbols_by_language: @@ -422,14 +427,18 @@ def collect_and_store_symbol_reachability_results(project, logger=None): continue result = { - "reachability_status": classify_reachability(vuln_evidence).value, - "vulnerable_symbols": sorted(vuln_evidence), - "fixed_symbols": sorted(fixed_evidence), - "evidence": vuln_evidence, - "patch": { - "vcs_url": vcs_url, - "commit_hash": commit_hash, - }, + "symbols_reachability": { + "patch": { + "vcs_url": vcs_url, + "commit_hash": commit_hash, + }, + "evidence": list(vuln_evidence.values()), + "fixed_symbols": sorted(fixed_evidence.keys()), + "vulnerable_symbols": sorted(vuln_evidence.keys()), + "reachability_status": classify_reachability( + vuln_evidence + ).value, + } } resource.update_extra_data( @@ -448,43 +457,147 @@ def collect_and_store_symbol_reachability_results(project, logger=None): pass -def build_resource_index(resource_text, language): +@dataclass +class SymbolNode: + qualified_name: str + node: Any # tree-sitter node + node_type: str + fingerprint: str + + +@dataclass +class CallGraph: + nodes: dict[str, SymbolNode] = field(default_factory=dict) + edges_qualified: dict[str, set[str]] = field(default_factory=dict) + imports: dict[str, str] = field(default_factory=dict) + + +@dataclass +class ResourceIndex: + definitions: set[str] = field(default_factory=set) + fingerprints: set[str] = field(default_factory=set) + call_graph: CallGraph | None = None + + def to_dict(self): + return { + "definitions": self.definitions, + "fingerprints": self.fingerprints, + "call_graph": { + "nodes": {k: vars(v) for k, v in self.call_graph.nodes.items()}, + "edges_qualified": self.call_graph.edges_qualified, + "imports": self.call_graph.imports, + } + if self.call_graph + else None, + } + + +class CallGraphBuilder: + def __init__(self, tree, language: str): + self.tree = tree + self.language = language + self.index = collect_definitions(tree.root_node, language) + self.imports = collect_imports(tree.root_node, language) + + self.nodes: dict[str, SymbolNode] = {} + self.definitions_by_name: dict[str, set[str]] = {} + self.class_methods: set[str] = set() + + def build(self) -> CallGraph: + """Main execution flow for building the call graph.""" + self._extract_nodes() + edges = self._resolve_all_edges() + + return CallGraph(nodes=self.nodes, edges_qualified=edges, imports=self.imports) + + def _extract_nodes(self): + """Extract all definitions and populate lookup maps.""" + sep = get_imports_language_config(self.language)["separator"] + + for definition in self.index.values(): + node = definition["node"] + qualified_name = qualified_name_from_index(node, self.index) + + if not qualified_name: + continue + + body_text = node.text.decode("utf-8", errors="replace") + fingerprint = create_sha256_fingerprint(body_text) + + self.nodes[qualified_name] = SymbolNode( + qualified_name=qualified_name, + node=node, + node_type=node.type, + fingerprint=fingerprint, + ) + + short_name = qualified_name.rsplit(sep, 1)[-1] + self.definitions_by_name.setdefault(short_name, set()).add(qualified_name) + + if node.type == "function_definition": + parent = node.parent + if parent is not None and parent.type == "class_definition": + self.class_methods.add(qualified_name) + + def _resolve_all_edges(self) -> dict[str, set[str]]: + """Map out all direct calls for every node in the graph.""" + edges_qualified = {} + + for qualified_name, symbol_node in self.nodes.items(): + direct_calls = extract_direct_calls(symbol_node.node, self.language) + resolved_callees = set() + + for receiver_name, callee_name in direct_calls: + resolved_callees |= resolve_callee( + receiver_name=receiver_name, + callee_name=callee_name, + owner_qn=qualified_name, + definitions_by_name=self.definitions_by_name, + class_methods=self.class_methods, + import_map=self.imports, + language=self.language, + ) + + edges_qualified[qualified_name] = resolved_callees + + return edges_qualified + + +def build_resource_index(resource_text, language) -> ResourceIndex | None: if not is_supported_language(language) or not resource_text: return None tree, _ = parse_code_to_ast(resource_text, language) - if tree is None: return None call_graph = build_call_graph(tree, language) - meta = ( - call_graph["nodes"] - if call_graph - else build_symbol_metadata( - extract_definitions(tree, language), - language, - ) - ) + if call_graph: + definitions = set(call_graph.nodes.keys()) + fingerprints = { + node.fingerprint for node in call_graph.nodes.values() if node.fingerprint + } + else: + meta = build_symbol_metadata(extract_definitions(tree, language), language) + definitions = {m["qualified_name"] for m in meta.values()} + fingerprints = {m["fingerprint"] for m in meta.values() if m["fingerprint"]} - return { - "definitions": {metadata["qualified_name"] for metadata in meta.values()}, - "fingerprints": { - metadata["fingerprint"] - for metadata in meta.values() - if metadata["fingerprint"] - }, - "call_graph": call_graph, - } + return ResourceIndex( + definitions=definitions, fingerprints=fingerprints, call_graph=call_graph + ) -def match_symbols_against_resource(patch_symbols_metadata, resource_index): +def match_symbols_against_resource( + patch_symbols_metadata: dict[str, Any], resource_index: "ResourceIndex" +) -> dict[str, Any]: if not patch_symbols_metadata or not resource_index: return {} - call_graph = resource_index.get("call_graph") or {} - imports = call_graph.get("imports", {}) + call_graph = resource_index.call_graph + imports = call_graph.imports if call_graph else {} + edges_qualified = call_graph.edges_qualified if call_graph else {} + imported_fq_names = set(imports.values()) target_qualified_names = { @@ -492,12 +605,12 @@ def match_symbols_against_resource(patch_symbols_metadata, resource_index): } reachable_callers, _ = compute_reachable_symbols( - call_graph, + edges_qualified, target_qualified_names, ) called_qualified_names = set() - for callees in call_graph.get("edges_qualified", {}).values(): + for callees in edges_qualified.values(): called_qualified_names |= set(callees) matched = {} @@ -505,9 +618,9 @@ def match_symbols_against_resource(patch_symbols_metadata, resource_index): qualified_name = metadata["qualified_name"] fingerprint = metadata["fingerprint"] - defined = qualified_name in resource_index.get("definitions", set()) + defined = qualified_name in resource_index.definitions fingerprint_hit = bool( - fingerprint and fingerprint in resource_index.get("fingerprints", set()) + fingerprint and fingerprint in resource_index.fingerprints ) imported = ( @@ -530,8 +643,9 @@ def match_symbols_against_resource(patch_symbols_metadata, resource_index): entry = matched.setdefault( qualified_name, { - "defined": False, + "symbol_name": qualified_name, "called": False, + "defined": False, "imported": False, "fingerprint": None, "reachable_from": [], @@ -540,15 +654,12 @@ def match_symbols_against_resource(patch_symbols_metadata, resource_index): if defined: entry["defined"] = True - if imported: entry["imported"] = True - if called: entry["called"] = True entry["reachable_from"] = sorted(reachable_callers) - - if fingerprint_hit: + if fingerprint: entry["fingerprint"] = fingerprint return matched @@ -558,8 +669,7 @@ def classify_reachability(evidence): if not evidence: return ReachabilityStatus.NOT_REACHABLE - highest_status = ReachabilityStatus.NOT_REACHABLE - + status = ReachabilityStatus.NOT_REACHABLE for item in evidence.values(): is_called = bool(item.get("called")) has_path = bool(item.get("reachable_from")) @@ -571,9 +681,9 @@ def classify_reachability(evidence): return ReachabilityStatus.REACHABLE if (is_imported or is_defined) and not is_exact: - highest_status = ReachabilityStatus.POTENTIALLY_REACHABLE + status = ReachabilityStatus.POTENTIALLY_REACHABLE - return highest_status + return status def build_symbol_metadata(nodes, language, index=None): @@ -582,15 +692,12 @@ def build_symbol_metadata(nodes, language, index=None): metadata = {} for node in nodes: - if is_nested_function(node, language): - continue - qualified_name = qualified_name_from_index(node, index) if not qualified_name: continue body_text = node.text.decode("utf-8", errors="replace") - fingerprints = create_exact_symbol_fingerprint(body_text) + fingerprints = create_sha256_fingerprint(body_text) key = qualified_name suffix = 1 @@ -609,65 +716,15 @@ def build_symbol_metadata(nodes, language, index=None): return metadata -def build_call_graph(tree, language): +def build_call_graph(tree, language) -> CallGraph | None: if tree is None or not is_supported_language(language): return None - index = collect_definitions(tree.root_node, language) - import_map = collect_imports(tree.root_node, language) + builder = CallGraphBuilder(tree, language) + return builder.build() - graph_meta = {} - for definition in index.values(): - node = definition["node"] - qualified_name = qualified_name_from_index(node, index) - - if not qualified_name: - continue - body_text = node.text.decode("utf-8", errors="replace") - fingerprint = create_exact_symbol_fingerprint(body_text) - graph_meta[qualified_name] = { - "qualified_name": qualified_name, - "node": node, - "node_type": node.type, - "fingerprint": fingerprint, - } - - definitions_by_name = {} - class_methods = set() - - for qualified_name, metadata in graph_meta.items(): - name = qualified_name.rsplit(".", 1)[-1] - definitions_by_name.setdefault(name, set()).add(qualified_name) - - if metadata["node_type"] == "function_definition" and "." in qualified_name: - class_methods.add(qualified_name) - - edges_qualified = {} - for qualified_name, metadata in graph_meta.items(): - direct_calls = extract_direct_calls(metadata["node"], language, index) - - resolved_callees = set() - for receiver_name, callee_name in direct_calls: - resolved_callees |= resolve_callee( - receiver_name=receiver_name, - callee_name=callee_name, - owner_qn=qualified_name, - definitions_by_name=definitions_by_name, - class_methods=class_methods, - import_map=import_map, - ) - - edges_qualified[qualified_name] = resolved_callees - - return { - "nodes": graph_meta, - "edges_qualified": edges_qualified, - "imports": import_map, - } - - -def extract_direct_calls(node, language, definition_index): +def extract_direct_calls(node, language): """ Return direct calls inside `node`, excluding calls inside nested definitions. @@ -684,54 +741,30 @@ def extract_direct_calls(node, language, definition_index): if query is None or node is None: return [] - definition_ids = set(definition_index) calls = [] + definition_types = {"function_definition", "class_definition"} for _, captures in query.matches(node): for callee_node in captures.get("callee", []): - if is_inside_nested_definition( - node=callee_node, - owner_node=node, - definition_ids=definition_ids, - ): + cur = callee_node.parent + inside_nested = False + while cur is not None and cur.id != node.id: + if cur.type in definition_types: + inside_nested = True + break + cur = cur.parent + + if inside_nested: continue receiver_name = get_call_receiver(callee_node) - callee_name = node_text(callee_node) + callee_name = callee_node.text.decode("utf-8", errors="replace") if callee_name: calls.append((receiver_name, callee_name)) - return calls -def is_inside_nested_definition(node, owner_node, definition_ids): - """ - Return True if node is inside a nested function/class within `owner_node`. - - Example: - def outer(): - foo() # belongs to outer - - def inner(): - bar() # nested; should not count as outer's call - - """ - current = node.parent - - while current is not None and current is not owner_node: - if current.id in definition_ids: - return True - - current = current.parent - - return False - - -def node_text(node): - return node.text.decode("utf-8", errors="replace") - - def get_call_receiver(callee_node): """ Return receiver name for attribute calls. @@ -751,50 +784,7 @@ def get_call_receiver(callee_node): if object_node is None: return None - return node_text(object_node) - - -def resolve_callee( - receiver_name, - callee_name, - owner_qn, - definitions_by_name, - class_methods, - import_map=None, -): - import_map = import_map or {} - - if receiver_name == "self": - owner_class = get_owner_class_name(owner_qn) - if owner_class: - method_qn = f"{owner_class}.{callee_name}" - if method_qn in class_methods: - return {method_qn} - - if callee_name in import_map: - return {import_map[callee_name]} - - if receiver_name is not None and receiver_name in import_map: - base = import_map[receiver_name] - return {f"{base}.{callee_name}"} - - return set(definitions_by_name.get(callee_name, set())) - - -def get_owner_class_name(owner_qn): - """ - Return enclosing class name from a qualified name. - - Examples: - "User.save" -> "User" - "User.Inner.save" -> "User.Inner" - "save" -> None - - """ - if "." not in owner_qn: - return None - - return owner_qn.rsplit(".", 1)[0] + return object_node.text.decode("utf-8", errors="replace") def compute_reachable_symbols(call_graph, target_qualified_names): @@ -828,46 +818,141 @@ def compute_reachable_symbols(call_graph, target_qualified_names): return reachable, bool(direct) -def collect_imports(root_node, language: str): - """ - Return a dict mapping local names/aliases to their absolute import path. +def get_imports_language_config(language: str) -> dict: + configs = { + "Python": {"self_keyword": "self", "separator": ".", "wildcard_symbol": "*"}, + "Javascript": { + "self_keyword": "this", + "separator": ".", + "wildcard_symbol": "*", + }, + "Java": {"self_keyword": "this", "separator": ".", "wildcard_symbol": "*"}, + "C++": {"self_keyword": "this", "separator": "::", "wildcard_symbol": None}, + "PHP": {"self_keyword": "$this", "separator": "::", "wildcard_symbol": None}, + "Go": {"self_keyword": None, "separator": "/", "wildcard_symbol": None}, + "Ruby": {"self_keyword": "self", "separator": "::", "wildcard_symbol": None}, + } + return configs.get( + language, {"self_keyword": None, "separator": ".", "wildcard_symbol": None} + ) - Examples: - 'from django.db import models' -> {'models': 'django.db.models'} - 'import os.path' -> {'os.path': 'os.path'} - 'import numpy as np' -> {'np': 'numpy'} - 'from a.b import c as d' -> {'d': 'a.b.c'} +def resolve_callee( + receiver_name: str | None, + callee_name: str, + owner_qn: str, + definitions_by_name: dict[str, set[str]], + class_methods: set[str], + language: str, + import_map: dict | None = None, +) -> set[str]: + config = get_imports_language_config(language) + sep = config["separator"] + self_kw = config["self_keyword"] + import_map = import_map or {} + + if receiver_name: + receiver_name = receiver_name.strip("'\"") + if callee_name: + callee_name = callee_name.strip("'\"") + + if self_kw is not None and receiver_name == self_kw: + owner_class = owner_qn.rsplit(sep, 1)[0] if sep in owner_qn else owner_qn + return {f"{owner_class}{sep}{callee_name}"} + + if callee_name in import_map: + return {import_map[callee_name]} + + if receiver_name is not None and receiver_name != self_kw: + candidates = set() + + if receiver_name in import_map and receiver_name != "*": + base = import_map[receiver_name] + candidates.add(f"{base}{sep}{callee_name}") + + static_fqn = f"{receiver_name}{sep}{callee_name}" + if static_fqn in class_methods: + candidates.add(static_fqn) + + return candidates + + local_defs = definitions_by_name.get(callee_name, set()) + if local_defs: + return local_defs + + if "*" in import_map: + return {f"{mod}{sep}{callee_name}" for mod in import_map["*"]} + + return set() + + +def collect_imports(root_node, language: str) -> dict[str, str | list[str]]: + """ + Extract import statements from a Tree-sitter AST and map every local alias to its absolute imported path. """ - import_map = {} + config = get_imports_language_config(language) + separator = config["separator"] + wildcard_sym = config.get("wildcard_symbol") + query = get_query(language, "imports") if not query or not root_node: - return import_map + return {} + + import_map: dict[str, str | list[str]] = {} + wildcard_modules: list[str] = [] for _pattern_index, captures in query.matches(root_node): + flat_captures = [] + for tag, nodes in captures.items(): + for n in nodes: + flat_captures.append( + ( + n.start_byte, + tag, + n.text.decode("utf-8", errors="replace").strip("'\""), + ) + ) + + flat_captures.sort(key=lambda x: x[0]) + module_name = None - import_name = None - alias = None + current_import = None + pairs = [] - for node_name, nodes in captures.items(): - if not nodes: + for _, tag, text in flat_captures: + if tag == "module_name": + module_name = text + elif tag == "import_name": + if current_import is not None: + pairs.append((current_import, None)) + current_import = text + elif tag == "alias": + pairs.append((current_import, text)) + current_import = None + + if current_import is not None: + pairs.append((current_import, None)) + + for imp_name, alias in pairs: + if not imp_name: continue - text = nodes[0].text.decode("utf-8", errors="replace") - if node_name == "module_name": - module_name = text - elif node_name == "import_name": - import_name = text - elif node_name == "alias": - alias = text + if wildcard_sym is not None and imp_name == wildcard_sym: + if module_name: + wildcard_modules.append(module_name) + continue - if not import_name: - continue + local_name = alias or imp_name + + if not alias and separator in imp_name: + local_name = imp_name.split(separator)[0] + + absolute_path = ( + f"{module_name}{separator}{imp_name}" if module_name else imp_name + ) + import_map[local_name] = absolute_path - local_name = alias or import_name - if module_name: - import_map[local_name] = f"{module_name}.{import_name}" - else: - import_map[local_name] = import_name + if wildcard_modules: + import_map["*"] = wildcard_modules return import_map diff --git a/scanpipe/pipes/symbols.py b/scanpipe/pipes/symbols.py index 6637bbf1c2..b4dd67fb44 100644 --- a/scanpipe/pipes/symbols.py +++ b/scanpipe/pipes/symbols.py @@ -207,19 +207,23 @@ def _collect_and_store_tree_sitter_symbols_and_strings(resource): attribute: (identifier) @callee)) """, "imports": """ - (import_statement name: (dotted_name) @import_name) - (import_statement - name: (aliased_import - name: (dotted_name) @import_name - alias: (identifier) @alias)) - (import_from_statement - module_name: (dotted_name) @module_name - name: (dotted_name) @import_name) + (import_statement name: [ + (dotted_name) @import_name + (aliased_import name: (dotted_name) @import_name alias: (identifier) @alias) + ]) + + ; Combine explicit, relative, aliased, and wildcard from_imports (import_from_statement - module_name: (dotted_name) @module_name - name: (aliased_import + module_name: [ + (dotted_name) @module_name + (relative_import) @module_name + ] + [ name: (dotted_name) @import_name - alias: (identifier) @alias)) + name: (aliased_import name: (dotted_name) @import_name alias: (identifier) @alias) + (wildcard_import) @import_name + ] + ) """, }, } @@ -274,61 +278,12 @@ def run_query(query: Query, root_node): yield def_nodes[0], name -def query_captures(language, kind, node): - """Re-run a definition query on the root of node's tree.""" - query = get_query(language, kind) - return list(run_query(query, _root_of(node))) - - def _root_of(node): while node.parent is not None: node = node.parent return node -def is_nested_function(node, language): - function_nodes = { - captured_node - for captured_node, _ in query_captures(language, "functions", node) - } - class_nodes = { - captured_node for captured_node, _ in query_captures(language, "classes", node) - } - - if node not in function_nodes: - return False - - function_types = {captured_node.type for captured_node in function_nodes} - class_types = {captured_node.type for captured_node in class_nodes} - - parent = node.parent - - while parent is not None: - if parent.type in function_types: - return True - - if parent.type in class_types: - return False - - parent = parent.parent - - return False - - -def extract_calls_in_node(node, language: str): - query = get_query(language, "calls") - if query is None or node is None: - return set() - - names = set() - for _pattern_index, captures in query.matches(node): - for callee_node in captures.get("callee", []): - name = callee_node.text.decode("utf-8", errors="replace") - if name: - names.add(name) - return names - - def collect_definitions(root_node, language: str): index: dict[int, dict] = {} for kind in ("functions", "classes"): @@ -381,7 +336,7 @@ def qualified_name_from_index(node, index): return ".".join(reversed(parts)) -def create_exact_symbol_fingerprint(text): +def create_sha256_fingerprint(text): if text is None: return None diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index 1c633c3cc0..27b3dbb1e3 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -107,23 +107,36 @@ def test_collect_and_store_symbol_reachability_results( self.assertEqual( results, { - "patch": { - "vcs_url": "https://github.com/aboutcode-org/test", - "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", - }, - "evidence": { - "serve_report": { - "called": False, - "defined": True, - "imported": False, - "fingerprint": "d7675efb263896da2a3c0067951183" - "3553907e7e6ea619115a6dfc8625c3457e", - "reachable_from": [], - } - }, - "fixed_symbols": ["serve_report"], - "vulnerable_symbols": ["serve_report"], - "reachability_status": "REACHABLE", + "symbols_reachability": { + "patch": { + "vcs_url": "https://github.com/aboutcode-org/test", + "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", + }, + "evidence": [ + { + "called": False, + "defined": True, + "imported": False, + "fingerprint": "d7675efb263896da2a3c00679511833553907e7e6ea619115a6dfc8625c3457e", + "symbol_name": "serve_report", + "reachable_from": [], + }, + { + "called": False, + "defined": True, + "imported": False, + "fingerprint": "762e4f7d03b1bf4359c3ca364e558140239913bfabcc5aa77156460c2eb0a355", + "symbol_name": "serve_report.build_file_path", + "reachable_from": [], + }, + ], + "fixed_symbols": ["serve_report", "serve_report.build_file_path"], + "vulnerable_symbols": [ + "serve_report", + "serve_report.build_file_path", + ], + "reachability_status": "REACHABLE", + } }, ) @@ -479,6 +492,9 @@ def test_collect_imports(self): import os.path import numpy as np from a.b import c as d +from . import utils +from ..core import engine +from math import * """.strip() tree, _ = parse_code_to_ast(source_code, "Python") @@ -487,10 +503,14 @@ def test_collect_imports(self): expected_map = { "models": "django.db.models", - "os.path": "os.path", + "os": "os.path", "np": "numpy", "d": "a.b.c", + "utils": "..utils", + "engine": "..core.engine", + "*": ["math"], } + self.assertEqual(result, expected_map) def test_extract_direct(self): @@ -506,5 +526,8 @@ def clean_function(): tree, _ = parse_code_to_ast(source_code, "Python") functions = extract_definitions(tree, "Python", kinds=("functions",)) - result = extract_direct_calls(functions[1], "Python", []) - self.assertEqual(result, [(None, "hello")]) + result = extract_direct_calls(functions[1], "Python") + self.assertEqual( + result, + [(None, "hello")], + ) From 4428760a2e7ee0f4e7526a9c244a298d40a133e1 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 24 Jun 2026 13:45:42 +0300 Subject: [PATCH 08/25] Add support for vulnerablecode reachability option Signed-off-by: ziad hany --- scanpipe/pipes/reachability.py | 160 +++++++++++++++---------------- scanpipe/pipes/vulnerablecode.py | 1 + 2 files changed, 76 insertions(+), 85 deletions(-) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index 1e52eadeff..de070b373e 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -54,20 +54,6 @@ class ReachabilityStatus(str, Enum): NOT_REACHABLE = "NOT_REACHABLE" -def api_mocker(): - """TODO: Remove this once the API patch url is done""" - return [ - { - "vcs_url": "https://github.com/pallets/flask", - "commit_hash": "089cb86dd22bff589a4eafb7ab8e42dc357623b4", - }, - # { - # "vcs_url": "https://github.com/aio-libs/aiohttp", - # "commit_hash": "0c2e9da51126238a421568eb7c5b53e5b5d17b36", - # } - ] - - def clone_repo(vcs_url, commit_hash=None): repo_path = tempfile.mkdtemp(prefix="symbol-reachability-") @@ -381,80 +367,96 @@ def collect_and_store_symbol_reachability_results(project, logger=None): is_media=False, ) - for patch in api_mocker(): - vcs_url = patch["vcs_url"] - commit_hash = patch["commit_hash"] - try: - # repo_path = clone_repo(vcs_url, commit_hash) - # repo = Repo(repo_path) + vulnerabilities = project.package_vulnerabilities + + for vulnerability in vulnerabilities: + patches = vulnerability.get("fixed_in_patches", []) - repo = Repo("/home/ziad-hany/PycharmProjects/flask") - # repo = Repo("/home/ziad-hany/PycharmProjects/vulnerablecode") - patch_symbols_by_language = collect_patch_symbols(repo, commit_hash) + cloned_repos = {} + for patch in patches: + vcs_url = patch.get("vcs_url") + commit_hash = patch.get("commit_hash") - if not patch_symbols_by_language: + if not vcs_url or not commit_hash: continue - for resource in candidate_resources: - resource_language = resource.programming_language - if resource_language not in patch_symbols_by_language: - continue + try: + if vcs_url not in cloned_repos: + cloned_repos[vcs_url] = clone_repo(vcs_url, commit_hash) - resource_text = resource.file_content - if not resource_text: - continue + repo_path = cloned_repos[vcs_url] + repo = Repo(repo_path) - patch_symbols = patch_symbols_by_language[resource_language] - resource_index = build_resource_index( - resource_text, - resource_language, - ) + patch_symbols_by_language = collect_patch_symbols(repo, commit_hash) - if not resource_index: + if not patch_symbols_by_language: continue - vuln_evidence = match_symbols_against_resource( - patch_symbols["vulnerable"], - resource_index, - ) + for resource in candidate_resources: + resource_language = resource.programming_language + if resource_language not in patch_symbols_by_language: + continue - fixed_evidence = match_symbols_against_resource( - patch_symbols["fixed"], - resource_index, - ) + resource_text = resource.file_content + if not resource_text: + continue - if not vuln_evidence and not fixed_evidence: - continue + patch_symbols = patch_symbols_by_language[resource_language] + resource_index = build_resource_index( + resource_text, + resource_language, + ) - result = { - "symbols_reachability": { - "patch": { - "vcs_url": vcs_url, - "commit_hash": commit_hash, - }, - "evidence": list(vuln_evidence.values()), - "fixed_symbols": sorted(fixed_evidence.keys()), - "vulnerable_symbols": sorted(vuln_evidence.keys()), - "reachability_status": classify_reachability( - vuln_evidence - ).value, - } - } + if not resource_index: + continue + + vuln_evidence = match_symbols_against_resource( + patch_symbols["vulnerable"], + resource_index, + ) - resource.update_extra_data( - { - "symbols_reachability": result, + fixed_evidence = match_symbols_against_resource( + patch_symbols["fixed"], + resource_index, + ) + + if not vuln_evidence and not fixed_evidence: + continue + + result = { + "symbols_reachability": { + "patch": { + "vcs_url": vcs_url, + "commit_hash": commit_hash, + }, + "evidence": list(vuln_evidence.values()), + "fixed_symbols": sorted(fixed_evidence.keys()), + "vulnerable_symbols": sorted(vuln_evidence.keys()), + "reachability_status": classify_reachability( + vuln_evidence + ).value, + } } - ) - except Exception as e: - logger( - f"Failed to collect symbol reachability for " - f"{vcs_url}@{commit_hash}: {e}" - ) - finally: - # cleanup_repo(repo_path) - pass + resource.update_extra_data( + { + "symbols_reachability": result, + } + ) + print(result) + except Exception as e: + if logger: + logger( + f"Failed to collect symbol reachability for " + f"{vcs_url}@{commit_hash}: {e}" + ) + + for url, path in cloned_repos.items(): + try: + cleanup_repo(path) + except Exception as e: + if logger: + logger(f"Failed to clean up repo {url} at {path}: {e}") @dataclass @@ -742,21 +744,9 @@ def extract_direct_calls(node, language): return [] calls = [] - definition_types = {"function_definition", "class_definition"} for _, captures in query.matches(node): for callee_node in captures.get("callee", []): - cur = callee_node.parent - inside_nested = False - while cur is not None and cur.id != node.id: - if cur.type in definition_types: - inside_nested = True - break - cur = cur.parent - - if inside_nested: - continue - receiver_name = get_call_receiver(callee_node) callee_name = callee_node.text.decode("utf-8", errors="replace") diff --git a/scanpipe/pipes/vulnerablecode.py b/scanpipe/pipes/vulnerablecode.py index 20f5e69b5f..4deaefc704 100644 --- a/scanpipe/pipes/vulnerablecode.py +++ b/scanpipe/pipes/vulnerablecode.py @@ -118,6 +118,7 @@ def bulk_search_by_purl( data = { "purls": purls, "details": True, + "reachability": True, } logger.debug(f"VulnerableCode: url={url} purls_count={len(purls)}") From e1a2d8651056362db6b2eb1005b6937a4c0e13c3 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 24 Jun 2026 17:58:43 +0300 Subject: [PATCH 09/25] Fix the tests and improve patch extraction performance Signed-off-by: ziad hany --- scanpipe/pipes/reachability.py | 120 ++++++++-------- .../tests/pipes/test_symbols_reachability.py | 128 ++++++++++-------- 2 files changed, 130 insertions(+), 118 deletions(-) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index de070b373e..4a156c9b60 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -182,58 +182,6 @@ def get_changed_files(parent_commit, commit): return files -def get_changed_lines(diff_text, file_path): - """ - Return `(removed_lines, added_lines)` for one file. - - For pure-insertion hunks (no removed lines) we anchor the vulnerable side - to the hunk's source location so the enclosing old symbol is still found. - For pure-deletion hunks we do the mirror image on the added side. - """ - removed = [] - added = [] - - if not diff_text: - return removed, added - - for patched_file in PatchSet.from_string(diff_text): - candidates = { - patched_file.path, - (patched_file.source_file or "").removeprefix("a/"), - (patched_file.target_file or "").removeprefix("b/"), - } - if file_path not in candidates: - continue - - for hunk in patched_file: - hunk_removed = [ - line.source_line_no - for line in hunk - if line.is_removed and line.source_line_no - ] - hunk_added = [ - line.target_line_no - for line in hunk - if line.is_added and line.target_line_no - ] - - # Pure insertion: nothing removed -> anchor old side to the - # line just before the insertion point in the source file. - if hunk_added and not hunk_removed: - anchor = max(hunk.source_start, 1) - hunk_removed = [anchor] - - # Pure deletion: nothing added -> anchor new side similarly. - if hunk_removed and not hunk_added: - anchor = max(hunk.target_start, 1) - hunk_added = [anchor] - - removed.extend(hunk_removed) - added.extend(hunk_added) - - return removed, added - - def diff_changed_symbols(vuln_meta, fixed_meta): """ Keep only symbols whose body actually differs between vulnerable and fixed @@ -254,7 +202,9 @@ def diff_changed_symbols(vuln_meta, fixed_meta): return vuln_only, fixed_only -def analyze_patched_file(vulnerable_text, fixed_text, diff_text, file_path): +def analyze_patched_file( + vulnerable_text, fixed_text, removed_lines, added_lines, file_path +): """ Return `(vuln_metadata, fixed_metadata, language)` for one changed file, restricted to symbols actually touched by the patch. @@ -282,8 +232,6 @@ def analyze_patched_file(vulnerable_text, fixed_text, diff_text, file_path): if vuln_tree is None and fixed_tree is None: return {}, {}, language - removed_lines, added_lines = get_changed_lines(diff_text, file_path) - vuln_nodes = ( extract_symbols(vuln_tree, removed_lines, language) if vuln_tree else [] ) @@ -300,6 +248,57 @@ def analyze_patched_file(vulnerable_text, fixed_text, diff_text, file_path): return vuln_meta, fixed_meta, language +def parse_diff_lines(diff_text) -> dict[str, tuple[list[int], list[int]]]: + """ + Parses the entire unified diff text once and maps each file path to its tuple of (removed_lines, added_lines). + """ + diff_map = {} + if not diff_text: + return diff_map + + for patched_file in PatchSet.from_string(diff_text): + removed = [] + added = [] + + for hunk in patched_file: + hunk_removed = [ + line.source_line_no + for line in hunk + if line.is_removed and line.source_line_no + ] + hunk_added = [ + line.target_line_no + for line in hunk + if line.is_added and line.target_line_no + ] + + # Pure insertion anchor + if hunk_added and not hunk_removed: + anchor = max(hunk.source_start, 1) + hunk_removed = [anchor] + + # Pure deletion anchor + if hunk_removed and not hunk_added: + anchor = max(hunk.target_start, 1) + hunk_added = [anchor] + + removed.extend(hunk_removed) + added.extend(hunk_added) + + # Register the line tracking data against all naming variations of this file + candidates = { + patched_file.path, + (patched_file.source_file or "").removeprefix("a/"), + (patched_file.target_file or "").removeprefix("b/"), + } + + for candidate in candidates: + if candidate: + diff_map[candidate] = (removed, added) + + return diff_map + + def collect_patch_symbols(repo, commit_hash): """ Return: @@ -319,17 +318,21 @@ def collect_patch_symbols(repo, commit_hash): """ commit, parent = get_commit_and_parent(repo=repo, commit_hash=commit_hash) - diff_text = get_commit_diff_text(repo, parent, commit) - changed = get_changed_files(parent, commit) + diff_text = get_commit_diff_text(repo=repo, parent_commit=parent, commit=commit) + changed = get_changed_files(parent_commit=parent, commit=commit) + diff_line_map = parse_diff_lines(diff_text=diff_text) by_language = {} for file_path, texts in changed.items(): vulnerable_text = texts["vulnerable_text"] fixed_text = texts["fixed_text"] + removed_lines, added_lines = diff_line_map.get(file_path, ([], [])) + vuln_meta, fixed_meta, language = analyze_patched_file( vulnerable_text=vulnerable_text, fixed_text=fixed_text, - diff_text=diff_text, + removed_lines=removed_lines, + added_lines=added_lines, file_path=file_path, ) @@ -443,7 +446,6 @@ def collect_and_store_symbol_reachability_results(project, logger=None): "symbols_reachability": result, } ) - print(result) except Exception as e: if logger: logger( diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index 27b3dbb1e3..93937dc857 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -21,6 +21,7 @@ # Visit https://github.com/nexB/scancode.io for support and download. from pathlib import Path +from unittest.mock import PropertyMock from unittest.mock import patch from django.test import TestCase @@ -36,7 +37,7 @@ from scanpipe.pipes.reachability import compute_reachable_symbols from scanpipe.pipes.reachability import diff_changed_symbols from scanpipe.pipes.reachability import extract_direct_calls -from scanpipe.pipes.reachability import get_changed_lines +from scanpipe.pipes.reachability import parse_diff_lines from scanpipe.pipes.symbols import collect_definitions from scanpipe.pipes.symbols import extract_definitions from scanpipe.pipes.symbols import extract_symbols @@ -51,31 +52,41 @@ def setUp(self): self.project1 = Project.objects.create(name="Analysis") self.project1.codebase_path.mkdir(parents=True, exist_ok=True) - @patch("scanpipe.pipes.reachability.Repo") @patch("scanpipe.pipes.reachability.clone_repo") - @patch("scanpipe.pipes.reachability.api_mocker") @patch("scanpipe.pipes.reachability.collect_patch_symbols") + @patch.object(Project, "package_vulnerabilities", new_callable=PropertyMock) def test_collect_and_store_symbol_reachability_results( - self, mock_collect_symbols, mock_api, mock_clone_repo, mock_repo + self, + mock_package_vulnerabilities, + mock_collect_symbols, + mock_clone_repo, ): app_text = (self.data / "app.py").read_text() vuln_text = (self.data / "vuln-app.py").read_text() fixed_text = (self.data / "fixed-app.py").read_text() diff_text = (self.data / "diff-app.patch").read_text() + diff_line_map = parse_diff_lines(diff_text=diff_text) + file_path = "app.py" + removed_lines, added_lines = diff_line_map.get(file_path, ([], [])) vuln_meta, fixed_meta, lang = analyze_patched_file( vulnerable_text=vuln_text, fixed_text=fixed_text, - diff_text=diff_text, - file_path="app.py", + removed_lines=removed_lines, + added_lines=added_lines, + file_path=file_path, ) self.assertTrue(lang) self.assertTrue(vuln_meta or fixed_meta) - mock_api.return_value = [ + mock_package_vulnerabilities.return_value = [ { - "vcs_url": "https://github.com/aboutcode-org/test", - "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", + "fixed_in_patches": [ + { + "vcs_url": "https://github.com/aboutcode-org/test", + "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", + } + ] } ] @@ -99,7 +110,8 @@ def test_collect_and_store_symbol_reachability_results( resource.programming_language = lang resource.save() - collect_and_store_symbol_reachability_results(self.project1) + with patch("scanpipe.pipes.reachability.Repo"): + collect_and_store_symbol_reachability_results(self.project1) resource.refresh_from_db() results = resource.extra_data.get("symbols_reachability") @@ -114,19 +126,20 @@ def test_collect_and_store_symbol_reachability_results( }, "evidence": [ { + "symbol_name": "serve_report", "called": False, "defined": True, "imported": False, "fingerprint": "d7675efb263896da2a3c00679511833553907e7e6ea619115a6dfc8625c3457e", - "symbol_name": "serve_report", "reachable_from": [], }, { - "called": False, + "symbol_name": "serve_report.build_file_path", + "called": True, "defined": True, "imported": False, - "fingerprint": "762e4f7d03b1bf4359c3ca364e558140239913bfabcc5aa77156460c2eb0a355", - "symbol_name": "serve_report.build_file_path", + "fingerprint": "762e4f7d03b1bf4359c3ca364e5581402" + "39913bfabcc5aa77156460c2eb0a355", "reachable_from": [], }, ], @@ -239,14 +252,6 @@ def test_classify_reachability(self): ReachabilityStatus.NOT_REACHABLE, ) - def test_get_changed_lines(self): - data = Path(__file__).parent.parent / "data" / "reachability" - diff_text = (data / "diff-app.patch").read_text(encoding="utf-8") - - removed, added = get_changed_lines(diff_text, "app.py") - self.assertEqual(removed, [17, 18, 19, 24]) - self.assertEqual(added, [17, 18, 19, 20, 21, 22, 27, 28, 29, 30]) - def test_build_symbol_metadata_processing(self): source_code = """ class Controller: @@ -268,21 +273,24 @@ def process_data(payload): { "Controller.process_data": { "qualified_name": "Controller.process_data", - "text": "def process_data(payload):\n" - " def inner_helper():\n" - " return True\n" - " return payload.strip()", - "fingerprint": "b0d0ad9a92209a6d79b84e932ce302" - "a8bc9054a405131adf7dc21e06e2e7c0c1", + "text": "def process_data(payload):\n def inner_helper():\n return True\n return payload.strip()", + "fingerprint": "b0d0ad9a92209a6d79b84e932ce302a8bc9054a405131adf7dc21e06e2e7c0c1", "start_line": 3, "end_line": 6, "node_type": "function_definition", }, + "Controller.process_data.inner_helper": { + "qualified_name": "Controller.process_data.inner_helper", + "text": "def inner_helper():\n return True", + "fingerprint": "ee2e246e01e960826cb39a9466e58095d209fdd1cbf8458630be430b3371d6a3", + "start_line": 4, + "end_line": 5, + "node_type": "function_definition", + }, "process_data": { "qualified_name": "process_data", "text": "def process_data(payload):\n return payload", - "fingerprint": "9b2797712c9ab60ea8452a441396" - "5c94d1b2f63739cab7de695e7b1dc0cf439a", + "fingerprint": "9b2797712c9ab60ea8452a4413965c94d1b2f63739cab7de695e7b1dc0cf439a", "start_line": 9, "end_line": 10, "node_type": "function_definition", @@ -359,11 +367,15 @@ def test_analyze_patched_file(self): vuln_text = (self.data / "vuln-app.py").read_text(encoding="utf-8") fixed_text = (self.data / "fixed-app.py").read_text(encoding="utf-8") diff_text = (self.data / "diff-app.patch").read_text(encoding="utf-8") + diff_line_map = parse_diff_lines(diff_text=diff_text) + file_path = "app.py" + removed_lines, added_lines = diff_line_map.get(file_path, ([], [])) vuln_meta, fixed_meta, lang = analyze_patched_file( vulnerable_text=vuln_text, fixed_text=fixed_text, - diff_text=diff_text, + removed_lines=removed_lines, + added_lines=added_lines, file_path="app.py", ) @@ -379,8 +391,7 @@ def test_analyze_patched_file(self): " # Helper function nested inside serve_report\n" " def build_file_path(filename):\n" " # VULNERABLE: Direct concatenation allows Path Traversal\n" - ' # An attacker passing "../../etc/passwd"' - " could read system files.\n" + ' # An attacker passing "../../etc/passwd" could read system files.\n' " return os.path.join(generator.base_dir, filename)\n\n" " if not requested_file:\n" ' return "Error: No file specified"\n\n' @@ -388,12 +399,23 @@ def test_analyze_patched_file(self): " if os.path.exists(target_path):\n" ' return f"Serving content of {target_path}"\n\n' ' return "Error: File not found"', - "fingerprint": "d7675efb263896da2a3c0067951183" - "3553907e7e6ea619115a6dfc8625c3457e", + "fingerprint": "d7675efb263896da2a3c006795118" + "33553907e7e6ea619115a6dfc8625c3457e", "start_line": 11, "end_line": 30, "node_type": "function_definition", - } + }, + "serve_report.build_file_path": { + "qualified_name": "serve_report.build_file_path", + "text": "def build_file_path(filename):\n" + " # VULNERABLE: Direct concatenation allows Path Traversal\n" + ' # An attacker passing "../../etc/passwd" could read system files.\n' + " return os.path.join(generator.base_dir, filename)", + "fingerprint": "762e4f7d03b1bf4359c3ca364e558140239913bfabcc5aa77156460c2eb0a355", + "start_line": 17, + "end_line": 20, + "node_type": "function_definition", + }, }, ) @@ -404,32 +426,20 @@ def test_analyze_patched_file(self): "qualified_name": "serve_report", "text": "def serve_report(request_payload):\n" ' """Top-level function handling a request."""\n' - ' generator = ReportGenerator("/var/reports")\n' - ' requested_file = request_payload.get("file")\n\n' - " # Helper function nested inside serve_report\n" - " def build_file_path(filename):\n" - " # FIXED: Validate that the resolved" - " path stays within the base_dir\n" - " base = os.path.abspath(generator.base_dir)\n" - " target = os.path.abspath(os.path.join(base, filename))\n" - " if not target.startswith(base):\n" - ' raise ValueError("Path Traversal Detected")\n' - " return target\n\n" - " if not requested_file:\n " - ' return "Error: No file specified"\n\n' - " try:\n" - " target_path = build_file_path(requested_file)\n" - " except ValueError:\n" - ' return "Error: Invalid path"\n\n ' - " if os.path.exists(target_path):\n" - ' return f"Serving content of {target_path}"\n\n' - ' return "Error: File not found"', - "fingerprint": "2deedb21d5f9b1409c59f0b1e5512d7" - "3d9afdfc3f469ccf86e8835915d240e76", + ' generator = ReportGenerator("/var/reports")\n requested_file = request_payload.get("file")\n\n # Helper function nested inside serve_report\n def build_file_path(filename):\n # FIXED: Validate that the resolved path stays within the base_dir\n base = os.path.abspath(generator.base_dir)\n target = os.path.abspath(os.path.join(base, filename))\n if not target.startswith(base):\n raise ValueError("Path Traversal Detected")\n return target\n\n if not requested_file:\n return "Error: No file specified"\n\n try:\n target_path = build_file_path(requested_file)\n except ValueError:\n return "Error: Invalid path"\n\n if os.path.exists(target_path):\n return f"Serving content of {target_path}"\n\n return "Error: File not found"', + "fingerprint": "2deedb21d5f9b1409c59f0b1e5512d73d9afdfc3f469ccf86e8835915d240e76", "start_line": 11, "end_line": 36, "node_type": "function_definition", - } + }, + "serve_report.build_file_path": { + "qualified_name": "serve_report.build_file_path", + "text": 'def build_file_path(filename):\n # FIXED: Validate that the resolved path stays within the base_dir\n base = os.path.abspath(generator.base_dir)\n target = os.path.abspath(os.path.join(base, filename))\n if not target.startswith(base):\n raise ValueError("Path Traversal Detected")\n return target', + "fingerprint": "646743b5d5497f6ea3b96f860bcbeb38096ce008ad16d2b9a9c3f77a98faca80", + "start_line": 17, + "end_line": 23, + "node_type": "function_definition", + }, }, ) From bf94dc81db3ef4cea4a5ac063505a0b1a7b3239c Mon Sep 17 00:00:00 2001 From: ziad hany Date: Thu, 25 Jun 2026 15:28:07 +0300 Subject: [PATCH 10/25] Fix formatting and linting errors. Signed-off-by: ziad hany --- scanpipe/pipes/reachability.py | 254 +++++++++--------- scanpipe/pipes/symbols.py | 6 +- .../tests/pipes/test_symbols_reachability.py | 94 +++++-- 3 files changed, 195 insertions(+), 159 deletions(-) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index 4a156c9b60..880d48946b 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -250,7 +250,8 @@ def analyze_patched_file( def parse_diff_lines(diff_text) -> dict[str, tuple[list[int], list[int]]]: """ - Parses the entire unified diff text once and maps each file path to its tuple of (removed_lines, added_lines). + Parse the entire unified diff text once and map each file path + to its tuple of (removed_lines, added_lines). """ diff_map = {} if not diff_text: @@ -358,7 +359,75 @@ def collect_patch_symbols(repo, commit_hash): return by_language -def collect_and_store_symbol_reachability_results(project, logger=None): +def generate_reachability_report(patch, candidate_resources, cloned_repos, logger=None): + vcs_url = patch.get("vcs_url") + commit_hash = patch.get("commit_hash") + + try: + if vcs_url not in cloned_repos: + cloned_repos[vcs_url] = clone_repo(vcs_url, commit_hash) + + repo_path = cloned_repos[vcs_url] + patch_symbols_by_language = collect_patch_symbols(Repo(repo_path), commit_hash) + + if not patch_symbols_by_language: + return + + for resource in candidate_resources: + resource_language = resource.programming_language + patch_symbols = patch_symbols_by_language.get(resource_language) + + if not all([patch_symbols, resource.file_content]): + continue + + resource_index = build_resource_index( + resource.file_content, + resource_language, + ) + + if not resource_index: + continue + + vuln_evidence = match_symbols_against_resource( + patch_symbols["vulnerable"], + resource_index, + ) + + fixed_evidence = match_symbols_against_resource( + patch_symbols["fixed"], + resource_index, + ) + + if not any([vuln_evidence, fixed_evidence]): + continue + + result = { + "symbols_reachability": { + "patch": { + "vcs_url": vcs_url, + "commit_hash": commit_hash, + }, + "evidence": list(vuln_evidence.values()), + "fixed_symbols": sorted(fixed_evidence.keys()), + "vulnerable_symbols": sorted(vuln_evidence.keys()), + "reachability_status": classify_reachability(vuln_evidence).value, + } + } + + resource.update_extra_data( + { + "symbols_reachability": result, + } + ) + except Exception as e: + if logger: + logger( + f"Failed to collect symbol reachability for " + f"{vcs_url}@{commit_hash}: {e}" + ) + + +def get_symbol_reachability_results(project, logger=None): """ For each known patch commit, determine whether each project codebase resource is reachable to the vulnerable code by comparing tree-sitter ASTs @@ -370,88 +439,13 @@ def collect_and_store_symbol_reachability_results(project, logger=None): is_media=False, ) - vulnerabilities = project.package_vulnerabilities - - for vulnerability in vulnerabilities: - patches = vulnerability.get("fixed_in_patches", []) - + for vulnerability in project.package_vulnerabilities: cloned_repos = {} - for patch in patches: - vcs_url = patch.get("vcs_url") - commit_hash = patch.get("commit_hash") - - if not vcs_url or not commit_hash: - continue - - try: - if vcs_url not in cloned_repos: - cloned_repos[vcs_url] = clone_repo(vcs_url, commit_hash) - - repo_path = cloned_repos[vcs_url] - repo = Repo(repo_path) - - patch_symbols_by_language = collect_patch_symbols(repo, commit_hash) - - if not patch_symbols_by_language: - continue - - for resource in candidate_resources: - resource_language = resource.programming_language - if resource_language not in patch_symbols_by_language: - continue - - resource_text = resource.file_content - if not resource_text: - continue - - patch_symbols = patch_symbols_by_language[resource_language] - resource_index = build_resource_index( - resource_text, - resource_language, - ) - - if not resource_index: - continue - - vuln_evidence = match_symbols_against_resource( - patch_symbols["vulnerable"], - resource_index, - ) - - fixed_evidence = match_symbols_against_resource( - patch_symbols["fixed"], - resource_index, - ) - - if not vuln_evidence and not fixed_evidence: - continue - - result = { - "symbols_reachability": { - "patch": { - "vcs_url": vcs_url, - "commit_hash": commit_hash, - }, - "evidence": list(vuln_evidence.values()), - "fixed_symbols": sorted(fixed_evidence.keys()), - "vulnerable_symbols": sorted(vuln_evidence.keys()), - "reachability_status": classify_reachability( - vuln_evidence - ).value, - } - } - - resource.update_extra_data( - { - "symbols_reachability": result, - } - ) - except Exception as e: - if logger: - logger( - f"Failed to collect symbol reachability for " - f"{vcs_url}@{commit_hash}: {e}" - ) + for patch in vulnerability.get("fixed_in_patches", []): + if patch.get("vcs_url") and patch.get("commit_hash"): + generate_reachability_report( + patch, candidate_resources, cloned_repos, logger + ) for url, path in cloned_repos.items(): try: @@ -508,7 +502,6 @@ def __init__(self, tree, language: str): self.class_methods: set[str] = set() def build(self) -> CallGraph: - """Main execution flow for building the call graph.""" self._extract_nodes() edges = self._resolve_all_edges() @@ -609,8 +602,8 @@ def match_symbols_against_resource( } reachable_callers, _ = compute_reachable_symbols( - edges_qualified, - target_qualified_names, + edges_qualified=edges_qualified, + target_qualified_names=target_qualified_names, ) called_qualified_names = set() @@ -730,11 +723,7 @@ def build_call_graph(tree, language) -> CallGraph | None: def extract_direct_calls(node, language): """ - Return direct calls inside `node`, excluding calls inside nested definitions. - - Returns: - list of (receiver_name, callee_name) - + Return direct calls inside `node` Examples: foo() -> (None, "foo") self.foo() -> ("self", "foo") @@ -779,17 +768,13 @@ def get_call_receiver(callee_node): return object_node.text.decode("utf-8", errors="replace") -def compute_reachable_symbols(call_graph, target_qualified_names): +def compute_reachable_symbols(edges_qualified, target_qualified_names): """Transitive callers using resolved qualified-name edges.""" - if not call_graph or not target_qualified_names: - return set(), False - - edges = call_graph.get("edges_qualified") - if not edges: + if not edges_qualified or not target_qualified_names: return set(), False callers_of = {} - for caller, callees in edges.items(): + for caller, callees in edges_qualified.items(): for callee in callees: callers_of.setdefault(callee, set()).add(caller) @@ -878,9 +863,49 @@ def resolve_callee( return set() +def extract_imports(captures: dict) -> tuple[str | None, list[tuple[str, str | None]]]: + """ + Flatten and sort tree-sitter captures, then extract the module name + and a list of import/alias pairs. + """ + flat_captures = [] + for tag, nodes in captures.items(): + for n in nodes: + flat_captures.append( + ( + n.start_byte, + tag, + n.text.decode("utf-8", errors="replace").strip("'\""), + ) + ) + + flat_captures.sort(key=lambda x: x[0]) + + module_name = None + current_import = None + pairs = [] + + for _, tag, text in flat_captures: + if tag == "module_name": + module_name = text + elif tag == "import_name": + if current_import is not None: + pairs.append((current_import, None)) + current_import = text + elif tag == "alias": + pairs.append((current_import, text)) + current_import = None + + if current_import is not None: + pairs.append((current_import, None)) + + return module_name, pairs + + def collect_imports(root_node, language: str) -> dict[str, str | list[str]]: """ - Extract import statements from a Tree-sitter AST and map every local alias to its absolute imported path. + Extract import statements from a Tree-sitter AST and map every + local alias to its absolute imported path. """ config = get_imports_language_config(language) separator = config["separator"] @@ -894,36 +919,7 @@ def collect_imports(root_node, language: str) -> dict[str, str | list[str]]: wildcard_modules: list[str] = [] for _pattern_index, captures in query.matches(root_node): - flat_captures = [] - for tag, nodes in captures.items(): - for n in nodes: - flat_captures.append( - ( - n.start_byte, - tag, - n.text.decode("utf-8", errors="replace").strip("'\""), - ) - ) - - flat_captures.sort(key=lambda x: x[0]) - - module_name = None - current_import = None - pairs = [] - - for _, tag, text in flat_captures: - if tag == "module_name": - module_name = text - elif tag == "import_name": - if current_import is not None: - pairs.append((current_import, None)) - current_import = text - elif tag == "alias": - pairs.append((current_import, text)) - current_import = None - - if current_import is not None: - pairs.append((current_import, None)) + module_name, pairs = extract_imports(captures) for imp_name, alias in pairs: if not imp_name: diff --git a/scanpipe/pipes/symbols.py b/scanpipe/pipes/symbols.py index b4dd67fb44..1cc142057a 100644 --- a/scanpipe/pipes/symbols.py +++ b/scanpipe/pipes/symbols.py @@ -209,7 +209,8 @@ def _collect_and_store_tree_sitter_symbols_and_strings(resource): "imports": """ (import_statement name: [ (dotted_name) @import_name - (aliased_import name: (dotted_name) @import_name alias: (identifier) @alias) + (aliased_import name: (dotted_name) + @import_name alias: (identifier) @alias) ]) ; Combine explicit, relative, aliased, and wildcard from_imports @@ -220,7 +221,8 @@ def _collect_and_store_tree_sitter_symbols_and_strings(resource): ] [ name: (dotted_name) @import_name - name: (aliased_import name: (dotted_name) @import_name alias: (identifier) @alias) + name: (aliased_import name: (dotted_name) + @import_name alias: (identifier) @alias) (wildcard_import) @import_name ] ) diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index 93937dc857..572dd782e6 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -32,11 +32,11 @@ from scanpipe.pipes.reachability import analyze_patched_file from scanpipe.pipes.reachability import build_symbol_metadata from scanpipe.pipes.reachability import classify_reachability -from scanpipe.pipes.reachability import collect_and_store_symbol_reachability_results from scanpipe.pipes.reachability import collect_imports from scanpipe.pipes.reachability import compute_reachable_symbols from scanpipe.pipes.reachability import diff_changed_symbols from scanpipe.pipes.reachability import extract_direct_calls +from scanpipe.pipes.reachability import get_symbol_reachability_results from scanpipe.pipes.reachability import parse_diff_lines from scanpipe.pipes.symbols import collect_definitions from scanpipe.pipes.symbols import extract_definitions @@ -55,7 +55,7 @@ def setUp(self): @patch("scanpipe.pipes.reachability.clone_repo") @patch("scanpipe.pipes.reachability.collect_patch_symbols") @patch.object(Project, "package_vulnerabilities", new_callable=PropertyMock) - def test_collect_and_store_symbol_reachability_results( + def test_get_symbol_reachability_results( self, mock_package_vulnerabilities, mock_collect_symbols, @@ -111,7 +111,7 @@ def test_collect_and_store_symbol_reachability_results( resource.save() with patch("scanpipe.pipes.reachability.Repo"): - collect_and_store_symbol_reachability_results(self.project1) + get_symbol_reachability_results(self.project1) resource.refresh_from_db() results = resource.extra_data.get("symbols_reachability") @@ -126,21 +126,22 @@ def test_collect_and_store_symbol_reachability_results( }, "evidence": [ { - "symbol_name": "serve_report", "called": False, "defined": True, "imported": False, - "fingerprint": "d7675efb263896da2a3c00679511833553907e7e6ea619115a6dfc8625c3457e", + "fingerprint": "d7675efb263896da2a3c00679511833" + "553907e7e6ea619115a6dfc8625c3457e", + "symbol_name": "serve_report", "reachable_from": [], }, { - "symbol_name": "serve_report.build_file_path", "called": True, "defined": True, "imported": False, - "fingerprint": "762e4f7d03b1bf4359c3ca364e5581402" - "39913bfabcc5aa77156460c2eb0a355", - "reachable_from": [], + "fingerprint": "762e4f7d03b1bf4359c3ca364" + "e558140239913bfabcc5aa77156460c2eb0a355", + "symbol_name": "serve_report.build_file_path", + "reachable_from": ["serve_report"], }, ], "fixed_symbols": ["serve_report", "serve_report.build_file_path"], @@ -273,8 +274,12 @@ def process_data(payload): { "Controller.process_data": { "qualified_name": "Controller.process_data", - "text": "def process_data(payload):\n def inner_helper():\n return True\n return payload.strip()", - "fingerprint": "b0d0ad9a92209a6d79b84e932ce302a8bc9054a405131adf7dc21e06e2e7c0c1", + "text": "def process_data(payload):\n" + " def inner_helper():\n" + " return True\n " + " return payload.strip()", + "fingerprint": "b0d0ad9a92209a6d79b84e932ce302" + "a8bc9054a405131adf7dc21e06e2e7c0c1", "start_line": 3, "end_line": 6, "node_type": "function_definition", @@ -282,7 +287,8 @@ def process_data(payload): "Controller.process_data.inner_helper": { "qualified_name": "Controller.process_data.inner_helper", "text": "def inner_helper():\n return True", - "fingerprint": "ee2e246e01e960826cb39a9466e58095d209fdd1cbf8458630be430b3371d6a3", + "fingerprint": "ee2e246e01e960826cb39a9466e5" + "8095d209fdd1cbf8458630be430b3371d6a3", "start_line": 4, "end_line": 5, "node_type": "function_definition", @@ -290,7 +296,8 @@ def process_data(payload): "process_data": { "qualified_name": "process_data", "text": "def process_data(payload):\n return payload", - "fingerprint": "9b2797712c9ab60ea8452a4413965c94d1b2f63739cab7de695e7b1dc0cf439a", + "fingerprint": "9b2797712c9ab60ea8452a4413965" + "c94d1b2f63739cab7de695e7b1dc0cf439a", "start_line": 9, "end_line": 10, "node_type": "function_definition", @@ -391,7 +398,9 @@ def test_analyze_patched_file(self): " # Helper function nested inside serve_report\n" " def build_file_path(filename):\n" " # VULNERABLE: Direct concatenation allows Path Traversal\n" - ' # An attacker passing "../../etc/passwd" could read system files.\n' + " # An attacker passing " + '"../../etc/passwd" ' + "could read system files.\n" " return os.path.join(generator.base_dir, filename)\n\n" " if not requested_file:\n" ' return "Error: No file specified"\n\n' @@ -409,9 +418,11 @@ def test_analyze_patched_file(self): "qualified_name": "serve_report.build_file_path", "text": "def build_file_path(filename):\n" " # VULNERABLE: Direct concatenation allows Path Traversal\n" - ' # An attacker passing "../../etc/passwd" could read system files.\n' + ' # An attacker passing "../../etc/passwd"' + " could read system files.\n" " return os.path.join(generator.base_dir, filename)", - "fingerprint": "762e4f7d03b1bf4359c3ca364e558140239913bfabcc5aa77156460c2eb0a355", + "fingerprint": "762e4f7d03b1bf4359c3ca364e5581" + "40239913bfabcc5aa77156460c2eb0a355", "start_line": 17, "end_line": 20, "node_type": "function_definition", @@ -426,16 +437,44 @@ def test_analyze_patched_file(self): "qualified_name": "serve_report", "text": "def serve_report(request_payload):\n" ' """Top-level function handling a request."""\n' - ' generator = ReportGenerator("/var/reports")\n requested_file = request_payload.get("file")\n\n # Helper function nested inside serve_report\n def build_file_path(filename):\n # FIXED: Validate that the resolved path stays within the base_dir\n base = os.path.abspath(generator.base_dir)\n target = os.path.abspath(os.path.join(base, filename))\n if not target.startswith(base):\n raise ValueError("Path Traversal Detected")\n return target\n\n if not requested_file:\n return "Error: No file specified"\n\n try:\n target_path = build_file_path(requested_file)\n except ValueError:\n return "Error: Invalid path"\n\n if os.path.exists(target_path):\n return f"Serving content of {target_path}"\n\n return "Error: File not found"', - "fingerprint": "2deedb21d5f9b1409c59f0b1e5512d73d9afdfc3f469ccf86e8835915d240e76", + ' generator = ReportGenerator("/var/reports")\n' + ' requested_file = request_payload.get("file")\n\n' + " # Helper function nested inside serve_report\n" + " def build_file_path(filename):\n" + " # FIXED: Validate that the resolved " + "path stays within the base_dir\n" + " base = os.path.abspath(generator.base_dir)\n" + " target = os.path.abspath(os.path.join(base, filename))\n" + " if not target.startswith(base):\n" + ' raise ValueError("Path Traversal Detected")\n' + " return target\n\n" + " if not requested_file:\n" + ' return "Error: No file specified"\n\n' + " try:\n" + " target_path = build_file_path(requested_file)\n" + " except ValueError:\n" + ' return "Error: Invalid path"\n\n' + " if os.path.exists(target_path):\n" + ' return f"Serving content of {target_path}"\n\n' + ' return "Error: File not found"', + "fingerprint": "2deedb21d5f9b1409c59f0b1e5512d" + "73d9afdfc3f469ccf86e8835915d240e76", "start_line": 11, "end_line": 36, "node_type": "function_definition", }, "serve_report.build_file_path": { "qualified_name": "serve_report.build_file_path", - "text": 'def build_file_path(filename):\n # FIXED: Validate that the resolved path stays within the base_dir\n base = os.path.abspath(generator.base_dir)\n target = os.path.abspath(os.path.join(base, filename))\n if not target.startswith(base):\n raise ValueError("Path Traversal Detected")\n return target', - "fingerprint": "646743b5d5497f6ea3b96f860bcbeb38096ce008ad16d2b9a9c3f77a98faca80", + "text": "def build_file_path(filename):\n" + " # FIXED: Validate that the resolved path" + " stays within the base_dir\n" + " base = os.path.abspath(generator.base_dir)\n" + " target = os.path.abspath(os.path.join(base, filename))\n" + " if not target.startswith(base):\n" + ' raise ValueError("Path Traversal Detected")\n' + " return target", + "fingerprint": "646743b5d5497f6ea3b96f860bcbe" + "b38096ce008ad16d2b9a9c3f77a98faca80", "start_line": 17, "end_line": 23, "node_type": "function_definition", @@ -480,17 +519,16 @@ def test_extract_symbols_deduplication(self): self.assertEqual(enclosing_symbols[0].type, "function_definition") def test_compute_reachable_symbols(self): - call_graph = { - "edges_qualified": { - "app.main": {"app.helper", "app.safe_func"}, - "app.helper": {"app.vuln_func"}, - "app.direct_caller": {"app.vuln_func"}, - "app.unrelated": {"app.safe_func"}, - } + edges_qualified = { + "app.main": {"app.helper", "app.safe_func"}, + "app.helper": {"app.vuln_func"}, + "app.direct_caller": {"app.vuln_func"}, + "app.unrelated": {"app.safe_func"}, } target_qns = ["app.vuln_func"] - reachable, has_direct = compute_reachable_symbols(call_graph, target_qns) + reachable, has_direct = compute_reachable_symbols(edges_qualified, target_qns) + self.assertTrue(has_direct) expected_reachable = {"app.main", "app.helper", "app.direct_caller"} From c9b96527d31369d773670b347bf51556c48d99c9 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Sat, 27 Jun 2026 00:29:07 +0300 Subject: [PATCH 11/25] Refactor and simplify reachability pipeline Improve pipeline structure Signed-off-by: ziad hany --- scanpipe/pipes/reachability.py | 1133 ++++++----------- scanpipe/pipes/symbols.py | 355 ++++-- .../tests/pipes/test_symbols_reachability.py | 288 +++-- 3 files changed, 791 insertions(+), 985 deletions(-) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index 880d48946b..b6bf86c837 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -23,29 +23,19 @@ import os import shutil import tempfile -from dataclasses import dataclass -from dataclasses import field from enum import Enum from pathlib import Path from typing import Any from git import Repo from git.diff import NULL_TREE -from git.exc import BadName from scancode.api import get_file_info from unidiff import PatchSet from scanpipe.pipes.symbols import TS_QUERIES -from scanpipe.pipes.symbols import _root_of -from scanpipe.pipes.symbols import collect_definitions +from scanpipe.pipes.symbols import SymbolExtractor from scanpipe.pipes.symbols import create_sha256_fingerprint -from scanpipe.pipes.symbols import extract_definitions -from scanpipe.pipes.symbols import extract_symbols -from scanpipe.pipes.symbols import get_query -from scanpipe.pipes.symbols import parse_code_to_ast -from scanpipe.pipes.symbols import qualified_name_from_index - -EMPTY_TREE_SHA = "4b825dc642cb6eb9a060e54bf8b8e6f9b79b4d2b" +from scanpipe.pipes.symbols import is_supported_language class ReachabilityStatus(str, Enum): @@ -54,31 +44,6 @@ class ReachabilityStatus(str, Enum): NOT_REACHABLE = "NOT_REACHABLE" -def clone_repo(vcs_url, commit_hash=None): - repo_path = tempfile.mkdtemp(prefix="symbol-reachability-") - - try: - repo = Repo.clone_from(vcs_url, repo_path) - - if commit_hash: - repo.git.checkout(commit_hash) - - return repo_path - - except BadName as exc: - cleanup_repo(repo_path) - raise ValueError(f"Commit {commit_hash} not found") from exc - - except Exception: - cleanup_repo(repo_path) - raise - - -def cleanup_repo(repo_path): - if repo_path and os.path.exists(repo_path): - shutil.rmtree(repo_path, ignore_errors=True) - - def normalize_text(content): if content is None: return "" @@ -89,17 +54,7 @@ def normalize_text(content): return str(content) -def is_supported_language(language): - """Return True if the language is supported by tree-sitter queries.""" - return bool(language) and language in TS_QUERIES - - def detect_language_with_scancode(file_path, content): - """ - Write `content` to a temp file preserving `file_path`'s basename - so the extension is meaningful, then ask ScanCode's `get_file_info` - to return the programming language. - """ content = normalize_text(content) if not content: @@ -118,257 +73,287 @@ def detect_language_with_scancode(file_path, content): shutil.rmtree(tmp_dir, ignore_errors=True) -def get_commit_and_parent(repo, commit_hash): - commit = repo.commit(commit_hash) - parent = commit.parents[0] if commit.parents else None - return commit, parent +class GitRepositoryContext: + def __init__(self, vcs_url: str, commit_hash: str = None): + self.vcs_url = vcs_url + self.commit_hash = commit_hash + self.repo_path = None + self._repo = None + + def __enter__(self) -> "GitRepositoryContext": + self.repo_path = tempfile.mkdtemp(prefix="symbol-reachability-") + try: + self._repo = Repo.clone_from(self.vcs_url, self.repo_path) + if self.commit_hash: + self._repo.git.checkout(self.commit_hash) + return self + except Exception as exc: + self._cleanup() + raise ValueError( + f"Failed to clone/checkout {self.vcs_url}@{self.commit_hash}" + ) from exc + + def __exit__(self, exc_type, exc_val, exc_tb): + self._cleanup() + + @property + def repo(self) -> Repo: + return self._repo + + def _cleanup(self): + if self.repo_path and os.path.exists(self.repo_path): + shutil.rmtree(self.repo_path, ignore_errors=True) + + +class PatchAnalyzer: + EMPTY_TREE_SHA = "4b825dc642cb6eb9a060e54bf8b8e6f9b79b4d2b" + + def __init__(self, repo: Repo, commit_hash: str): + self.repo = repo + self.commit = repo.commit(commit_hash) + self.parent_commit = self.commit.parents[0] if self.commit.parents else None + + def get_changed_files(self): + diffs = ( + self.parent_commit.diff(self.commit, create_patch=False) + if self.parent_commit + else self.commit.diff(NULL_TREE, create_patch=False) + ) + files = {} + for diff in diffs: + change_type = diff.change_type + old_path = diff.a_path if change_type in ("D", "M", "R") else None + new_path = diff.b_path if change_type in ("A", "M", "R") else None + path_key = new_path or old_path -def get_commit_diff_text(repo, parent_commit, commit): - """Whole-commit unified diff (used to extract changed line numbers).""" - base = parent_commit.hexsha if parent_commit else EMPTY_TREE_SHA - return repo.git.diff(base, commit.hexsha, unified=3) + if not path_key: + continue + entry = files.setdefault( + path_key, {"vulnerable_text": "", "fixed_text": ""} + ) -def get_changed_files(parent_commit, commit): - """ - Return: - { - file_path: { - "vulnerable_text": "...", - "fixed_text": "...", - } - } + if old_path and self.parent_commit: + entry["vulnerable_text"] = ( + (self.parent_commit.tree / old_path) + .data_stream.read() + .decode("utf-8", errors="replace") + ) - """ - diffs = ( - parent_commit.diff(commit, create_patch=False) - if parent_commit - else commit.diff(NULL_TREE, create_patch=False) - ) + if new_path: + entry["fixed_text"] = ( + (self.commit.tree / new_path) + .data_stream.read() + .decode("utf-8", errors="replace") + ) - files = {} - for diff in diffs: - change_type = diff.change_type - old_path = diff.a_path if change_type in ("D", "M", "R") else None - new_path = diff.b_path if change_type in ("A", "M", "R") else None - path_key = new_path or old_path - - if not path_key: - continue - - entry = files.setdefault( - path_key, - { - "vulnerable_text": "", - "fixed_text": "", - }, - ) + return files - if old_path and parent_commit: - entry["vulnerable_text"] = ( - (parent_commit.tree / old_path) - .data_stream.read() - .decode("utf-8", errors="replace") - ) + def get_commit_diff_text(self): + base = self.parent_commit.hexsha if self.parent_commit else self.EMPTY_TREE_SHA + return self.repo.git.diff(base, self.commit.hexsha, unified=3) - if new_path: - entry["fixed_text"] = ( - (commit.tree / new_path) - .data_stream.read() - .decode("utf-8", errors="replace") - ) + @classmethod + def diff_changed_symbols(cls, vuln_meta, fixed_meta): + vuln_only = { + key: metadata + for key, metadata in vuln_meta.items() + if fixed_meta.get(key, {}).get("text") != metadata["text"] + } + fixed_only = { + key: metadata + for key, metadata in fixed_meta.items() + if vuln_meta.get(key, {}).get("text") != metadata["text"] + } + return vuln_only, fixed_only + + @classmethod + def parse_diff_lines(cls, diff_text): + diff_map = {} + if not diff_text: + return diff_map + + for patched_file in PatchSet.from_string(diff_text): + removed = [] + added = [] + + for hunk in patched_file: + hunk_removed = [ + line.source_line_no + for line in hunk + if line.is_removed and line.source_line_no + ] + hunk_added = [ + line.target_line_no + for line in hunk + if line.is_added and line.target_line_no + ] + + if hunk_added and not hunk_removed: + anchor = max(hunk.source_start, 1) + hunk_removed = [anchor] + + if hunk_removed and not hunk_added: + anchor = max(hunk.target_start, 1) + hunk_added = [anchor] + + removed.extend(hunk_removed) + added.extend(hunk_added) + + candidates = { + patched_file.path, + (patched_file.source_file or "").removeprefix("a/"), + (patched_file.target_file or "").removeprefix("b/"), + } - return files - - -def diff_changed_symbols(vuln_meta, fixed_meta): - """ - Keep only symbols whose body actually differs between vulnerable and fixed - versions. Utilizes the unique suffix keys generated by build_symbol_metadata. - """ - vuln_only = { - key: metadata - for key, metadata in vuln_meta.items() - if fixed_meta.get(key, {}).get("text") != metadata["text"] - } - - fixed_only = { - key: metadata - for key, metadata in fixed_meta.items() - if vuln_meta.get(key, {}).get("text") != metadata["text"] - } - - return vuln_only, fixed_only - - -def analyze_patched_file( - vulnerable_text, fixed_text, removed_lines, added_lines, file_path -): - """ - Return `(vuln_metadata, fixed_metadata, language)` for one changed file, - restricted to symbols actually touched by the patch. - """ - vulnerable_text = normalize_text(vulnerable_text) - fixed_text = normalize_text(fixed_text) - - language = detect_language_with_scancode( - file_path, fixed_text - ) or detect_language_with_scancode(file_path, vulnerable_text) - - if not is_supported_language(language): - return {}, {}, language - - vuln_tree, _ = ( - parse_code_to_ast(vulnerable_text, language) - if vulnerable_text - else (None, None) - ) + for candidate in candidates: + if candidate: + diff_map[candidate] = (removed, added) - fixed_tree, _ = ( - parse_code_to_ast(fixed_text, language) if fixed_text else (None, None) - ) + return diff_map - if vuln_tree is None and fixed_tree is None: - return {}, {}, language + def collect_patch_symbols(self): + diff_text = self.get_commit_diff_text() + changed_files = self.get_changed_files() + diff_line_map = self.parse_diff_lines(diff_text=diff_text) + + by_language = {} + for file_path, texts in changed_files.items(): + vulnerable_text = texts["vulnerable_text"] + fixed_text = texts["fixed_text"] + removed_lines, added_lines = diff_line_map.get(file_path, ([], [])) + + vuln_meta, fixed_meta, language = self.analyze( + vulnerable_text=vulnerable_text, + fixed_text=fixed_text, + removed_lines=removed_lines, + added_lines=added_lines, + file_path=file_path, + ) - vuln_nodes = ( - extract_symbols(vuln_tree, removed_lines, language) if vuln_tree else [] - ) + if not language or not (vuln_meta or fixed_meta): + continue - fixed_nodes = ( - extract_symbols(fixed_tree, added_lines, language) if fixed_tree else [] - ) + language_bucket = by_language.setdefault( + language, {"vulnerable": {}, "fixed": {}} + ) - vuln_meta, fixed_meta = diff_changed_symbols( - build_symbol_metadata(vuln_nodes, language), - build_symbol_metadata(fixed_nodes, language), - ) + language_bucket["vulnerable"].update( + {f"{file_path}::{key}": metadata for key, metadata in vuln_meta.items()} + ) + language_bucket["fixed"].update( + { + f"{file_path}::{key}": metadata + for key, metadata in fixed_meta.items() + } + ) - return vuln_meta, fixed_meta, language + return by_language + @classmethod + def build_symbol_metadata( + cls, nodes, extractor: SymbolExtractor, index: dict = None + ): + if not nodes or not extractor: + return {} -def parse_diff_lines(diff_text) -> dict[str, tuple[list[int], list[int]]]: - """ - Parse the entire unified diff text once and map each file path - to its tuple of (removed_lines, added_lines). - """ - diff_map = {} - if not diff_text: - return diff_map + if index is None: + index = extractor.extract_definitions_index() - for patched_file in PatchSet.from_string(diff_text): - removed = [] - added = [] - - for hunk in patched_file: - hunk_removed = [ - line.source_line_no - for line in hunk - if line.is_removed and line.source_line_no - ] - hunk_added = [ - line.target_line_no - for line in hunk - if line.is_added and line.target_line_no - ] - - # Pure insertion anchor - if hunk_added and not hunk_removed: - anchor = max(hunk.source_start, 1) - hunk_removed = [anchor] - - # Pure deletion anchor - if hunk_removed and not hunk_added: - anchor = max(hunk.target_start, 1) - hunk_added = [anchor] - - removed.extend(hunk_removed) - added.extend(hunk_added) - - # Register the line tracking data against all naming variations of this file - candidates = { - patched_file.path, - (patched_file.source_file or "").removeprefix("a/"), - (patched_file.target_file or "").removeprefix("b/"), - } + metadata = {} + for node in nodes: + qualified_name = extractor._build_qualified_name(node, index) + if not qualified_name: + continue - for candidate in candidates: - if candidate: - diff_map[candidate] = (removed, added) + body_text = node.text.decode("utf-8", errors="replace") + fingerprints = create_sha256_fingerprint(body_text) + + key = qualified_name + suffix = 1 + while key in metadata: + suffix += 1 + key = f"{qualified_name}#{suffix}" + + metadata[key] = { + "qualified_name": qualified_name, + "text": body_text, + "fingerprint": fingerprints, + "start_line": node.start_point[0] + 1, + "end_line": node.end_point[0] + 1, + "node_type": node.type, + } + return metadata - return diff_map + @classmethod + def analyze( + self, vulnerable_text, fixed_text, removed_lines, added_lines, file_path + ): + vulnerable_text = normalize_text(vulnerable_text) + fixed_text = normalize_text(fixed_text) + language = detect_language_with_scancode( + file_path, fixed_text + ) or detect_language_with_scancode(file_path, vulnerable_text) -def collect_patch_symbols(repo, commit_hash): - """ - Return: - { - language: { - "vulnerable": { - "file_path::symbol_key": metadata, - ... - }, - "fixed": { - "file_path::symbol_key": metadata, - ... - }, - }, - ... - } + if not is_supported_language(language): + return {}, {}, language - """ - commit, parent = get_commit_and_parent(repo=repo, commit_hash=commit_hash) - diff_text = get_commit_diff_text(repo=repo, parent_commit=parent, commit=commit) - changed = get_changed_files(parent_commit=parent, commit=commit) - diff_line_map = parse_diff_lines(diff_text=diff_text) - - by_language = {} - for file_path, texts in changed.items(): - vulnerable_text = texts["vulnerable_text"] - fixed_text = texts["fixed_text"] - removed_lines, added_lines = diff_line_map.get(file_path, ([], [])) - - vuln_meta, fixed_meta, language = analyze_patched_file( - vulnerable_text=vulnerable_text, - fixed_text=fixed_text, - removed_lines=removed_lines, - added_lines=added_lines, - file_path=file_path, + lang_query = TS_QUERIES[language]() + + vuln_tree, _ = ( + lang_query.parse_code_to_ast(code_text=vulnerable_text) + if vulnerable_text + else (None, None) + ) + fixed_tree, _ = ( + lang_query.parse_code_to_ast(code_text=fixed_text) + if fixed_text + else (None, None) ) - if not language or not (vuln_meta or fixed_meta): - continue + if vuln_tree is None and fixed_tree is None: + return {}, {}, language - language_bucket = by_language.setdefault( - language, - { - "vulnerable": {}, - "fixed": {}, - }, - ) + vuln_meta_all = {} + fixed_meta_all = {} - language_bucket["vulnerable"].update( - {f"{file_path}::{key}": metadata for key, metadata in vuln_meta.items()} - ) + if vuln_tree: + vuln_extractor = SymbolExtractor( + lang_query=lang_query, root_node=vuln_tree.root_node + ) + vuln_nodes = vuln_extractor.extract_changed_symbols( + changed_lines=removed_lines + ) + vuln_meta_all = self.build_symbol_metadata( + nodes=vuln_nodes, extractor=vuln_extractor + ) - language_bucket["fixed"].update( - {f"{file_path}::{key}": metadata for key, metadata in fixed_meta.items()} - ) + if fixed_tree: + fixed_extractor = SymbolExtractor( + lang_query=lang_query, root_node=fixed_tree.root_node + ) + fixed_nodes = fixed_extractor.extract_changed_symbols( + changed_lines=added_lines + ) + fixed_meta_all = self.build_symbol_metadata( + fixed_nodes, extractor=fixed_extractor + ) - return by_language + vuln_meta, fixed_meta = self.diff_changed_symbols( + vuln_meta=vuln_meta_all, fixed_meta=fixed_meta_all + ) + return vuln_meta, fixed_meta, language -def generate_reachability_report(patch, candidate_resources, cloned_repos, logger=None): +def generate_reachability_report(patch, repo, candidate_resources, logger=None): vcs_url = patch.get("vcs_url") commit_hash = patch.get("commit_hash") try: - if vcs_url not in cloned_repos: - cloned_repos[vcs_url] = clone_repo(vcs_url, commit_hash) - - repo_path = cloned_repos[vcs_url] - patch_symbols_by_language = collect_patch_symbols(Repo(repo_path), commit_hash) + patch_analyzer = PatchAnalyzer(repo=repo, commit_hash=commit_hash) + patch_symbols_by_language = patch_analyzer.collect_patch_symbols() if not patch_symbols_by_language: return @@ -377,31 +362,29 @@ def generate_reachability_report(patch, candidate_resources, cloned_repos, logge resource_language = resource.programming_language patch_symbols = patch_symbols_by_language.get(resource_language) - if not all([patch_symbols, resource.file_content]): + if not patch_symbols: + continue + + file_content = normalize_text(resource.file_content) + if not file_content: continue - resource_index = build_resource_index( - resource.file_content, - resource_language, + resource_analyzer = ResourceAnalyzer( + resource_text=file_content, language=resource_language ) + resource_index = resource_analyzer.build_index() if not resource_index: continue - vuln_evidence = match_symbols_against_resource( - patch_symbols["vulnerable"], - resource_index, - ) - - fixed_evidence = match_symbols_against_resource( - patch_symbols["fixed"], - resource_index, - ) + matcher = ResourcePatchMatcher(resource_index=resource_index) + vuln_evidence = matcher.match(patch_symbols["vulnerable"]) + fixed_evidence = matcher.match(patch_symbols["fixed"]) if not any([vuln_evidence, fixed_evidence]): continue - result = { + report = { "symbols_reachability": { "patch": { "vcs_url": vcs_url, @@ -413,12 +396,17 @@ def generate_reachability_report(patch, candidate_resources, cloned_repos, logge "reachability_status": classify_reachability(vuln_evidence).value, } } + print(report) + + existing = resource.extra_data.get("symbols_reachability") + reports = existing if isinstance(existing, list) else [] + + if not any( + r.get("patch", {}).get("commit_hash") == commit_hash for r in reports + ): + reports.append(report) + resource.update_extra_data({"symbols_reachability": reports}) - resource.update_extra_data( - { - "symbols_reachability": result, - } - ) except Exception as e: if logger: logger( @@ -427,239 +415,34 @@ def generate_reachability_report(patch, candidate_resources, cloned_repos, logge ) -def get_symbol_reachability_results(project, logger=None): - """ - For each known patch commit, determine whether each project codebase - resource is reachable to the vulnerable code by comparing tree-sitter ASTs - of the patch versus the resource. - """ +def collect_and_store_symbol_reachability_results(project, logger=None): candidate_resources = project.codebaseresources.files().filter( - is_binary=False, - is_archive=False, - is_media=False, + is_binary=False, is_archive=False, is_media=False ) for vulnerability in project.package_vulnerabilities: - cloned_repos = {} + patches_by_repo = {} for patch in vulnerability.get("fixed_in_patches", []): - if patch.get("vcs_url") and patch.get("commit_hash"): - generate_reachability_report( - patch, candidate_resources, cloned_repos, logger - ) + vcs_url = patch.get("vcs_url") + commit_hash = patch.get("commit_hash") + if vcs_url and commit_hash: + patches_by_repo.setdefault(vcs_url, []).append(patch) - for url, path in cloned_repos.items(): + for vcs_url, patches in patches_by_repo.items(): try: - cleanup_repo(path) + with GitRepositoryContext(vcs_url) as git_context: + for patch in patches: + commit_hash = patch.get("commit_hash") + git_context.repo.git.checkout(commit_hash) + generate_reachability_report( + patch, git_context.repo, candidate_resources, logger + ) except Exception as e: if logger: - logger(f"Failed to clean up repo {url} at {path}: {e}") - - -@dataclass -class SymbolNode: - qualified_name: str - node: Any # tree-sitter node - node_type: str - fingerprint: str - - -@dataclass -class CallGraph: - nodes: dict[str, SymbolNode] = field(default_factory=dict) - edges_qualified: dict[str, set[str]] = field(default_factory=dict) - imports: dict[str, str] = field(default_factory=dict) - - -@dataclass -class ResourceIndex: - definitions: set[str] = field(default_factory=set) - fingerprints: set[str] = field(default_factory=set) - call_graph: CallGraph | None = None - - def to_dict(self): - return { - "definitions": self.definitions, - "fingerprints": self.fingerprints, - "call_graph": { - "nodes": {k: vars(v) for k, v in self.call_graph.nodes.items()}, - "edges_qualified": self.call_graph.edges_qualified, - "imports": self.call_graph.imports, - } - if self.call_graph - else None, - } - - -class CallGraphBuilder: - def __init__(self, tree, language: str): - self.tree = tree - self.language = language - self.index = collect_definitions(tree.root_node, language) - self.imports = collect_imports(tree.root_node, language) - - self.nodes: dict[str, SymbolNode] = {} - self.definitions_by_name: dict[str, set[str]] = {} - self.class_methods: set[str] = set() - - def build(self) -> CallGraph: - self._extract_nodes() - edges = self._resolve_all_edges() - - return CallGraph(nodes=self.nodes, edges_qualified=edges, imports=self.imports) - - def _extract_nodes(self): - """Extract all definitions and populate lookup maps.""" - sep = get_imports_language_config(self.language)["separator"] - - for definition in self.index.values(): - node = definition["node"] - qualified_name = qualified_name_from_index(node, self.index) - - if not qualified_name: - continue - - body_text = node.text.decode("utf-8", errors="replace") - fingerprint = create_sha256_fingerprint(body_text) - - self.nodes[qualified_name] = SymbolNode( - qualified_name=qualified_name, - node=node, - node_type=node.type, - fingerprint=fingerprint, - ) - - short_name = qualified_name.rsplit(sep, 1)[-1] - self.definitions_by_name.setdefault(short_name, set()).add(qualified_name) - - if node.type == "function_definition": - parent = node.parent - if parent is not None and parent.type == "class_definition": - self.class_methods.add(qualified_name) - - def _resolve_all_edges(self) -> dict[str, set[str]]: - """Map out all direct calls for every node in the graph.""" - edges_qualified = {} - - for qualified_name, symbol_node in self.nodes.items(): - direct_calls = extract_direct_calls(symbol_node.node, self.language) - resolved_callees = set() - - for receiver_name, callee_name in direct_calls: - resolved_callees |= resolve_callee( - receiver_name=receiver_name, - callee_name=callee_name, - owner_qn=qualified_name, - definitions_by_name=self.definitions_by_name, - class_methods=self.class_methods, - import_map=self.imports, - language=self.language, - ) - - edges_qualified[qualified_name] = resolved_callees - - return edges_qualified - - -def build_resource_index(resource_text, language) -> ResourceIndex | None: - if not is_supported_language(language) or not resource_text: - return None - - tree, _ = parse_code_to_ast(resource_text, language) - if tree is None: - return None - - call_graph = build_call_graph(tree, language) - - if call_graph: - definitions = set(call_graph.nodes.keys()) - fingerprints = { - node.fingerprint for node in call_graph.nodes.values() if node.fingerprint - } - else: - meta = build_symbol_metadata(extract_definitions(tree, language), language) - definitions = {m["qualified_name"] for m in meta.values()} - fingerprints = {m["fingerprint"] for m in meta.values() if m["fingerprint"]} - - return ResourceIndex( - definitions=definitions, fingerprints=fingerprints, call_graph=call_graph - ) - - -def match_symbols_against_resource( - patch_symbols_metadata: dict[str, Any], resource_index: "ResourceIndex" -) -> dict[str, Any]: - if not patch_symbols_metadata or not resource_index: - return {} - - call_graph = resource_index.call_graph - imports = call_graph.imports if call_graph else {} - edges_qualified = call_graph.edges_qualified if call_graph else {} - - imported_fq_names = set(imports.values()) - - target_qualified_names = { - metadata["qualified_name"] for metadata in patch_symbols_metadata.values() - } - - reachable_callers, _ = compute_reachable_symbols( - edges_qualified=edges_qualified, - target_qualified_names=target_qualified_names, - ) - - called_qualified_names = set() - for callees in edges_qualified.values(): - called_qualified_names |= set(callees) - - matched = {} - for metadata in patch_symbols_metadata.values(): - qualified_name = metadata["qualified_name"] - fingerprint = metadata["fingerprint"] - - defined = qualified_name in resource_index.definitions - fingerprint_hit = bool( - fingerprint and fingerprint in resource_index.fingerprints - ) - - imported = ( - qualified_name in imports - or qualified_name in imported_fq_names - or any( - fq == qualified_name or fq.endswith("." + qualified_name) - for fq in imported_fq_names - ) - ) - - called = any( - fq_name == qualified_name or fq_name.endswith("." + qualified_name) - for fq_name in called_qualified_names - ) - - if not (defined or fingerprint_hit or called or imported): - continue - - entry = matched.setdefault( - qualified_name, - { - "symbol_name": qualified_name, - "called": False, - "defined": False, - "imported": False, - "fingerprint": None, - "reachable_from": [], - }, - ) - - if defined: - entry["defined"] = True - if imported: - entry["imported"] = True - if called: - entry["called"] = True - entry["reachable_from"] = sorted(reachable_callers) - if fingerprint: - entry["fingerprint"] = fingerprint - - return matched + logger( + f"Failed to process repository {vcs_url} " + f"for symbol reachability: {e}" + ) def classify_reachability(evidence): @@ -683,264 +466,126 @@ def classify_reachability(evidence): return status -def build_symbol_metadata(nodes, language, index=None): - if index is None and nodes: - index = collect_definitions(_root_of(nodes[0]), language) - - metadata = {} - for node in nodes: - qualified_name = qualified_name_from_index(node, index) - if not qualified_name: - continue - - body_text = node.text.decode("utf-8", errors="replace") - fingerprints = create_sha256_fingerprint(body_text) - - key = qualified_name - suffix = 1 - while key in metadata: - suffix += 1 - key = f"{qualified_name}#{suffix}" - - metadata[key] = { - "qualified_name": qualified_name, - "text": body_text, - "fingerprint": fingerprints, - "start_line": node.start_point[0] + 1, - "end_line": node.end_point[0] + 1, - "node_type": node.type, - } - return metadata - - -def build_call_graph(tree, language) -> CallGraph | None: - if tree is None or not is_supported_language(language): - return None - - builder = CallGraphBuilder(tree, language) - return builder.build() - - -def extract_direct_calls(node, language): - """ - Return direct calls inside `node` - Examples: - foo() -> (None, "foo") - self.foo() -> ("self", "foo") - obj.foo() -> ("obj", "foo") +class ResourceAnalyzer: + def __init__(self, resource_text: str, language: str): + self.resource_text = resource_text + self.language = language - """ - query = get_query(language, "calls") - if query is None or node is None: - return [] + def build_index(self) -> dict | None: + text = normalize_text(self.resource_text) + if not is_supported_language(self.language) or not text: + return None - calls = [] + lang_query = TS_QUERIES[self.language]() + tree, _ = lang_query.parse_code_to_ast(text) - for _, captures in query.matches(node): - for callee_node in captures.get("callee", []): - receiver_name = get_call_receiver(callee_node) - callee_name = callee_node.text.decode("utf-8", errors="replace") + if tree is None: + return None - if callee_name: - calls.append((receiver_name, callee_name)) - return calls + extractor = SymbolExtractor(lang_query=lang_query, root_node=tree.root_node) + definitions_index = extractor.extract_definitions_index() + imports_map = extractor.extract_imports() + separator = extractor.syntax_config.get("separator", ".") + definitions = set() + fingerprints = set() + callers_of = {} # callee_name -> set of caller_qualified_names -def get_call_receiver(callee_node): - """ - Return receiver name for attribute calls. + for node, _ in lang_query.get_functions(tree.root_node): + qualified_name = extractor._build_qualified_name(node, definitions_index) + if not qualified_name: + continue - Examples: - foo() -> None - self.foo() -> "self" - obj.foo() -> "obj" + definitions.add(qualified_name) + body_text = node.text.decode("utf-8", errors="replace") + fingerprint = create_sha256_fingerprint(body_text) + if fingerprint: + fingerprints.add(fingerprint) - """ - parent = callee_node.parent + for _, callee_name in extractor.extract_calls(node): + callers_of.setdefault(callee_name, set()).add(qualified_name) - if parent is None or parent.type != "attribute": - return None + for node, _ in lang_query.get_classes(tree.root_node): + qualified_name = extractor._build_qualified_name(node, definitions_index) + if not qualified_name: + continue - object_node = parent.child_by_field_name("object") - if object_node is None: - return None + definitions.add(qualified_name) + body_text = node.text.decode("utf-8", errors="replace") + fingerprint = create_sha256_fingerprint(body_text) + if fingerprint: + fingerprints.add(fingerprint) - return object_node.text.decode("utf-8", errors="replace") - - -def compute_reachable_symbols(edges_qualified, target_qualified_names): - """Transitive callers using resolved qualified-name edges.""" - if not edges_qualified or not target_qualified_names: - return set(), False - - callers_of = {} - for caller, callees in edges_qualified.items(): - for callee in callees: - callers_of.setdefault(callee, set()).add(caller) - - targets = set(target_qualified_names) - direct = set() - for target in targets: - direct |= callers_of.get(target, set()) - - reachable = set(direct) - frontier = list(direct) - while frontier: - cur = frontier.pop() - for parent in callers_of.get(cur, ()): - if parent not in reachable: - reachable.add(parent) - frontier.append(parent) - - return reachable, bool(direct) - - -def get_imports_language_config(language: str) -> dict: - configs = { - "Python": {"self_keyword": "self", "separator": ".", "wildcard_symbol": "*"}, - "Javascript": { - "self_keyword": "this", - "separator": ".", - "wildcard_symbol": "*", - }, - "Java": {"self_keyword": "this", "separator": ".", "wildcard_symbol": "*"}, - "C++": {"self_keyword": "this", "separator": "::", "wildcard_symbol": None}, - "PHP": {"self_keyword": "$this", "separator": "::", "wildcard_symbol": None}, - "Go": {"self_keyword": None, "separator": "/", "wildcard_symbol": None}, - "Ruby": {"self_keyword": "self", "separator": "::", "wildcard_symbol": None}, - } - return configs.get( - language, {"self_keyword": None, "separator": ".", "wildcard_symbol": None} - ) + return { + "definitions": definitions, + "fingerprints": fingerprints, + "imports": imports_map, + "callers_of": callers_of, + "separator": separator, + } -def resolve_callee( - receiver_name: str | None, - callee_name: str, - owner_qn: str, - definitions_by_name: dict[str, set[str]], - class_methods: set[str], - language: str, - import_map: dict | None = None, -) -> set[str]: - config = get_imports_language_config(language) - sep = config["separator"] - self_kw = config["self_keyword"] - import_map = import_map or {} - - if receiver_name: - receiver_name = receiver_name.strip("'\"") - if callee_name: - callee_name = callee_name.strip("'\"") - - if self_kw is not None and receiver_name == self_kw: - owner_class = owner_qn.rsplit(sep, 1)[0] if sep in owner_qn else owner_qn - return {f"{owner_class}{sep}{callee_name}"} - - if callee_name in import_map: - return {import_map[callee_name]} - - if receiver_name is not None and receiver_name != self_kw: - candidates = set() - - if receiver_name in import_map and receiver_name != "*": - base = import_map[receiver_name] - candidates.add(f"{base}{sep}{callee_name}") - - static_fqn = f"{receiver_name}{sep}{callee_name}" - if static_fqn in class_methods: - candidates.add(static_fqn) - - return candidates - - local_defs = definitions_by_name.get(callee_name, set()) - if local_defs: - return local_defs - - if "*" in import_map: - return {f"{mod}{sep}{callee_name}" for mod in import_map["*"]} - - return set() - - -def extract_imports(captures: dict) -> tuple[str | None, list[tuple[str, str | None]]]: - """ - Flatten and sort tree-sitter captures, then extract the module name - and a list of import/alias pairs. - """ - flat_captures = [] - for tag, nodes in captures.items(): - for n in nodes: - flat_captures.append( - ( - n.start_byte, - tag, - n.text.decode("utf-8", errors="replace").strip("'\""), - ) +class ResourcePatchMatcher: + def __init__(self, resource_index: dict): + self.resource_index = resource_index + self.definitions = resource_index.get("definitions", set()) + self.fingerprints = resource_index.get("fingerprints", set()) + self.imports = resource_index.get("imports", {}) + self.callers_of = resource_index.get("callers_of", {}) + self.separator = resource_index.get("separator", ".") + + def match(self, patch_symbols_metadata: dict[str, Any]) -> dict[str, Any]: + if not patch_symbols_metadata or not self.resource_index: + return {} + + matched = {} + for metadata in patch_symbols_metadata.values(): + qualified_name = metadata["qualified_name"] + fingerprint = metadata["fingerprint"] + + short_name = ( + qualified_name.rsplit(self.separator, 1)[-1] + if self.separator in qualified_name + else qualified_name ) - flat_captures.sort(key=lambda x: x[0]) - - module_name = None - current_import = None - pairs = [] - - for _, tag, text in flat_captures: - if tag == "module_name": - module_name = text - elif tag == "import_name": - if current_import is not None: - pairs.append((current_import, None)) - current_import = text - elif tag == "alias": - pairs.append((current_import, text)) - current_import = None - - if current_import is not None: - pairs.append((current_import, None)) - - return module_name, pairs - + defined = qualified_name in self.definitions + fingerprint_hit = bool(fingerprint and fingerprint in self.fingerprints) -def collect_imports(root_node, language: str) -> dict[str, str | list[str]]: - """ - Extract import statements from a Tree-sitter AST and map every - local alias to its absolute imported path. - """ - config = get_imports_language_config(language) - separator = config["separator"] - wildcard_sym = config.get("wildcard_symbol") - - query = get_query(language, "imports") - if not query or not root_node: - return {} - - import_map: dict[str, str | list[str]] = {} - wildcard_modules: list[str] = [] - - for _pattern_index, captures in query.matches(root_node): - module_name, pairs = extract_imports(captures) + imported = ( + qualified_name in self.imports + or qualified_name in self.imports.values() + ) - for imp_name, alias in pairs: - if not imp_name: - continue + callers = set() + callers.update(self.callers_of.get(short_name, set())) + callers.update(self.callers_of.get(qualified_name, set())) + called = bool(callers) - if wildcard_sym is not None and imp_name == wildcard_sym: - if module_name: - wildcard_modules.append(module_name) + if not (defined or fingerprint_hit or called or imported): continue - local_name = alias or imp_name - - if not alias and separator in imp_name: - local_name = imp_name.split(separator)[0] - - absolute_path = ( - f"{module_name}{separator}{imp_name}" if module_name else imp_name + entry = matched.setdefault( + qualified_name, + { + "symbol_name": qualified_name, + "called": False, + "defined": False, + "imported": False, + "fingerprint": None, + "reachable_from": [], + }, ) - import_map[local_name] = absolute_path - if wildcard_modules: - import_map["*"] = wildcard_modules + if defined: + entry["defined"] = True + if imported: + entry["imported"] = True + if called: + entry["called"] = True + entry["reachable_from"] = sorted(callers) + + if fingerprint_hit: + entry["fingerprint"] = fingerprint - return import_map + return matched diff --git a/scanpipe/pipes/symbols.py b/scanpipe/pipes/symbols.py index 1cc142057a..855b481fed 100644 --- a/scanpipe/pipes/symbols.py +++ b/scanpipe/pipes/symbols.py @@ -22,6 +22,7 @@ import hashlib import importlib +from abc import ABC from functools import cache from django.db.models import Q @@ -192,44 +193,6 @@ def _collect_and_store_tree_sitter_symbols_and_strings(resource): "pygments": symbols_pygments.get_pygments_symbols, } -TS_QUERIES = { - "Python": { - "functions": """ - (function_definition name: (identifier) @name) @function - """, - "classes": """ - (class_definition name: (identifier) @name) @class - """, - "calls": """ - (call function: (identifier) @callee) - (call function: (attribute - object: (_) @receiver - attribute: (identifier) @callee)) - """, - "imports": """ - (import_statement name: [ - (dotted_name) @import_name - (aliased_import name: (dotted_name) - @import_name alias: (identifier) @alias) - ]) - - ; Combine explicit, relative, aliased, and wildcard from_imports - (import_from_statement - module_name: [ - (dotted_name) @module_name - (relative_import) @module_name - ] - [ - name: (dotted_name) @import_name - name: (aliased_import name: (dotted_name) - @import_name alias: (identifier) @alias) - (wildcard_import) @import_name - ] - ) - """, - }, -} - @cache def load_language(language: str) -> Language: @@ -246,101 +209,265 @@ def load_language(language: str) -> Language: return Language(grammar.language()) -@cache -def get_query(language: str, kind: str) -> Query | None: - source = TS_QUERIES.get(language, {}).get(kind, "").strip() - if not source: +def create_sha256_fingerprint(text): + if not text: return None - return Query(load_language(language), source) - -def parse_code_to_ast(code_text: str, language: str): - if not code_text or not language or language not in TS_LANGUAGE_WHEELS: - return None, None - - ts_language = load_language(language) - parser = Parser(language=ts_language) - return parser.parse(code_text.encode("utf-8")), TS_LANGUAGE_WHEELS[language] - - -def run_query(query: Query, root_node): - """Yield ``(definition_node, name)`` pairs for function/class queries.""" - if query is None: - return + text = text.encode("utf-8", errors="replace") + return hashlib.sha256(text).hexdigest() - for _pattern_index, captures in query.matches(root_node): - def_nodes = captures.get("function") or captures.get("class") or [] - if not def_nodes: - continue - name_nodes = captures.get("name") or [] - name = ( - name_nodes[0].text.decode("utf-8", errors="replace") if name_nodes else None - ) - yield def_nodes[0], name +class LanguageQuery(ABC): + language_name: str = "" + functions_query: str = "" + classes_query: str = "" + calls_query: str = "" + imports_query: str = "" + syntax_config: dict = { + "self_keyword": None, + "separator": ".", + "wildcard_symbol": None, + } + + def __init__(self): + self.ts_language = load_language(self.language_name) + self._compiled_queries = {} + + for kind in ("functions", "classes", "calls", "imports"): + source = getattr(self, f"{kind}_query", "").strip() + self._compiled_queries[kind] = ( + Query(self.ts_language, source) if source else None + ) + def parse_code_to_ast(self, code_text: str): + if not code_text: + return None, None + parser = Parser(language=self.ts_language) + return parser.parse(code_text.encode("utf-8")), TS_LANGUAGE_WHEELS[ + self.language_name + ] + + def run_query(self, kind: str, root_node): + query = self._compiled_queries.get(kind) + return query.matches(root_node) if query else [] + + def get_functions(self, root_node): + for _, captures in self.run_query("functions", root_node): + def_nodes = captures.get("function") + if not def_nodes: + continue + name_nodes = captures.get("name") + name = ( + name_nodes[0].text.decode("utf-8", errors="replace") + if name_nodes + else None + ) + yield def_nodes[0], name + + def get_classes(self, root_node): + for _, captures in self.run_query("classes", root_node): + def_nodes = captures.get("class") + if not def_nodes: + continue + name_nodes = captures.get("name") + name = ( + name_nodes[0].text.decode("utf-8", errors="replace") + if name_nodes + else None + ) + yield def_nodes[0], name + + def get_calls(self, node): + """Yield raw (receiver_node, callee_node).""" + seen_callees = set() + for _, captures in self.run_query("calls", node): + for callee_node in captures.get("callee", []): + if callee_node.id in seen_callees: + continue + seen_callees.add(callee_node.id) + + receiver_nodes = captures.get("receiver") + receiver_node = receiver_nodes[0] if receiver_nodes else None + yield receiver_node, callee_node + + def get_imports(self, root_node): + """Yield raw (module_name, [(import_name, alias), ...]).""" + for _, captures in self.run_query("imports", root_node): + flat_captures = [] + for tag, nodes in captures.items(): + for n in nodes: + text = n.text.decode("utf-8", errors="replace").strip("'\"") + flat_captures.append((n.start_byte, tag, text)) + + flat_captures.sort(key=lambda x: x[0]) + module_name, current_import = None, None + pairs = [] + + for _, tag, text in flat_captures: + if tag == "module_name": + module_name = text + elif tag == "import_name": + if current_import is not None: + pairs.append((current_import, None)) + current_import = text + elif tag == "alias": + pairs.append((current_import, text)) + current_import = None + + if current_import is not None: + pairs.append((current_import, None)) + + yield module_name, pairs + + +class PythonTreeSitterQuery(LanguageQuery): + language_name = "Python" + functions_query = "(function_definition name: (identifier) @name) @function" + classes_query = "(class_definition name: (identifier) @name) @class" + calls_query = """ + (call function: (identifier) @callee) + (call function: (attribute object: (_) @receiver + attribute: (identifier) @callee)) + """ + imports_query = """ + (import_statement name: (dotted_name) @import_name) + (import_statement name: (aliased_import + name: (dotted_name) @import_name + alias: (identifier) @alias)) + + (import_from_statement + module_name: [(dotted_name) (relative_import)] @module_name + name: [ + (dotted_name) @import_name + (aliased_import name: (dotted_name) @import_name + alias: (identifier) @alias) + ]) + + (import_from_statement + module_name: [(dotted_name) (relative_import)] @module_name + (wildcard_import) @import_name) + """ + syntax_config = {"self_keyword": "self", "separator": ".", "wildcard_symbol": "*"} -def _root_of(node): - while node.parent is not None: - node = node.parent - return node +TS_QUERIES = { + "Python": PythonTreeSitterQuery, +} -def collect_definitions(root_node, language: str): - index: dict[int, dict] = {} - for kind in ("functions", "classes"): - query = get_query(language, kind) - for node, name in run_query(query, root_node): - index[node.id] = {"node": node, "name": name, "kind": kind} - return index +class SymbolExtractor: + def __init__(self, lang_query: LanguageQuery, root_node): + self.lang_query = lang_query + self.root_node = root_node + self.syntax_config = lang_query.syntax_config + + def _build_qualified_name(self, node, index: dict) -> str: + """ + Build fully qualified names internally + (e.g., ClassName.function_name or ClassName::function_name). + """ + parts = [] + curr = node + while curr is not None: + definition = index.get(curr.id) + if definition is not None and definition["name"]: + parts.append(definition["name"]) + curr = curr.parent + + separator = self.syntax_config.get("separator", ".") + return separator.join(reversed(parts)) + + def extract_definitions_index(self): + """Build the index of definitions with fully qualified names.""" + index: dict[int, dict] = {} + + for node, name in self.lang_query.get_functions(self.root_node): + index[node.id] = {"node": node, "name": name, "kind": "functions"} + + for node, name in self.lang_query.get_classes(self.root_node): + index[node.id] = {"node": node, "name": name, "kind": "classes"} + + for def_info in index.values(): + def_info["qualified_name"] = self._build_qualified_name( + def_info["node"], index + ) -def extract_definitions(tree, language: str, kinds=("functions", "classes")): - if tree is None: - return [] - index = collect_definitions(tree.root_node, language) - return [d["node"] for d in index.values() if d["kind"] in kinds] + return index + + def extract_changed_symbols(self, changed_lines: list[int]): + """Map changed line numbers to their enclosing symbol nodes.""" + if self.root_node is None or not changed_lines: + return [] + + definition_ids = set(self.extract_definitions_index().keys()) + if not definition_ids: + return [] + + seen = set() + enclosing = [] + + for line in changed_lines: + row = max(0, line - 1) + node = self.root_node.descendant_for_point_range((row, 0), (row, 0)) + + while node is not None: + if node.id in definition_ids and node.id not in seen: + seen.add(node.id) + enclosing.append(node) + break + node = node.parent + + return enclosing + + def extract_calls(self, node): + """Extract direct calls into (receiver_name, callee_name) text format.""" + calls = [] + for receiver_node, callee_node in self.lang_query.get_calls(node): + receiver_name = ( + receiver_node.text.decode("utf-8", errors="replace") + if receiver_node + else None + ) + callee_name = callee_node.text.decode("utf-8", errors="replace") + if callee_name: + calls.append((receiver_name, callee_name)) -def extract_symbols(tree, changed_lines: list[int], language: str): - if tree is None or not changed_lines: - return [] + return calls - definition_ids = set(collect_definitions(tree.root_node, language).keys()) - if not definition_ids: - return [] + def extract_imports(self): + """Map every local alias to its absolute imported path.""" + separator = self.syntax_config.get("separator", ".") + wildcard_sym = self.syntax_config.get("wildcard_symbol") - seen = set() - enclosing = [] + import_map: dict[str, str | list[str]] = {} + wildcard_modules: list[str] = [] - for line in changed_lines: - row = max(0, line - 1) - node = tree.root_node.descendant_for_point_range((row, 0), (row, 0)) + for module_name, pairs in self.lang_query.get_imports(self.root_node): + for imp_name, alias in pairs: + if not imp_name: + continue - while node is not None: - if node.id in definition_ids and node.id not in seen: - seen.add(node.id) - enclosing.append(node) - break - node = node.parent + if wildcard_sym is not None and imp_name == wildcard_sym: + if module_name: + wildcard_modules.append(module_name) + continue - return enclosing + local_name = alias or imp_name + if not alias and separator in imp_name: + local_name = imp_name.split(separator)[0] + absolute_path = ( + f"{module_name}{separator}{imp_name}" if module_name else imp_name + ) + import_map[local_name] = absolute_path -def qualified_name_from_index(node, index): - parts = [] - curr = node - while curr is not None: - definition = index.get(curr.id) - if definition is not None and definition["name"]: - parts.append(definition["name"]) - curr = curr.parent - return ".".join(reversed(parts)) + if wildcard_modules: + import_map["*"] = wildcard_modules + return import_map -def create_sha256_fingerprint(text): - if text is None: - return None - text = text.encode("utf-8", errors="replace") - return hashlib.sha256(text).hexdigest() +def is_supported_language(language): + """Return True if the language is supported by tree-sitter queries.""" + return bool(language) and language in TS_QUERIES diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index 572dd782e6..eeeb0560a3 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -21,6 +21,7 @@ # Visit https://github.com/nexB/scancode.io for support and download. from pathlib import Path +from unittest.mock import MagicMock from unittest.mock import PropertyMock from unittest.mock import patch @@ -28,21 +29,12 @@ from scanpipe.models import Project from scanpipe.pipes import collect_and_create_codebase_resources +from scanpipe.pipes.reachability import PatchAnalyzer from scanpipe.pipes.reachability import ReachabilityStatus -from scanpipe.pipes.reachability import analyze_patched_file -from scanpipe.pipes.reachability import build_symbol_metadata from scanpipe.pipes.reachability import classify_reachability -from scanpipe.pipes.reachability import collect_imports -from scanpipe.pipes.reachability import compute_reachable_symbols -from scanpipe.pipes.reachability import diff_changed_symbols -from scanpipe.pipes.reachability import extract_direct_calls -from scanpipe.pipes.reachability import get_symbol_reachability_results -from scanpipe.pipes.reachability import parse_diff_lines -from scanpipe.pipes.symbols import collect_definitions -from scanpipe.pipes.symbols import extract_definitions -from scanpipe.pipes.symbols import extract_symbols -from scanpipe.pipes.symbols import parse_code_to_ast -from scanpipe.pipes.symbols import qualified_name_from_index +from scanpipe.pipes.reachability import collect_and_store_symbol_reachability_results +from scanpipe.pipes.symbols import TS_QUERIES +from scanpipe.pipes.symbols import SymbolExtractor class SymbolReachabilityPipesTest(TestCase): @@ -52,24 +44,26 @@ def setUp(self): self.project1 = Project.objects.create(name="Analysis") self.project1.codebase_path.mkdir(parents=True, exist_ok=True) - @patch("scanpipe.pipes.reachability.clone_repo") - @patch("scanpipe.pipes.reachability.collect_patch_symbols") + @patch("scanpipe.pipes.reachability.GitRepositoryContext") + @patch("scanpipe.pipes.reachability.PatchAnalyzer.collect_patch_symbols") @patch.object(Project, "package_vulnerabilities", new_callable=PropertyMock) def test_get_symbol_reachability_results( self, mock_package_vulnerabilities, mock_collect_symbols, - mock_clone_repo, + mock_git_context, ): app_text = (self.data / "app.py").read_text() vuln_text = (self.data / "vuln-app.py").read_text() fixed_text = (self.data / "fixed-app.py").read_text() diff_text = (self.data / "diff-app.patch").read_text() - diff_line_map = parse_diff_lines(diff_text=diff_text) + + analyzer = PatchAnalyzer(repo=MagicMock(), commit_hash="dummy") + diff_line_map = analyzer.parse_diff_lines(diff_text=diff_text) file_path = "app.py" removed_lines, added_lines = diff_line_map.get(file_path, ([], [])) - vuln_meta, fixed_meta, lang = analyze_patched_file( + vuln_meta, fixed_meta, lang = analyzer.analyze( vulnerable_text=vuln_text, fixed_text=fixed_text, removed_lines=removed_lines, @@ -90,7 +84,7 @@ def test_get_symbol_reachability_results( } ] - mock_clone_repo.return_value = str(self.project1.codebase_path) + mock_git_context.return_value.__enter__.return_value.repo = MagicMock() mock_collect_symbols.return_value = { lang: { "vulnerable": { @@ -110,48 +104,52 @@ def test_get_symbol_reachability_results( resource.programming_language = lang resource.save() - with patch("scanpipe.pipes.reachability.Repo"): - get_symbol_reachability_results(self.project1) + collect_and_store_symbol_reachability_results(self.project1) resource.refresh_from_db() results = resource.extra_data.get("symbols_reachability") self.assertEqual( results, - { - "symbols_reachability": { - "patch": { - "vcs_url": "https://github.com/aboutcode-org/test", - "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", - }, - "evidence": [ - { - "called": False, - "defined": True, - "imported": False, - "fingerprint": "d7675efb263896da2a3c00679511833" - "553907e7e6ea619115a6dfc8625c3457e", - "symbol_name": "serve_report", - "reachable_from": [], - }, - { - "called": True, - "defined": True, - "imported": False, - "fingerprint": "762e4f7d03b1bf4359c3ca364" - "e558140239913bfabcc5aa77156460c2eb0a355", - "symbol_name": "serve_report.build_file_path", - "reachable_from": ["serve_report"], + [ + { + "symbols_reachability": { + "patch": { + "vcs_url": "https://github.com/aboutcode-org/test", + "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", }, - ], - "fixed_symbols": ["serve_report", "serve_report.build_file_path"], - "vulnerable_symbols": [ - "serve_report", - "serve_report.build_file_path", - ], - "reachability_status": "REACHABLE", + "evidence": [ + { + "called": False, + "defined": True, + "imported": False, + "fingerprint": "d7675efb263896da2a3c00679511833" + "553907e7e6ea619115a6dfc8625c3457e", + "symbol_name": "serve_report", + "reachable_from": [], + }, + { + "called": True, + "defined": True, + "imported": False, + "fingerprint": "762e4f7d03b1bf4359c3ca364" + "e558140239913bfabcc5aa77156460c2eb0a355", + "symbol_name": "serve_report.build_file_path", + "reachable_from": ["serve_report"], + }, + ], + "fixed_symbols": [ + "serve_report", + "serve_report.build_file_path", + ], + "vulnerable_symbols": [ + "serve_report", + "serve_report.build_file_path", + ], + "reachability_status": "REACHABLE", + } } - }, + ], ) def test_extract_definitions(self): @@ -169,27 +167,32 @@ def calculate_discount(price): class InventoryItem: pass """ - tree, _ = parse_code_to_ast(source_code, "Python") - functions = extract_definitions(tree, "Python", kinds=("functions",)) + + lang_query = TS_QUERIES["Python"]() + tree, _ = lang_query.parse_code_to_ast(code_text=source_code) + functions = list(lang_query.get_functions(tree.root_node)) + self.assertEqual( len(functions), 3 ) # '__init__', 'process_payment', and 'calculate_discount' - self.assertEqual(functions[0].type, "function_definition") - first_func_text = functions[0].text.decode("utf-8") + self.assertEqual(functions[0][0].type, "function_definition") + first_func_text = functions[0][0].text.decode("utf-8") self.assertIn("def __init__", first_func_text) - classes = extract_definitions(tree, "Python", kinds=("classes",)) + classes = list(lang_query.get_classes(tree.root_node)) self.assertEqual(len(classes), 2) - second_class_text = classes[1].text.decode("utf-8") + second_class_text = classes[1][0].text.decode("utf-8") self.assertIn("class InventoryItem", second_class_text) def test_extract_definitions_empty(self): - tree, _ = parse_code_to_ast("", "Python") - self.assertEqual(extract_definitions(tree, "Python", kinds=("functions",)), []) - self.assertEqual(extract_definitions(tree, "Python", kinds=("functions",)), []) - self.assertEqual(extract_definitions(None, "Python", kinds=("classes",)), []) - self.assertEqual(extract_definitions(None, "Python", kinds=("classes",)), []) + lang_query = TS_QUERIES["Python"]() + tree, _ = lang_query.parse_code_to_ast("") + + self.assertIsNone(tree) + + tree_none, _ = lang_query.parse_code_to_ast(None) + self.assertIsNone(tree_none) def test_get_qualified_name_functions(self): source_code = """ @@ -202,14 +205,17 @@ def global_utility(): pass """ - tree, _ = parse_code_to_ast(source_code, "Python") - index = collect_definitions(tree.root_node, "Python") + lang_query = TS_QUERIES["Python"]() + tree, _ = lang_query.parse_code_to_ast(source_code) + + extractor = SymbolExtractor(lang_query=lang_query, root_node=tree.root_node) + index = extractor.extract_definitions_index() - functions = extract_definitions(tree, "Python", kinds=("functions",)) + functions = list(lang_query.get_functions(tree.root_node)) self.assertEqual(len(functions), 2) - outer_function_name = qualified_name_from_index(functions[0], index) - inner_function_name = qualified_name_from_index(functions[1], index) + outer_function_name = extractor._build_qualified_name(functions[0][0], index) + inner_function_name = extractor._build_qualified_name(functions[1][0], index) self.assertEqual(outer_function_name, "CoreService.Validator.validate_payload") self.assertEqual(inner_function_name, "global_utility") @@ -220,14 +226,17 @@ class FleetManagement: class DroneController: pass """ - tree, _ = parse_code_to_ast(source_code, "Python") - index = collect_definitions(tree.root_node, "Python") + lang_query = TS_QUERIES["Python"]() + tree, _ = lang_query.parse_code_to_ast(source_code) + + extractor = SymbolExtractor(lang_query=lang_query, root_node=tree.root_node) + index = extractor.extract_definitions_index() - classes = extract_definitions(tree, "Python", kinds=("classes",)) + classes = list(lang_query.get_classes(tree.root_node)) self.assertEqual(len(classes), 2) - outer_class_name = qualified_name_from_index(classes[0], index) - inner_class_name = qualified_name_from_index(classes[1], index) + outer_class_name = extractor._build_qualified_name(classes[0][0], index) + inner_class_name = extractor._build_qualified_name(classes[1][0], index) self.assertEqual(outer_class_name, "FleetManagement") self.assertEqual(inner_class_name, "FleetManagement.DroneController") @@ -235,6 +244,9 @@ class DroneController: def test_classify_reachability(self): self.assertEqual(classify_reachability(None), ReachabilityStatus.NOT_REACHABLE) self.assertEqual(classify_reachability({}), ReachabilityStatus.NOT_REACHABLE) + self.assertEqual( + classify_reachability({"evidence": {}}), ReachabilityStatus.NOT_REACHABLE + ) self.assertEqual( classify_reachability({"evidence": {"fingerprint": "hash123"}}), ReachabilityStatus.REACHABLE, @@ -265,21 +277,40 @@ def inner_helper(): def process_data(payload): return payload """ - tree, _ = parse_code_to_ast(source_code, "Python") - nodes = extract_definitions(tree, "Python", kinds=("functions",)) - - metadata = build_symbol_metadata(nodes, "Python") + lang_query = TS_QUERIES["Python"]() + tree, _ = lang_query.parse_code_to_ast(source_code) + extractor = SymbolExtractor(lang_query=lang_query, root_node=tree.root_node) + index = extractor.extract_definitions_index() + vuln_nodes = extractor.extract_changed_symbols( + changed_lines=[1, 2, 3, 4, 5, 6, 7, 8, 9] + ) + metadata = PatchAnalyzer.build_symbol_metadata( + nodes=vuln_nodes, extractor=extractor, index=index + ) self.assertEqual( metadata, { + "Controller": { + "qualified_name": "Controller", + "text": "class Controller:\n" + " def process_data(payload):\n" + " def inner_helper():\n " + " return True\n " + " return payload.strip()", + "fingerprint": "de81abd637e27302d8e19c41eab8f4" + "fb6b8abdd9fc4f1fb31d354bc7b23f6d4d", + "start_line": 2, + "end_line": 6, + "node_type": "class_definition", + }, "Controller.process_data": { "qualified_name": "Controller.process_data", "text": "def process_data(payload):\n" " def inner_helper():\n" - " return True\n " - " return payload.strip()", - "fingerprint": "b0d0ad9a92209a6d79b84e932ce302" - "a8bc9054a405131adf7dc21e06e2e7c0c1", + " return True\n" + " return payload.strip()", + "fingerprint": "b0d0ad9a92209a6d79b84e932ce3" + "02a8bc9054a405131adf7dc21e06e2e7c0c1", "start_line": 3, "end_line": 6, "node_type": "function_definition", @@ -287,21 +318,12 @@ def process_data(payload): "Controller.process_data.inner_helper": { "qualified_name": "Controller.process_data.inner_helper", "text": "def inner_helper():\n return True", - "fingerprint": "ee2e246e01e960826cb39a9466e5" - "8095d209fdd1cbf8458630be430b3371d6a3", + "fingerprint": "ee2e246e01e960826cb39a9466e58095" + "d209fdd1cbf8458630be430b3371d6a3", "start_line": 4, "end_line": 5, "node_type": "function_definition", }, - "process_data": { - "qualified_name": "process_data", - "text": "def process_data(payload):\n return payload", - "fingerprint": "9b2797712c9ab60ea8452a4413965" - "c94d1b2f63739cab7de695e7b1dc0cf439a", - "start_line": 9, - "end_line": 10, - "node_type": "function_definition", - }, }, ) @@ -339,8 +361,9 @@ def test_diff_changed_symbols(self): }, } - vuln_only, fixed_only = diff_changed_symbols(vuln_meta, fixed_meta) - + vuln_only, fixed_only = PatchAnalyzer.diff_changed_symbols( + vuln_meta, fixed_meta + ) self.assertEqual( vuln_only, { @@ -374,11 +397,12 @@ def test_analyze_patched_file(self): vuln_text = (self.data / "vuln-app.py").read_text(encoding="utf-8") fixed_text = (self.data / "fixed-app.py").read_text(encoding="utf-8") diff_text = (self.data / "diff-app.patch").read_text(encoding="utf-8") - diff_line_map = parse_diff_lines(diff_text=diff_text) + + diff_line_map = PatchAnalyzer.parse_diff_lines(diff_text=diff_text) file_path = "app.py" removed_lines, added_lines = diff_line_map.get(file_path, ([], [])) - vuln_meta, fixed_meta, lang = analyze_patched_file( + vuln_meta, fixed_meta, lang = PatchAnalyzer.analyze( vulnerable_text=vuln_text, fixed_text=fixed_text, removed_lines=removed_lines, @@ -491,13 +515,15 @@ def test_extract_symbols(self): " return build_path(request)\n" # Line 5 (Row 4) ) - tree, _ = parse_code_to_ast(source_code, "Python") + lang_query = TS_QUERIES["Python"]() + tree, _ = lang_query.parse_code_to_ast(source_code) changed_lines = [4] - enclosing_symbols = extract_symbols(tree, changed_lines, "Python") + extractor = SymbolExtractor(lang_query=lang_query, root_node=tree.root_node) + changed_symbols = extractor.extract_changed_symbols(changed_lines) - self.assertEqual(len(enclosing_symbols), 1) - target_node = enclosing_symbols[0] + self.assertEqual(len(changed_symbols), 1) + target_node = changed_symbols[0] self.assertEqual(target_node.type, "function_definition") node_text = target_node.text.decode("utf-8") @@ -511,29 +537,17 @@ def test_extract_symbols_deduplication(self): " return price + amount\n" # Line 3 -> Changed ) - tree, _ = parse_code_to_ast(source_code, "Python") - changed_lines = [2, 3] + lang_query = TS_QUERIES["Python"]() + tree, _ = lang_query.parse_code_to_ast(source_code) - enclosing_symbols = extract_symbols(tree, changed_lines, "Python") + changed_lines = [2, 3] + symbol_extractor = SymbolExtractor( + lang_query=lang_query, root_node=tree.root_node + ) + enclosing_symbols = symbol_extractor.extract_changed_symbols(changed_lines) self.assertEqual(len(enclosing_symbols), 1) self.assertEqual(enclosing_symbols[0].type, "function_definition") - def test_compute_reachable_symbols(self): - edges_qualified = { - "app.main": {"app.helper", "app.safe_func"}, - "app.helper": {"app.vuln_func"}, - "app.direct_caller": {"app.vuln_func"}, - "app.unrelated": {"app.safe_func"}, - } - - target_qns = ["app.vuln_func"] - reachable, has_direct = compute_reachable_symbols(edges_qualified, target_qns) - - self.assertTrue(has_direct) - - expected_reachable = {"app.main", "app.helper", "app.direct_caller"} - self.assertEqual(reachable, expected_reachable) - def test_collect_imports(self): source_code = """ from django.db import models @@ -545,9 +559,10 @@ def test_collect_imports(self): from math import * """.strip() - tree, _ = parse_code_to_ast(source_code, "Python") - real_root_node = tree.root_node - result = collect_imports(real_root_node, language="Python") + lang_query = TS_QUERIES["Python"]() + tree, _ = lang_query.parse_code_to_ast(code_text=source_code) + extractor = SymbolExtractor(lang_query=lang_query, root_node=tree.root_node) + result = extractor.extract_imports() expected_map = { "models": "django.db.models", @@ -571,11 +586,30 @@ def clean_function(): return hello() + x + y """.strip() - tree, _ = parse_code_to_ast(source_code, "Python") - functions = extract_definitions(tree, "Python", kinds=("functions",)) - - result = extract_direct_calls(functions[1], "Python") + lang_query = TS_QUERIES["Python"]() + tree, _ = lang_query.parse_code_to_ast(code_text=source_code) + extractor = SymbolExtractor(lang_query=lang_query, root_node=tree.root_node) + result = extractor.extract_calls(node=tree.root_node) self.assertEqual( result, [(None, "hello")], ) + + def test_extract_direct_calls(self): + python_source = """ +self.update() +process_data() +user.save() + """.strip() + + lang_query = TS_QUERIES["Python"]() + tree, _ = lang_query.parse_code_to_ast(code_text=python_source) + extractor = SymbolExtractor(lang_query=lang_query, root_node=tree.root_node) + python_calls = extractor.extract_calls(node=tree.root_node) + + expected_python = [ + ("self", "update"), + (None, "process_data"), + ("user", "save"), + ] + self.assertEqual(expected_python, python_calls) From c4ac68ac91594d6f71d94233e590ca57e90e3396 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 1 Jul 2026 16:03:05 +0300 Subject: [PATCH 12/25] Add support for getting constant symbols Fix formating tests Signed-off-by: ziad hany --- scanpipe/pipes/symbols.py | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/scanpipe/pipes/symbols.py b/scanpipe/pipes/symbols.py index 855b481fed..94c8d5c936 100644 --- a/scanpipe/pipes/symbols.py +++ b/scanpipe/pipes/symbols.py @@ -219,6 +219,7 @@ def create_sha256_fingerprint(text): class LanguageQuery(ABC): language_name: str = "" + constants_query: str = "" functions_query: str = "" classes_query: str = "" calls_query: str = "" @@ -233,7 +234,7 @@ def __init__(self): self.ts_language = load_language(self.language_name) self._compiled_queries = {} - for kind in ("functions", "classes", "calls", "imports"): + for kind in ("constants", "functions", "classes", "calls", "imports"): source = getattr(self, f"{kind}_query", "").strip() self._compiled_queries[kind] = ( Query(self.ts_language, source) if source else None @@ -319,9 +320,30 @@ def get_imports(self, root_node): yield module_name, pairs + def get_constants(self, root_node): + """Yield raw (constant_node, name).""" + for _, captures in self.run_query("constants", root_node): + def_nodes = captures.get("constant") + if not def_nodes: + continue + name_nodes = captures.get("name") + name = ( + name_nodes[0].text.decode("utf-8", errors="replace") + if name_nodes + else None + ) + yield def_nodes[0], name + class PythonTreeSitterQuery(LanguageQuery): language_name = "Python" + constants_query = """ + (assignment + left: (identifier) @name) @constant + + (assignment + left: (pattern_list (identifier) @name)) @constant + """ functions_query = "(function_definition name: (identifier) @name) @function" classes_query = "(class_definition name: (identifier) @name) @class" calls_query = """ @@ -334,7 +356,6 @@ class PythonTreeSitterQuery(LanguageQuery): (import_statement name: (aliased_import name: (dotted_name) @import_name alias: (identifier) @alias)) - (import_from_statement module_name: [(dotted_name) (relative_import)] @module_name name: [ @@ -342,7 +363,6 @@ class PythonTreeSitterQuery(LanguageQuery): (aliased_import name: (dotted_name) @import_name alias: (identifier) @alias) ]) - (import_from_statement module_name: [(dotted_name) (relative_import)] @module_name (wildcard_import) @import_name) @@ -387,6 +407,9 @@ def extract_definitions_index(self): for node, name in self.lang_query.get_classes(self.root_node): index[node.id] = {"node": node, "name": name, "kind": "classes"} + for node, name in self.lang_query.get_constants(self.root_node): + index[node.id] = {"node": node, "name": name, "kind": "constants"} + for def_info in index.values(): def_info["qualified_name"] = self._build_qualified_name( def_info["node"], index From d48c18e3d843d318138b30e8d45d4777b1879a5e Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 1 Jul 2026 16:52:17 +0300 Subject: [PATCH 13/25] Remove dead code and fix the test Signed-off-by: ziad hany --- scanpipe/pipes/reachability.py | 15 ++++++--------- scanpipe/pipes/symbols.py | 29 ++++++++++------------------- 2 files changed, 16 insertions(+), 28 deletions(-) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index b6bf86c837..3a51372332 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -74,9 +74,8 @@ def detect_language_with_scancode(file_path, content): class GitRepositoryContext: - def __init__(self, vcs_url: str, commit_hash: str = None): + def __init__(self, vcs_url: str): self.vcs_url = vcs_url - self.commit_hash = commit_hash self.repo_path = None self._repo = None @@ -84,13 +83,11 @@ def __enter__(self) -> "GitRepositoryContext": self.repo_path = tempfile.mkdtemp(prefix="symbol-reachability-") try: self._repo = Repo.clone_from(self.vcs_url, self.repo_path) - if self.commit_hash: - self._repo.git.checkout(self.commit_hash) return self except Exception as exc: self._cleanup() raise ValueError( - f"Failed to clone/checkout {self.vcs_url}@{self.commit_hash}" + f"Failed to clone/checkout {self.vcs_url}" ) from exc def __exit__(self, exc_type, exc_val, exc_tb): @@ -288,7 +285,7 @@ def build_symbol_metadata( @classmethod def analyze( - self, vulnerable_text, fixed_text, removed_lines, added_lines, file_path + cls, vulnerable_text, fixed_text, removed_lines, added_lines, file_path ): vulnerable_text = normalize_text(vulnerable_text) fixed_text = normalize_text(fixed_text) @@ -326,7 +323,7 @@ def analyze( vuln_nodes = vuln_extractor.extract_changed_symbols( changed_lines=removed_lines ) - vuln_meta_all = self.build_symbol_metadata( + vuln_meta_all = cls.build_symbol_metadata( nodes=vuln_nodes, extractor=vuln_extractor ) @@ -337,11 +334,11 @@ def analyze( fixed_nodes = fixed_extractor.extract_changed_symbols( changed_lines=added_lines ) - fixed_meta_all = self.build_symbol_metadata( + fixed_meta_all = cls.build_symbol_metadata( fixed_nodes, extractor=fixed_extractor ) - vuln_meta, fixed_meta = self.diff_changed_symbols( + vuln_meta, fixed_meta = cls.diff_changed_symbols( vuln_meta=vuln_meta_all, fixed_meta=fixed_meta_all ) return vuln_meta, fixed_meta, language diff --git a/scanpipe/pipes/symbols.py b/scanpipe/pipes/symbols.py index 94c8d5c936..adf50980ee 100644 --- a/scanpipe/pipes/symbols.py +++ b/scanpipe/pipes/symbols.py @@ -26,16 +26,6 @@ from functools import cache from django.db.models import Q - -from source_inspector import symbols_ctags -from source_inspector import symbols_pygments -from source_inspector import symbols_tree_sitter -from source_inspector.symbols_tree_sitter import TS_LANGUAGE_WHEELS -from source_inspector.symbols_tree_sitter import TreeSitterWheelNotInstalled -from tree_sitter import Language -from tree_sitter import Parser -from tree_sitter import Query - from aboutcode.pipeline import LoopProgress @@ -186,16 +176,12 @@ def _collect_and_store_tree_sitter_symbols_and_strings(resource): } ) - -SYMBOLS_TYPE_SUPPORTED = { - "ctags": symbols_ctags.get_symbols, - "tree_sitter": symbols_tree_sitter.get_treesitter_symbols, - "pygments": symbols_pygments.get_pygments_symbols, -} - - @cache -def load_language(language: str) -> Language: +def load_language(language: str): + from source_inspector.symbols_tree_sitter import TS_LANGUAGE_WHEELS + from source_inspector.symbols_tree_sitter import TreeSitterWheelNotInstalled + from tree_sitter import Language + if language not in TS_LANGUAGE_WHEELS: raise ValueError(f"Unsupported language: {language}") @@ -231,6 +217,8 @@ class LanguageQuery(ABC): } def __init__(self): + from tree_sitter import Query + self.ts_language = load_language(self.language_name) self._compiled_queries = {} @@ -241,6 +229,9 @@ def __init__(self): ) def parse_code_to_ast(self, code_text: str): + from source_inspector.symbols_tree_sitter import TS_LANGUAGE_WHEELS + from tree_sitter import Parser + if not code_text: return None, None parser = Parser(language=self.ts_language) From 66864870a40c0a5960fa134c2a26ff646b51a464 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 1 Jul 2026 17:13:44 +0300 Subject: [PATCH 14/25] Add unidiff to uv.lock Fix formating Signed-off-by: ziad hany --- scanpipe/pipes/reachability.py | 4 +--- scanpipe/pipes/symbols.py | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index 3a51372332..635941da20 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -86,9 +86,7 @@ def __enter__(self) -> "GitRepositoryContext": return self except Exception as exc: self._cleanup() - raise ValueError( - f"Failed to clone/checkout {self.vcs_url}" - ) from exc + raise ValueError(f"Failed to clone/checkout {self.vcs_url}") from exc def __exit__(self, exc_type, exc_val, exc_tb): self._cleanup() diff --git a/scanpipe/pipes/symbols.py b/scanpipe/pipes/symbols.py index adf50980ee..d5ed4bf5b9 100644 --- a/scanpipe/pipes/symbols.py +++ b/scanpipe/pipes/symbols.py @@ -26,6 +26,7 @@ from functools import cache from django.db.models import Q + from aboutcode.pipeline import LoopProgress @@ -176,6 +177,7 @@ def _collect_and_store_tree_sitter_symbols_and_strings(resource): } ) + @cache def load_language(language: str): from source_inspector.symbols_tree_sitter import TS_LANGUAGE_WHEELS From a9076d37b1a4ddfc60eb01dd4e43a1bba8aca1d1 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 1 Jul 2026 18:11:32 +0300 Subject: [PATCH 15/25] Add support constant to resource analyzer Signed-off-by: ziad hany --- scanpipe/pipes/reachability.py | 59 ++++++++++++++++++++-------------- scanpipe/pipes/symbols.py | 16 ++++----- 2 files changed, 42 insertions(+), 33 deletions(-) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index 635941da20..0e1f7f59b5 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -463,16 +463,32 @@ def classify_reachability(evidence): class ResourceAnalyzer: def __init__(self, resource_text: str, language: str): - self.resource_text = resource_text + self.resource_text = normalize_text(resource_text) self.language = language + def process_node( + self, node, extractor, definitions_index, definitions: set, fingerprints: set + ) -> str | None: + """Extract the qualified name, update definitions, and fingerprint""" + qualified_name = extractor._build_qualified_name(node, definitions_index) + if not qualified_name: + return None + + definitions.add(qualified_name) + body_text = node.text.decode("utf-8", errors="replace") + fingerprint = create_sha256_fingerprint(body_text) + + if fingerprint: + fingerprints.add(fingerprint) + + return qualified_name + def build_index(self) -> dict | None: - text = normalize_text(self.resource_text) - if not is_supported_language(self.language) or not text: + if not is_supported_language(self.language) or not self.resource_text: return None lang_query = TS_QUERIES[self.language]() - tree, _ = lang_query.parse_code_to_ast(text) + tree, _ = lang_query.parse_code_to_ast(self.resource_text) if tree is None: return None @@ -487,29 +503,24 @@ def build_index(self) -> dict | None: callers_of = {} # callee_name -> set of caller_qualified_names for node, _ in lang_query.get_functions(tree.root_node): - qualified_name = extractor._build_qualified_name(node, definitions_index) - if not qualified_name: - continue - - definitions.add(qualified_name) - body_text = node.text.decode("utf-8", errors="replace") - fingerprint = create_sha256_fingerprint(body_text) - if fingerprint: - fingerprints.add(fingerprint) - - for _, callee_name in extractor.extract_calls(node): - callers_of.setdefault(callee_name, set()).add(qualified_name) + qualified_name = self.process_node( + node, extractor, definitions_index, definitions, fingerprints + ) + if qualified_name: + for _, callee_name in extractor.extract_calls(node): + callers_of.setdefault(callee_name, set()).add(qualified_name) for node, _ in lang_query.get_classes(tree.root_node): - qualified_name = extractor._build_qualified_name(node, definitions_index) - if not qualified_name: - continue + self.process_node( + node, extractor, definitions_index, definitions, fingerprints + ) - definitions.add(qualified_name) - body_text = node.text.decode("utf-8", errors="replace") - fingerprint = create_sha256_fingerprint(body_text) - if fingerprint: - fingerprints.add(fingerprint) + for node, _ in lang_query.get_constants(tree.root_node): + # Skip constants defined inside functions, classes, or other blocks + if node.parent == tree.root_node: + self.process_node( + node, extractor, definitions_index, definitions, fingerprints + ) return { "definitions": definitions, diff --git a/scanpipe/pipes/symbols.py b/scanpipe/pipes/symbols.py index d5ed4bf5b9..b32d8bd955 100644 --- a/scanpipe/pipes/symbols.py +++ b/scanpipe/pipes/symbols.py @@ -180,21 +180,19 @@ def _collect_and_store_tree_sitter_symbols_and_strings(resource): @cache def load_language(language: str): - from source_inspector.symbols_tree_sitter import TS_LANGUAGE_WHEELS - from source_inspector.symbols_tree_sitter import TreeSitterWheelNotInstalled - from tree_sitter import Language + from source_inspector import symbols_tree_sitter - if language not in TS_LANGUAGE_WHEELS: + if language not in symbols_tree_sitter.TS_LANGUAGE_WHEELS: raise ValueError(f"Unsupported language: {language}") - wheel = TS_LANGUAGE_WHEELS[language]["wheel"] + wheel = symbols_tree_sitter.TS_LANGUAGE_WHEELS[language]["wheel"] try: grammar = importlib.import_module(wheel) except ModuleNotFoundError as exc: - raise TreeSitterWheelNotInstalled( + raise symbols_tree_sitter.TreeSitterWheelNotInstalled( f"Grammar wheel '{wheel}' is not installed." ) from exc - return Language(grammar.language()) + return symbols_tree_sitter.Language(grammar.language()) def create_sha256_fingerprint(text): @@ -231,13 +229,13 @@ def __init__(self): ) def parse_code_to_ast(self, code_text: str): - from source_inspector.symbols_tree_sitter import TS_LANGUAGE_WHEELS + from source_inspector import symbols_tree_sitter from tree_sitter import Parser if not code_text: return None, None parser = Parser(language=self.ts_language) - return parser.parse(code_text.encode("utf-8")), TS_LANGUAGE_WHEELS[ + return parser.parse(code_text.encode("utf-8")), symbols_tree_sitter.TS_LANGUAGE_WHEELS[ self.language_name ] From 28676436397046584c68f06eb635e52f27b354bd Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 1 Jul 2026 18:16:48 +0300 Subject: [PATCH 16/25] Fix formating error in CI Signed-off-by: ziad hany --- scanpipe/pipes/symbols.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scanpipe/pipes/symbols.py b/scanpipe/pipes/symbols.py index b32d8bd955..9c517d3ff6 100644 --- a/scanpipe/pipes/symbols.py +++ b/scanpipe/pipes/symbols.py @@ -235,9 +235,9 @@ def parse_code_to_ast(self, code_text: str): if not code_text: return None, None parser = Parser(language=self.ts_language) - return parser.parse(code_text.encode("utf-8")), symbols_tree_sitter.TS_LANGUAGE_WHEELS[ - self.language_name - ] + return parser.parse( + code_text.encode("utf-8") + ), symbols_tree_sitter.TS_LANGUAGE_WHEELS[self.language_name] def run_query(self, kind: str, root_node): query = self._compiled_queries.get(kind) From 4b48ad72d7d131cc5563e85907871c03861d01d3 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Thu, 2 Jul 2026 03:57:22 +0300 Subject: [PATCH 17/25] Add a test for constants, extract_imports,collect_imports Fix a bug in extract_imports Signed-off-by: ziad hany --- scanpipe/pipes/reachability.py | 20 +++-------- scanpipe/pipes/symbols.py | 27 +++++++-------- .../tests/pipes/test_symbols_reachability.py | 34 ++++++++++++++++++- 3 files changed, 49 insertions(+), 32 deletions(-) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index 0e1f7f59b5..d9e75f4cfa 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -19,7 +19,6 @@ # # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. - import os import shutil import tempfile @@ -174,27 +173,16 @@ def parse_diff_lines(cls, diff_text): added = [] for hunk in patched_file: - hunk_removed = [ + removed.extend( line.source_line_no for line in hunk if line.is_removed and line.source_line_no - ] - hunk_added = [ + ) + added.extend( line.target_line_no for line in hunk if line.is_added and line.target_line_no - ] - - if hunk_added and not hunk_removed: - anchor = max(hunk.source_start, 1) - hunk_removed = [anchor] - - if hunk_removed and not hunk_added: - anchor = max(hunk.target_start, 1) - hunk_added = [anchor] - - removed.extend(hunk_removed) - added.extend(hunk_added) + ) candidates = { patched_file.path, diff --git a/scanpipe/pipes/symbols.py b/scanpipe/pipes/symbols.py index 9c517d3ff6..0f25819d51 100644 --- a/scanpipe/pipes/symbols.py +++ b/scanpipe/pipes/symbols.py @@ -287,14 +287,12 @@ def get_imports(self, root_node): for _, captures in self.run_query("imports", root_node): flat_captures = [] for tag, nodes in captures.items(): - for n in nodes: - text = n.text.decode("utf-8", errors="replace").strip("'\"") - flat_captures.append((n.start_byte, tag, text)) + for node in nodes: + text = node.text.decode("utf-8", errors="replace").strip("'\"") + flat_captures.append((node.start_byte, tag, text)) - flat_captures.sort(key=lambda x: x[0]) module_name, current_import = None, None pairs = [] - for _, tag, text in flat_captures: if tag == "module_name": module_name = text @@ -328,13 +326,7 @@ def get_constants(self, root_node): class PythonTreeSitterQuery(LanguageQuery): language_name = "Python" - constants_query = """ - (assignment - left: (identifier) @name) @constant - - (assignment - left: (pattern_list (identifier) @name)) @constant - """ + constants_query = "(assignment left: (identifier) @name) @constant" functions_query = "(function_definition name: (identifier) @name) @function" classes_query = "(class_definition name: (identifier) @name) @class" calls_query = """ @@ -471,9 +463,14 @@ def extract_imports(self): if not alias and separator in imp_name: local_name = imp_name.split(separator)[0] - absolute_path = ( - f"{module_name}{separator}{imp_name}" if module_name else imp_name - ) + if module_name: + if module_name == separator: + absolute_path = f"{separator}{imp_name}" + else: + absolute_path = f"{module_name}{separator}{imp_name}" + else: + absolute_path = imp_name + import_map[local_name] = absolute_path if wildcard_modules: diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index eeeb0560a3..620e886c05 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -154,6 +154,7 @@ def test_get_symbol_reachability_results( def test_extract_definitions(self): source_code = """ +price = 0 class OrderManager: def __init__(self, order_id): self.order_id = order_id @@ -185,6 +186,9 @@ class InventoryItem: second_class_text = classes[1][0].text.decode("utf-8") self.assertIn("class InventoryItem", second_class_text) + constants = list(lang_query.get_constants(tree.root_node)) + self.assertEqual(len(constants), 1) + def test_extract_definitions_empty(self): lang_query = TS_QUERIES["Python"]() tree, _ = lang_query.parse_code_to_ast("") @@ -569,7 +573,7 @@ def test_collect_imports(self): "os": "os.path", "np": "numpy", "d": "a.b.c", - "utils": "..utils", + "utils": ".utils", "engine": "..core.engine", "*": ["math"], } @@ -613,3 +617,31 @@ def test_extract_direct_calls(self): ("user", "save"), ] self.assertEqual(expected_python, python_calls) + + def test_extract_imports(self): + source_code = """ +from django.db import models +import os.path +import numpy as np +from a.b import c as d +from . import utils +from ..core import engine +from math import * + """.strip() + + lang_query = TS_QUERIES["Python"]() + tree, _ = lang_query.parse_code_to_ast(code_text=source_code) + extractor = SymbolExtractor(lang_query=lang_query, root_node=tree.root_node) + result = extractor.extract_imports() + + expected_map = { + "models": "django.db.models", + "os": "os.path", + "np": "numpy", + "d": "a.b.c", + "utils": ".utils", + "engine": "..core.engine", + "*": ["math"], + } + + self.assertEqual(result, expected_map) From 1bd85f674a8d7e20606b7ecd494689ee19dfec3c Mon Sep 17 00:00:00 2001 From: ziad hany Date: Thu, 16 Jul 2026 02:02:04 +0300 Subject: [PATCH 18/25] Remove dependency and copy only the required file Signed-off-by: ziad hany --- scanpipe/pipes/unidiff/patch.py | 667 ++++++++++++++++++++++++ scanpipe/pipes/unidiff/patch.py.ABOUT | 13 + scanpipe/pipes/unidiff/patch.py.LICENSE | 20 + 3 files changed, 700 insertions(+) create mode 100644 scanpipe/pipes/unidiff/patch.py create mode 100644 scanpipe/pipes/unidiff/patch.py.ABOUT create mode 100644 scanpipe/pipes/unidiff/patch.py.LICENSE diff --git a/scanpipe/pipes/unidiff/patch.py b/scanpipe/pipes/unidiff/patch.py new file mode 100644 index 0000000000..c51bdbef79 --- /dev/null +++ b/scanpipe/pipes/unidiff/patch.py @@ -0,0 +1,667 @@ +# Extracted essential patch code analyzer and modified from the original unidiff library: +# https://github.com/matiasb/python-unidiff/blob/2771a878f7bc6619e625feb4dbad3427f57f5237/unidiff/patch.py + +# -*- coding: utf-8 -*- + +# The MIT License (MIT) +# Copyright (c) 2014-2023 Matias Bordese +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE +# OR OTHER DEALINGS IN THE SOFTWARE. + +from __future__ import unicode_literals +import re +from io import StringIO +from typing import Iterable, Optional, Union + +class UnidiffParseError(Exception): ... + +open_file = open +make_str = str +implements_to_string = lambda x: x +unicode = str +basestring = str + +RE_SOURCE_FILENAME = re.compile( + r'^--- (?P"?[^\t\n]+"?)(?:\t(?P[^\n]+))?') +RE_TARGET_FILENAME = re.compile( + r'^\+\+\+ (?P"?[^\t\n]+"?)(?:\t(?P[^\n]+))?') + + +# check diff git line for git renamed files support +RE_DIFF_GIT_HEADER = re.compile( + r'^diff --git (?P"?a/[^\t\n]+"?) (?P"?b/[^\t\n]+"?)') +RE_DIFF_GIT_HEADER_URI_LIKE = re.compile( + r'^diff --git (?P.*://[^\t\n]+) (?P.*://[^\t\n]+)') +RE_DIFF_GIT_HEADER_NO_PREFIX = re.compile( + r'^diff --git (?P[^\t\n]+) (?P[^\t\n]+)') + +# check diff git new file marker `deleted file mode 100644` +RE_DIFF_GIT_DELETED_FILE = re.compile(r'^deleted file mode \d+$') + +# check diff git new file marker `new file mode 100644` +RE_DIFF_GIT_NEW_FILE = re.compile(r'^new file mode \d+$') + + +# @@ (source offset, length) (target offset, length) @@ (section header) +RE_HUNK_HEADER = re.compile( + r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))?\ @@[ ]?(.*)") + +# kept line (context) +# \n empty line (treat like context) +# + added line +# - deleted line +# \ No newline case +RE_HUNK_BODY_LINE = re.compile( + r'^(?P[- \+\\])(?P.*)', re.DOTALL) +RE_HUNK_EMPTY_BODY_LINE = re.compile( + r'^(?P[- \+\\]?)(?P[\r\n]{1,2})', re.DOTALL) + +RE_NO_NEWLINE_MARKER = re.compile(r'^\\ No newline at end of file') + +RE_BINARY_DIFF = re.compile( + r'^Binary files? ' + r'(?P[^\t]+?)(?:\t(?P[\s0-9:\+-]+))?' + r'(?: and (?P[^\t]+?)(?:\t(?P[\s0-9:\+-]+))?)? (differ|has changed)') + +DEFAULT_ENCODING = 'UTF-8' + +DEV_NULL = '/dev/null' +LINE_TYPE_ADDED = '+' +LINE_TYPE_REMOVED = '-' +LINE_TYPE_CONTEXT = ' ' +LINE_TYPE_EMPTY = '' +LINE_TYPE_NO_NEWLINE = '\\' +LINE_VALUE_NO_NEWLINE = ' No newline at end of file' + +@implements_to_string +class Line(object): + """A diff line.""" + + def __init__(self, value, line_type, + source_line_no=None, target_line_no=None, diff_line_no=None): + # type: (str, str, Optional[int], Optional[int], Optional[int]) -> None + super(Line, self).__init__() + self.source_line_no = source_line_no + self.target_line_no = target_line_no + self.diff_line_no = diff_line_no + self.line_type = line_type + self.value = value + + def __repr__(self): + # type: () -> str + return make_str("") % (self.line_type, self.value) + + def __str__(self): + # type: () -> str + return "%s%s" % (self.line_type, self.value) + + def __eq__(self, other): + # type: (Line) -> bool + return (self.source_line_no == other.source_line_no and + self.target_line_no == other.target_line_no and + self.diff_line_no == other.diff_line_no and + self.line_type == other.line_type and + self.value == other.value) + + @property + def is_added(self): + # type: () -> bool + return self.line_type == LINE_TYPE_ADDED + + @property + def is_removed(self): + # type: () -> bool + return self.line_type == LINE_TYPE_REMOVED + + @property + def is_context(self): + # type: () -> bool + return self.line_type == LINE_TYPE_CONTEXT + + +@implements_to_string +class PatchInfo(list): + """Lines with extended patch info. + + Format of this info is not documented and it very much depends on + patch producer. + + """ + + def __repr__(self): + # type: () -> str + value = "" % self[0].strip() + return make_str(value) + + def __str__(self): + # type: () -> str + return ''.join(unicode(line) for line in self) + + +@implements_to_string +class Hunk(list): + """Each of the modified blocks of a file.""" + + def __init__(self, src_start=0, src_len=0, tgt_start=0, tgt_len=0, + section_header=''): + # type: (int, int, int, int, str) -> None + super(Hunk, self).__init__() + if src_len is None: + src_len = 1 + if tgt_len is None: + tgt_len = 1 + self.source_start = int(src_start) + self.source_length = int(src_len) + self.target_start = int(tgt_start) + self.target_length = int(tgt_len) + self.section_header = section_header + self._added = None # Optional[int] + self._removed = None # Optional[int] + + def __repr__(self): + # type: () -> str + value = "" % (self.source_start, + self.source_length, + self.target_start, + self.target_length, + self.section_header) + return make_str(value) + + def __str__(self): + # type: () -> str + # section header is optional and thus we output it only if it's present + head = "@@ -%d,%d +%d,%d @@%s\n" % ( + self.source_start, self.source_length, + self.target_start, self.target_length, + ' ' + self.section_header if self.section_header else '') + content = ''.join(unicode(line) for line in self) + return head + content + + def append(self, line): + # type: (Line) -> None + """Append the line to hunk, and keep track of source/target lines.""" + # Make sure the line is encoded correctly. This is a no-op except for + # potentially raising a UnicodeDecodeError. + str(line) + super(Hunk, self).append(line) + + @property + def added(self): + # type: () -> Optional[int] + if self._added is not None: + return self._added + # re-calculate each time to allow for hunk modifications + # (which should mean metadata_only switch wasn't used) + return sum(1 for line in self if line.is_added) + + @property + def removed(self): + # type: () -> Optional[int] + if self._removed is not None: + return self._removed + # re-calculate each time to allow for hunk modifications + # (which should mean metadata_only switch wasn't used) + return sum(1 for line in self if line.is_removed) + + def is_valid(self): + # type: () -> bool + """Check hunk header data matches entered lines info.""" + return (len(self.source) == self.source_length and + len(self.target) == self.target_length) + + def source_lines(self): + # type: () -> Iterable[Line] + """Hunk lines from source file (generator).""" + return (l for l in self if l.is_context or l.is_removed) + + @property + def source(self): + # type: () -> Iterable[str] + return [str(l) for l in self.source_lines()] + + def target_lines(self): + # type: () -> Iterable[Line] + """Hunk lines from target file (generator).""" + return (l for l in self if l.is_context or l.is_added) + + @property + def target(self): + # type: () -> Iterable[str] + return [str(l) for l in self.target_lines()] + + +class PatchedFile(list): + """Patch updated file, it is a list of Hunks.""" + + def __init__(self, patch_info=None, source='', target='', + source_timestamp=None, target_timestamp=None, + is_binary_file=False): + # type: (Optional[PatchInfo], str, str, Optional[str], Optional[str], bool, bool) -> None + super(PatchedFile, self).__init__() + self.patch_info = patch_info + self.source_file = source + self.source_timestamp = source_timestamp + self.target_file = target + self.target_timestamp = target_timestamp + self.is_binary_file = is_binary_file + + def __repr__(self): + # type: () -> str + return make_str("") % make_str(self.path) + + def __str__(self): + # type: () -> str + source = '' + target = '' + # patch info is optional + info = '' if self.patch_info is None else str(self.patch_info) + if not self.is_binary_file and self: + source = "--- %s%s\n" % ( + self.source_file, + '\t' + self.source_timestamp if self.source_timestamp else '') + target = "+++ %s%s\n" % ( + self.target_file, + '\t' + self.target_timestamp if self.target_timestamp else '') + hunks = ''.join(unicode(hunk) for hunk in self) + return info + source + target + hunks + + def _parse_hunk(self, header, diff, encoding, metadata_only): + # type: (str, enumerate[str], Optional[str], bool) -> None + """Parse hunk details.""" + header_info = RE_HUNK_HEADER.match(header) + hunk_info = header_info.groups() + hunk = Hunk(*hunk_info) + + source_line_no = hunk.source_start + target_line_no = hunk.target_start + expected_source_end = source_line_no + hunk.source_length + expected_target_end = target_line_no + hunk.target_length + added = 0 + removed = 0 + + for diff_line_no, line in diff: + if encoding is not None: + line = line.decode(encoding) + + if metadata_only: + # quick line type detection, no regex required + line_type = line[0] if line else LINE_TYPE_CONTEXT + if line_type not in (LINE_TYPE_ADDED, + LINE_TYPE_REMOVED, + LINE_TYPE_CONTEXT, + LINE_TYPE_NO_NEWLINE): + raise UnidiffParseError( + 'Hunk diff line expected: %s' % line) + + if line_type == LINE_TYPE_ADDED: + target_line_no += 1 + added += 1 + elif line_type == LINE_TYPE_REMOVED: + source_line_no += 1 + removed += 1 + elif line_type == LINE_TYPE_CONTEXT: + target_line_no += 1 + source_line_no += 1 + + # no file content tracking + original_line = None + + else: + # parse diff line content + valid_line = RE_HUNK_BODY_LINE.match(line) + if not valid_line: + valid_line = RE_HUNK_EMPTY_BODY_LINE.match(line) + + if not valid_line: + raise UnidiffParseError( + 'Hunk diff line expected: %s' % line) + + line_type = valid_line.group('line_type') + if line_type == LINE_TYPE_EMPTY: + line_type = LINE_TYPE_CONTEXT + + value = valid_line.group('value') # type: str + original_line = Line(value, line_type=line_type) + + if line_type == LINE_TYPE_ADDED: + original_line.target_line_no = target_line_no + target_line_no += 1 + elif line_type == LINE_TYPE_REMOVED: + original_line.source_line_no = source_line_no + source_line_no += 1 + elif line_type == LINE_TYPE_CONTEXT: + original_line.target_line_no = target_line_no + original_line.source_line_no = source_line_no + target_line_no += 1 + source_line_no += 1 + elif line_type == LINE_TYPE_NO_NEWLINE: + pass + else: + original_line = None + + # stop parsing if we got past expected number of lines + if (source_line_no > expected_source_end or + target_line_no > expected_target_end): + raise UnidiffParseError('Hunk is longer than expected') + + if original_line: + original_line.diff_line_no = diff_line_no + hunk.append(original_line) + + # if hunk source/target lengths are ok, hunk is complete + if (source_line_no == expected_source_end and + target_line_no == expected_target_end): + break + + # report an error if we haven't got expected number of lines + if (source_line_no < expected_source_end or + target_line_no < expected_target_end): + raise UnidiffParseError('Hunk is shorter than expected') + + if metadata_only: + # HACK: set fixed calculated values when metadata_only is enabled + hunk._added = added + hunk._removed = removed + + self.append(hunk) + + def _add_no_newline_marker_to_last_hunk(self): + # type: () -> None + if not self: + raise UnidiffParseError( + 'Unexpected marker:' + LINE_VALUE_NO_NEWLINE) + last_hunk = self[-1] + last_hunk.append( + Line(LINE_VALUE_NO_NEWLINE + '\n', line_type=LINE_TYPE_NO_NEWLINE)) + + def _append_trailing_empty_line(self): + # type: () -> None + if not self: + raise UnidiffParseError('Unexpected trailing newline character') + last_hunk = self[-1] + last_hunk.append(Line('\n', line_type=LINE_TYPE_EMPTY)) + + @property + def path(self): + # type: () -> str + """Return the file path abstracted from VCS.""" + filepath = self.source_file + if filepath in (None, DEV_NULL) or ( + self.is_rename and self.target_file not in (None, DEV_NULL)): + # if this is a rename, prefer the target filename + filepath = self.target_file + + quoted = filepath.startswith('"') and filepath.endswith('"') + if quoted: + filepath = filepath[1:-1] + + if filepath.startswith('a/') or filepath.startswith('b/'): + filepath = filepath[2:] + + if quoted: + filepath = '"{}"'.format(filepath) + + return filepath + + @property + def added(self): + # type: () -> int + """Return the file total added lines.""" + return sum([hunk.added for hunk in self]) + + @property + def removed(self): + # type: () -> int + """Return the file total removed lines.""" + return sum([hunk.removed for hunk in self]) + + @property + def is_rename(self): + return (self.source_file != DEV_NULL + and self.target_file != DEV_NULL + and self.source_file[2:] != self.target_file[2:]) + + @property + def is_added_file(self): + # type: () -> bool + """Return True if this patch adds the file.""" + if self.source_file == DEV_NULL: + return True + return (len(self) == 1 and self[0].source_start == 0 and + self[0].source_length == 0) + + @property + def is_removed_file(self): + # type: () -> bool + """Return True if this patch removes the file.""" + if self.target_file == DEV_NULL: + return True + return (len(self) == 1 and self[0].target_start == 0 and + self[0].target_length == 0) + + @property + def is_modified_file(self): + # type: () -> bool + """Return True if this patch modifies the file.""" + return not (self.is_added_file or self.is_removed_file) + + +@implements_to_string +class PatchSet(list): + """A list of PatchedFiles.""" + + def __init__(self, f, encoding=None, metadata_only=False): + # type: (Union[StringIO, str], Optional[str], bool) -> None + super(PatchSet, self).__init__() + + # convert string inputs to StringIO objects + if isinstance(f, basestring): + f = self._convert_string(f, encoding) # type: StringIO + + # make sure we pass an iterator object to parse + data = iter(f) + # if encoding is None, assume we are reading unicode data + # when metadata_only is True, only perform a minimal metadata parsing + # (ie. hunks without content) which is around 2.5-6 times faster; + # it will still validate the diff metadata consistency and get counts + self._parse(data, encoding=encoding, metadata_only=metadata_only) + + def __repr__(self): + # type: () -> str + return make_str('') % super(PatchSet, self).__repr__() + + def __str__(self): + # type: () -> str + return ''.join(unicode(patched_file) for patched_file in self) + + def _parse(self, diff, encoding, metadata_only): + # type: (StringIO, Optional[str], bool) -> None + current_file = None + patch_info = None + + diff = enumerate(diff, 1) + for unused_diff_line_no, line in diff: + if encoding is not None: + line = line.decode(encoding) + + # check for a git file rename + is_diff_git_header = RE_DIFF_GIT_HEADER.match(line) or \ + RE_DIFF_GIT_HEADER_URI_LIKE.match(line) or \ + RE_DIFF_GIT_HEADER_NO_PREFIX.match(line) + if is_diff_git_header: + patch_info = PatchInfo() + source_file = is_diff_git_header.group('source') + target_file = is_diff_git_header.group('target') + current_file = PatchedFile( + patch_info, source_file, target_file, None, None) + self.append(current_file) + patch_info.append(line) + continue + + # check for a git new file + is_diff_git_new_file = RE_DIFF_GIT_NEW_FILE.match(line) + if is_diff_git_new_file: + if current_file is None or patch_info is None: + raise UnidiffParseError('Unexpected new file found: %s' % line) + current_file.source_file = DEV_NULL + patch_info.append(line) + continue + + # check for a git deleted file + is_diff_git_deleted_file = RE_DIFF_GIT_DELETED_FILE.match(line) + if is_diff_git_deleted_file: + if current_file is None or patch_info is None: + raise UnidiffParseError('Unexpected deleted file found: %s' % line) + current_file.target_file = DEV_NULL + patch_info.append(line) + continue + + # check for source file header + is_source_filename = RE_SOURCE_FILENAME.match(line) + if is_source_filename: + source_file = is_source_filename.group('filename') + source_timestamp = is_source_filename.group('timestamp') + # reset current file, unless we are processing a rename + # (in that case, source files should match) + if current_file is not None and not ( + current_file.source_file == source_file): + current_file = None + elif current_file is not None: + current_file.source_timestamp = source_timestamp + continue + + # check for target file header + is_target_filename = RE_TARGET_FILENAME.match(line) + if is_target_filename: + target_file = is_target_filename.group('filename') + target_timestamp = is_target_filename.group('timestamp') + if current_file is not None and not (current_file.target_file == target_file): + raise UnidiffParseError('Target without source: %s' % line) + if current_file is None: + # add current file to PatchSet + current_file = PatchedFile( + patch_info, source_file, target_file, + source_timestamp, target_timestamp) + self.append(current_file) + patch_info = None + else: + current_file.target_timestamp = target_timestamp + continue + + # check for hunk header + is_hunk_header = RE_HUNK_HEADER.match(line) + if is_hunk_header: + patch_info = None + if current_file is None: + raise UnidiffParseError('Unexpected hunk found: %s' % line) + current_file._parse_hunk(line, diff, encoding, metadata_only) + continue + + # check for no newline marker + is_no_newline = RE_NO_NEWLINE_MARKER.match(line) + if is_no_newline: + if current_file is None: + raise UnidiffParseError('Unexpected marker: %s' % line) + current_file._add_no_newline_marker_to_last_hunk() + continue + + # sometimes hunks can be followed by empty lines + if line == '\n' and current_file is not None: + current_file._append_trailing_empty_line() + continue + + # if nothing has matched above then this line is a patch info + if patch_info is None: + current_file = None + patch_info = PatchInfo() + + is_binary_diff = RE_BINARY_DIFF.match(line) + if is_binary_diff: + source_file = is_binary_diff.group('source_filename') + target_file = is_binary_diff.group('target_filename') + patch_info.append(line) + if current_file is not None: + current_file.is_binary_file = True + else: + current_file = PatchedFile( + patch_info, source_file, target_file, is_binary_file=True) + self.append(current_file) + patch_info = None + current_file = None + continue + + if line == 'GIT binary patch\n': + current_file.is_binary_file = True + patch_info = None + current_file = None + continue + + patch_info.append(line) + + @classmethod + def from_filename(cls, filename, encoding=DEFAULT_ENCODING, errors=None, newline=None): + # type: (str, str, Optional[str]) -> PatchSet + """Return a PatchSet instance given a diff filename.""" + with open_file(filename, 'r', encoding=encoding, errors=errors, newline=newline) as f: + instance = cls(f) + return instance + + @staticmethod + def _convert_string(data, encoding=None, errors='strict'): + # type: (Union[str, bytes], str, str) -> StringIO + if encoding is not None: + # if encoding is given, assume bytes and decode + data = unicode(data, encoding=encoding, errors=errors) + return StringIO(data) + + @classmethod + def from_string(cls, data, encoding=None, errors='strict'): + # type: (str, str, Optional[str]) -> PatchSet + """Return a PatchSet instance given a diff string.""" + return cls(cls._convert_string(data, encoding, errors)) + + @property + def added_files(self): + # type: () -> list[PatchedFile] + """Return patch added files as a list.""" + return [f for f in self if f.is_added_file] + + @property + def removed_files(self): + # type: () -> list[PatchedFile] + """Return patch removed files as a list.""" + return [f for f in self if f.is_removed_file] + + @property + def modified_files(self): + # type: () -> list[PatchedFile] + """Return patch modified files as a list.""" + return [f for f in self if f.is_modified_file] + + @property + def added(self): + # type: () -> int + """Return the patch total added lines.""" + return sum([f.added for f in self]) + + @property + def removed(self): + # type: () -> int + """Return the patch total removed lines.""" + return sum([f.removed for f in self]) \ No newline at end of file diff --git a/scanpipe/pipes/unidiff/patch.py.ABOUT b/scanpipe/pipes/unidiff/patch.py.ABOUT new file mode 100644 index 0000000000..3118ed9643 --- /dev/null +++ b/scanpipe/pipes/unidiff/patch.py.ABOUT @@ -0,0 +1,13 @@ +about_resource: patch.py constants.py errors.py +name: patch +version: 0.7.5 +download_url: https://github.com/matiasb/python-unidiff/archive/refs/tags/v0.7.5.zip +description: Simple Python library to parse and interact with unified diff data. +homepage_url: https://github.com/matiasb/python-unidiff +license_expression: mit +attribute: yes +package_url: pkg:pypi/unidiff@0.7.5 +licenses: + - key: mit + name: MIT License + file: mit.LICENSE \ No newline at end of file diff --git a/scanpipe/pipes/unidiff/patch.py.LICENSE b/scanpipe/pipes/unidiff/patch.py.LICENSE new file mode 100644 index 0000000000..ca2c04c202 --- /dev/null +++ b/scanpipe/pipes/unidiff/patch.py.LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) +Copyright (c) 2012 Matias Bordese + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE +OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file From 09cfa5f95f4a8f2cdab58fc8e104c63f6c212b69 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Thu, 23 Jul 2026 02:50:28 +0300 Subject: [PATCH 19/25] Update pipeline/functions name Get the dependency file and included in the PR Signed-off-by: ziad hany --- pyproject.toml | 2 +- ...ity.py => analyze_symbols_reachability.py} | 6 +- scanpipe/pipelines/find_vulnerabilities.py | 12 +- scanpipe/pipes/reachability.py | 12 +- scanpipe/pipes/symbols.py | 28 ++ scanpipe/pipes/unidiff/patch.py | 309 +++++++++++------- scanpipe/pipes/vulnerablecode.py | 7 +- .../tests/pipes/test_symbols_reachability.py | 4 +- 8 files changed, 237 insertions(+), 143 deletions(-) rename scanpipe/pipelines/{collect_symbols_reachability.py => analyze_symbols_reachability.py} (85%) diff --git a/pyproject.toml b/pyproject.toml index 50a4286150..d5dfbae829 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -152,7 +152,7 @@ run = "scancodeio:combined_run" analyze_docker_image = "scanpipe.pipelines.analyze_docker:Docker" analyze_root_filesystem_or_vm_image = "scanpipe.pipelines.analyze_root_filesystem:RootFS" analyze_windows_docker_image = "scanpipe.pipelines.analyze_docker_windows:DockerWindows" -analyze_symbols_reachability = "scanpipe.pipelines.collect_symbols_reachability:SymbolReachability" +analyze_symbols_reachability = "scanpipe.pipelines.analyze_symbols_reachability:SymbolReachability" benchmark_purls = "scanpipe.pipelines.benchmark_purls:BenchmarkPurls" collect_strings_gettext = "scanpipe.pipelines.collect_strings_gettext:CollectStringsGettext" collect_symbols_ctags = "scanpipe.pipelines.collect_symbols_ctags:CollectSymbolsCtags" diff --git a/scanpipe/pipelines/collect_symbols_reachability.py b/scanpipe/pipelines/analyze_symbols_reachability.py similarity index 85% rename from scanpipe/pipelines/collect_symbols_reachability.py rename to scanpipe/pipelines/analyze_symbols_reachability.py index c1d5fb11c4..2489df9c49 100644 --- a/scanpipe/pipelines/collect_symbols_reachability.py +++ b/scanpipe/pipelines/analyze_symbols_reachability.py @@ -20,14 +20,14 @@ class SymbolReachability(Pipeline): @classmethod def steps(cls): - return (cls.analyze_and_store_symbol_reachability,) + return (cls.analyze_symbol_reachability,) - def analyze_and_store_symbol_reachability(self): + def analyze_symbol_reachability(self): """ Perform symbol-level reachability analysis for each patch. This step compares the AST of patched/vulnerable files against the codebase resources. Results are stored directly in the 'extra_data' of each CodebaseResource. """ - reachability.collect_and_store_symbol_reachability_results( + reachability.analyze_and_store_symbol_reachability_results( project=self.project, logger=self.log ) diff --git a/scanpipe/pipelines/find_vulnerabilities.py b/scanpipe/pipelines/find_vulnerabilities.py index b0c8066b9a..1bdf8af6bc 100644 --- a/scanpipe/pipelines/find_vulnerabilities.py +++ b/scanpipe/pipelines/find_vulnerabilities.py @@ -19,7 +19,7 @@ # # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. - +from aboutcode.pipeline import optional_step from scanpipe.pipelines import Pipeline from scanpipe.pipes import vulnerablecode @@ -34,11 +34,13 @@ class FindVulnerabilities(Pipeline): download_inputs = False is_addon = True results_url = "/project/{slug}/packages/?is_vulnerable=yes" + reachability = False @classmethod def steps(cls): return ( cls.check_vulnerablecode_service_availability, + cls.enable_reachability_analysis, cls.lookup_packages_vulnerabilities, cls.lookup_dependencies_vulnerabilities, ) @@ -48,6 +50,12 @@ def get_availability(cls): if not vulnerablecode.is_configured(): return "VulnerableCode is not configured." + @optional_step("reachability") + def enable_reachability_analysis(self): + """Enable the reachability flag for vulnerability lookups.""" + self.reachability = True + self.log("Reachability analysis is ENABLED.") + def check_vulnerablecode_service_availability(self): """Check if the VulnerableCode service if configured and available.""" if not vulnerablecode.is_configured(): @@ -62,6 +70,7 @@ def lookup_packages_vulnerabilities(self): vulnerablecode.fetch_vulnerabilities( packages=packages, ignore_set=self.project.ignored_vulnerabilities_set, + reachability=self.reachability, logger=self.log, ) @@ -71,5 +80,6 @@ def lookup_dependencies_vulnerabilities(self): vulnerablecode.fetch_vulnerabilities( packages=dependencies, ignore_set=self.project.ignored_vulnerabilities_set, + reachability=self.reachability, logger=self.log, ) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index d9e75f4cfa..46cda929ee 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -19,6 +19,8 @@ # # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. +# + import os import shutil import tempfile @@ -29,12 +31,12 @@ from git import Repo from git.diff import NULL_TREE from scancode.api import get_file_info -from unidiff import PatchSet from scanpipe.pipes.symbols import TS_QUERIES from scanpipe.pipes.symbols import SymbolExtractor from scanpipe.pipes.symbols import create_sha256_fingerprint from scanpipe.pipes.symbols import is_supported_language +from scanpipe.pipes.unidiff.patch import PatchSet class ReachabilityStatus(str, Enum): @@ -100,8 +102,6 @@ def _cleanup(self): class PatchAnalyzer: - EMPTY_TREE_SHA = "4b825dc642cb6eb9a060e54bf8b8e6f9b79b4d2b" - def __init__(self, repo: Repo, commit_hash: str): self.repo = repo self.commit = repo.commit(commit_hash) @@ -145,7 +145,8 @@ def get_changed_files(self): return files def get_commit_diff_text(self): - base = self.parent_commit.hexsha if self.parent_commit else self.EMPTY_TREE_SHA + """Get the diff text, falling back to an empty tree if no parent exists.""" + base = self.parent_commit.hexsha if self.parent_commit else NULL_TREE return self.repo.git.diff(base, self.commit.hexsha, unified=3) @classmethod @@ -379,7 +380,6 @@ def generate_reachability_report(patch, repo, candidate_resources, logger=None): "reachability_status": classify_reachability(vuln_evidence).value, } } - print(report) existing = resource.extra_data.get("symbols_reachability") reports = existing if isinstance(existing, list) else [] @@ -398,7 +398,7 @@ def generate_reachability_report(patch, repo, candidate_resources, logger=None): ) -def collect_and_store_symbol_reachability_results(project, logger=None): +def analyze_and_store_symbol_reachability_results(project, logger=None): candidate_resources = project.codebaseresources.files().filter( is_binary=False, is_archive=False, is_media=False ) diff --git a/scanpipe/pipes/symbols.py b/scanpipe/pipes/symbols.py index 0f25819d51..f86630b4f5 100644 --- a/scanpipe/pipes/symbols.py +++ b/scanpipe/pipes/symbols.py @@ -353,8 +353,36 @@ class PythonTreeSitterQuery(LanguageQuery): syntax_config = {"self_keyword": "self", "separator": ".", "wildcard_symbol": "*"} +class JavaTreeSitterQuery(LanguageQuery): + language_name = "Java" + constants_query = ( + "(field_declaration declarator: " + "(variable_declarator name: (identifier) @name)) @constant" + ) + functions_query = """ + [(method_declaration name: (identifier) @name) + (constructor_declaration name: (identifier) @name)] @function + """ + classes_query = """ + [(class_declaration name: (identifier) @name) + (interface_declaration name: (identifier) @name) + (record_declaration name: (identifier) @name) + (enum_declaration name: (identifier) @name)] @class + """ + calls_query = """ + (method_invocation name: (identifier) @callee) + (method_invocation object: (_) @receiver name: (identifier) @callee) + """ + imports_query = """ + (import_declaration (scoped_identifier) @import_name) + (import_declaration (scoped_identifier) @module_name (asterisk) @import_name) + """ + syntax_config = {"self_keyword": "this", "separator": ".", "wildcard_symbol": "*"} + + TS_QUERIES = { "Python": PythonTreeSitterQuery, + "Java": JavaTreeSitterQuery, } diff --git a/scanpipe/pipes/unidiff/patch.py b/scanpipe/pipes/unidiff/patch.py index c51bdbef79..8075a8cc77 100644 --- a/scanpipe/pipes/unidiff/patch.py +++ b/scanpipe/pipes/unidiff/patch.py @@ -1,4 +1,5 @@ -# Extracted essential patch code analyzer and modified from the original unidiff library: +# Extracted essential patch code analyzer and modified +# from the original unidiff library: # https://github.com/matiasb/python-unidiff/blob/2771a878f7bc6619e625feb4dbad3427f57f5237/unidiff/patch.py # -*- coding: utf-8 -*- @@ -24,13 +25,13 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE # OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import unicode_literals import re from io import StringIO -from typing import Iterable, Optional, Union + class UnidiffParseError(Exception): ... + open_file = open make_str = str implements_to_string = lambda x: x @@ -38,65 +39,77 @@ class UnidiffParseError(Exception): ... basestring = str RE_SOURCE_FILENAME = re.compile( - r'^--- (?P"?[^\t\n]+"?)(?:\t(?P[^\n]+))?') + r'^--- (?P"?[^\t\n]+"?)(?:\t(?P[^\n]+))?' +) RE_TARGET_FILENAME = re.compile( - r'^\+\+\+ (?P"?[^\t\n]+"?)(?:\t(?P[^\n]+))?') + r'^\+\+\+ (?P"?[^\t\n]+"?)(?:\t(?P[^\n]+))?' +) # check diff git line for git renamed files support RE_DIFF_GIT_HEADER = re.compile( - r'^diff --git (?P"?a/[^\t\n]+"?) (?P"?b/[^\t\n]+"?)') + r'^diff --git (?P"?a/[^\t\n]+"?) (?P"?b/[^\t\n]+"?)' +) RE_DIFF_GIT_HEADER_URI_LIKE = re.compile( - r'^diff --git (?P.*://[^\t\n]+) (?P.*://[^\t\n]+)') + r"^diff --git (?P.*://[^\t\n]+) (?P.*://[^\t\n]+)" +) RE_DIFF_GIT_HEADER_NO_PREFIX = re.compile( - r'^diff --git (?P[^\t\n]+) (?P[^\t\n]+)') + r"^diff --git (?P[^\t\n]+) (?P[^\t\n]+)" +) # check diff git new file marker `deleted file mode 100644` -RE_DIFF_GIT_DELETED_FILE = re.compile(r'^deleted file mode \d+$') +RE_DIFF_GIT_DELETED_FILE = re.compile(r"^deleted file mode \d+$") # check diff git new file marker `new file mode 100644` -RE_DIFF_GIT_NEW_FILE = re.compile(r'^new file mode \d+$') +RE_DIFF_GIT_NEW_FILE = re.compile(r"^new file mode \d+$") # @@ (source offset, length) (target offset, length) @@ (section header) -RE_HUNK_HEADER = re.compile( - r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))?\ @@[ ]?(.*)") +RE_HUNK_HEADER = re.compile(r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))?\ @@[ ]?(.*)") # kept line (context) # \n empty line (treat like context) # + added line # - deleted line # \ No newline case -RE_HUNK_BODY_LINE = re.compile( - r'^(?P[- \+\\])(?P.*)', re.DOTALL) +RE_HUNK_BODY_LINE = re.compile(r"^(?P[- \+\\])(?P.*)", re.DOTALL) RE_HUNK_EMPTY_BODY_LINE = re.compile( - r'^(?P[- \+\\]?)(?P[\r\n]{1,2})', re.DOTALL) + r"^(?P[- \+\\]?)(?P[\r\n]{1,2})", re.DOTALL +) -RE_NO_NEWLINE_MARKER = re.compile(r'^\\ No newline at end of file') +RE_NO_NEWLINE_MARKER = re.compile(r"^\\ No newline at end of file") RE_BINARY_DIFF = re.compile( - r'^Binary files? ' - r'(?P[^\t]+?)(?:\t(?P[\s0-9:\+-]+))?' - r'(?: and (?P[^\t]+?)(?:\t(?P[\s0-9:\+-]+))?)? (differ|has changed)') + r"^Binary files? " + r"(?P[^\t]+?)(?:\t(?P[\s0-9:\+-]+))?" + r"(?: and (?P[^\t]+?)(?:\t(?P[\s0-9:\+-]+))?)? (differ|has changed)" +) + +DEFAULT_ENCODING = "UTF-8" -DEFAULT_ENCODING = 'UTF-8' +DEV_NULL = "/dev/null" +LINE_TYPE_ADDED = "+" +LINE_TYPE_REMOVED = "-" +LINE_TYPE_CONTEXT = " " +LINE_TYPE_EMPTY = "" +LINE_TYPE_NO_NEWLINE = "\\" +LINE_VALUE_NO_NEWLINE = " No newline at end of file" -DEV_NULL = '/dev/null' -LINE_TYPE_ADDED = '+' -LINE_TYPE_REMOVED = '-' -LINE_TYPE_CONTEXT = ' ' -LINE_TYPE_EMPTY = '' -LINE_TYPE_NO_NEWLINE = '\\' -LINE_VALUE_NO_NEWLINE = ' No newline at end of file' @implements_to_string -class Line(object): +class Line: """A diff line.""" - def __init__(self, value, line_type, - source_line_no=None, target_line_no=None, diff_line_no=None): + def __init__( + self, + value, + line_type, + source_line_no=None, + target_line_no=None, + diff_line_no=None, + ): # type: (str, str, Optional[int], Optional[int], Optional[int]) -> None - super(Line, self).__init__() + super().__init__() self.source_line_no = source_line_no self.target_line_no = target_line_no self.diff_line_no = diff_line_no @@ -113,11 +126,13 @@ def __str__(self): def __eq__(self, other): # type: (Line) -> bool - return (self.source_line_no == other.source_line_no and - self.target_line_no == other.target_line_no and - self.diff_line_no == other.diff_line_no and - self.line_type == other.line_type and - self.value == other.value) + return ( + self.source_line_no == other.source_line_no + and self.target_line_no == other.target_line_no + and self.diff_line_no == other.diff_line_no + and self.line_type == other.line_type + and self.value == other.value + ) @property def is_added(self): @@ -137,7 +152,8 @@ def is_context(self): @implements_to_string class PatchInfo(list): - """Lines with extended patch info. + """ + Lines with extended patch info. Format of this info is not documented and it very much depends on patch producer. @@ -151,17 +167,18 @@ def __repr__(self): def __str__(self): # type: () -> str - return ''.join(unicode(line) for line in self) + return "".join(unicode(line) for line in self) @implements_to_string class Hunk(list): """Each of the modified blocks of a file.""" - def __init__(self, src_start=0, src_len=0, tgt_start=0, tgt_len=0, - section_header=''): + def __init__( + self, src_start=0, src_len=0, tgt_start=0, tgt_len=0, section_header="" + ): # type: (int, int, int, int, str) -> None - super(Hunk, self).__init__() + super().__init__() if src_len is None: src_len = 1 if tgt_len is None: @@ -176,21 +193,26 @@ def __init__(self, src_start=0, src_len=0, tgt_start=0, tgt_len=0, def __repr__(self): # type: () -> str - value = "" % (self.source_start, - self.source_length, - self.target_start, - self.target_length, - self.section_header) + value = "" % ( + self.source_start, + self.source_length, + self.target_start, + self.target_length, + self.section_header, + ) return make_str(value) def __str__(self): # type: () -> str # section header is optional and thus we output it only if it's present head = "@@ -%d,%d +%d,%d @@%s\n" % ( - self.source_start, self.source_length, - self.target_start, self.target_length, - ' ' + self.section_header if self.section_header else '') - content = ''.join(unicode(line) for line in self) + self.source_start, + self.source_length, + self.target_start, + self.target_length, + " " + self.section_header if self.section_header else "", + ) + content = "".join(unicode(line) for line in self) return head + content def append(self, line): @@ -199,7 +221,7 @@ def append(self, line): # Make sure the line is encoded correctly. This is a no-op except for # potentially raising a UnicodeDecodeError. str(line) - super(Hunk, self).append(line) + super().append(line) @property def added(self): @@ -222,8 +244,10 @@ def removed(self): def is_valid(self): # type: () -> bool """Check hunk header data matches entered lines info.""" - return (len(self.source) == self.source_length and - len(self.target) == self.target_length) + return ( + len(self.source) == self.source_length + and len(self.target) == self.target_length + ) def source_lines(self): # type: () -> Iterable[Line] @@ -249,11 +273,17 @@ def target(self): class PatchedFile(list): """Patch updated file, it is a list of Hunks.""" - def __init__(self, patch_info=None, source='', target='', - source_timestamp=None, target_timestamp=None, - is_binary_file=False): + def __init__( + self, + patch_info=None, + source="", + target="", + source_timestamp=None, + target_timestamp=None, + is_binary_file=False, + ): # type: (Optional[PatchInfo], str, str, Optional[str], Optional[str], bool, bool) -> None - super(PatchedFile, self).__init__() + super().__init__() self.patch_info = patch_info self.source_file = source self.source_timestamp = source_timestamp @@ -267,18 +297,20 @@ def __repr__(self): def __str__(self): # type: () -> str - source = '' - target = '' + source = "" + target = "" # patch info is optional - info = '' if self.patch_info is None else str(self.patch_info) + info = "" if self.patch_info is None else str(self.patch_info) if not self.is_binary_file and self: source = "--- %s%s\n" % ( self.source_file, - '\t' + self.source_timestamp if self.source_timestamp else '') + "\t" + self.source_timestamp if self.source_timestamp else "", + ) target = "+++ %s%s\n" % ( self.target_file, - '\t' + self.target_timestamp if self.target_timestamp else '') - hunks = ''.join(unicode(hunk) for hunk in self) + "\t" + self.target_timestamp if self.target_timestamp else "", + ) + hunks = "".join(unicode(hunk) for hunk in self) return info + source + target + hunks def _parse_hunk(self, header, diff, encoding, metadata_only): @@ -302,12 +334,13 @@ def _parse_hunk(self, header, diff, encoding, metadata_only): if metadata_only: # quick line type detection, no regex required line_type = line[0] if line else LINE_TYPE_CONTEXT - if line_type not in (LINE_TYPE_ADDED, - LINE_TYPE_REMOVED, - LINE_TYPE_CONTEXT, - LINE_TYPE_NO_NEWLINE): - raise UnidiffParseError( - 'Hunk diff line expected: %s' % line) + if line_type not in ( + LINE_TYPE_ADDED, + LINE_TYPE_REMOVED, + LINE_TYPE_CONTEXT, + LINE_TYPE_NO_NEWLINE, + ): + raise UnidiffParseError("Hunk diff line expected: %s" % line) if line_type == LINE_TYPE_ADDED: target_line_no += 1 @@ -329,14 +362,13 @@ def _parse_hunk(self, header, diff, encoding, metadata_only): valid_line = RE_HUNK_EMPTY_BODY_LINE.match(line) if not valid_line: - raise UnidiffParseError( - 'Hunk diff line expected: %s' % line) + raise UnidiffParseError("Hunk diff line expected: %s" % line) - line_type = valid_line.group('line_type') + line_type = valid_line.group("line_type") if line_type == LINE_TYPE_EMPTY: line_type = LINE_TYPE_CONTEXT - value = valid_line.group('value') # type: str + value = valid_line.group("value") # type: str original_line = Line(value, line_type=line_type) if line_type == LINE_TYPE_ADDED: @@ -356,23 +388,26 @@ def _parse_hunk(self, header, diff, encoding, metadata_only): original_line = None # stop parsing if we got past expected number of lines - if (source_line_no > expected_source_end or - target_line_no > expected_target_end): - raise UnidiffParseError('Hunk is longer than expected') + if ( + source_line_no > expected_source_end + or target_line_no > expected_target_end + ): + raise UnidiffParseError("Hunk is longer than expected") if original_line: original_line.diff_line_no = diff_line_no hunk.append(original_line) # if hunk source/target lengths are ok, hunk is complete - if (source_line_no == expected_source_end and - target_line_no == expected_target_end): + if ( + source_line_no == expected_source_end + and target_line_no == expected_target_end + ): break # report an error if we haven't got expected number of lines - if (source_line_no < expected_source_end or - target_line_no < expected_target_end): - raise UnidiffParseError('Hunk is shorter than expected') + if source_line_no < expected_source_end or target_line_no < expected_target_end: + raise UnidiffParseError("Hunk is shorter than expected") if metadata_only: # HACK: set fixed calculated values when metadata_only is enabled @@ -384,18 +419,18 @@ def _parse_hunk(self, header, diff, encoding, metadata_only): def _add_no_newline_marker_to_last_hunk(self): # type: () -> None if not self: - raise UnidiffParseError( - 'Unexpected marker:' + LINE_VALUE_NO_NEWLINE) + raise UnidiffParseError("Unexpected marker:" + LINE_VALUE_NO_NEWLINE) last_hunk = self[-1] last_hunk.append( - Line(LINE_VALUE_NO_NEWLINE + '\n', line_type=LINE_TYPE_NO_NEWLINE)) + Line(LINE_VALUE_NO_NEWLINE + "\n", line_type=LINE_TYPE_NO_NEWLINE) + ) def _append_trailing_empty_line(self): # type: () -> None if not self: - raise UnidiffParseError('Unexpected trailing newline character') + raise UnidiffParseError("Unexpected trailing newline character") last_hunk = self[-1] - last_hunk.append(Line('\n', line_type=LINE_TYPE_EMPTY)) + last_hunk.append(Line("\n", line_type=LINE_TYPE_EMPTY)) @property def path(self): @@ -403,7 +438,8 @@ def path(self): """Return the file path abstracted from VCS.""" filepath = self.source_file if filepath in (None, DEV_NULL) or ( - self.is_rename and self.target_file not in (None, DEV_NULL)): + self.is_rename and self.target_file not in (None, DEV_NULL) + ): # if this is a rename, prefer the target filename filepath = self.target_file @@ -411,11 +447,11 @@ def path(self): if quoted: filepath = filepath[1:-1] - if filepath.startswith('a/') or filepath.startswith('b/'): + if filepath.startswith("a/") or filepath.startswith("b/"): filepath = filepath[2:] if quoted: - filepath = '"{}"'.format(filepath) + filepath = f'"{filepath}"' return filepath @@ -433,9 +469,11 @@ def removed(self): @property def is_rename(self): - return (self.source_file != DEV_NULL + return ( + self.source_file != DEV_NULL and self.target_file != DEV_NULL - and self.source_file[2:] != self.target_file[2:]) + and self.source_file[2:] != self.target_file[2:] + ) @property def is_added_file(self): @@ -443,8 +481,9 @@ def is_added_file(self): """Return True if this patch adds the file.""" if self.source_file == DEV_NULL: return True - return (len(self) == 1 and self[0].source_start == 0 and - self[0].source_length == 0) + return ( + len(self) == 1 and self[0].source_start == 0 and self[0].source_length == 0 + ) @property def is_removed_file(self): @@ -452,8 +491,9 @@ def is_removed_file(self): """Return True if this patch removes the file.""" if self.target_file == DEV_NULL: return True - return (len(self) == 1 and self[0].target_start == 0 and - self[0].target_length == 0) + return ( + len(self) == 1 and self[0].target_start == 0 and self[0].target_length == 0 + ) @property def is_modified_file(self): @@ -468,7 +508,7 @@ class PatchSet(list): def __init__(self, f, encoding=None, metadata_only=False): # type: (Union[StringIO, str], Optional[str], bool) -> None - super(PatchSet, self).__init__() + super().__init__() # convert string inputs to StringIO objects if isinstance(f, basestring): @@ -484,11 +524,11 @@ def __init__(self, f, encoding=None, metadata_only=False): def __repr__(self): # type: () -> str - return make_str('') % super(PatchSet, self).__repr__() + return make_str("") % super().__repr__() def __str__(self): # type: () -> str - return ''.join(unicode(patched_file) for patched_file in self) + return "".join(unicode(patched_file) for patched_file in self) def _parse(self, diff, encoding, metadata_only): # type: (StringIO, Optional[str], bool) -> None @@ -501,15 +541,18 @@ def _parse(self, diff, encoding, metadata_only): line = line.decode(encoding) # check for a git file rename - is_diff_git_header = RE_DIFF_GIT_HEADER.match(line) or \ - RE_DIFF_GIT_HEADER_URI_LIKE.match(line) or \ - RE_DIFF_GIT_HEADER_NO_PREFIX.match(line) + is_diff_git_header = ( + RE_DIFF_GIT_HEADER.match(line) + or RE_DIFF_GIT_HEADER_URI_LIKE.match(line) + or RE_DIFF_GIT_HEADER_NO_PREFIX.match(line) + ) if is_diff_git_header: patch_info = PatchInfo() - source_file = is_diff_git_header.group('source') - target_file = is_diff_git_header.group('target') + source_file = is_diff_git_header.group("source") + target_file = is_diff_git_header.group("target") current_file = PatchedFile( - patch_info, source_file, target_file, None, None) + patch_info, source_file, target_file, None, None + ) self.append(current_file) patch_info.append(line) continue @@ -518,7 +561,7 @@ def _parse(self, diff, encoding, metadata_only): is_diff_git_new_file = RE_DIFF_GIT_NEW_FILE.match(line) if is_diff_git_new_file: if current_file is None or patch_info is None: - raise UnidiffParseError('Unexpected new file found: %s' % line) + raise UnidiffParseError("Unexpected new file found: %s" % line) current_file.source_file = DEV_NULL patch_info.append(line) continue @@ -527,7 +570,7 @@ def _parse(self, diff, encoding, metadata_only): is_diff_git_deleted_file = RE_DIFF_GIT_DELETED_FILE.match(line) if is_diff_git_deleted_file: if current_file is None or patch_info is None: - raise UnidiffParseError('Unexpected deleted file found: %s' % line) + raise UnidiffParseError("Unexpected deleted file found: %s" % line) current_file.target_file = DEV_NULL patch_info.append(line) continue @@ -535,12 +578,13 @@ def _parse(self, diff, encoding, metadata_only): # check for source file header is_source_filename = RE_SOURCE_FILENAME.match(line) if is_source_filename: - source_file = is_source_filename.group('filename') - source_timestamp = is_source_filename.group('timestamp') + source_file = is_source_filename.group("filename") + source_timestamp = is_source_filename.group("timestamp") # reset current file, unless we are processing a rename # (in that case, source files should match) if current_file is not None and not ( - current_file.source_file == source_file): + current_file.source_file == source_file + ): current_file = None elif current_file is not None: current_file.source_timestamp = source_timestamp @@ -549,15 +593,21 @@ def _parse(self, diff, encoding, metadata_only): # check for target file header is_target_filename = RE_TARGET_FILENAME.match(line) if is_target_filename: - target_file = is_target_filename.group('filename') - target_timestamp = is_target_filename.group('timestamp') - if current_file is not None and not (current_file.target_file == target_file): - raise UnidiffParseError('Target without source: %s' % line) + target_file = is_target_filename.group("filename") + target_timestamp = is_target_filename.group("timestamp") + if current_file is not None and not ( + current_file.target_file == target_file + ): + raise UnidiffParseError("Target without source: %s" % line) if current_file is None: # add current file to PatchSet current_file = PatchedFile( - patch_info, source_file, target_file, - source_timestamp, target_timestamp) + patch_info, + source_file, + target_file, + source_timestamp, + target_timestamp, + ) self.append(current_file) patch_info = None else: @@ -569,7 +619,7 @@ def _parse(self, diff, encoding, metadata_only): if is_hunk_header: patch_info = None if current_file is None: - raise UnidiffParseError('Unexpected hunk found: %s' % line) + raise UnidiffParseError("Unexpected hunk found: %s" % line) current_file._parse_hunk(line, diff, encoding, metadata_only) continue @@ -577,12 +627,12 @@ def _parse(self, diff, encoding, metadata_only): is_no_newline = RE_NO_NEWLINE_MARKER.match(line) if is_no_newline: if current_file is None: - raise UnidiffParseError('Unexpected marker: %s' % line) + raise UnidiffParseError("Unexpected marker: %s" % line) current_file._add_no_newline_marker_to_last_hunk() continue # sometimes hunks can be followed by empty lines - if line == '\n' and current_file is not None: + if line == "\n" and current_file is not None: current_file._append_trailing_empty_line() continue @@ -593,20 +643,21 @@ def _parse(self, diff, encoding, metadata_only): is_binary_diff = RE_BINARY_DIFF.match(line) if is_binary_diff: - source_file = is_binary_diff.group('source_filename') - target_file = is_binary_diff.group('target_filename') + source_file = is_binary_diff.group("source_filename") + target_file = is_binary_diff.group("target_filename") patch_info.append(line) if current_file is not None: current_file.is_binary_file = True else: current_file = PatchedFile( - patch_info, source_file, target_file, is_binary_file=True) + patch_info, source_file, target_file, is_binary_file=True + ) self.append(current_file) patch_info = None current_file = None continue - if line == 'GIT binary patch\n': + if line == "GIT binary patch\n": current_file.is_binary_file = True patch_info = None current_file = None @@ -615,15 +666,19 @@ def _parse(self, diff, encoding, metadata_only): patch_info.append(line) @classmethod - def from_filename(cls, filename, encoding=DEFAULT_ENCODING, errors=None, newline=None): + def from_filename( + cls, filename, encoding=DEFAULT_ENCODING, errors=None, newline=None + ): # type: (str, str, Optional[str]) -> PatchSet """Return a PatchSet instance given a diff filename.""" - with open_file(filename, 'r', encoding=encoding, errors=errors, newline=newline) as f: + with open_file( + filename, "r", encoding=encoding, errors=errors, newline=newline + ) as f: instance = cls(f) return instance @staticmethod - def _convert_string(data, encoding=None, errors='strict'): + def _convert_string(data, encoding=None, errors="strict"): # type: (Union[str, bytes], str, str) -> StringIO if encoding is not None: # if encoding is given, assume bytes and decode @@ -631,7 +686,7 @@ def _convert_string(data, encoding=None, errors='strict'): return StringIO(data) @classmethod - def from_string(cls, data, encoding=None, errors='strict'): + def from_string(cls, data, encoding=None, errors="strict"): # type: (str, str, Optional[str]) -> PatchSet """Return a PatchSet instance given a diff string.""" return cls(cls._convert_string(data, encoding, errors)) @@ -664,4 +719,4 @@ def added(self): def removed(self): # type: () -> int """Return the patch total removed lines.""" - return sum([f.removed for f in self]) \ No newline at end of file + return sum([f.removed for f in self]) diff --git a/scanpipe/pipes/vulnerablecode.py b/scanpipe/pipes/vulnerablecode.py index 4deaefc704..57924bff70 100644 --- a/scanpipe/pipes/vulnerablecode.py +++ b/scanpipe/pipes/vulnerablecode.py @@ -109,6 +109,7 @@ def request_post( def bulk_search_by_purl( purls, + reachability=False, timeout=None, api_url=VULNERABLECODE_API_URL, ): @@ -118,7 +119,7 @@ def bulk_search_by_purl( data = { "purls": purls, "details": True, - "reachability": True, + "reachability": reachability, } logger.debug(f"VulnerableCode: url={url} purls_count={len(purls)}") @@ -137,7 +138,7 @@ def filter_vulnerabilities(vulnerabilities, ignore_set): def fetch_vulnerabilities( - packages, chunk_size=1000, logger=logger.info, ignore_set=None + packages, chunk_size=1000, logger=logger.info, ignore_set=None, reachability=False ): """ Fetch and store vulnerabilities for each provided `packages`. @@ -147,7 +148,7 @@ def fetch_vulnerabilities( for purls_batch in chunked(get_purls(packages), chunk_size): try: - response_data = bulk_search_by_purl(purls_batch) + response_data = bulk_search_by_purl(purls_batch, reachability=reachability) except (requests.RequestException, ValueError, TypeError) as exception: logger(f"{label} [Exception] {exception}") return diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index 620e886c05..ef054dd5a1 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -32,7 +32,7 @@ from scanpipe.pipes.reachability import PatchAnalyzer from scanpipe.pipes.reachability import ReachabilityStatus from scanpipe.pipes.reachability import classify_reachability -from scanpipe.pipes.reachability import collect_and_store_symbol_reachability_results +from scanpipe.pipes.reachability import analyze_and_store_symbol_reachability_results from scanpipe.pipes.symbols import TS_QUERIES from scanpipe.pipes.symbols import SymbolExtractor @@ -104,7 +104,7 @@ def test_get_symbol_reachability_results( resource.programming_language = lang resource.save() - collect_and_store_symbol_reachability_results(self.project1) + analyze_and_store_symbol_reachability_results(self.project1) resource.refresh_from_db() results = resource.extra_data.get("symbols_reachability") From 3bfa88aa371cff71242e1dfe66ef350237972bc8 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Thu, 23 Jul 2026 03:53:25 +0300 Subject: [PATCH 20/25] Fix CI files format Signed-off-by: ziad hany --- scanpipe/pipes/unidiff/patch.py | 75 +++++++++---------- .../tests/pipes/test_symbols_reachability.py | 2 +- 2 files changed, 38 insertions(+), 39 deletions(-) diff --git a/scanpipe/pipes/unidiff/patch.py b/scanpipe/pipes/unidiff/patch.py index 8075a8cc77..e53c7c7d94 100644 --- a/scanpipe/pipes/unidiff/patch.py +++ b/scanpipe/pipes/unidiff/patch.py @@ -34,7 +34,12 @@ class UnidiffParseError(Exception): ... open_file = open make_str = str -implements_to_string = lambda x: x + + +def implements_to_string(x): + return x + + unicode = str basestring = str @@ -82,7 +87,8 @@ class UnidiffParseError(Exception): ... RE_BINARY_DIFF = re.compile( r"^Binary files? " r"(?P[^\t]+?)(?:\t(?P[\s0-9:\+-]+))?" - r"(?: and (?P[^\t]+?)(?:\t(?P[\s0-9:\+-]+))?)? (differ|has changed)" + r"(?: and (?P[^\t]+?)(?:\t(?P[\s0-9:\+-]+))?)?" + r" (differ|has changed)" ) DEFAULT_ENCODING = "UTF-8" @@ -122,7 +128,7 @@ def __repr__(self): def __str__(self): # type: () -> str - return "%s%s" % (self.line_type, self.value) + return f"{self.line_type}{self.value}" def __eq__(self, other): # type: (Line) -> bool @@ -162,7 +168,7 @@ class PatchInfo(list): def __repr__(self): # type: () -> str - value = "" % self[0].strip() + value = f"" return make_str(value) def __str__(self): @@ -193,24 +199,19 @@ def __init__( def __repr__(self): # type: () -> str - value = "" % ( - self.source_start, - self.source_length, - self.target_start, - self.target_length, - self.section_header, + value = ( + f"" ) return make_str(value) def __str__(self): # type: () -> str # section header is optional and thus we output it only if it's present - head = "@@ -%d,%d +%d,%d @@%s\n" % ( - self.source_start, - self.source_length, - self.target_start, - self.target_length, - " " + self.section_header if self.section_header else "", + section_hdr = f" {self.section_header}" if self.section_header else "" + head = ( + f"@@ -{self.source_start},{self.source_length} " + f"+{self.target_start},{self.target_length} @@{section_hdr}\n" ) content = "".join(unicode(line) for line in self) return head + content @@ -252,22 +253,22 @@ def is_valid(self): def source_lines(self): # type: () -> Iterable[Line] """Hunk lines from source file (generator).""" - return (l for l in self if l.is_context or l.is_removed) + return (line for line in self if line.is_context or line.is_removed) @property def source(self): # type: () -> Iterable[str] - return [str(l) for l in self.source_lines()] + return [str(line) for line in self.source_lines()] def target_lines(self): # type: () -> Iterable[Line] """Hunk lines from target file (generator).""" - return (l for l in self if l.is_context or l.is_added) + return (line for line in self if line.is_context or line.is_added) @property def target(self): # type: () -> Iterable[str] - return [str(l) for l in self.target_lines()] + return [str(line) for line in self.target_lines()] class PatchedFile(list): @@ -302,18 +303,16 @@ def __str__(self): # patch info is optional info = "" if self.patch_info is None else str(self.patch_info) if not self.is_binary_file and self: - source = "--- %s%s\n" % ( - self.source_file, - "\t" + self.source_timestamp if self.source_timestamp else "", - ) - target = "+++ %s%s\n" % ( - self.target_file, - "\t" + self.target_timestamp if self.target_timestamp else "", - ) + source_ts = f"\t{self.source_timestamp}" if self.source_timestamp else "" + source = f"--- {self.source_file}{source_ts}\n" + + target_ts = f"\t{self.target_timestamp}" if self.target_timestamp else "" + target = f"+++ {self.target_file}{target_ts}\n" + hunks = "".join(unicode(hunk) for hunk in self) return info + source + target + hunks - def _parse_hunk(self, header, diff, encoding, metadata_only): + def _parse_hunk(self, header, diff, encoding, metadata_only): # noqa: C901 # type: (str, enumerate[str], Optional[str], bool) -> None """Parse hunk details.""" header_info = RE_HUNK_HEADER.match(header) @@ -340,7 +339,7 @@ def _parse_hunk(self, header, diff, encoding, metadata_only): LINE_TYPE_CONTEXT, LINE_TYPE_NO_NEWLINE, ): - raise UnidiffParseError("Hunk diff line expected: %s" % line) + raise UnidiffParseError(f"Hunk diff line expected: {line}") if line_type == LINE_TYPE_ADDED: target_line_no += 1 @@ -362,7 +361,7 @@ def _parse_hunk(self, header, diff, encoding, metadata_only): valid_line = RE_HUNK_EMPTY_BODY_LINE.match(line) if not valid_line: - raise UnidiffParseError("Hunk diff line expected: %s" % line) + raise UnidiffParseError(f"Hunk diff line expected: {line}") line_type = valid_line.group("line_type") if line_type == LINE_TYPE_EMPTY: @@ -410,7 +409,7 @@ def _parse_hunk(self, header, diff, encoding, metadata_only): raise UnidiffParseError("Hunk is shorter than expected") if metadata_only: - # HACK: set fixed calculated values when metadata_only is enabled + # set fixed calculated values when metadata_only is enabled hunk._added = added hunk._removed = removed @@ -530,7 +529,7 @@ def __str__(self): # type: () -> str return "".join(unicode(patched_file) for patched_file in self) - def _parse(self, diff, encoding, metadata_only): + def _parse(self, diff, encoding, metadata_only): # noqa: C901 # type: (StringIO, Optional[str], bool) -> None current_file = None patch_info = None @@ -561,7 +560,7 @@ def _parse(self, diff, encoding, metadata_only): is_diff_git_new_file = RE_DIFF_GIT_NEW_FILE.match(line) if is_diff_git_new_file: if current_file is None or patch_info is None: - raise UnidiffParseError("Unexpected new file found: %s" % line) + raise UnidiffParseError(f"Unexpected new file found: {line}") current_file.source_file = DEV_NULL patch_info.append(line) continue @@ -570,7 +569,7 @@ def _parse(self, diff, encoding, metadata_only): is_diff_git_deleted_file = RE_DIFF_GIT_DELETED_FILE.match(line) if is_diff_git_deleted_file: if current_file is None or patch_info is None: - raise UnidiffParseError("Unexpected deleted file found: %s" % line) + raise UnidiffParseError(f"Unexpected deleted file found: {line}") current_file.target_file = DEV_NULL patch_info.append(line) continue @@ -598,7 +597,7 @@ def _parse(self, diff, encoding, metadata_only): if current_file is not None and not ( current_file.target_file == target_file ): - raise UnidiffParseError("Target without source: %s" % line) + raise UnidiffParseError(f"Target without source: {line}") if current_file is None: # add current file to PatchSet current_file = PatchedFile( @@ -619,7 +618,7 @@ def _parse(self, diff, encoding, metadata_only): if is_hunk_header: patch_info = None if current_file is None: - raise UnidiffParseError("Unexpected hunk found: %s" % line) + raise UnidiffParseError(f"Unexpected hunk found: {line}") current_file._parse_hunk(line, diff, encoding, metadata_only) continue @@ -627,7 +626,7 @@ def _parse(self, diff, encoding, metadata_only): is_no_newline = RE_NO_NEWLINE_MARKER.match(line) if is_no_newline: if current_file is None: - raise UnidiffParseError("Unexpected marker: %s" % line) + raise UnidiffParseError(f"Unexpected marker: {line}") current_file._add_no_newline_marker_to_last_hunk() continue diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index ef054dd5a1..a86081b1f3 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -31,8 +31,8 @@ from scanpipe.pipes import collect_and_create_codebase_resources from scanpipe.pipes.reachability import PatchAnalyzer from scanpipe.pipes.reachability import ReachabilityStatus -from scanpipe.pipes.reachability import classify_reachability from scanpipe.pipes.reachability import analyze_and_store_symbol_reachability_results +from scanpipe.pipes.reachability import classify_reachability from scanpipe.pipes.symbols import TS_QUERIES from scanpipe.pipes.symbols import SymbolExtractor From ec799431d653d2f1540c6239a72a0ddce0990366 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Thu, 23 Jul 2026 04:02:18 +0300 Subject: [PATCH 21/25] Fix a typo in patch.py.ABOUT file Signed-off-by: ziad hany --- scanpipe/pipes/unidiff/patch.py.ABOUT | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanpipe/pipes/unidiff/patch.py.ABOUT b/scanpipe/pipes/unidiff/patch.py.ABOUT index 3118ed9643..75c3bb8ade 100644 --- a/scanpipe/pipes/unidiff/patch.py.ABOUT +++ b/scanpipe/pipes/unidiff/patch.py.ABOUT @@ -10,4 +10,4 @@ package_url: pkg:pypi/unidiff@0.7.5 licenses: - key: mit name: MIT License - file: mit.LICENSE \ No newline at end of file + file: patch.py.LICENSE \ No newline at end of file From 3744e2fc49afdab53c1d2ecade8cd6490d022ad1 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Thu, 23 Jul 2026 05:05:15 +0300 Subject: [PATCH 22/25] Remove the unidiff library from the dependencies Signed-off-by: ziad hany --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d5dfbae829..b309d0a955 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,7 +84,6 @@ dependencies = [ "urllib3==2.7.0", "idna==3.18", "GitPython==3.1.50", - "unidiff==0.7.5", "lxml==6.1.1", "certifi==2026.6.17", # Profiling From 128bfdfd7c013845fd4ab870071e3f9eb497b7a5 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Thu, 23 Jul 2026 05:27:26 +0300 Subject: [PATCH 23/25] Skip the test for macOS Signed-off-by: ziad hany --- scanpipe/tests/pipes/test_symbols_reachability.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index a86081b1f3..e24c42932b 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -19,8 +19,9 @@ # # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/scancode.io for support and download. - +import sys from pathlib import Path +from unittest import skipIf from unittest.mock import MagicMock from unittest.mock import PropertyMock from unittest.mock import patch @@ -37,6 +38,7 @@ from scanpipe.pipes.symbols import SymbolExtractor +@skipIf(sys.platform == "darwin", "Not supported on macOS") class SymbolReachabilityPipesTest(TestCase): data = Path(__file__).parent.parent / "data" / "reachability" From 7258c0a680733890dcb25ab540e4d38cb60e26d7 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 29 Jul 2026 03:17:06 +0300 Subject: [PATCH 24/25] Allow reachability by default, for vulnerabilities pipeline Signed-off-by: ziad hany --- scanpipe/pipelines/find_vulnerabilities.py | 12 +----------- scanpipe/pipes/vulnerablecode.py | 7 +++---- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/scanpipe/pipelines/find_vulnerabilities.py b/scanpipe/pipelines/find_vulnerabilities.py index 1bdf8af6bc..b0c8066b9a 100644 --- a/scanpipe/pipelines/find_vulnerabilities.py +++ b/scanpipe/pipelines/find_vulnerabilities.py @@ -19,7 +19,7 @@ # # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. -from aboutcode.pipeline import optional_step + from scanpipe.pipelines import Pipeline from scanpipe.pipes import vulnerablecode @@ -34,13 +34,11 @@ class FindVulnerabilities(Pipeline): download_inputs = False is_addon = True results_url = "/project/{slug}/packages/?is_vulnerable=yes" - reachability = False @classmethod def steps(cls): return ( cls.check_vulnerablecode_service_availability, - cls.enable_reachability_analysis, cls.lookup_packages_vulnerabilities, cls.lookup_dependencies_vulnerabilities, ) @@ -50,12 +48,6 @@ def get_availability(cls): if not vulnerablecode.is_configured(): return "VulnerableCode is not configured." - @optional_step("reachability") - def enable_reachability_analysis(self): - """Enable the reachability flag for vulnerability lookups.""" - self.reachability = True - self.log("Reachability analysis is ENABLED.") - def check_vulnerablecode_service_availability(self): """Check if the VulnerableCode service if configured and available.""" if not vulnerablecode.is_configured(): @@ -70,7 +62,6 @@ def lookup_packages_vulnerabilities(self): vulnerablecode.fetch_vulnerabilities( packages=packages, ignore_set=self.project.ignored_vulnerabilities_set, - reachability=self.reachability, logger=self.log, ) @@ -80,6 +71,5 @@ def lookup_dependencies_vulnerabilities(self): vulnerablecode.fetch_vulnerabilities( packages=dependencies, ignore_set=self.project.ignored_vulnerabilities_set, - reachability=self.reachability, logger=self.log, ) diff --git a/scanpipe/pipes/vulnerablecode.py b/scanpipe/pipes/vulnerablecode.py index 57924bff70..4deaefc704 100644 --- a/scanpipe/pipes/vulnerablecode.py +++ b/scanpipe/pipes/vulnerablecode.py @@ -109,7 +109,6 @@ def request_post( def bulk_search_by_purl( purls, - reachability=False, timeout=None, api_url=VULNERABLECODE_API_URL, ): @@ -119,7 +118,7 @@ def bulk_search_by_purl( data = { "purls": purls, "details": True, - "reachability": reachability, + "reachability": True, } logger.debug(f"VulnerableCode: url={url} purls_count={len(purls)}") @@ -138,7 +137,7 @@ def filter_vulnerabilities(vulnerabilities, ignore_set): def fetch_vulnerabilities( - packages, chunk_size=1000, logger=logger.info, ignore_set=None, reachability=False + packages, chunk_size=1000, logger=logger.info, ignore_set=None ): """ Fetch and store vulnerabilities for each provided `packages`. @@ -148,7 +147,7 @@ def fetch_vulnerabilities( for purls_batch in chunked(get_purls(packages), chunk_size): try: - response_data = bulk_search_by_purl(purls_batch, reachability=reachability) + response_data = bulk_search_by_purl(purls_batch) except (requests.RequestException, ValueError, TypeError) as exception: logger(f"{label} [Exception] {exception}") return From 27de9f1c915d2972fbcd271402620ab7b623f29b Mon Sep 17 00:00:00 2001 From: ziad hany Date: Thu, 30 Jul 2026 02:23:55 +0300 Subject: [PATCH 25/25] Update the code to difflib instead of unidiff library Fix a bug related to constant detections and add a test Signed-off-by: ziad hany --- scanpipe/pipes/reachability.py | 73 +- scanpipe/pipes/unidiff/patch.py | 721 ------------------ scanpipe/pipes/unidiff/patch.py.ABOUT | 13 - scanpipe/pipes/unidiff/patch.py.LICENSE | 20 - scanpipe/tests/data/reachability/app.py | 2 + .../tests/data/reachability/diff-app.patch | 39 - scanpipe/tests/data/reachability/fixed-app.py | 2 + scanpipe/tests/data/reachability/vuln-app.py | 2 + .../tests/pipes/test_symbols_reachability.py | 194 +++-- 9 files changed, 146 insertions(+), 920 deletions(-) delete mode 100644 scanpipe/pipes/unidiff/patch.py delete mode 100644 scanpipe/pipes/unidiff/patch.py.ABOUT delete mode 100644 scanpipe/pipes/unidiff/patch.py.LICENSE delete mode 100644 scanpipe/tests/data/reachability/diff-app.patch diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index 46cda929ee..6c7768dbad 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -20,7 +20,7 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. # - +import difflib import os import shutil import tempfile @@ -36,7 +36,6 @@ from scanpipe.pipes.symbols import SymbolExtractor from scanpipe.pipes.symbols import create_sha256_fingerprint from scanpipe.pipes.symbols import is_supported_language -from scanpipe.pipes.unidiff.patch import PatchSet class ReachabilityStatus(str, Enum): @@ -149,6 +148,26 @@ def get_commit_diff_text(self): base = self.parent_commit.hexsha if self.parent_commit else NULL_TREE return self.repo.git.diff(base, self.commit.hexsha, unified=3) + @classmethod + def compute_changed_lines(cls, vulnerable_text, fixed_text): + """Return the removed and added line numbers between two file contents.""" + matcher = difflib.SequenceMatcher( + a=vulnerable_text.splitlines(), + b=fixed_text.splitlines(), + autojunk=False, # otherwise difflib ignores lines that repeat often + ) + + removed_lines = [] + added_lines = [] + for tag, vuln_start, vuln_end, fixed_start, fixed_end in matcher.get_opcodes(): + if tag == "equal": + continue + # opcodes are 0-based and end-exclusive, line numbers are 1-based + removed_lines.extend(range(vuln_start + 1, vuln_end + 1)) + added_lines.extend(range(fixed_start + 1, fixed_end + 1)) + + return removed_lines, added_lines + @classmethod def diff_changed_symbols(cls, vuln_meta, fixed_meta): vuln_only = { @@ -163,50 +182,16 @@ def diff_changed_symbols(cls, vuln_meta, fixed_meta): } return vuln_only, fixed_only - @classmethod - def parse_diff_lines(cls, diff_text): - diff_map = {} - if not diff_text: - return diff_map - - for patched_file in PatchSet.from_string(diff_text): - removed = [] - added = [] - - for hunk in patched_file: - removed.extend( - line.source_line_no - for line in hunk - if line.is_removed and line.source_line_no - ) - added.extend( - line.target_line_no - for line in hunk - if line.is_added and line.target_line_no - ) - - candidates = { - patched_file.path, - (patched_file.source_file or "").removeprefix("a/"), - (patched_file.target_file or "").removeprefix("b/"), - } - - for candidate in candidates: - if candidate: - diff_map[candidate] = (removed, added) - - return diff_map - def collect_patch_symbols(self): - diff_text = self.get_commit_diff_text() + by_language = {} changed_files = self.get_changed_files() - diff_line_map = self.parse_diff_lines(diff_text=diff_text) - by_language = {} for file_path, texts in changed_files.items(): vulnerable_text = texts["vulnerable_text"] fixed_text = texts["fixed_text"] - removed_lines, added_lines = diff_line_map.get(file_path, ([], [])) + removed_lines, added_lines = self.compute_changed_lines( + vulnerable_text, fixed_text + ) vuln_meta, fixed_meta, language = self.analyze( vulnerable_text=vulnerable_text, @@ -504,11 +489,9 @@ def build_index(self) -> dict | None: ) for node, _ in lang_query.get_constants(tree.root_node): - # Skip constants defined inside functions, classes, or other blocks - if node.parent == tree.root_node: - self.process_node( - node, extractor, definitions_index, definitions, fingerprints - ) + self.process_node( + node, extractor, definitions_index, definitions, fingerprints + ) return { "definitions": definitions, diff --git a/scanpipe/pipes/unidiff/patch.py b/scanpipe/pipes/unidiff/patch.py deleted file mode 100644 index e53c7c7d94..0000000000 --- a/scanpipe/pipes/unidiff/patch.py +++ /dev/null @@ -1,721 +0,0 @@ -# Extracted essential patch code analyzer and modified -# from the original unidiff library: -# https://github.com/matiasb/python-unidiff/blob/2771a878f7bc6619e625feb4dbad3427f57f5237/unidiff/patch.py - -# -*- coding: utf-8 -*- - -# The MIT License (MIT) -# Copyright (c) 2014-2023 Matias Bordese -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE -# OR OTHER DEALINGS IN THE SOFTWARE. - -import re -from io import StringIO - - -class UnidiffParseError(Exception): ... - - -open_file = open -make_str = str - - -def implements_to_string(x): - return x - - -unicode = str -basestring = str - -RE_SOURCE_FILENAME = re.compile( - r'^--- (?P"?[^\t\n]+"?)(?:\t(?P[^\n]+))?' -) -RE_TARGET_FILENAME = re.compile( - r'^\+\+\+ (?P"?[^\t\n]+"?)(?:\t(?P[^\n]+))?' -) - - -# check diff git line for git renamed files support -RE_DIFF_GIT_HEADER = re.compile( - r'^diff --git (?P"?a/[^\t\n]+"?) (?P"?b/[^\t\n]+"?)' -) -RE_DIFF_GIT_HEADER_URI_LIKE = re.compile( - r"^diff --git (?P.*://[^\t\n]+) (?P.*://[^\t\n]+)" -) -RE_DIFF_GIT_HEADER_NO_PREFIX = re.compile( - r"^diff --git (?P[^\t\n]+) (?P[^\t\n]+)" -) - -# check diff git new file marker `deleted file mode 100644` -RE_DIFF_GIT_DELETED_FILE = re.compile(r"^deleted file mode \d+$") - -# check diff git new file marker `new file mode 100644` -RE_DIFF_GIT_NEW_FILE = re.compile(r"^new file mode \d+$") - - -# @@ (source offset, length) (target offset, length) @@ (section header) -RE_HUNK_HEADER = re.compile(r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))?\ @@[ ]?(.*)") - -# kept line (context) -# \n empty line (treat like context) -# + added line -# - deleted line -# \ No newline case -RE_HUNK_BODY_LINE = re.compile(r"^(?P[- \+\\])(?P.*)", re.DOTALL) -RE_HUNK_EMPTY_BODY_LINE = re.compile( - r"^(?P[- \+\\]?)(?P[\r\n]{1,2})", re.DOTALL -) - -RE_NO_NEWLINE_MARKER = re.compile(r"^\\ No newline at end of file") - -RE_BINARY_DIFF = re.compile( - r"^Binary files? " - r"(?P[^\t]+?)(?:\t(?P[\s0-9:\+-]+))?" - r"(?: and (?P[^\t]+?)(?:\t(?P[\s0-9:\+-]+))?)?" - r" (differ|has changed)" -) - -DEFAULT_ENCODING = "UTF-8" - -DEV_NULL = "/dev/null" -LINE_TYPE_ADDED = "+" -LINE_TYPE_REMOVED = "-" -LINE_TYPE_CONTEXT = " " -LINE_TYPE_EMPTY = "" -LINE_TYPE_NO_NEWLINE = "\\" -LINE_VALUE_NO_NEWLINE = " No newline at end of file" - - -@implements_to_string -class Line: - """A diff line.""" - - def __init__( - self, - value, - line_type, - source_line_no=None, - target_line_no=None, - diff_line_no=None, - ): - # type: (str, str, Optional[int], Optional[int], Optional[int]) -> None - super().__init__() - self.source_line_no = source_line_no - self.target_line_no = target_line_no - self.diff_line_no = diff_line_no - self.line_type = line_type - self.value = value - - def __repr__(self): - # type: () -> str - return make_str("") % (self.line_type, self.value) - - def __str__(self): - # type: () -> str - return f"{self.line_type}{self.value}" - - def __eq__(self, other): - # type: (Line) -> bool - return ( - self.source_line_no == other.source_line_no - and self.target_line_no == other.target_line_no - and self.diff_line_no == other.diff_line_no - and self.line_type == other.line_type - and self.value == other.value - ) - - @property - def is_added(self): - # type: () -> bool - return self.line_type == LINE_TYPE_ADDED - - @property - def is_removed(self): - # type: () -> bool - return self.line_type == LINE_TYPE_REMOVED - - @property - def is_context(self): - # type: () -> bool - return self.line_type == LINE_TYPE_CONTEXT - - -@implements_to_string -class PatchInfo(list): - """ - Lines with extended patch info. - - Format of this info is not documented and it very much depends on - patch producer. - - """ - - def __repr__(self): - # type: () -> str - value = f"" - return make_str(value) - - def __str__(self): - # type: () -> str - return "".join(unicode(line) for line in self) - - -@implements_to_string -class Hunk(list): - """Each of the modified blocks of a file.""" - - def __init__( - self, src_start=0, src_len=0, tgt_start=0, tgt_len=0, section_header="" - ): - # type: (int, int, int, int, str) -> None - super().__init__() - if src_len is None: - src_len = 1 - if tgt_len is None: - tgt_len = 1 - self.source_start = int(src_start) - self.source_length = int(src_len) - self.target_start = int(tgt_start) - self.target_length = int(tgt_len) - self.section_header = section_header - self._added = None # Optional[int] - self._removed = None # Optional[int] - - def __repr__(self): - # type: () -> str - value = ( - f"" - ) - return make_str(value) - - def __str__(self): - # type: () -> str - # section header is optional and thus we output it only if it's present - section_hdr = f" {self.section_header}" if self.section_header else "" - head = ( - f"@@ -{self.source_start},{self.source_length} " - f"+{self.target_start},{self.target_length} @@{section_hdr}\n" - ) - content = "".join(unicode(line) for line in self) - return head + content - - def append(self, line): - # type: (Line) -> None - """Append the line to hunk, and keep track of source/target lines.""" - # Make sure the line is encoded correctly. This is a no-op except for - # potentially raising a UnicodeDecodeError. - str(line) - super().append(line) - - @property - def added(self): - # type: () -> Optional[int] - if self._added is not None: - return self._added - # re-calculate each time to allow for hunk modifications - # (which should mean metadata_only switch wasn't used) - return sum(1 for line in self if line.is_added) - - @property - def removed(self): - # type: () -> Optional[int] - if self._removed is not None: - return self._removed - # re-calculate each time to allow for hunk modifications - # (which should mean metadata_only switch wasn't used) - return sum(1 for line in self if line.is_removed) - - def is_valid(self): - # type: () -> bool - """Check hunk header data matches entered lines info.""" - return ( - len(self.source) == self.source_length - and len(self.target) == self.target_length - ) - - def source_lines(self): - # type: () -> Iterable[Line] - """Hunk lines from source file (generator).""" - return (line for line in self if line.is_context or line.is_removed) - - @property - def source(self): - # type: () -> Iterable[str] - return [str(line) for line in self.source_lines()] - - def target_lines(self): - # type: () -> Iterable[Line] - """Hunk lines from target file (generator).""" - return (line for line in self if line.is_context or line.is_added) - - @property - def target(self): - # type: () -> Iterable[str] - return [str(line) for line in self.target_lines()] - - -class PatchedFile(list): - """Patch updated file, it is a list of Hunks.""" - - def __init__( - self, - patch_info=None, - source="", - target="", - source_timestamp=None, - target_timestamp=None, - is_binary_file=False, - ): - # type: (Optional[PatchInfo], str, str, Optional[str], Optional[str], bool, bool) -> None - super().__init__() - self.patch_info = patch_info - self.source_file = source - self.source_timestamp = source_timestamp - self.target_file = target - self.target_timestamp = target_timestamp - self.is_binary_file = is_binary_file - - def __repr__(self): - # type: () -> str - return make_str("") % make_str(self.path) - - def __str__(self): - # type: () -> str - source = "" - target = "" - # patch info is optional - info = "" if self.patch_info is None else str(self.patch_info) - if not self.is_binary_file and self: - source_ts = f"\t{self.source_timestamp}" if self.source_timestamp else "" - source = f"--- {self.source_file}{source_ts}\n" - - target_ts = f"\t{self.target_timestamp}" if self.target_timestamp else "" - target = f"+++ {self.target_file}{target_ts}\n" - - hunks = "".join(unicode(hunk) for hunk in self) - return info + source + target + hunks - - def _parse_hunk(self, header, diff, encoding, metadata_only): # noqa: C901 - # type: (str, enumerate[str], Optional[str], bool) -> None - """Parse hunk details.""" - header_info = RE_HUNK_HEADER.match(header) - hunk_info = header_info.groups() - hunk = Hunk(*hunk_info) - - source_line_no = hunk.source_start - target_line_no = hunk.target_start - expected_source_end = source_line_no + hunk.source_length - expected_target_end = target_line_no + hunk.target_length - added = 0 - removed = 0 - - for diff_line_no, line in diff: - if encoding is not None: - line = line.decode(encoding) - - if metadata_only: - # quick line type detection, no regex required - line_type = line[0] if line else LINE_TYPE_CONTEXT - if line_type not in ( - LINE_TYPE_ADDED, - LINE_TYPE_REMOVED, - LINE_TYPE_CONTEXT, - LINE_TYPE_NO_NEWLINE, - ): - raise UnidiffParseError(f"Hunk diff line expected: {line}") - - if line_type == LINE_TYPE_ADDED: - target_line_no += 1 - added += 1 - elif line_type == LINE_TYPE_REMOVED: - source_line_no += 1 - removed += 1 - elif line_type == LINE_TYPE_CONTEXT: - target_line_no += 1 - source_line_no += 1 - - # no file content tracking - original_line = None - - else: - # parse diff line content - valid_line = RE_HUNK_BODY_LINE.match(line) - if not valid_line: - valid_line = RE_HUNK_EMPTY_BODY_LINE.match(line) - - if not valid_line: - raise UnidiffParseError(f"Hunk diff line expected: {line}") - - line_type = valid_line.group("line_type") - if line_type == LINE_TYPE_EMPTY: - line_type = LINE_TYPE_CONTEXT - - value = valid_line.group("value") # type: str - original_line = Line(value, line_type=line_type) - - if line_type == LINE_TYPE_ADDED: - original_line.target_line_no = target_line_no - target_line_no += 1 - elif line_type == LINE_TYPE_REMOVED: - original_line.source_line_no = source_line_no - source_line_no += 1 - elif line_type == LINE_TYPE_CONTEXT: - original_line.target_line_no = target_line_no - original_line.source_line_no = source_line_no - target_line_no += 1 - source_line_no += 1 - elif line_type == LINE_TYPE_NO_NEWLINE: - pass - else: - original_line = None - - # stop parsing if we got past expected number of lines - if ( - source_line_no > expected_source_end - or target_line_no > expected_target_end - ): - raise UnidiffParseError("Hunk is longer than expected") - - if original_line: - original_line.diff_line_no = diff_line_no - hunk.append(original_line) - - # if hunk source/target lengths are ok, hunk is complete - if ( - source_line_no == expected_source_end - and target_line_no == expected_target_end - ): - break - - # report an error if we haven't got expected number of lines - if source_line_no < expected_source_end or target_line_no < expected_target_end: - raise UnidiffParseError("Hunk is shorter than expected") - - if metadata_only: - # set fixed calculated values when metadata_only is enabled - hunk._added = added - hunk._removed = removed - - self.append(hunk) - - def _add_no_newline_marker_to_last_hunk(self): - # type: () -> None - if not self: - raise UnidiffParseError("Unexpected marker:" + LINE_VALUE_NO_NEWLINE) - last_hunk = self[-1] - last_hunk.append( - Line(LINE_VALUE_NO_NEWLINE + "\n", line_type=LINE_TYPE_NO_NEWLINE) - ) - - def _append_trailing_empty_line(self): - # type: () -> None - if not self: - raise UnidiffParseError("Unexpected trailing newline character") - last_hunk = self[-1] - last_hunk.append(Line("\n", line_type=LINE_TYPE_EMPTY)) - - @property - def path(self): - # type: () -> str - """Return the file path abstracted from VCS.""" - filepath = self.source_file - if filepath in (None, DEV_NULL) or ( - self.is_rename and self.target_file not in (None, DEV_NULL) - ): - # if this is a rename, prefer the target filename - filepath = self.target_file - - quoted = filepath.startswith('"') and filepath.endswith('"') - if quoted: - filepath = filepath[1:-1] - - if filepath.startswith("a/") or filepath.startswith("b/"): - filepath = filepath[2:] - - if quoted: - filepath = f'"{filepath}"' - - return filepath - - @property - def added(self): - # type: () -> int - """Return the file total added lines.""" - return sum([hunk.added for hunk in self]) - - @property - def removed(self): - # type: () -> int - """Return the file total removed lines.""" - return sum([hunk.removed for hunk in self]) - - @property - def is_rename(self): - return ( - self.source_file != DEV_NULL - and self.target_file != DEV_NULL - and self.source_file[2:] != self.target_file[2:] - ) - - @property - def is_added_file(self): - # type: () -> bool - """Return True if this patch adds the file.""" - if self.source_file == DEV_NULL: - return True - return ( - len(self) == 1 and self[0].source_start == 0 and self[0].source_length == 0 - ) - - @property - def is_removed_file(self): - # type: () -> bool - """Return True if this patch removes the file.""" - if self.target_file == DEV_NULL: - return True - return ( - len(self) == 1 and self[0].target_start == 0 and self[0].target_length == 0 - ) - - @property - def is_modified_file(self): - # type: () -> bool - """Return True if this patch modifies the file.""" - return not (self.is_added_file or self.is_removed_file) - - -@implements_to_string -class PatchSet(list): - """A list of PatchedFiles.""" - - def __init__(self, f, encoding=None, metadata_only=False): - # type: (Union[StringIO, str], Optional[str], bool) -> None - super().__init__() - - # convert string inputs to StringIO objects - if isinstance(f, basestring): - f = self._convert_string(f, encoding) # type: StringIO - - # make sure we pass an iterator object to parse - data = iter(f) - # if encoding is None, assume we are reading unicode data - # when metadata_only is True, only perform a minimal metadata parsing - # (ie. hunks without content) which is around 2.5-6 times faster; - # it will still validate the diff metadata consistency and get counts - self._parse(data, encoding=encoding, metadata_only=metadata_only) - - def __repr__(self): - # type: () -> str - return make_str("") % super().__repr__() - - def __str__(self): - # type: () -> str - return "".join(unicode(patched_file) for patched_file in self) - - def _parse(self, diff, encoding, metadata_only): # noqa: C901 - # type: (StringIO, Optional[str], bool) -> None - current_file = None - patch_info = None - - diff = enumerate(diff, 1) - for unused_diff_line_no, line in diff: - if encoding is not None: - line = line.decode(encoding) - - # check for a git file rename - is_diff_git_header = ( - RE_DIFF_GIT_HEADER.match(line) - or RE_DIFF_GIT_HEADER_URI_LIKE.match(line) - or RE_DIFF_GIT_HEADER_NO_PREFIX.match(line) - ) - if is_diff_git_header: - patch_info = PatchInfo() - source_file = is_diff_git_header.group("source") - target_file = is_diff_git_header.group("target") - current_file = PatchedFile( - patch_info, source_file, target_file, None, None - ) - self.append(current_file) - patch_info.append(line) - continue - - # check for a git new file - is_diff_git_new_file = RE_DIFF_GIT_NEW_FILE.match(line) - if is_diff_git_new_file: - if current_file is None or patch_info is None: - raise UnidiffParseError(f"Unexpected new file found: {line}") - current_file.source_file = DEV_NULL - patch_info.append(line) - continue - - # check for a git deleted file - is_diff_git_deleted_file = RE_DIFF_GIT_DELETED_FILE.match(line) - if is_diff_git_deleted_file: - if current_file is None or patch_info is None: - raise UnidiffParseError(f"Unexpected deleted file found: {line}") - current_file.target_file = DEV_NULL - patch_info.append(line) - continue - - # check for source file header - is_source_filename = RE_SOURCE_FILENAME.match(line) - if is_source_filename: - source_file = is_source_filename.group("filename") - source_timestamp = is_source_filename.group("timestamp") - # reset current file, unless we are processing a rename - # (in that case, source files should match) - if current_file is not None and not ( - current_file.source_file == source_file - ): - current_file = None - elif current_file is not None: - current_file.source_timestamp = source_timestamp - continue - - # check for target file header - is_target_filename = RE_TARGET_FILENAME.match(line) - if is_target_filename: - target_file = is_target_filename.group("filename") - target_timestamp = is_target_filename.group("timestamp") - if current_file is not None and not ( - current_file.target_file == target_file - ): - raise UnidiffParseError(f"Target without source: {line}") - if current_file is None: - # add current file to PatchSet - current_file = PatchedFile( - patch_info, - source_file, - target_file, - source_timestamp, - target_timestamp, - ) - self.append(current_file) - patch_info = None - else: - current_file.target_timestamp = target_timestamp - continue - - # check for hunk header - is_hunk_header = RE_HUNK_HEADER.match(line) - if is_hunk_header: - patch_info = None - if current_file is None: - raise UnidiffParseError(f"Unexpected hunk found: {line}") - current_file._parse_hunk(line, diff, encoding, metadata_only) - continue - - # check for no newline marker - is_no_newline = RE_NO_NEWLINE_MARKER.match(line) - if is_no_newline: - if current_file is None: - raise UnidiffParseError(f"Unexpected marker: {line}") - current_file._add_no_newline_marker_to_last_hunk() - continue - - # sometimes hunks can be followed by empty lines - if line == "\n" and current_file is not None: - current_file._append_trailing_empty_line() - continue - - # if nothing has matched above then this line is a patch info - if patch_info is None: - current_file = None - patch_info = PatchInfo() - - is_binary_diff = RE_BINARY_DIFF.match(line) - if is_binary_diff: - source_file = is_binary_diff.group("source_filename") - target_file = is_binary_diff.group("target_filename") - patch_info.append(line) - if current_file is not None: - current_file.is_binary_file = True - else: - current_file = PatchedFile( - patch_info, source_file, target_file, is_binary_file=True - ) - self.append(current_file) - patch_info = None - current_file = None - continue - - if line == "GIT binary patch\n": - current_file.is_binary_file = True - patch_info = None - current_file = None - continue - - patch_info.append(line) - - @classmethod - def from_filename( - cls, filename, encoding=DEFAULT_ENCODING, errors=None, newline=None - ): - # type: (str, str, Optional[str]) -> PatchSet - """Return a PatchSet instance given a diff filename.""" - with open_file( - filename, "r", encoding=encoding, errors=errors, newline=newline - ) as f: - instance = cls(f) - return instance - - @staticmethod - def _convert_string(data, encoding=None, errors="strict"): - # type: (Union[str, bytes], str, str) -> StringIO - if encoding is not None: - # if encoding is given, assume bytes and decode - data = unicode(data, encoding=encoding, errors=errors) - return StringIO(data) - - @classmethod - def from_string(cls, data, encoding=None, errors="strict"): - # type: (str, str, Optional[str]) -> PatchSet - """Return a PatchSet instance given a diff string.""" - return cls(cls._convert_string(data, encoding, errors)) - - @property - def added_files(self): - # type: () -> list[PatchedFile] - """Return patch added files as a list.""" - return [f for f in self if f.is_added_file] - - @property - def removed_files(self): - # type: () -> list[PatchedFile] - """Return patch removed files as a list.""" - return [f for f in self if f.is_removed_file] - - @property - def modified_files(self): - # type: () -> list[PatchedFile] - """Return patch modified files as a list.""" - return [f for f in self if f.is_modified_file] - - @property - def added(self): - # type: () -> int - """Return the patch total added lines.""" - return sum([f.added for f in self]) - - @property - def removed(self): - # type: () -> int - """Return the patch total removed lines.""" - return sum([f.removed for f in self]) diff --git a/scanpipe/pipes/unidiff/patch.py.ABOUT b/scanpipe/pipes/unidiff/patch.py.ABOUT deleted file mode 100644 index 75c3bb8ade..0000000000 --- a/scanpipe/pipes/unidiff/patch.py.ABOUT +++ /dev/null @@ -1,13 +0,0 @@ -about_resource: patch.py constants.py errors.py -name: patch -version: 0.7.5 -download_url: https://github.com/matiasb/python-unidiff/archive/refs/tags/v0.7.5.zip -description: Simple Python library to parse and interact with unified diff data. -homepage_url: https://github.com/matiasb/python-unidiff -license_expression: mit -attribute: yes -package_url: pkg:pypi/unidiff@0.7.5 -licenses: - - key: mit - name: MIT License - file: patch.py.LICENSE \ No newline at end of file diff --git a/scanpipe/pipes/unidiff/patch.py.LICENSE b/scanpipe/pipes/unidiff/patch.py.LICENSE deleted file mode 100644 index ca2c04c202..0000000000 --- a/scanpipe/pipes/unidiff/patch.py.LICENSE +++ /dev/null @@ -1,20 +0,0 @@ -The MIT License (MIT) -Copyright (c) 2012 Matias Bordese - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE -OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/scanpipe/tests/data/reachability/app.py b/scanpipe/tests/data/reachability/app.py index b8c9eff5e0..32eebc2fa1 100644 --- a/scanpipe/tests/data/reachability/app.py +++ b/scanpipe/tests/data/reachability/app.py @@ -1,5 +1,7 @@ import os +debug = False + class ReportGenerator: """A dummy class to test AST class method parsing.""" diff --git a/scanpipe/tests/data/reachability/diff-app.patch b/scanpipe/tests/data/reachability/diff-app.patch deleted file mode 100644 index ccb86953a8..0000000000 --- a/scanpipe/tests/data/reachability/diff-app.patch +++ /dev/null @@ -1,39 +0,0 @@ -From 8f7b1c3d9a4e2b6f5d8c1a2e3f4b5c6d7e8f9a0b Mon Sep 17 00:00:00 2001 -From: Security Team -Date: Tue, 2 Jun 2026 10:00:00 +0000 -Subject: [PATCH] Fix path traversal vulnerability in report generator - -- Validates that target paths stay within the designated base_dir. -- Catches ValueError on invalid path resolution. ---- - app.py | 12 +++++++++--- - 1 file changed, 9 insertions(+), 3 deletions(-) - -diff --git a/app.py b/app.py -index a1b2c3d..e4f5g6h 100644 ---- a/app.py -+++ b/app.py -@@ -15,13 +15,19 @@ def serve_report(request_payload): - # Helper function nested inside serve_report - def build_file_path(filename): -- # VULNERABLE: Direct concatenation allows Path Traversal -- # An attacker passing "../../etc/passwd" could read system files. -- return os.path.join(generator.base_dir, filename) -+ # FIXED: Validate that the resolved path stays within the base_dir -+ base = os.path.abspath(generator.base_dir) -+ target = os.path.abspath(os.path.join(base, filename)) -+ if not target.startswith(base): -+ raise ValueError("Path Traversal Detected") -+ return target - - if not requested_file: - return "Error: No file specified" - -- target_path = build_file_path(requested_file) -+ try: -+ target_path = build_file_path(requested_file) -+ except ValueError: -+ return "Error: Invalid path" - - if os.path.exists(target_path): - return f"Serving content of {target_path}" \ No newline at end of file diff --git a/scanpipe/tests/data/reachability/fixed-app.py b/scanpipe/tests/data/reachability/fixed-app.py index ca5a6f4c8b..30470b6cda 100644 --- a/scanpipe/tests/data/reachability/fixed-app.py +++ b/scanpipe/tests/data/reachability/fixed-app.py @@ -1,5 +1,7 @@ import os +debug = True + class ReportGenerator: """A dummy class to test AST class method parsing.""" diff --git a/scanpipe/tests/data/reachability/vuln-app.py b/scanpipe/tests/data/reachability/vuln-app.py index b8c9eff5e0..32eebc2fa1 100644 --- a/scanpipe/tests/data/reachability/vuln-app.py +++ b/scanpipe/tests/data/reachability/vuln-app.py @@ -1,5 +1,7 @@ import os +debug = False + class ReportGenerator: """A dummy class to test AST class method parsing.""" diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index e24c42932b..0d3ff42fa1 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -55,16 +55,16 @@ def test_get_symbol_reachability_results( mock_collect_symbols, mock_git_context, ): - app_text = (self.data / "app.py").read_text() + file_path = "app.py" + app_text = (self.data / file_path).read_text() vuln_text = (self.data / "vuln-app.py").read_text() fixed_text = (self.data / "fixed-app.py").read_text() - diff_text = (self.data / "diff-app.patch").read_text() analyzer = PatchAnalyzer(repo=MagicMock(), commit_hash="dummy") - diff_line_map = analyzer.parse_diff_lines(diff_text=diff_text) - file_path = "app.py" - removed_lines, added_lines = diff_line_map.get(file_path, ([], [])) + removed_lines, added_lines = analyzer.compute_changed_lines( + vulnerable_text=vuln_text, fixed_text=fixed_text + ) vuln_meta, fixed_meta, lang = analyzer.analyze( vulnerable_text=vuln_text, fixed_text=fixed_text, @@ -90,10 +90,12 @@ def test_get_symbol_reachability_results( mock_collect_symbols.return_value = { lang: { "vulnerable": { - f"app.py::{key}": metadata for key, metadata in vuln_meta.items() + f"{file_path}::{key}": metadata + for key, metadata in vuln_meta.items() }, "fixed": { - f"app.py::{key}": metadata for key, metadata in fixed_meta.items() + f"{file_path}::{key}": metadata + for key, metadata in fixed_meta.items() }, } } @@ -125,9 +127,9 @@ def test_get_symbol_reachability_results( "called": False, "defined": True, "imported": False, - "fingerprint": "d7675efb263896da2a3c00679511833" - "553907e7e6ea619115a6dfc8625c3457e", - "symbol_name": "serve_report", + "fingerprint": "336908735214468b103dbde" + "11c3ffbd2f76ac9212b8514f831cfa078a67892df", + "symbol_name": "debug", "reachable_from": [], }, { @@ -139,12 +141,23 @@ def test_get_symbol_reachability_results( "symbol_name": "serve_report.build_file_path", "reachable_from": ["serve_report"], }, + { + "called": False, + "defined": True, + "imported": False, + "fingerprint": "d7675efb263896da2a3c0067951183" + "3553907e7e6ea619115a6dfc8625c3457e", + "symbol_name": "serve_report", + "reachable_from": [], + }, ], "fixed_symbols": [ + "debug", "serve_report", "serve_report.build_file_path", ], "vulnerable_symbols": [ + "debug", "serve_report", "serve_report.build_file_path", ], @@ -402,59 +415,68 @@ def test_diff_changed_symbols(self): def test_analyze_patched_file(self): vuln_text = (self.data / "vuln-app.py").read_text(encoding="utf-8") fixed_text = (self.data / "fixed-app.py").read_text(encoding="utf-8") - diff_text = (self.data / "diff-app.patch").read_text(encoding="utf-8") - - diff_line_map = PatchAnalyzer.parse_diff_lines(diff_text=diff_text) file_path = "app.py" - removed_lines, added_lines = diff_line_map.get(file_path, ([], [])) + removed_lines, added_lines = PatchAnalyzer.compute_changed_lines( + vuln_text, fixed_text + ) vuln_meta, fixed_meta, lang = PatchAnalyzer.analyze( vulnerable_text=vuln_text, fixed_text=fixed_text, removed_lines=removed_lines, added_lines=added_lines, - file_path="app.py", + file_path=file_path, ) self.assertEqual( vuln_meta, { + "debug": { + "qualified_name": "debug", + "text": "debug = False", + "fingerprint": "336908735214468b103dbde11c3ff" + "bd2f76ac9212b8514f831cfa078a67892df", + "start_line": 3, + "end_line": 3, + "node_type": "assignment", + }, + "serve_report.build_file_path": { + "qualified_name": "serve_report.build_file_path", + "text": "def build_file_path(filename):\n" + " # VULNERABLE: Direct concatenation " + "allows Path Traversal\n " + ' # An attacker passing "../../etc/passwd" ' + "could read system files.\n" + " return os.path.join(generator.base_dir, filename)", + "fingerprint": "762e4f7d03b1bf4359c3ca364e55814" + "0239913bfabcc5aa77156460c2eb0a355", + "start_line": 19, + "end_line": 22, + "node_type": "function_definition", + }, "serve_report": { "qualified_name": "serve_report", - "text": "def serve_report(request_payload):\n" - ' """Top-level function handling a request."""\n' + "text": "def serve_report(request_payload):\n " + ' """Top-level function handling a request."""\n' ' generator = ReportGenerator("/var/reports")\n' ' requested_file = request_payload.get("file")\n\n' " # Helper function nested inside serve_report\n" - " def build_file_path(filename):\n" - " # VULNERABLE: Direct concatenation allows Path Traversal\n" - " # An attacker passing " - '"../../etc/passwd" ' - "could read system files.\n" + " def build_file_path(filename):\n " + " # VULNERABLE: Direct " + "concatenation allows Path Traversal\n " + " # An attacker passing " + '"../../etc/passwd" could read system files.\n' " return os.path.join(generator.base_dir, filename)\n\n" - " if not requested_file:\n" - ' return "Error: No file specified"\n\n' - " target_path = build_file_path(requested_file)\n\n" - " if os.path.exists(target_path):\n" - ' return f"Serving content of {target_path}"\n\n' - ' return "Error: File not found"', + " if not requested_file:\n " + ' return "Error: No file specified"\n\n ' + " target_path = build_file_path(requested_file)\n\n " + " if os.path.exists(target_path):\n " + ' return f"Serving content of {target_path}"\n\n ' + ' return "Error: File not found"', "fingerprint": "d7675efb263896da2a3c006795118" "33553907e7e6ea619115a6dfc8625c3457e", - "start_line": 11, - "end_line": 30, - "node_type": "function_definition", - }, - "serve_report.build_file_path": { - "qualified_name": "serve_report.build_file_path", - "text": "def build_file_path(filename):\n" - " # VULNERABLE: Direct concatenation allows Path Traversal\n" - ' # An attacker passing "../../etc/passwd"' - " could read system files.\n" - " return os.path.join(generator.base_dir, filename)", - "fingerprint": "762e4f7d03b1bf4359c3ca364e5581" - "40239913bfabcc5aa77156460c2eb0a355", - "start_line": 17, - "end_line": 20, + "start_line": 13, + "end_line": 32, "node_type": "function_definition", }, }, @@ -463,50 +485,58 @@ def test_analyze_patched_file(self): self.assertEqual( fixed_meta, { - "serve_report": { - "qualified_name": "serve_report", - "text": "def serve_report(request_payload):\n" - ' """Top-level function handling a request."""\n' - ' generator = ReportGenerator("/var/reports")\n' - ' requested_file = request_payload.get("file")\n\n' - " # Helper function nested inside serve_report\n" - " def build_file_path(filename):\n" - " # FIXED: Validate that the resolved " - "path stays within the base_dir\n" - " base = os.path.abspath(generator.base_dir)\n" - " target = os.path.abspath(os.path.join(base, filename))\n" - " if not target.startswith(base):\n" - ' raise ValueError("Path Traversal Detected")\n' - " return target\n\n" - " if not requested_file:\n" - ' return "Error: No file specified"\n\n' - " try:\n" - " target_path = build_file_path(requested_file)\n" - " except ValueError:\n" - ' return "Error: Invalid path"\n\n' - " if os.path.exists(target_path):\n" - ' return f"Serving content of {target_path}"\n\n' - ' return "Error: File not found"', - "fingerprint": "2deedb21d5f9b1409c59f0b1e5512d" - "73d9afdfc3f469ccf86e8835915d240e76", - "start_line": 11, - "end_line": 36, - "node_type": "function_definition", + "debug": { + "qualified_name": "debug", + "text": "debug = True", + "fingerprint": "55d2e2010de610fd32f0c28bc49f535" + "3d6ac60afc70adc5713aa4b675646590e", + "start_line": 3, + "end_line": 3, + "node_type": "assignment", }, "serve_report.build_file_path": { "qualified_name": "serve_report.build_file_path", - "text": "def build_file_path(filename):\n" - " # FIXED: Validate that the resolved path" - " stays within the base_dir\n" - " base = os.path.abspath(generator.base_dir)\n" - " target = os.path.abspath(os.path.join(base, filename))\n" - " if not target.startswith(base):\n" - ' raise ValueError("Path Traversal Detected")\n' - " return target", + "text": "def build_file_path(filename):\n " + " # FIXED: Validate that the resolved" + " path stays within the base_dir\n " + " base = os.path.abspath(generator.base_dir)\n " + " target = os.path.abspath(os.path.join(base, filename))\n " + " if not target.startswith(base):\n " + ' raise ValueError("Path Traversal Detected")\n ' + " return target", "fingerprint": "646743b5d5497f6ea3b96f860bcbe" "b38096ce008ad16d2b9a9c3f77a98faca80", - "start_line": 17, - "end_line": 23, + "start_line": 19, + "end_line": 25, + "node_type": "function_definition", + }, + "serve_report": { + "qualified_name": "serve_report", + "text": "def serve_report(request_payload):\n " + ' """Top-level function handling a request."""\n ' + ' generator = ReportGenerator("/var/reports")\n ' + ' requested_file = request_payload.get("file")\n\n ' + " # Helper function nested inside serve_report\n " + "def build_file_path(filename):\n " + " # FIXED: Validate that the" + " resolved path stays within the base_dir\n " + " base = os.path.abspath(generator.base_dir)\n " + " target = os.path.abspath(os.path.join(base, filename))\n" + " if not target.startswith(base):\n " + ' raise ValueError("Path Traversal Detected")\n ' + " return target\n\n " + " if not requested_file:\n " + ' return "Error: No file specified"\n\n try:\n ' + " target_path = build_file_path(requested_file)\n" + " except ValueError:\n " + ' return "Error: Invalid path"\n\n ' + " if os.path.exists(target_path):\n " + ' return f"Serving content of {target_path}"\n\n ' + ' return "Error: File not found"', + "fingerprint": "2deedb21d5f9b1409c59f0b1e55" + "12d73d9afdfc3f469ccf86e8835915d240e76", + "start_line": 13, + "end_line": 38, "node_type": "function_definition", }, },