diff --git a/.claude/FACADE_DECISIONS.md b/.claude/FACADE_DECISIONS.md new file mode 100644 index 00000000..6757841a --- /dev/null +++ b/.claude/FACADE_DECISIONS.md @@ -0,0 +1,53 @@ +# Facade decisions — python-sdk + +Decision log for SDK-surface design (per designing-cldk-changes, sdk-facade-design-loop). +One line per locked decision; newest section first. + +## 2026-07-27 — L3/L4 verb wiring is staged per release + +- **Staging:** rc.1 wires the five slice/flow verbs on the Python facade only (local + Neo4j); + rc.2 adds TypeScript; rc.3 adds the Java honest-degrade leg; rc.4 (Go) and rc.5 (C++) are + new-language legs entering via designing-cldk-changes (the C++ leg absorbs the existing C + facade); 2.0.0 final swaps the Rust query core (#279) in under the verbs, restores the + all-language DoD, and closes #270. +- **URI minting:** providers mint body-vertex URIs as `@`, + matching the analyzers' own param_in/param_out vocabulary (the real 1.0.2 Neo4j emitter stores + these can:// ids directly on PyCFGNode). Local keys never escape a provider. +- **source_slice contract:** existing span-less (synthetic) vertex → `(module_path, None)`; + unknown vertex → `(None, None)`; never derive a line from key shape. Both backends identical. +- **Rust hand-off unchanged:** the Rust query core (#279) replaces the Python engine underneath + these verbs post-M1; the facade surface added here is the stable contract it must satisfy. + +## 2026-07-21 — Rust query engine (fluent API core) + +- **M1 scope:** Epic B success criteria — the two Odoo PoE audit queries (#155), single-language, + in-memory + Neo4j backends, `.explain()` reproduces the manual audit evidence. Cross-service + (services/gRPC/proto, the RFC's boutique examples) is Epic E, out of scope; requires an + analyzer-side schema design that has not happened. +- **Identity scheme:** `can://` (what analyzers emit today), extended as needed. The 2026-07-09 + fluent-query spec's `cldk://` is amended to `can://`; no parallel `service://`/`proto://` + schemes — Epic E extends the `can://` grammar instead. +- **Plan algebra:** redesigned fresh, taking the 2026-07-09 spec's six primitives + (Descend/Ascend/Relate/Filter/PathQuery/Project) and the RFC's LogicalOp sketch as inputs. + Deliverable: an algebra ADR locked before the Rust core builds. +- **Data plane:** `cldk-query-core` consumes schema-2.0.0 `analysis.json` natively (serde CPG + models) AND speaks Bolt directly (neo4rs) for the Neo4j backend. Core tests are cargo-only on + fixture JSONs; no Python in the core. +- **Opaque(fn):** plan-split semantics — Rust executes the prefix, returns URIs, Python applies + the lambda, execution re-enters Rust for remaining steps; `explain()` marks the split point. +- **Packaging:** fat wheel — `cldk` itself becomes a maturin/PyO3 platform wheel (abi3). + Consequence accepted: `cldk` is no longer pure-Python; release workflow becomes a per-platform + build matrix; platforms without a prebuilt wheel need a Rust toolchain for the sdist. +- **L3/L4 slicer:** the Rust core REPLACES the `cldk.graph` slice engine (#270/#271); the Python + engine is deprecated once the Rust slicer passes the same exact-set gates. Single dataflow + semantics owner; replacement staged post-M1. +- **Extraction boundary:** no PyO3 types/exceptions/callbacks in `cldk-query-core`; versioned + `PlanEnvelope` wire format (semver string, house convention, not u32); language-neutral result + structs; extraction only when independently consumed/released (per RFC criteria). +- **Repo layout (amends the RFC's `rust/crates/` sketch):** root-level `crates/` with the + workspace `Cargo.toml` at the repo root (polars/ruff idiom; canonical Cargo layout, zero-config + rust-analyzer, maturin driven from the root pyproject via + `tool.maturin.manifest-path = "crates/cldk-python/Cargo.toml"`). The extension module compiles + to the private submodule `cldk._native` — users import `cldk.query`; the public namespace never + admits Rust exists. Crate names unchanged: `cldk-query-core` (survives extraction), + `cldk-python` (bindings). diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ad75552a..47681606 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -65,28 +65,66 @@ jobs: - name: Build Package run: uv build - - name: Read Changelog Entry - id: changelog_reader - uses: mindsers/changelog-reader-action@v2 - with: - validation_level: warn - version: ${{ steps.tag_name.outputs.current_version }} - path: ./CHANGELOG.md - - - name: Build Changelog - id: gen_changelog - uses: mikepenz/release-changelog-builder-action@v5 - with: - failOnError: "true" - configuration: .github/workflows/release_config.json - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Verify the codeanalyzer JAR is bundled + # Guard against the hatchling/.gitignore regression (issue #284): a jarless wheel + # installs fine but fails at runtime with "codeanalyzer jar not found". Fail the + # release here rather than publish a broken artifact to PyPI. + # + # The listing is captured before grepping: piping `tar tzf` (which decompresses the + # whole 32MB sdist) straight into `grep -q` lets grep close the pipe on first match, + # SIGPIPE-killing tar and — under `pipefail` — reporting a false "missing JAR". + run: | + set -euo pipefail + jar_re='codeanalyzer/jar/codeanalyzer-[0-9][^/]*\.jar$' + fail=0 + for f in dist/*.whl dist/*.tar.gz; do + case "$f" in + *.whl) listing=$(unzip -l "$f") ;; + *.tar.gz) listing=$(tar tzf "$f") ;; + esac + if grep -qE "$jar_re" <<<"$listing"; then + echo " ✓ $f" + else + echo "::error::$f is missing the codeanalyzer JAR" + grep -i '\.jar' <<<"$listing" || echo " (no .jar entries at all)" + fail=1 + fi + done + if [ "$fail" -ne 0 ]; then + echo "Refusing to publish a jarless release."; exit 1 + fi + echo "codeanalyzer JAR present in wheel and sdist ✓" + + - name: Extract release notes from CHANGELOG.md + id: notes + # Source the release body from the hand-written CHANGELOG.md section for this tag — + # deterministic and independent of PR labels — and refuse to publish/announce a blank + # body. The previous label-based changelog scraper emitted nothing for unlabeled PRs, + # which blanked the release and crashed the org announcement. See issue #289. + run: | + set -euo pipefail + version="${GITHUB_REF#refs/tags/}" # e.g. v1.4.4 — matches the "## [v1.4.4]" heading + notes=$(awk -v h="## [$version]" ' + !seen && index($0, h) == 1 { seen = 1; next } + seen && index($0, "## [") == 1 { exit } + seen { print } + ' CHANGELOG.md | sed '/./,$!d' | tac | sed '/./,$!d' | tac) # strip blank edges + if [ -z "$notes" ]; then + echo "::error::No CHANGELOG.md entry for $version — refusing to publish a blank release." + exit 1 + fi + { + echo "notes<<__CHANGELOG_EOF__" + echo "$notes" + echo "__CHANGELOG_EOF__" + } >> "$GITHUB_OUTPUT" + echo "Release notes for $version:"; echo "$notes" - name: Publish Release on GitHub uses: softprops/action-gh-release@v2 with: files: dist/* - body: ${{ steps.gen_changelog.outputs.changelog }} + body: ${{ steps.notes.outputs.notes }} # Auto-open a repo-level Discussion linked to this release, seeded with # the same notes. Requires Discussions enabled and this category to exist. discussion_category_name: Announcements @@ -96,13 +134,13 @@ jobs: # Mirror the release announcement into the ORG-level discussions, which are # backed by codellm-devkit/.github. GITHUB_TOKEN can't write cross-repo, so # this uses a PAT (ORG_DISCUSSIONS_TOKEN) with repo scope, and posts via the - # createDiscussion GraphQL mutation. The body (the generated changelog) is + # createDiscussion GraphQL mutation. The body (the CHANGELOG.md notes) is # passed via env to avoid shell-injection, matching the repo-level post. - name: Announce in org-level discussions (codellm-devkit/.github) continue-on-error: true # a failed org post must not fail an otherwise-good release env: GH_TOKEN: ${{ secrets.ORG_DISCUSSIONS_TOKEN }} - BODY: ${{ steps.gen_changelog.outputs.changelog }} + BODY: ${{ steps.notes.outputs.notes }} run: | set -uo pipefail VERSION="${GITHUB_REF#refs/tags/v}" diff --git a/.github/workflows/release_config.json b/.github/workflows/release_config.json deleted file mode 100644 index 200120c7..00000000 --- a/.github/workflows/release_config.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "categories": [ - { - "title": "## ✨ Release", - "labels": [ - "release" - ] - }, - { - "title": "## 🚀 Features", - "labels": [ - "kind/feature", - "enhancement" - ] - }, - { - "title": "## 🐛 Fixes", - "labels": [ - "fix", - "bug" - ] - }, - { - "title": "## ♻️ Refactoring", - "labels": [ - "refactoring" - ] - }, - { - "title": "## ⚡️ Performance Improvements", - "labels": [ - "performance" - ] - }, - { - "title": "## \uD83D\uDCDA Documentation", - "labels": [ - "documentation", - "doc" - ] - }, - { - "title": "## \uD83D\uDEA6 Tests", - "labels": [ - "test" - ] - }, - { - "title": "## \uD83D\uDEE0 Other Updates", - "labels": [ - "other", - "kind/dependency-change" - ] - }, - { - "title": "## 🚨 Breaking Changes", - "labels": [ - "breaking" - ] - } - ], - "ignore_labels": [ - "ignore" - ] -} \ No newline at end of file diff --git a/.gitignore b/.gitignore index b7fe8d82..ba6d4fae 100644 --- a/.gitignore +++ b/.gitignore @@ -74,3 +74,8 @@ poetry.lock !CLAUDE.md !AGENTS.md !GEMINI.md + +# Track the design decision log (overrides the global .claude ignore; everything else in .claude/ stays ignored) +!.claude/ +.claude/* +!.claude/FACADE_DECISIONS.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 20787d8f..b7a89070 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,53 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed +- **Published wheels bundle the `codeanalyzer-java` JAR again.** `2.0.0-rc.1` (like the 1.2.0–1.4.3 + line) shipped without the bundled JAR, so `CLDK.java(...)` after a plain `pip install` raised + `CodeanalyzerExecutionException: codeanalyzer jar not found`. Hatchling applied the root + `.gitignore` `*.jar` rule at build time but not the nested `!codeanalyzer-*.jar` negation that + keeps the JAR tracked in git; a `[tool.hatch.build] artifacts` rule force-includes it, and the + release workflow now fails fast if a built artifact is missing the JAR. (#284) + +## [v2.0.0-rc.1] - 2026-07-16 + +First release candidate for 2.0.0 — the schema-v2 release. Both the TypeScript and Python +facades now speak the analyzers' **schema 2.0.0** end to end and fail fast on any other version. + +### Changed +- **BREAKING (Python): the SDK speaks analysis schema 2.0.0 and requires `codeanalyzer-python>=1.0.2`.** + The re-exported Python models follow the schema-v2 renames: `PyModule.classes` → `types`, + `PyClass.methods`/`inner_classes` → `callables`/`types`, `PyCallable.inner_callables`/`inner_classes` + → `callables`/`types`, and `PyCallEdge` is now `src`/`dst`/`prov`. A callable's source text no + longer lives on a `code` field — the SDK recovers it by slicing `PyModule.source` with the + callable's byte-offset `span` (accessor behavior such as `get_method_bodies` is unchanged). + Call-graph node keys remain dotted signatures: schema v2's CanNode (`can://`) edge endpoints are + translated back to signatures; unresolved externals keep their raw `can://` ids. Both Python + backends fail fast on a schema mismatch: the in-process backend checks the `Analysis` envelope's + `schema_version`, and the Neo4j backend checks the stamp on the scoped `:PyApplication` node — + re-analyze / re-emit with `codeanalyzer-python>=1.0.2` if you hit `CldkSchemaMismatchException`. +- **BREAKING (TypeScript): the SDK speaks graph schema 2.0.0 and requires `codeanalyzer-typescript 1.0.0`.** + TS-prefixed graph vocabulary, CanNode keys, and a fail-fast `schema_version` check on the + `:Application` node. Accessors whose vocabulary is not projected into graph schema 2.0.0 + (decorators, fields, imports/exports, variables) raise `NotImplementedError` on the Neo4j + backend instead of silently returning wrong data. (#268) + +### Added +- **Canonical schema-v2 CPG models** (`cldk.models.cpg`): the shared, language-neutral + `Application` model tree for schema-2.0.0 analyzer output, modeled once and validated against + real L1–L4 samples from multiple analyzers. (#240, #274) +- **L3/L4 program-slice engine core** (`cldk.graph`): forward/backward slicing over schema-v2 + CFG/CDG/DDG program graphs. (#270, #271) + +### Fixed +- **TypeScript Neo4j backend guards ambiguous application matches** instead of silently merging + two applications' module scopes. (#268) + +### Dependencies +- `codeanalyzer-python` 0.3.1 → **1.0.2** (schema 2.0.0; includes the emitter fix for null `code` + node properties, codellm-devkit/codeanalyzer-python#104) +- `codeanalyzer-typescript` 0.4.3 → **1.0.0** (graph schema 2.0.0) + ## [v1.4.3] - 2026-07-14 ### Fixed diff --git a/cldk/analysis/__init__.py b/cldk/analysis/__init__.py index 7735d3c3..9ff4598b 100644 --- a/cldk/analysis/__init__.py +++ b/cldk/analysis/__init__.py @@ -28,3 +28,26 @@ class AnalysisLevel(str, Enum): call_graph = "call graph" program_dependency_graph = "program dependency graph" system_dependency_graph = "system dependency graph" + + +def to_analysis_level(value) -> AnalysisLevel: + """Normalize a level given as an AnalysisLevel, its value ("call graph"), + or its name ("call_graph") — both spellings are in the wild.""" + if isinstance(value, AnalysisLevel): + return value + try: + return AnalysisLevel(value) + except ValueError: + try: + return AnalysisLevel[value] + except KeyError: + raise ValueError(f"unknown analysis level: {value!r}") from None + + +#: Facade-level vocabulary → the analyzers' integer analysis level (schema 2.0 ``max_level``). +ANALYSIS_LEVEL_TO_INT = { + AnalysisLevel.symbol_table: 1, + AnalysisLevel.call_graph: 2, + AnalysisLevel.program_dependency_graph: 3, + AnalysisLevel.system_dependency_graph: 4, +} diff --git a/cldk/analysis/python/backend.py b/cldk/analysis/python/backend.py index c0c64ad5..70ea4526 100644 --- a/cldk/analysis/python/backend.py +++ b/cldk/analysis/python/backend.py @@ -34,6 +34,7 @@ import networkx as nx +from cldk.graph.provider import ProgramGraphProvider from cldk.models.python import ( PyApplication, PyCallable, @@ -45,7 +46,7 @@ ) -class PythonAnalysisBackend(ABC): +class PythonAnalysisBackend(ProgramGraphProvider, ABC): """Abstract base every Python analysis backend implements. A backend owns all indexing and query logic for a Python application; the diff --git a/cldk/analysis/python/codeanalyzer/codeanalyzer.py b/cldk/analysis/python/codeanalyzer/codeanalyzer.py index 8ddaced1..e242b9b2 100644 --- a/cldk/analysis/python/codeanalyzer/codeanalyzer.py +++ b/cldk/analysis/python/codeanalyzer/codeanalyzer.py @@ -60,8 +60,10 @@ from codeanalyzer.options import AnalysisOptions from codeanalyzer.schema import model_dump_json -from cldk.analysis import AnalysisLevel +from cldk.analysis import ANALYSIS_LEVEL_TO_INT, AnalysisLevel, to_analysis_level from cldk.analysis.python.backend import PythonAnalysisBackend +from cldk.graph._cpg_local import CpgLocalProviderMixin +from cldk.utils.exceptions import CldkSchemaMismatchException from cldk.models.python import ( PyApplication, PyCallEdge, @@ -77,6 +79,18 @@ logger = logging.getLogger(__name__) +def _code_of(module: PyModule, c: PyCallable) -> str: + """Slice a callable's source text out of its module's ``source`` using ``span.bytes``. + + Schema 2.0.0 stores source once per module; callables carry a byte-offset ``Span`` + instead of a duplicated ``code`` string. + """ + if not module.source or c.span is None: + return "" + start, end = c.span.bytes + return module.source.encode("utf-8")[start:end].decode("utf-8") + + def _overview(c: PyCallable, class_signature: str | None, kind: str) -> PyCallableOverview: """Project a :class:`PyCallable` into a lightweight :class:`PyCallableOverview`.""" return PyCallableOverview( @@ -91,7 +105,7 @@ def _overview(c: PyCallable, class_signature: str | None, kind: str) -> PyCallab ) -class PyCodeanalyzer(PythonAnalysisBackend): +class PyCodeanalyzer(CpgLocalProviderMixin, PythonAnalysisBackend): """In-process driver for the ``codeanalyzer-python`` analysis backend. This class serves as the primary interface to the codeanalyzer-python @@ -120,6 +134,10 @@ class PyCodeanalyzer(PythonAnalysisBackend): - :class:`~cldk.analysis.python.PythonAnalysis`: High-level facade. """ + #: The ``codeanalyzer-python`` analysis schema this backend speaks. ``_run_analyzer`` + #: fails fast when the analyzer's ``Analysis.schema_version`` differs. + SUPPORTED_ANALYSIS_SCHEMA = "2.0.0" + def __init__( self, project_dir: Union[str, Path], @@ -177,6 +195,7 @@ def __init__( if not self.project_dir.is_dir(): raise ValueError(f"project_dir does not exist or is not a directory: {self.project_dir}") self.analysis_level = analysis_level + self._level_int = ANALYSIS_LEVEL_TO_INT[to_analysis_level(analysis_level)] self.eager_analysis = eager_analysis self.target_files = target_files self.use_ray = use_ray @@ -191,11 +210,11 @@ def __init__( # Class-signature → file path lookup, built once. self._class_to_file: Dict[str, str] = {} for file_path, module in self.application.symbol_table.items(): - for class_sig in module.classes: + for class_sig in module.types: self._class_to_file[class_sig] = file_path - if analysis_level == AnalysisLevel.call_graph: - self.call_graph: nx.DiGraph | None = self._build_call_graph(self.application.call_graph) + if self._level_int >= 2: + self.call_graph: nx.DiGraph | None = self._build_call_graph(self.application.call_graph, self._id_to_signature()) else: self.call_graph = None @@ -210,7 +229,13 @@ def _run_analyzer(self) -> PyApplication: Returns: A :class:`~cldk.models.python.PyApplication` object containing the complete analysis results, including the symbol table and - call graph edges. + call graph edges. The analyzer returns a schema-v2 ``Analysis`` + envelope; this method verifies its ``schema_version`` and unwraps + the ``application`` payload. + + Raises: + CldkSchemaMismatchException: If the analyzer's ``schema_version`` + is not :attr:`SUPPORTED_ANALYSIS_SCHEMA`. Note: If ``target_files`` contains multiple files, only the first @@ -223,24 +248,58 @@ def _run_analyzer(self) -> PyApplication: logger.warning("codeanalyzer-python supports only a single target file; using the first.") target_file = Path(self.target_files[0]) - options = AnalysisOptions( - input=self.project_dir, - output=self.analysis_json_path, - format=OutputFormat.JSON, - using_ray=self.use_ray, - rebuild_analysis=self.eager_analysis, - skip_tests=True, - file_name=target_file, - cache_dir=self.cache_dir, - clear_cache=False, - verbosity=0, - ) + opts = { + "input": self.project_dir, + "output": self.analysis_json_path, + "format": OutputFormat.JSON, + "using_ray": self.use_ray, + "rebuild_analysis": self.eager_analysis, + "skip_tests": True, + "file_name": target_file, + "cache_dir": self.cache_dir, + "clear_cache": False, + "verbosity": 0, + } + # Add analysis_level if _level_int is available (set in __init__). The guard IS + # load-bearing, not dead defensiveness: tests/analysis/python/test_python_schema_contract.py + # exercises this method on a `PyCodeanalyzer.__new__` instance that never ran `__init__` + # (and so never set `_level_int`) to isolate the schema-envelope gate below from analyzer + # construction — removing the guard would make those tests raise AttributeError. + if hasattr(self, "_level_int"): + opts["analysis_level"] = self._level_int + + options = AnalysisOptions(**opts) with Codeanalyzer(options) as analyzer: - return analyzer.analyze() + analysis = analyzer.analyze() + if analysis.schema_version != self.SUPPORTED_ANALYSIS_SCHEMA: + raise CldkSchemaMismatchException( + f"analysis schema {analysis.schema_version!r} from codeanalyzer-python, this SDK speaks " + f"{self.SUPPORTED_ANALYSIS_SCHEMA!r} — align the pinned codeanalyzer-python with the SDK" + ) + # The envelope reports what the analyzer actually computed; the capability gate + # (cldk/graph/capability.py) reads it via max_level() — recorded, never sniffed. + # Schema 2.0.0+'s Analysis envelope always carries max_level (default 1), so capturing + # it is unconditional here; an envelope that somehow lacks the attribute should fail + # loudly rather than silently fabricate a level. + self._max_level: int = analysis.max_level + return analysis.application + + def max_level(self) -> int: + """The analysis level of the underlying run (1-4), as reported by the analyzer.""" + return self._max_level + + def _id_to_signature(self) -> Dict[str, str]: + """Map every symbol-table callable's ``can://`` id to its dotted signature. + + Schema 2.0.0 call edges reference callables by CanNode id (``PyCallEdge.src``/``dst``); + the SDK's public call-graph vocabulary stays dotted signatures (matching the Neo4j + backend, whose ``:PySymbol`` nodes carry the dotted ``signature`` property). + """ + return {c.id: c.signature for c, _, _, _ in self._iter_callables() if c.id} @staticmethod - def _build_call_graph(edges: List[PyCallEdge]) -> nx.DiGraph: + def _build_call_graph(edges: List[PyCallEdge], id_to_signature: Dict[str, str]) -> nx.DiGraph: """Convert a list of call edges into a NetworkX directed graph. Transforms the flat list of :class:`PyCallEdge` objects from the @@ -250,6 +309,9 @@ def _build_call_graph(edges: List[PyCallEdge]) -> nx.DiGraph: Args: edges: List of :class:`~cldk.models.python.PyCallEdge` objects representing call relationships between methods/functions. + id_to_signature: ``can://`` id → dotted signature for symbol-table callables + (see :meth:`_id_to_signature`). Ids with no entry — external targets — + keep their raw ``can://`` id as the node key. Returns: A ``networkx.DiGraph`` where: @@ -259,7 +321,15 @@ def _build_call_graph(edges: List[PyCallEdge]) -> nx.DiGraph: """ graph = nx.DiGraph() for edge in edges: - graph.add_edge(edge.source, edge.target, type=edge.type, weight=edge.weight, provenance=tuple(edge.provenance)) + # Schema 2.0.0 dropped the edge's `type` field; "CALL_DEP" is the fixed edge kind + # (matching the Neo4j backend). nx attribute names stay the SDK's public shape. + graph.add_edge( + id_to_signature.get(edge.src, edge.src), + id_to_signature.get(edge.dst, edge.dst), + type="CALL_DEP", + weight=edge.weight, + provenance=tuple(edge.prov), + ) return graph # --------------------------------------------------------- application @@ -300,7 +370,7 @@ def get_call_graph(self) -> nx.DiGraph: relationships across the project. """ if self.call_graph is None: - self.call_graph = self._build_call_graph(self.application.call_graph) + self.call_graph = self._build_call_graph(self.application.call_graph, self._id_to_signature()) return self.call_graph def get_call_graph_json(self) -> str: @@ -348,7 +418,7 @@ def get_all_classes(self) -> Dict[str, PyClass]: """ result: Dict[str, PyClass] = {} for module in self.application.symbol_table.values(): - result.update(module.classes) + result.update(module.types) return result def get_class(self, qualified_class_name: str) -> PyClass | None: @@ -376,7 +446,7 @@ def get_all_nested_classes(self, qualified_class_name: str) -> List[PyClass]: not found. """ cls = self.get_class(qualified_class_name) - return list(cls.inner_classes.values()) if cls else [] + return list(cls.types.values()) if cls else [] def get_all_sub_classes(self, qualified_class_name: str) -> Dict[str, PyClass]: """Return all classes that inherit from a specific class. @@ -447,8 +517,8 @@ def get_all_methods_in_application(self) -> Dict[str, Dict[str, PyCallable]]: """ result: Dict[str, Dict[str, PyCallable]] = {} for module in self.application.symbol_table.values(): - for class_sig, cls in module.classes.items(): - result[class_sig] = dict(cls.methods) + for class_sig, cls in module.types.items(): + result[class_sig] = dict(cls.callables) if module.functions: result.setdefault(module.module_name, {}).update(module.functions) return result @@ -465,7 +535,7 @@ def get_all_methods_in_class(self, qualified_class_name: str) -> Dict[str, PyCal Returns empty dict if class not found. """ cls = self.get_class(qualified_class_name) - return dict(cls.methods) if cls else {} + return dict(cls.callables) if cls else {} def get_method(self, qualified_class_name: str, qualified_method_name: str) -> PyCallable | None: """Return a specific method or module-level function by scope and name. @@ -478,7 +548,7 @@ def get_method(self, qualified_class_name: str, qualified_method_name: str) -> P attribute. Note: - Callables nested inside another callable (``inner_callables``) are not reachable via + Callables nested inside another callable (``PyCallable.callables``) are not reachable via this lookup — only top-level class methods and top-level module functions are. Note: @@ -546,60 +616,61 @@ def get_all_fields(self, qualified_class_name: str) -> List[PyClassAttribute]: return list(cls.attributes.values()) if cls else [] # ----------------------------------------------------------- bulk / projected accessors - def _iter_callables(self) -> Iterator[Tuple[PyCallable, "str | None", str]]: - """Yield ``(callable, class_signature, kind)`` for every callable in the application. + def _iter_callables(self) -> Iterator[Tuple[PyCallable, "str | None", str, PyModule]]: + """Yield ``(callable, class_signature, kind, module)`` for every callable in the application. Walks the in-memory symbol table the same way the Neo4j backend's ``MATCH (c:PyCallable)`` sees nodes: a callable is a ``"method"`` only when a class declares it directly (mirroring ``PY_HAS_METHOD``); module-level functions and functions nested inside a callable are ``"function"`` with a ``None`` class signature. The two backends therefore enumerate the - same set. + same set. The declaring module rides along so consumers can slice source text from + ``module.source`` (see :func:`_code_of`). """ - def from_callable(c: PyCallable): - for inner in c.inner_callables.values(): - yield inner, None, "function" - yield from from_callable(inner) - for inner_cls in c.inner_classes.values(): - yield from from_class(inner_cls) + def from_callable(c: PyCallable, module: PyModule): + for inner in c.callables.values(): + yield inner, None, "function", module + yield from from_callable(inner, module) + for inner_cls in c.types.values(): + yield from from_class(inner_cls, module) - def from_class(cls: PyClass): - for m in cls.methods.values(): - yield m, cls.signature, "method" - yield from from_callable(m) - for inner_cls in cls.inner_classes.values(): - yield from from_class(inner_cls) + def from_class(cls: PyClass, module: PyModule): + for m in cls.callables.values(): + yield m, cls.signature, "method", module + yield from from_callable(m, module) + for inner_cls in cls.types.values(): + yield from from_class(inner_cls, module) for module in self.application.symbol_table.values(): - for cls in module.classes.values(): - yield from from_class(cls) + for cls in module.types.values(): + yield from from_class(cls, module) for fn in module.functions.values(): - yield fn, None, "function" - yield from from_callable(fn) + yield fn, None, "function", module + yield from from_callable(fn, module) def get_callables_overview(self) -> List[PyCallableOverview]: """Return a lightweight overview of every callable in the application (see :meth:`PythonAnalysisBackend.get_callables_overview`).""" - return [_overview(c, class_sig, kind) for c, class_sig, kind in self._iter_callables()] + return [_overview(c, class_sig, kind) for c, class_sig, kind, _ in self._iter_callables()] def get_method_bodies(self, signatures: List[str]) -> Dict[str, str]: """Return ``{signature: code}`` for the requested signatures that exist.""" wanted = set(signatures) - return {c.signature: c.code for c, _, _ in self._iter_callables() if c.signature in wanted} + return {c.signature: _code_of(module, c) for c, _, _, module in self._iter_callables() if c.signature in wanted} def get_decorated_callables(self, markers: List[str]) -> List[PyCallableOverview]: """Return overviews of callables decorated with any of ``markers``.""" marker_set = set(markers) return [ _overview(c, class_sig, kind) - for c, class_sig, kind in self._iter_callables() + for c, class_sig, kind, _ in self._iter_callables() if marker_set.intersection(c.decorators or []) ] def get_callsites_for(self, signatures: List[str]) -> Dict[str, List[PyCallsite]]: """Return ``{signature: call_sites}`` for the requested signatures that exist.""" wanted = set(signatures) - return {c.signature: list(c.call_sites) for c, _, _ in self._iter_callables() if c.signature in wanted} + return {c.signature: list(c.call_sites) for c, _, _, _ in self._iter_callables() if c.signature in wanted} # ----------------------------------------------------------- callers/callees def get_all_callers(self, target_class_name: str, target_method_declaration: str) -> Dict: @@ -708,7 +779,7 @@ def get_class_call_graph( return [] return list(nx.edge_dfs(graph, source=method.signature)) edges: List[Tuple[str, str]] = [] - for method in cls.methods.values(): + for method in cls.callables.values(): if method.signature in graph: edges.extend(nx.edge_dfs(graph, source=method.signature)) return edges diff --git a/cldk/analysis/python/neo4j/neo4j_backend.py b/cldk/analysis/python/neo4j/neo4j_backend.py index e01e8176..b0cec195 100644 --- a/cldk/analysis/python/neo4j/neo4j_backend.py +++ b/cldk/analysis/python/neo4j/neo4j_backend.py @@ -17,9 +17,12 @@ """Neo4j-backed Python analysis backend (read-only Cypher client). A drop-in alternative to :class:`~cldk.analysis.python.codeanalyzer.PyCodeanalyzer`: it exposes the -**same query method surface** (the 21 methods of :class:`PythonAnalysisBackend`) so the -:class:`~cldk.analysis.python.PythonAnalysis` facade can delegate to either one, but every method -answers by running **Cypher over a live Neo4j graph** instead of walking the in-memory +**same query method surface** — the 21 :class:`PythonAnalysisBackend` accessors plus the six +:class:`~cldk.graph.provider.ProgramGraphProvider` primitives (``program_graph``, ``sdg_edges``, +``resolve_location``, ``source_slice``, ``callable_of``, ``max_level``) that ABC now also requires +(#270) — so the :class:`~cldk.analysis.python.PythonAnalysis` facade and the slice/flow +:class:`~cldk.graph.engine.Engine` can both delegate to either backend interchangeably. Every +method answers by running **Cypher over a live Neo4j graph** instead of walking the in-memory pydantic / NetworkX structures. Mirrors :class:`~cldk.analysis.typescript.neo4j.TSNeo4jBackend`. This class is purely a **query client**: it never builds the graph and has no dependency on the @@ -37,7 +40,7 @@ (also carrying its specific label ``:PyClass`` / ``:PyCallable`` / ``:PyExternal``); * a module is a ``:PyModule`` keyed by ``file_key`` (which equals the original ``PyModule.file_path`` and the symbol-table key); -* call-graph edges are ``(:PyCallable|:PyExternal)-[:PY_CALLS {weight, provenance}]->(...)`` with a +* call-graph edges are ``(:PyCallable|:PyExternal)-[:PY_CALLS {weight, prov}]->(...)`` with a constant ``CALL_DEP`` type; * class inheritance is ``(:PyClass)-[:PY_EXTENDS]->(:PyClass)`` (plus a ``base_classes`` property); * every project-owned node carries a ``_module`` provenance property, so a single database may hold @@ -45,8 +48,8 @@ ``(:PyApplication {name})-[:PY_HAS_MODULE]->(:PyModule)``. In-memory dict keys this backend reproduces exactly (the projection stores nodes by ``signature`` -only, so the keys are rebuilt from node properties): ``module.classes`` / ``inner_classes`` → -``signature``; ``module.functions`` / ``methods`` / ``inner_callables`` → short ``name``; +only, so the keys are rebuilt from node properties): ``module.types`` / nested ``types`` → +``signature``; ``module.functions`` / class ``callables`` / nested ``callables`` → short ``name``; ``attributes`` → ``name``. ``get_all_classes`` / ``get_class`` return **top-level** classes only (``PyModule-[:PY_DECLARES]->PyClass``), matching the in-memory backend. @@ -71,13 +74,14 @@ from __future__ import annotations import logging -from typing import Any, Dict, List, Tuple +from typing import Any, Dict, Iterable, List, Optional, Tuple import networkx as nx from codeanalyzer.schema import model_dump_json from cldk.analysis.python.backend import PythonAnalysisBackend from cldk.analysis.python.neo4j import reconstruct as R +from cldk.models.cpg import Edge as CpgEdge from cldk.models.python import ( PyApplication, PyCallEdge, @@ -88,7 +92,7 @@ PyClassAttribute, PyModule, ) -from cldk.utils.exceptions.exceptions import CodeanalyzerExecutionException +from cldk.utils.exceptions.exceptions import CldkSchemaMismatchException, CodeanalyzerExecutionException logger = logging.getLogger(__name__) @@ -108,6 +112,11 @@ class PyNeo4jBackend(PythonAnalysisBackend): ``--app-name`` the graph was loaded with (defaults to the project directory name). """ + #: The Neo4j graph schema this backend speaks — ``codeanalyzer-python`` stamps its + #: ``SCHEMA_VERSION`` onto every emitted ``:PyApplication`` node; construction fails fast on + #: any mismatch (see :meth:`_check_schema_version`), mirroring :class:`TSNeo4jBackend`. + SUPPORTED_GRAPH_SCHEMA = "2.0.0" + def __init__( self, neo4j_uri: str, @@ -133,6 +142,7 @@ def __init__( # fan-out, so reopening a session per query added real per-call overhead. Created lazily. self._session_obj: Any | None = None + self._check_schema_version() # The application's module file_keys, used to scope every query to this app. self._modules: List[str] = self._load_module_keys() # Lazily-built call graph cache (mirrors PyCodeanalyzer.call_graph). @@ -176,6 +186,25 @@ def _run(self, query: str, **params: Any) -> List[Dict[str, Any]]: self._close_session() raise + def _check_schema_version(self, expected: str | None = None, found: str | None = None) -> None: + """Fail fast unless the persisted graph's ``schema_version`` is the one this SDK speaks. + + Reads the scoped ``(:PyApplication).schema_version`` once (unless ``found`` is supplied, + as the tests do) and raises :class:`CldkSchemaMismatchException` on any mismatch — + including a graph whose application node carries no version at all (pre-2.0 emitters). + """ + expected = expected or self.SUPPORTED_GRAPH_SCHEMA + if found is None: + rows = self._run( + "MATCH (a:PyApplication {name: $app}) RETURN a.schema_version AS v LIMIT 1", + app=self.application_name, + ) + found = rows[0]["v"] if rows else None + if found != expected: + raise CldkSchemaMismatchException( + f"graph schema {found!r} in database, this SDK speaks {expected!r} — re-emit with codeanalyzer-python>=1.0.1" + ) + def _load_module_keys(self) -> List[str]: """The application's module ``file_key``s — the scope key for every other query.""" rows = self._run( @@ -184,6 +213,215 @@ def _load_module_keys(self) -> List[str]: ) return [r["k"] for r in rows] + # ===================================================================================== + # ProgramGraphProvider primitives (#270) — same app-scoping idiom as every other accessor + # in this file: anchor on (:PyApplication {name: $app})-[:PY_HAS_MODULE]->(:PyModule) and + # filter node-carrying queries by that module set. Folded into a single ``WITH ... AS mods`` + # prelude per query (rather than reusing the cached ``self._modules`` list other accessors + # build in ``__init__``) so every primitive stays a single round trip. + # ===================================================================================== + _MODULES_CTE = "MATCH (:PyApplication {name: $app})-[:PY_HAS_MODULE]->(m:PyModule) WITH collect(m.file_key) AS mods " + + def _sig_to_can(self) -> Dict[str, str]: + """Lazy, cached dotted-signature -> can:// id map, built from this app's ``PyCallable`` + nodes (each carries its minted ``can://...`` id in ``.id``). Only feeds ``_to_uri``'s + defensive ``#``-form fallback (see its docstring) — the real emitter's ``PyCFGNode.id`` + needs no such translation, but ``PyCallable.signature`` (dotted) -> ``PyCallable.id`` + (can://) is also how ``program_graph``/``source_slice`` resolve a caller-supplied can:// + callable id back to the ``signature`` a Cypher ``MATCH`` needs. + """ + m = getattr(self, "_sig_can_map", None) + if m is None: + rows = self._run( + self._MODULES_CTE + "MATCH (c:PyCallable) WHERE c._module IN mods RETURN c.signature AS sig, c.id AS id", + app=self.application_name, + ) + m = {r["sig"]: r["id"] for r in rows if r.get("id")} + self._sig_can_map = m + self._can_sig_map = {v: k for k, v in m.items()} + return m + + def _to_uri(self, cfg_node_id: str) -> str: + """Translate a raw ``PyCFGNode.id`` into the minted ``can://...@`` vertex id + the local backend's mixin would produce. + + Handles two shapes defensively: + + * The **real** codeanalyzer-python 1.0.2 emitter (verified against a live Neo4j + instance populated by the real analyzer+emitter — see #295) stores ``PyCFGNode.id`` + as the already-minted ``can://...@`` URI directly, with no ``#`` anywhere + in it. That id *is* the answer; no translation is needed or possible (a dotted-sig + lookup would only ever miss). + * The dotted ``"#"`` form documented in the analyzer repo's + ``schema.py`` comment (and this file's original brief) has not been observed on any + real graph. Kept as a defensive fallback in case a future/older emitter version + actually produces it. + """ + if "#" not in cfg_node_id: + return cfg_node_id + sig, _, key = cfg_node_id.partition("#") + can = self._sig_to_can().get(sig, sig) + return f"{can}@{key.removeprefix('@')}" + + def max_level(self) -> int: + """The deepest overlay actually present in the graph for this application (see the + module docstring's :class:`PyNeo4jBackend` — persisted ``max_level`` is upstream gap + codellm-devkit/codeanalyzer-python, filed in #270's Task 5).""" + scope = self._MODULES_CTE + app = self.application_name + if self._run( + scope + "MATCH (a)-[r:PY_PARAM_IN|PY_PARAM_OUT|PY_SUMMARY]->() WHERE a._module IN mods RETURN 1 AS one LIMIT 1", + app=app, + ): + return 4 + if self._run(scope + "MATCH (n:PyCFGNode) WHERE n._module IN mods RETURN 1 AS one LIMIT 1", app=app): + return 3 + if self._run(scope + "MATCH (s:PySymbol)-[r:PY_CALLS]->() WHERE s._module IN mods RETURN 1 AS one LIMIT 1", app=app): + return 2 + return 1 + + def program_graph(self, callable_uri: str) -> nx.MultiDiGraph: + """The per-callable CFG/CDG/DDG overlay, translated to can:// vertex ids. Parallel + cfg/cdg/ddg edges between the same vertex pair stay distinct (MultiDiGraph auto-keys + each ``add_edge``), matching :class:`~cldk.graph.provider.ProgramGraphProvider`'s + contract and the local backend's mixin. + """ + self._sig_to_can() + sig = self._can_sig_map.get(callable_uri) + g = nx.MultiDiGraph() + if sig is None: + return g + scope = self._MODULES_CTE + app = self.application_name + rows = self._run( + scope + "MATCH (c:PyCallable {signature:$sig})-[:PY_HAS_CFG_NODE]->(n:PyCFGNode) " + "WHERE c._module IN mods " + "RETURN n.id AS id, n.kind AS kind, n.start_line AS sl, n.end_line AS el " + "ORDER BY n.id", + app=app, + sig=sig, + ) + for r in rows: + g.add_node(self._to_uri(r["id"]), kind=r["kind"], span=(r["sl"], r["el"])) + for fam, rel in (("cfg", "PY_CFG_NEXT"), ("cdg", "PY_CDG"), ("ddg", "PY_DDG")): + edge_rows = self._run( + scope + f"MATCH (c:PyCallable {{signature:$sig}})-[:PY_HAS_CFG_NODE]->(a)-[r:{rel}]->(b) " + "WHERE c._module IN mods AND (c)-[:PY_HAS_CFG_NODE]->(b) " + "RETURN a.id AS src, b.id AS dst, r.kind AS kind, r.var AS var, r.prov AS prov " + "ORDER BY a.id, b.id", + app=app, + sig=sig, + ) + for r in edge_rows: + g.add_edge( + self._to_uri(r["src"]), + self._to_uri(r["dst"]), + family=fam, + kind=r.get("kind"), + var=r.get("var"), + prov=list(r.get("prov") or []), + ) + return g + + def sdg_edges(self) -> Iterable[CpgEdge]: + """Every application-level interprocedural edge (``PY_PARAM_IN``/``PY_PARAM_OUT``/ + ``PY_SUMMARY``), translated to can:// vertex ids and kinded (real graph edges carry no + ``kind`` of their own — the family name doubles as the kind, mirroring the local + backend's mixin so a boundary hop never surfaces as opaque "sdg").""" + self._sig_to_can() + scope = self._MODULES_CTE + app = self.application_name + out: List[CpgEdge] = [] + for kind, rel in (("param_in", "PY_PARAM_IN"), ("param_out", "PY_PARAM_OUT"), ("summary", "PY_SUMMARY")): + rows = self._run( + scope + f"MATCH (a:PyCFGNode)-[r:{rel}]->(b:PyCFGNode) " + "WHERE a._module IN mods AND b._module IN mods " + "RETURN a.id AS src, b.id AS dst, r.var AS var " + "ORDER BY a.id, b.id", + app=app, + ) + out.extend( + CpgEdge(src=self._to_uri(r["src"]), dst=self._to_uri(r["dst"]), kind=kind, var=r.get("var"), prov=[]) for r in rows + ) + return out + + def resolve_location(self, file: str, line: int, col: Optional[int] = None) -> List[str]: + """Vertex ids at a source location, ordered by parsed column — the local key encodes + ``line:col`` (e.g. ``"3:8"``), so column is recovered by parsing the key rather than + from a stored property, matching the local backend's ordering exactly. + + Parses the key out of ``_to_uri``'s *output* (always ``"@"``, no + ``#``), not the raw ``r["id"]`` — the real emitter's raw ``PyCFGNode.id`` already *is* + that translated form (see ``_to_uri``), so partitioning the raw id on ``#`` (as a first + version of this method did) would never find a separator and always parse an empty key. + + Accepts ``file`` as either the full stored ``_module`` key or just its basename/suffix + (``"mod.py"`` matching ``"pkg/mod.py"``) — the same latitude the local backend's mixin + gives a caller, so an identical seed string behaves identically on both backends. + """ + scope = self._MODULES_CTE + rows = self._run( + scope + "MATCH (n:PyCFGNode) WHERE n._module IN mods " + "AND (n._module = $file OR n._module ENDS WITH $suffix) AND n.start_line = $line " + "RETURN n.id AS id", + app=self.application_name, + file=file, + suffix="/" + file, + line=line, + ) + hits = [] + for r in rows: + uri = self._to_uri(r["id"]) + key = uri.partition("@")[2] + head = key.split("/")[0] + c = int(head.split(":")[1]) if ":" in head else -1 + if col is None or c == col: + hits.append(((line, c), uri)) + return [u for _, u in sorted(hits)] + + def source_slice(self, vertex_uri: str) -> Tuple[Optional[str], Optional[str]]: + """Lossy by contract: ``PyCFGNode`` carries no byte offsets and modules carry no source + in the graph, so ``code`` is always ``None`` (Task 6's parity harness treats this as the + documented exception). The location half follows the same three-way contract as the + local backend's mixin — never fabricated by parsing the vertex key's shape: + + * the vertex doesn't exist (unknown callable, or no matching ``PyCFGNode``) → ``(None, None)``; + * it exists but carries no ``start_line`` (a synthetic ``@entry``/``@exit``/ + ``@formal_in:N``/``@actual_*`` port) → ``(module_path, None)``; + * it exists with a ``start_line`` → ``(f"{module_path}:{start_line}", None)``. + + Matches the EXACT vertex by comparing ``_to_uri(n.id)`` (which bridges both the real + emitter's already-minted can:// ``PyCFGNode.id`` and the defensive dotted-sig ``#`` form) + against ``vertex_uri`` — scoped to the owning callable so this stays a single, cheap + round trip rather than a whole-application node scan. + """ + self._sig_to_can() + cid = self.callable_of(vertex_uri) + sig = self._can_sig_map.get(cid) + if sig is None: + return (None, None) + scope = self._MODULES_CTE + rows = self._run( + scope + "MATCH (c:PyCallable {signature:$sig})-[:PY_HAS_CFG_NODE]->(n:PyCFGNode) " + "WHERE c._module IN mods " + "RETURN n.id AS id, n.start_line AS sl, n._module AS mod", + app=self.application_name, + sig=sig, + ) + for r in rows: + if self._to_uri(r["id"]) != vertex_uri: + continue + if r.get("sl") is None: + return (r["mod"], None) + return (f"{r['mod']}:{r['sl']}", None) + return (None, None) + + def callable_of(self, vertex_uri: str) -> Optional[str]: + """The owning callable's can:// id — vertex ids are ``"@"``, so this + is a partition at the first ``@`` (can:// ids never contain one themselves).""" + head, sep, _ = vertex_uri.partition("@") + return head if sep else vertex_uri + # ===================================================================================== # Reconstruction helpers — fetch a node's children over Cypher, then assemble via R. # ===================================================================================== @@ -239,7 +477,7 @@ def _module_full(self, props: Dict[str, Any]) -> PyModule: classes: Dict[str, PyClass] = {} for r in self._run("MATCH (:PyModule {file_key: $fk})-[:PY_DECLARES]->(c:PyClass) RETURN properties(c) AS p", fk=file_key): c = self._class_full(r["p"]) - classes[c.signature] = c # module.classes keyed by signature + classes[c.signature] = c # module.types keyed by signature functions: Dict[str, PyCallable] = {} for r in self._run("MATCH (:PyModule {file_key: $fk})-[:PY_DECLARES]->(f:PyCallable) RETURN properties(f) AS p", fk=file_key): fn = self._callable_full(r["p"]) @@ -305,7 +543,7 @@ def get_python_module(self, file_path: str) -> PyModule | None: return self._module_full(rows[0]["p"]) if rows else None def get_python_file(self, qualified_class_name: str) -> str | None: - # Only top-level classes are in the in-memory _class_to_file map (module.classes). + # Only top-level classes are in the in-memory _class_to_file map (module.types). rows = self._run( "MATCH (:PyModule)-[:PY_DECLARES]->(c:PyClass {signature: $sig}) WHERE c._module IN $mods RETURN c._module AS fk LIMIT 1", sig=qualified_class_name, @@ -320,10 +558,10 @@ def _call_edges(self) -> List[PyCallEdge]: """The application's call edges as ``PyCallEdge`` records (``PyApplication.call_graph``).""" return [ PyCallEdge( - source=r["src"], - target=r["tgt"], + src=r["src"], + dst=r["tgt"], weight=r["p"].get("weight", 1), - provenance=list(r["p"].get("provenance", []) or []), + prov=list(r["p"].get("prov", []) or []), ) for r in self._call_rows() ] @@ -332,7 +570,7 @@ def _build_call_graph(self) -> nx.DiGraph: graph = nx.DiGraph() for r in self._call_rows(): p = r["p"] - graph.add_edge(r["src"], r["tgt"], type="CALL_DEP", weight=p.get("weight", 1), provenance=tuple(p.get("provenance", []) or [])) + graph.add_edge(r["src"], r["tgt"], type="CALL_DEP", weight=p.get("weight", 1), provenance=tuple(p.get("prov", []) or [])) return graph def get_call_graph(self) -> nx.DiGraph: @@ -370,7 +608,7 @@ def get_class_call_graph(self, qualified_class_name: str, method_signature: str return [] return list(nx.edge_dfs(graph, source=method.signature)) edges: List[Tuple[str, str]] = [] - for method in cls.methods.values(): + for method in cls.callables.values(): if method.signature in graph: edges.extend(nx.edge_dfs(graph, source=method.signature)) return edges @@ -399,7 +637,7 @@ def get_class(self, qualified_class_name: str) -> PyClass | None: def get_all_nested_classes(self, qualified_class_name: str) -> List[PyClass]: cls = self.get_class(qualified_class_name) - return list(cls.inner_classes.values()) if cls else [] + return list(cls.types.values()) if cls else [] def get_all_sub_classes(self, qualified_class_name: str) -> Dict[str, PyClass]: cls = self.get_class(qualified_class_name) @@ -426,15 +664,15 @@ def get_extended_classes(self, qualified_class_name: str) -> List[str]: def get_all_methods_in_application(self) -> Dict[str, Dict[str, PyCallable]]: result: Dict[str, Dict[str, PyCallable]] = {} for module in self.get_symbol_table().values(): - for class_sig, cls in module.classes.items(): - result[class_sig] = dict(cls.methods) + for class_sig, cls in module.types.items(): + result[class_sig] = dict(cls.callables) if module.functions: result.setdefault(module.module_name, {}).update(module.functions) return result def get_all_methods_in_class(self, qualified_class_name: str) -> Dict[str, PyCallable]: cls = self.get_class(qualified_class_name) - return dict(cls.methods) if cls else {} + return dict(cls.callables) if cls else {} def _get_module_functions(self, module_name: str) -> Dict[str, PyCallable]: """Fetch a module's top-level functions by ``module_name`` (not ``file_key``) — the scope @@ -458,7 +696,7 @@ def get_method(self, qualified_class_name: str, qualified_method_name: str) -> P is treated as a module name and resolved against that module's top-level functions. """ cls = self.get_class(qualified_class_name) - methods = dict(cls.methods) if cls is not None else self._get_module_functions(qualified_class_name) + methods = dict(cls.callables) if cls is not None else self._get_module_functions(qualified_class_name) if qualified_method_name in methods: return methods[qualified_method_name] for sig, callable_ in methods.items(): diff --git a/cldk/analysis/python/neo4j/reconstruct.py b/cldk/analysis/python/neo4j/reconstruct.py index 6c681763..1dd6dbe9 100644 --- a/cldk/analysis/python/neo4j/reconstruct.py +++ b/cldk/analysis/python/neo4j/reconstruct.py @@ -162,7 +162,11 @@ def callable_( inner_classes: Dict[str, PyClass] | None = None, local_variables: List[PyVariableDeclaration] | None = None, ) -> PyCallable: - """Rebuild a :class:`PyCallable` from a ``:PyCallable`` node plus its fetched children.""" + """Rebuild a :class:`PyCallable` from a ``:PyCallable`` node plus its fetched children. + + The node's ``code`` property (source text) has no schema-2.0.0 model field — source lives + once on the module, sliced by ``span`` — so it is not reconstructed here. + """ return PyCallable( name=props.get("name", ""), path=props.get("path", ""), @@ -171,14 +175,13 @@ def callable_( decorators=list(props.get("decorators", []) or []), parameters=parameters(props), return_type=props.get("return_type"), - code=props.get("code"), start_line=props.get("start_line", -1), end_line=props.get("end_line", -1), code_start_line=props.get("code_start_line", -1), accessed_symbols=accessed_symbols(props), call_sites=call_sites or [], - inner_callables=inner_callables or {}, - inner_classes=inner_classes or {}, + callables=inner_callables or {}, + types=inner_classes or {}, local_variables=local_variables or [], cyclomatic_complexity=props.get("cyclomatic_complexity", 0), ) @@ -196,11 +199,10 @@ def class_( name=props.get("name", ""), signature=props.get("signature", ""), comments=comments(props), - code=props.get("code"), base_classes=list(props.get("base_classes", []) or []), - methods=methods or {}, + callables=methods or {}, attributes=attributes or {}, - inner_classes=inner_classes or {}, + types=inner_classes or {}, start_line=props.get("start_line", -1), end_line=props.get("end_line", -1), ) @@ -225,7 +227,7 @@ def module( module_name=props.get("module_name", ""), imports=imports or [], comments=[], - classes=classes or {}, + types=classes or {}, functions=functions or {}, variables=variables or [], content_hash=props.get("content_hash"), diff --git a/cldk/analysis/python/python_analysis.py b/cldk/analysis/python/python_analysis.py index 32623095..41dd7322 100644 --- a/cldk/analysis/python/python_analysis.py +++ b/cldk/analysis/python/python_analysis.py @@ -57,6 +57,7 @@ from cldk.analysis.python.backend import PythonAnalysisBackend from cldk.analysis.python.codeanalyzer import PyCodeanalyzer from cldk.analysis.python.neo4j import PyNeo4jBackend +from cldk.graph import Engine from cldk.models.python import ( PyApplication, PyCallable, @@ -234,6 +235,31 @@ def get_raw_ast(self, source_code: str) -> Tree: """ return self.treesitter_python.get_raw_ast(source_code) + # -----[ L3/L4 slice & flow verbs (#270) ]----- + def slice_backward(self, seed, *, edges=("cfg", "cdg", "ddg"), + interprocedural=None, strict=False): + """Backward program slice from ``seed`` ('file:line[:col]', can:// URI, or body node).""" + return Engine(self.backend).slice_backward( + seed, edges=edges, interprocedural=interprocedural, strict=strict) + + def slice_forward(self, seed, *, edges=("cfg", "cdg", "ddg"), + interprocedural=None, strict=False): + """Forward program slice from ``seed``.""" + return Engine(self.backend).slice_forward( + seed, edges=edges, interprocedural=interprocedural, strict=strict) + + def flows_to(self, source_seed, sink_seed, *, strict=False): + """Dataflow reachability with witness paths (interprocedural at L4).""" + return Engine(self.backend).flows_to(source_seed, sink_seed, strict=strict) + + def def_use(self, seed, *, strict=False): + """Definitions/uses reachable from ``seed`` over the dataflow graph.""" + return Engine(self.backend).def_use(seed, strict=strict) + + def control_deps(self, seed, *, strict=False): + """The control-dependence ancestors of ``seed`` (intraprocedural by design).""" + return Engine(self.backend).control_deps(seed, strict=strict) + # -----[ application view ]----- def get_application_view(self) -> PyApplication: """Return the complete analyzed application model. diff --git a/cldk/analysis/typescript/neo4j/neo4j_backend.py b/cldk/analysis/typescript/neo4j/neo4j_backend.py index 3025e1ca..a79633ac 100644 --- a/cldk/analysis/typescript/neo4j/neo4j_backend.py +++ b/cldk/analysis/typescript/neo4j/neo4j_backend.py @@ -28,25 +28,39 @@ deployment wants, where a third-party job (e.g. inside Kubernetes) loads the graph out of band and the SDK only reads it. -The graph is the one ``codeanalyzer-typescript`` emits with ``--emit neo4j`` -(schema: ``codeanalyzer-ts/schema.neo4j.json``). Populating it always happens out -of band — never from this backend. - -Identity model (must match the in-memory backend): - -* a callable/class/interface/enum/type-alias is a ``:Symbol`` keyed by ``signature``; -* call-graph edges are ``(:Symbol)-[:CALLS]->(:Symbol|:External)``; -* every project-owned node carries a ``_module`` provenance property, so a single - database may hold several applications — all queries here are scoped to this - backend's application by the set of its module ``file_key``s. - -Parity caveats (inherent to what the projection stores, not bugs): - -* ``CALLS`` edge ``tags`` only round-trip the three keys the projection keeps +The graph is the one ``codeanalyzer-typescript>=1.0.0`` emits with ``--emit +neo4j`` (**graph schema 2.0.0**). Populating it always happens out of band — +never from this backend. On first use the backend reads +``(:Application).schema_version`` and fails fast if it is not the schema this SDK +speaks (see :meth:`_check_schema_version`). + +Identity model (graph schema 2.0.0 — must match the in-memory backend): + +* every projected node is a ``:CanNode`` carrying a canonical ``id`` (a ``can://`` + URI) as its merge key; TypeScript nodes wear a twin ``:TS*`` label + (``:CanNode:TSClass``, ``:CanNode:TSCallable``, ...); +* a callable/class/interface/enum/type-alias still carries a ``signature`` + *property*, which is what the SDK's public accessors key on; +* call-graph edges are ``(:TSCallable)-[:TS_CALLS]->(:TSCallable|:TSExternal)``; +* call *sites* are body nodes — ``(:TSBodyNode {kind: "call"})`` reached via + ``TS_HAS_BODY_NODE`` and resolved through ``TS_RESOLVES_TO`` — not standalone + call-site nodes; +* module nodes carry a ``_module`` property (the project-relative path); this + backend further *assumes* — to-verify against a live 1.0.0 graph, see the + ``VERIFY(2.0.0-e2e)`` markers — that every project-owned node carries the same + ``_module`` provenance property, so a single database may hold several + applications and all queries here can be scoped to this backend's application + by the set of its module ``_module`` keys. + +Parity caveats (inherent to what schema 2.0.0 projects, not bugs): + +* ``TS_CALLS`` edge ``tags`` only round-trip the three keys the projection keeps (``ts.dispatch`` / ``ts.external`` / ``ts.module``); -* ``get_imports`` / ``get_all_exports`` are reconstructed from the *aggregated* - ``IMPORTS`` / ``RE_EXPORTS`` edges (individual bindings, aliases and positions - are not stored); +* decorators, class/interface attributes, module imports/exports and variable + declarations are **not projected** into graph schema 2.0.0 — the accessors for + them raise :class:`NotImplementedError` (there is no in-memory/JSON backend to + fall back to from a read-only Cypher client), and the reconstructed + ``TSClass`` / ``TSModule`` objects carry empty collections for those fields; * comments collapse to a single docstring, type-parameters keep only their names. """ @@ -79,11 +93,16 @@ TSTypeAlias, TSVariableDeclaration, ) -from cldk.utils.exceptions.exceptions import CodeanalyzerExecutionException +from cldk.utils.exceptions.exceptions import CldkSchemaMismatchException, CodeanalyzerExecutionException, CodeanalyzerUsageException logger = logging.getLogger(__name__) +def _unprojected(feature: str) -> NotImplementedError: + """The uniform error for an accessor whose vocabulary graph schema 2.0.0 does not project.""" + return NotImplementedError(f"{feature} is not projected in graph schema {TSNeo4jBackend.SUPPORTED_GRAPH_SCHEMA} — use the in-memory (JSON) backend") + + class TSNeo4jBackend(TSAnalysisBackend): """Query the application view of a TypeScript project over Neo4j (Cypher), read-only. @@ -95,10 +114,14 @@ class TSNeo4jBackend(TSAnalysisBackend): neo4j_uri: Bolt URI of the Neo4j server (e.g. ``bolt://localhost:7687``). neo4j_username / neo4j_password: Credentials (read-only is sufficient). neo4j_database: Database name (None ⇒ server default). - application_name: The ``:Application`` anchor name to scope every query to. Matches the - ``--app-name`` the graph was loaded with (defaults to the project directory name). + application_name: The ``:Application`` anchor to scope every query to. Matched against the + tail of the application's ``can://`` ``id`` (the ``--app-name`` the graph was loaded + with; defaults to the project directory name). """ + #: The graph schema version this backend speaks; enforced on first use. + SUPPORTED_GRAPH_SCHEMA = "2.0.0" + def __init__( self, neo4j_uri: str, @@ -118,7 +141,17 @@ def __init__( self._database = neo4j_database self._driver = GraphDatabase.driver(neo4j_uri, auth=(neo4j_username, neo4j_password)) - # The application's module file_keys, used to scope every query to this app. + # Fail fast if the persisted graph isn't the schema this SDK speaks. This runs *before* + # the application-id resolution on purpose: a pre-2.0.0 graph has no `can://` ids, so + # resolving first would report "no application found" instead of the (more actionable) + # schema mismatch. + self._check_schema_version() + + # Resolve `application_name` to exactly one Application `id` (guards against a shared + # database where several apps' ids share the same trailing path segment). + self._app_id: str = self._resolve_application_id() + + # The application's module `_module` keys, used to scope every query to this app. self._modules: List[str] = self._load_module_keys() # -----[ lifecycle ]----- @@ -137,43 +170,73 @@ def _run(self, query: str, **params: Any) -> List[Dict[str, Any]]: with self._driver.session(database=self._database) as session: return [record.data() for record in session.run(query, **params)] - def _load_module_keys(self) -> List[str]: - rows = self._run( - "MATCH (:Application {name: $app})-[:HAS_MODULE]->(m:Module) RETURN m.file_key AS k", - app=self.application_name, - ) - return [r["k"] for r in rows] + def _check_schema_version(self, expected: str | None = None, found: str | None = None) -> None: + """Fail fast unless the persisted graph's ``schema_version`` is the one this SDK speaks. - # -----[ child-fetch helpers (reconstruction) ]----- - def _decorators_of(self, signature: str) -> List[TSDecorator]: + Reads ``(:Application).schema_version`` once (unless ``found`` is supplied, as the tests + do) and raises :class:`CldkSchemaMismatchException` on any mismatch — including a graph + with no ``:Application`` node at all. + """ + expected = expected or self.SUPPORTED_GRAPH_SCHEMA + if found is None: + rows = self._run("MATCH (a:Application) RETURN a.schema_version AS v LIMIT 1") + found = rows[0]["v"] if rows else None + if found != expected: + raise CldkSchemaMismatchException( + f"graph schema {found!r} in database, this SDK speaks {expected!r} — re-analyze with codeanalyzer-typescript>=1.0.0" + ) + + def _resolve_application_id(self) -> str: + """Resolve ``application_name`` to exactly one ``:Application`` node's ``can://`` id. + + The suffix match (``a.id ENDS WITH "/" + $app``) can bind multiple applications in a + shared database — e.g. two repos whose ids both end in ``/frontend`` — which would + silently merge their module scopes. Raise :class:`CodeanalyzerUsageException` unless the + match is unique; the caller disambiguates by passing a longer trailing path (any suffix + of the ``can://`` id starting at a ``/`` boundary works as ``application_name``). + """ rows = self._run( - "MATCH (s:Symbol {signature: $sig})-[r:DECORATED_BY]->(d:Decorator) " "RETURN properties(d) AS node, properties(r) AS edge ORDER BY r.start_line", - sig=signature, + 'MATCH (a:Application) WHERE a.id ENDS WITH "/" + $app RETURN a.id AS id ORDER BY id', + app=self.application_name, ) - return [R.decorator(r["node"], r["edge"]) for r in rows] + if not rows: + raise CodeanalyzerUsageException( + f"no :Application found whose id ends with '/{self.application_name}' — check application_name (the --app-name the graph was loaded with)." + ) + if len(rows) > 1: + candidates = ", ".join(r["id"] for r in rows) + raise CodeanalyzerUsageException( + f"application_name '{self.application_name}' is ambiguous: it matches {len(rows)} applications in this database ({candidates}). " + "Pass a longer trailing path of the intended application's id to disambiguate." + ) + return rows[0]["id"] - def _attribute_decorators(self, attr_id: str) -> List[TSDecorator]: + def _load_module_keys(self) -> List[str]: + # VERIFY(2.0.0-e2e): every project-owned node is assumed to carry a `_module` provenance + # property (as in the pre-2.0.0 projection); all the `x._module IN $mods` scoping below + # depends on it — validate against a live 1.0.0 graph (Task 9). rows = self._run( - "MATCH (a:Attribute {id: $id})-[r:DECORATED_BY]->(d:Decorator) " "RETURN properties(d) AS node, properties(r) AS edge ORDER BY r.start_line", - id=attr_id, + "MATCH (a:Application {id: $app_id})-[:TS_HAS_MODULE]->(m:TSModule) RETURN m._module AS k", + app_id=self._app_id, ) - return [R.decorator(r["node"], r["edge"]) for r in rows] + return [r["k"] for r in rows] + # -----[ child-fetch helpers (reconstruction) ]----- def _callsites_of(self, signature: str) -> List[TSCallsite]: rows = self._run( - "MATCH (c:Callable {signature: $sig})-[:HAS_CALLSITE]->(cs:CallSite) " "RETURN properties(cs) AS p ORDER BY cs.start_line, cs.start_column", + "MATCH (c:TSCallable {signature: $sig})-[:TS_HAS_BODY_NODE]->(cs:TSBodyNode {kind: 'call'}) " "RETURN properties(cs) AS p ORDER BY cs.start_line, cs.start_column", sig=signature, ) return [R.callsite(r["p"]) for r in rows] def _callable_full(self, props: Dict[str, Any]) -> TSCallable: sig = props["signature"] - # Symbol-keyed containers are keyed by signature (matching the analyzer's dict keys). - inner_callables = {p["signature"]: self._callable_full(p) for p in self._children(sig, "DECLARES", "Callable")} - inner_classes = {p["signature"]: self._class_full(p) for p in self._children(sig, "DECLARES", "Class")} + # Nested-declaration containers are keyed by signature (matching the analyzer's dict keys). + inner_callables = {p["signature"]: self._callable_full(p) for p in self._children(sig, "TS_DECLARES", "TSCallable")} + inner_classes = {p["signature"]: self._class_full(p) for p in self._children(sig, "TS_DECLARES", "TSClass")} return R.callable_( props, - decorators=self._decorators_of(sig), + decorators=[], # decorators are not projected in graph schema 2.0.0 call_sites=self._callsites_of(sig), inner_callables=inner_callables, inner_classes=inner_classes, @@ -193,41 +256,37 @@ def _method_key(props: Dict[str, Any]) -> str: def _class_full(self, props: Dict[str, Any]) -> TSClass: sig = props["signature"] - # methods keyed by the analyzer's method-key; inner_classes by signature; attributes by name. - methods = {self._method_key(p): self._callable_full(p) for p in self._members(sig, "HAS_METHOD", "Callable")} - attributes: Dict[str, TSClassAttribute] = {} - for p in self._members(sig, "HAS_ATTRIBUTE", "Attribute"): - attributes[p["name"]] = R.attribute(p, self._attribute_decorators(p.get("id", ""))) - inner_classes = {p["signature"]: self._class_full(p) for p in self._children(sig, "DECLARES", "Class")} + # methods keyed by the analyzer's method-key; inner_classes by signature. Attributes and + # decorators are not projected in graph schema 2.0.0, so those collections stay empty. + methods = {self._method_key(p): self._callable_full(p) for p in self._members(sig, "TS_HAS_METHOD", "TSCallable")} + inner_classes = {p["signature"]: self._class_full(p) for p in self._children(sig, "TS_DECLARES", "TSClass")} return R.class_( props, - decorators=self._decorators_of(sig), + decorators=[], methods=methods, - attributes=attributes, + attributes={}, inner_classes=inner_classes, ) def _interface_full(self, props: Dict[str, Any]) -> TSInterface: sig = props["signature"] - methods = {self._method_key(p): self._callable_full(p) for p in self._members(sig, "HAS_METHOD", "Callable")} - properties: Dict[str, TSClassAttribute] = {} - for p in self._members(sig, "HAS_ATTRIBUTE", "Attribute"): - properties[p["name"]] = R.attribute(p, self._attribute_decorators(p.get("id", ""))) - return R.interface(props, methods=methods, properties=properties) + methods = {self._method_key(p): self._callable_full(p) for p in self._members(sig, "TS_HAS_METHOD", "TSCallable")} + # interface properties are attributes — not projected in graph schema 2.0.0. + return R.interface(props, methods=methods, properties={}) def _children(self, signature: str, rel: str, label: str) -> List[Dict[str, Any]]: """Property maps of ``label`` nodes reached from a symbol via ``rel`` (one hop), in declaration (source) order.""" rows = self._run( - f"MATCH (s:Symbol {{signature: $sig}})-[:{rel}]->(n:{label}) " "RETURN properties(n) AS p ORDER BY n.start_line, n.name", + f"MATCH (s:CanNode {{signature: $sig}})-[:{rel}]->(n:{label}) " "RETURN properties(n) AS p ORDER BY n.start_line, n.name", sig=signature, ) return [r["p"] for r in rows] def _members(self, signature: str, rel: str, label: str) -> List[Dict[str, Any]]: - """Property maps of member ``label`` nodes (methods/attributes), in declaration order.""" + """Property maps of member ``label`` nodes (methods), in declaration order.""" rows = self._run( - f"MATCH (s:Symbol {{signature: $sig}})-[:{rel}]->(n:{label}) " "RETURN properties(n) AS p ORDER BY n.start_line, n.name", + f"MATCH (s:CanNode {{signature: $sig}})-[:{rel}]->(n:{label}) " "RETURN properties(n) AS p ORDER BY n.start_line, n.name", sig=signature, ) return [r["p"] for r in rows] @@ -255,47 +314,45 @@ def get_modules(self) -> List[TSModule]: def get_external_symbols(self) -> Dict[str, TSExternalSymbol]: rows = self._run( - "MATCH (s:Symbol)-[:CALLS]->(e:External) WHERE s._module IN $mods " + "MATCH (s:CanNode)-[:TS_CALLS]->(e:TSExternal) WHERE s._module IN $mods " "RETURN DISTINCT properties(e) AS p " "UNION " - "MATCH (cs:CallSite)-[:RESOLVES_TO]->(e:External) WHERE cs._module IN $mods " + "MATCH (cs:TSBodyNode {kind: 'call'})-[:TS_RESOLVES_TO]->(e:TSExternal) WHERE cs._module IN $mods " "RETURN DISTINCT properties(e) AS p", mods=self._modules, ) return {r["p"]["signature"]: R.external(r["p"]) for r in rows} def get_synthesized_callables(self) -> Dict[str, TSSynthesizedCallable]: - """Anonymous-callback endpoints minted as ``:AnonymousCallable`` nodes (keyed by signature), - scoped to this application's modules.""" + """Anonymous-callback endpoints minted as ``:TSAnonymousCallable`` nodes (keyed by + signature), scoped to this application's modules.""" rows = self._run( - "MATCH (a:AnonymousCallable) WHERE a._module IN $mods RETURN DISTINCT properties(a) AS p", + "MATCH (a:TSAnonymousCallable) WHERE a._module IN $mods RETURN DISTINCT properties(a) AS p", mods=self._modules, ) return {r["p"]["signature"]: R.synthesized(r["p"]) for r in rows} def get_typescript_file(self, qualified_name: str) -> str | None: rows = self._run( - "MATCH (s:Symbol {signature: $sig}) WHERE s._module IN $mods RETURN s._module AS m LIMIT 1", + "MATCH (s:CanNode {signature: $sig}) WHERE s._module IN $mods RETURN s._module AS m LIMIT 1", sig=qualified_name, mods=self._modules, ) return rows[0]["m"] if rows else None def get_typescript_module(self, file_path: str) -> TSModule | None: - rows = self._run("MATCH (m:Module {file_key: $key}) RETURN properties(m) AS p", key=file_path) + rows = self._run("MATCH (m:TSModule {_module: $key}) RETURN properties(m) AS p", key=file_path) if not rows: return None props = rows[0]["p"] - # All symbol containers are keyed by signature (matching the analyzer's dict keys). - classes = {p["signature"]: self._class_full(p) for p in self._module_decls(file_path, "Class")} - interfaces = {p["signature"]: self._interface_full(p) for p in self._module_decls(file_path, "Interface")} - enums = {p["signature"]: R.enum(p) for p in self._module_decls(file_path, "Enum")} - type_aliases = {p["signature"]: R.type_alias(p) for p in self._module_decls(file_path, "TypeAlias")} - functions = {p["signature"]: self._callable_full(p) for p in self._module_decls(file_path, "Callable")} - namespaces = {p["signature"]: self._namespace_full(p) for p in self._module_decls(file_path, "Namespace")} - variables = self._module_variables(file_path) - imports = self._module_imports(file_path) - exports = self._module_exports(file_path) + # All declaration containers are keyed by signature (matching the analyzer's dict keys). + classes = {p["signature"]: self._class_full(p) for p in self._module_decls(file_path, "TSClass")} + interfaces = {p["signature"]: self._interface_full(p) for p in self._module_decls(file_path, "TSInterface")} + enums = {p["signature"]: R.enum(p) for p in self._module_decls(file_path, "TSEnum")} + type_aliases = {p["signature"]: R.type_alias(p) for p in self._module_decls(file_path, "TSTypeAlias")} + functions = {p["signature"]: self._callable_full(p) for p in self._module_decls(file_path, "TSCallable")} + namespaces = {p["signature"]: self._namespace_full(p) for p in self._module_decls(file_path, "TSNamespace")} + # variables / imports / exports are not projected in graph schema 2.0.0. return R.module( props, classes=classes, @@ -304,31 +361,27 @@ def get_typescript_module(self, file_path: str) -> TSModule | None: type_aliases=type_aliases, functions=functions, namespaces=namespaces, - variables=variables, - imports=imports, - exports=exports, + variables=[], + imports=[], + exports=[], ) - def _module_decls(self, file_key: str, label: str) -> List[Dict[str, Any]]: + def _module_decls(self, module_key: str, label: str) -> List[Dict[str, Any]]: rows = self._run( - f"MATCH (m:Module {{file_key: $key}})-[:DECLARES]->(n:{label}) " "RETURN properties(n) AS p ORDER BY n.start_line, n.name", - key=file_key, + f"MATCH (m:TSModule {{_module: $key}})-[:TS_DECLARES]->(n:{label}) " "RETURN properties(n) AS p ORDER BY n.start_line, n.name", + key=module_key, ) return [r["p"] for r in rows] def _namespace_full(self, props: Dict[str, Any]): sig = props["signature"] - classes = {p["signature"]: self._class_full(p) for p in self._children(sig, "DECLARES", "Class")} - interfaces = {p["signature"]: self._interface_full(p) for p in self._children(sig, "DECLARES", "Interface")} - enums = {p["signature"]: R.enum(p) for p in self._children(sig, "DECLARES", "Enum")} - type_aliases = {p["signature"]: R.type_alias(p) for p in self._children(sig, "DECLARES", "TypeAlias")} - functions = {p["signature"]: self._callable_full(p) for p in self._children(sig, "DECLARES", "Callable")} - namespaces = {p["signature"]: self._namespace_full(p) for p in self._children(sig, "DECLARES", "Namespace")} - rows = self._run( - "MATCH (s:Symbol {signature: $sig})-[:DECLARES_VAR]->(v:Variable) RETURN properties(v) AS p", - sig=sig, - ) - variables = [R.variable(r["p"]) for r in rows] + classes = {p["signature"]: self._class_full(p) for p in self._children(sig, "TS_DECLARES", "TSClass")} + interfaces = {p["signature"]: self._interface_full(p) for p in self._children(sig, "TS_DECLARES", "TSInterface")} + enums = {p["signature"]: R.enum(p) for p in self._children(sig, "TS_DECLARES", "TSEnum")} + type_aliases = {p["signature"]: R.type_alias(p) for p in self._children(sig, "TS_DECLARES", "TSTypeAlias")} + functions = {p["signature"]: self._callable_full(p) for p in self._children(sig, "TS_DECLARES", "TSCallable")} + namespaces = {p["signature"]: self._namespace_full(p) for p in self._children(sig, "TS_DECLARES", "TSNamespace")} + # namespace-level variables are not projected in graph schema 2.0.0. return R.namespace( props, classes=classes, @@ -337,52 +390,13 @@ def _namespace_full(self, props: Dict[str, Any]): type_aliases=type_aliases, functions=functions, namespaces=namespaces, - variables=variables, - ) - - def _module_variables(self, file_key: str) -> List[TSVariableDeclaration]: - rows = self._run( - "MATCH (m:Module {file_key: $key})-[:DECLARES_VAR]->(v:Variable) RETURN properties(v) AS p", - key=file_key, - ) - return [R.variable(r["p"]) for r in rows] - - def _module_imports(self, file_key: str) -> List[TSImport]: - """Best-effort: synthesize one TSImport per imported name on each aggregated IMPORTS edge. - - The projection collapses every binding to a module-pair into a single edge carrying - ``imported_names`` / ``import_kinds`` / ``is_type_only``, so per-binding aliases, kinds - and positions are not recoverable. - """ - rows = self._run( - "MATCH (m:Module {file_key: $key})-[r:IMPORTS]->(t) " "RETURN coalesce(t.file_key, t.name) AS target, properties(r) AS edge", - key=file_key, - ) - out: List[TSImport] = [] - for r in rows: - edge = r["edge"] - kinds = edge.get("import_kinds", []) or [] - kind = kinds[0] if len(kinds) == 1 else "named" - type_only = edge.get("is_type_only", False) - names = edge.get("imported_names", []) or [] - if not names: - out.append(TSImport(module=r["target"], name="", import_kind=kind, is_type_only=type_only)) - for name in names: - out.append(TSImport(module=r["target"], name=name, import_kind=kind, is_type_only=type_only)) - return out - - def _module_exports(self, file_key: str) -> List[TSExport]: - """Best-effort: only re-exports survive as edges (local exports become ``is_exported`` props).""" - rows = self._run( - "MATCH (m:Module {file_key: $key})-[:RE_EXPORTS]->(t) " "RETURN coalesce(t.file_key, t.name) AS target", - key=file_key, + variables=[], ) - return [TSExport(module=r["target"], name="*", export_kind="re_export") for r in rows] # -----[ call graph ]----- def _call_edges(self) -> List[TSCallEdge]: rows = self._run( - "MATCH (s:Symbol)-[r:CALLS]->(t:Symbol) WHERE s._module IN $mods " "RETURN s.signature AS src, t.signature AS tgt, properties(r) AS edge", + "MATCH (s:CanNode)-[r:TS_CALLS]->(t:CanNode) WHERE s._module IN $mods " "RETURN s.signature AS src, t.signature AS tgt, properties(r) AS edge", mods=self._modules, ) return [ @@ -398,7 +412,7 @@ def _call_edges(self) -> List[TSCallEdge]: @staticmethod def _edge_tags(edge: Dict[str, Any]) -> Dict[str, str]: - """Invert the flattened CALLS-edge tag props back into the ``ts.*`` tag dict.""" + """Invert the flattened TS_CALLS-edge tag props back into the ``ts.*`` tag dict.""" tags: Dict[str, str] = {} if edge.get("dispatch") is not None: tags["ts.dispatch"] = edge["dispatch"] @@ -409,7 +423,7 @@ def _edge_tags(edge: Dict[str, Any]) -> Dict[str, str]: return tags def get_call_graph(self) -> nx.DiGraph: - """NetworkX DiGraph of callable signatures (+ phantom external symbols) and CALLS edges.""" + """NetworkX DiGraph of callable signatures (+ phantom external symbols) and TS_CALLS edges.""" graph = nx.DiGraph() # Internal callable nodes (with the reconstructed callable, matching the in-memory backend). for props in self._all_callable_props(): @@ -419,7 +433,7 @@ def get_call_graph(self) -> nx.DiGraph: graph.add_node(sig, external=True, module=ext.module, name=ext.name) # Edges (auto-create any endpoint not added above, matching nx.add_edge semantics). for r in self._run( - "MATCH (s:Symbol)-[r:CALLS]->(t:Symbol) WHERE s._module IN $mods " "RETURN s.signature AS src, t.signature AS tgt, properties(r) AS edge", + "MATCH (s:CanNode)-[r:TS_CALLS]->(t:CanNode) WHERE s._module IN $mods " "RETURN s.signature AS src, t.signature AS tgt, properties(r) AS edge", mods=self._modules, ): edge = r["edge"] @@ -437,7 +451,7 @@ def get_call_graph_json(self) -> str: return self.get_application().model_dump_json() def _all_callable_props(self) -> List[Dict[str, Any]]: - rows = self._run("MATCH (c:Callable) WHERE c._module IN $mods RETURN properties(c) AS p", mods=self._modules) + rows = self._run("MATCH (c:TSCallable) WHERE c._module IN $mods RETURN properties(c) AS p", mods=self._modules) return [r["p"] for r in rows] def _resolve_signature(self, class_or_sig: str, member: str | None = None) -> str: @@ -445,20 +459,20 @@ def _resolve_signature(self, class_or_sig: str, member: str | None = None) -> st if member is None: return class_or_sig rows = self._run( - "MATCH (o:Symbol {signature: $owner})-[:HAS_METHOD]->(m:Callable {name: $name}) " "RETURN m.signature AS sig LIMIT 1", + "MATCH (o:CanNode {signature: $owner})-[:TS_HAS_METHOD]->(m:TSCallable {name: $name}) " "RETURN m.signature AS sig LIMIT 1", owner=class_or_sig, name=member, ) if rows: return rows[0]["sig"] composed = f"{class_or_sig}.{member}" - rows = self._run("MATCH (c:Callable {signature: $sig}) RETURN c.signature AS sig LIMIT 1", sig=composed) + rows = self._run("MATCH (c:TSCallable {signature: $sig}) RETURN c.signature AS sig LIMIT 1", sig=composed) return rows[0]["sig"] if rows else composed def get_all_callers(self, target_class_name: str, target_method_declaration: str | None = None) -> Dict: target = self._resolve_signature(target_class_name, target_method_declaration) rows = self._run( - "MATCH (src:Symbol)-[r:CALLS]->(t:Symbol {signature: $target}) WHERE src._module IN $mods " "RETURN src.signature AS caller, properties(r) AS edge", + "MATCH (src:CanNode)-[r:TS_CALLS]->(t:CanNode {signature: $target}) WHERE src._module IN $mods " "RETURN src.signature AS caller, properties(r) AS edge", target=target, mods=self._modules, ) @@ -468,7 +482,7 @@ def get_all_callers(self, target_class_name: str, target_method_declaration: str def get_all_callees(self, source_class_name: str, source_method_declaration: str | None = None) -> Dict: source = self._resolve_signature(source_class_name, source_method_declaration) rows = self._run( - "MATCH (s:Symbol {signature: $source})-[r:CALLS]->(tgt:Symbol) " "RETURN tgt.signature AS callee, properties(r) AS edge", + "MATCH (s:CanNode {signature: $source})-[r:TS_CALLS]->(tgt:CanNode) " "RETURN tgt.signature AS callee, properties(r) AS edge", source=source, ) callee_details = [{"callee_signature": r["callee"], "edge": self._edge_dict(r["edge"])} for r in rows] @@ -487,7 +501,7 @@ def get_class_call_graph(self, qualified_class_name: str, method_signature: str """Call-graph edges reachable (BFS) from a class (or one of its methods).""" adjacency: Dict[str, List[str]] = {} for r in self._run( - "MATCH (s:Symbol)-[:CALLS]->(t:Symbol) WHERE s._module IN $mods " "RETURN s.signature AS src, t.signature AS tgt ORDER BY src, tgt", + "MATCH (s:CanNode)-[:TS_CALLS]->(t:CanNode) WHERE s._module IN $mods " "RETURN s.signature AS src, t.signature AS tgt ORDER BY src, tgt", mods=self._modules, ): adjacency.setdefault(r["src"], []).append(r["tgt"]) @@ -495,7 +509,7 @@ def get_class_call_graph(self, qualified_class_name: str, method_signature: str if method_signature is not None: seeds = [method_signature] else: - seeds = [p["signature"] for p in self._members(qualified_class_name, "HAS_METHOD", "Callable")] + seeds = [p["signature"] for p in self._members(qualified_class_name, "TS_HAS_METHOD", "TSCallable")] edges: List[Tuple[str, str]] = [] seen = set(seeds) queue = deque(seeds) @@ -510,9 +524,13 @@ def get_class_call_graph(self, qualified_class_name: str, method_signature: str def get_class_hierarchy(self) -> nx.DiGraph: """Inheritance/implementation graph: an edge child → base for every base_class.""" + # VERIFY(2.0.0-e2e): hierarchy is read from the `base_classes` / `implements_types` array + # properties (as pre-2.0.0), not the TS_EXTENDS / TS_IMPLEMENTS edges — validate against a + # live 1.0.0 graph (Task 9); this also covers get_extended_classes, + # get_implemented_interfaces and get_all_sub_classes. graph = nx.DiGraph() rows = self._run( - "MATCH (n:Symbol) WHERE (n:Class OR n:Interface) AND n._module IN $mods " "RETURN n.signature AS sig, n.base_classes AS bases", + "MATCH (n:CanNode) WHERE (n:TSClass OR n:TSInterface) AND n._module IN $mods " "RETURN n.signature AS sig, n.base_classes AS bases", mods=self._modules, ) for r in rows: @@ -528,54 +546,57 @@ def get_call_sites(self, qualified_callable_name: str) -> List[TSCallsite]: def get_calling_lines(self, target_signature: str) -> List[int]: rows = self._run( - "MATCH (cs:CallSite) WHERE cs._module IN $mods AND cs.callee_signature = $sig " "AND cs.start_line >= 0 RETURN DISTINCT cs.start_line AS line ORDER BY line", + "MATCH (cs:TSBodyNode {kind: 'call'}) WHERE cs._module IN $mods AND cs.callee = $sig " "AND cs.start_line >= 0 RETURN DISTINCT cs.start_line AS line ORDER BY line", mods=self._modules, sig=target_signature, ) return [r["line"] for r in rows] def get_call_targets(self, source_signature: str) -> Set[str]: + # VERIFY(2.0.0-e2e): falls back to `cs.method_name` when a call body node has no resolved + # `callee`; whether TSBodyNode carries `method_name` is unconfirmed — validate against a + # live 1.0.0 graph (Task 9). rows = self._run( - "MATCH (c:Callable {signature: $sig})-[:HAS_CALLSITE]->(cs:CallSite) " "RETURN cs.callee_signature AS cosig, cs.method_name AS mn", + "MATCH (c:TSCallable {signature: $sig})-[:TS_HAS_BODY_NODE]->(cs:TSBodyNode {kind: 'call'}) " "RETURN cs.callee AS cosig, cs.method_name AS mn", sig=source_signature, ) return {(r["cosig"] or r["mn"]) for r in rows} # -----[ classes / interfaces / enums / type-aliases ]----- def get_all_classes(self) -> Dict[str, TSClass]: - rows = self._run("MATCH (c:Class) WHERE c._module IN $mods RETURN properties(c) AS p", mods=self._modules) + rows = self._run("MATCH (c:TSClass) WHERE c._module IN $mods RETURN properties(c) AS p", mods=self._modules) return {r["p"]["signature"]: self._class_full(r["p"]) for r in rows} def get_class(self, qualified_class_name: str) -> TSClass | None: rows = self._run( - "MATCH (c:Class {signature: $sig}) WHERE c._module IN $mods RETURN properties(c) AS p", + "MATCH (c:TSClass {signature: $sig}) WHERE c._module IN $mods RETURN properties(c) AS p", sig=qualified_class_name, mods=self._modules, ) return self._class_full(rows[0]["p"]) if rows else None def get_all_interfaces(self) -> Dict[str, TSInterface]: - rows = self._run("MATCH (i:Interface) WHERE i._module IN $mods RETURN properties(i) AS p", mods=self._modules) + rows = self._run("MATCH (i:TSInterface) WHERE i._module IN $mods RETURN properties(i) AS p", mods=self._modules) return {r["p"]["signature"]: self._interface_full(r["p"]) for r in rows} def get_all_enums(self) -> Dict[str, TSEnum]: - rows = self._run("MATCH (e:Enum) WHERE e._module IN $mods RETURN properties(e) AS p", mods=self._modules) + rows = self._run("MATCH (e:TSEnum) WHERE e._module IN $mods RETURN properties(e) AS p", mods=self._modules) return {r["p"]["signature"]: R.enum(r["p"]) for r in rows} def get_enum_members(self, qualified_enum_name: str) -> List[TSEnumMember]: - rows = self._run("MATCH (e:Enum {signature: $sig}) RETURN properties(e) AS p", sig=qualified_enum_name) + rows = self._run("MATCH (e:TSEnum {signature: $sig}) RETURN properties(e) AS p", sig=qualified_enum_name) return R.enum(rows[0]["p"]).members if rows else [] def get_all_type_aliases(self) -> Dict[str, TSTypeAlias]: - rows = self._run("MATCH (t:TypeAlias) WHERE t._module IN $mods RETURN properties(t) AS p", mods=self._modules) + rows = self._run("MATCH (t:TSTypeAlias) WHERE t._module IN $mods RETURN properties(t) AS p", mods=self._modules) return {r["p"]["signature"]: R.type_alias(r["p"]) for r in rows} def get_all_nested_classes(self, qualified_class_name: str) -> List[TSClass]: - return [self._class_full(p) for p in self._children(qualified_class_name, "DECLARES", "Class")] + return [self._class_full(p) for p in self._children(qualified_class_name, "TS_DECLARES", "TSClass")] def get_all_sub_classes(self, qualified_class_name: str) -> Dict[str, TSClass]: rows = self._run( - "MATCH (c:Class) WHERE c._module IN $mods AND $sig IN c.base_classes " "RETURN properties(c) AS p", + "MATCH (c:TSClass) WHERE c._module IN $mods AND $sig IN c.base_classes " "RETURN properties(c) AS p", sig=qualified_class_name, mods=self._modules, ) @@ -583,7 +604,7 @@ def get_all_sub_classes(self, qualified_class_name: str) -> Dict[str, TSClass]: def get_extended_classes(self, qualified_class_name: str) -> List[str]: rows = self._run( - "MATCH (c:Class {signature: $sig}) RETURN c.base_classes AS bases, c.implements_types AS impl", + "MATCH (c:TSClass {signature: $sig}) RETURN c.base_classes AS bases, c.implements_types AS impl", sig=qualified_class_name, ) if not rows: @@ -593,7 +614,7 @@ def get_extended_classes(self, qualified_class_name: str) -> List[str]: return [b for b in bases if b not in impl] def get_implemented_interfaces(self, qualified_class_name: str) -> List[str]: - rows = self._run("MATCH (c:Class {signature: $sig}) RETURN c.implements_types AS impl", sig=qualified_class_name) + rows = self._run("MATCH (c:TSClass {signature: $sig}) RETURN c.implements_types AS impl", sig=qualified_class_name) return list(rows[0]["impl"] or []) if rows else [] # -----[ methods / functions / fields ]----- @@ -602,30 +623,30 @@ def get_all_methods_in_application(self) -> Dict[str, Dict[str, TSCallable]]: # (even those with no methods), each keyed by the method's short name. out: Dict[str, Dict[str, TSCallable]] = {} for r in self._run( - "MATCH (n:Symbol) WHERE (n:Class OR n:Interface) AND n._module IN $mods " "RETURN n.signature AS sig", + "MATCH (n:CanNode) WHERE (n:TSClass OR n:TSInterface) AND n._module IN $mods " "RETURN n.signature AS sig", mods=self._modules, ): out[r["sig"]] = {} for r in self._run( - "MATCH (owner:Symbol)-[:HAS_METHOD]->(m:Callable) WHERE owner._module IN $mods " "RETURN owner.signature AS owner, properties(m) AS p", + "MATCH (owner:CanNode)-[:TS_HAS_METHOD]->(m:TSCallable) WHERE owner._module IN $mods " "RETURN owner.signature AS owner, properties(m) AS p", mods=self._modules, ): out.setdefault(r["owner"], {})[r["p"]["name"]] = self._callable_full(r["p"]) return out def get_all_methods_in_class(self, qualified_class_name: str) -> Dict[str, TSCallable]: - return {p["name"]: self._callable_full(p) for p in self._members(qualified_class_name, "HAS_METHOD", "Callable")} + return {p["name"]: self._callable_full(p) for p in self._members(qualified_class_name, "TS_HAS_METHOD", "TSCallable")} def get_method(self, qualified_class_name: str, qualified_method_name: str) -> TSCallable | None: rows = self._run( - "MATCH (o:Symbol {signature: $sig})-[:HAS_METHOD]->(m:Callable {name: $name}) " "RETURN properties(m) AS p LIMIT 1", + "MATCH (o:CanNode {signature: $sig})-[:TS_HAS_METHOD]->(m:TSCallable {name: $name}) " "RETURN properties(m) AS p LIMIT 1", sig=qualified_class_name, name=qualified_method_name, ) if rows: return self._callable_full(rows[0]["p"]) # Class lookup missed (or the scope isn't a class at all): fall back to module/namespace - # -level functions via DECLARES, mirroring get_all_functions. + # -level functions via TS_DECLARES, mirroring get_all_functions. return self._resolve_function(qualified_class_name, qualified_method_name) def _resolve_function(self, scope: str, name: str) -> TSCallable | None: @@ -634,8 +655,8 @@ def _resolve_function(self, scope: str, name: str) -> TSCallable | None: ``scope`` (handles functions nested in a namespace the caller doesn't know the full path of).""" rows = self._run( - "MATCH (parent)-[:DECLARES]->(c:Callable {signature: $sig}) " - "WHERE (parent:Module OR parent:Namespace) AND c._module IN $mods " + "MATCH (parent)-[:TS_DECLARES]->(c:TSCallable {signature: $sig}) " + "WHERE (parent:TSModule OR parent:TSNamespace) AND c._module IN $mods " "RETURN properties(c) AS p LIMIT 1", sig=name, mods=self._modules, @@ -643,8 +664,8 @@ def _resolve_function(self, scope: str, name: str) -> TSCallable | None: if rows: return self._callable_full(rows[0]["p"]) rows = self._run( - "MATCH (parent)-[:DECLARES]->(c:Callable {name: $name}) " - "WHERE (parent:Module OR parent:Namespace) AND c._module IN $mods AND c.signature STARTS WITH $prefix " + "MATCH (parent)-[:TS_DECLARES]->(c:TSCallable {name: $name}) " + "WHERE (parent:TSModule OR parent:TSNamespace) AND c._module IN $mods AND c.signature STARTS WITH $prefix " "RETURN properties(c) AS p LIMIT 1", name=name, mods=self._modules, @@ -657,56 +678,49 @@ def get_method_parameters(self, qualified_class_name: str, qualified_method_name return [p.name for p in method.parameters] if method else [] def get_all_constructors(self, qualified_class_name: str) -> Dict[str, TSCallable]: - return {p["name"]: self._callable_full(p) for p in self._members(qualified_class_name, "HAS_METHOD", "Callable") if p.get("kind") == "constructor"} + return {p["name"]: self._callable_full(p) for p in self._members(qualified_class_name, "TS_HAS_METHOD", "TSCallable") if p.get("kind") == "constructor"} def get_all_functions(self) -> Dict[str, TSCallable]: rows = self._run( - "MATCH (parent)-[:DECLARES]->(c:Callable) " "WHERE (parent:Module OR parent:Namespace) AND c._module IN $mods " "RETURN properties(c) AS p", + "MATCH (parent)-[:TS_DECLARES]->(c:TSCallable) " "WHERE (parent:TSModule OR parent:TSNamespace) AND c._module IN $mods " "RETURN properties(c) AS p", mods=self._modules, ) return {r["p"]["signature"]: self._callable_full(r["p"]) for r in rows} def get_all_fields(self, qualified_class_name: str) -> List[TSClassAttribute]: - return [R.attribute(p, self._attribute_decorators(p.get("id", ""))) for p in self._members(qualified_class_name, "HAS_ATTRIBUTE", "Attribute")] + # Class attributes are not projected as nodes in graph schema 2.0.0. + raise _unprojected("get_all_fields") def get_interface_properties(self, qualified_interface_name: str) -> List[TSClassAttribute]: - return [R.attribute(p, self._attribute_decorators(p.get("id", ""))) for p in self._members(qualified_interface_name, "HAS_ATTRIBUTE", "Attribute")] + # Interface properties are attributes — not projected in graph schema 2.0.0. + raise _unprojected("get_interface_properties") # -----[ imports / exports / variables ]----- def get_imports(self) -> Dict[str, List[TSImport]]: - return {key: self._module_imports(key) for key in self._modules} + # Module imports are not projected in graph schema 2.0.0. + raise _unprojected("get_imports") def get_all_exports(self) -> Dict[str, List[TSExport]]: - return {key: self._module_exports(key) for key in self._modules} + # Module (re-)exports are not projected in graph schema 2.0.0. + raise _unprojected("get_all_exports") def get_all_variables(self) -> Dict[str, List[TSVariableDeclaration]]: - return {key: self._module_variables(key) for key in self._modules} + # Variable declarations are not projected in graph schema 2.0.0. + raise _unprojected("get_all_variables") # -----[ decorators ]----- def get_decorators(self, qualified_callable_name: str) -> List[TSDecorator]: - return self._decorators_of(qualified_callable_name) + # Decorators are not projected in graph schema 2.0.0. + raise _unprojected("get_decorators") def get_class_decorators(self, qualified_class_name: str) -> List[TSDecorator]: - return self._decorators_of(qualified_class_name) + # Decorators are not projected in graph schema 2.0.0. + raise _unprojected("get_class_decorators") def get_methods_with_decorators(self, decorators: List[str]) -> Dict[str, List[str]]: - result: Dict[str, List[str]] = {d: [] for d in decorators} - rows = self._run( - "MATCH (c:Callable)-[:DECORATED_BY]->(d:Decorator) " "WHERE c._module IN $mods AND d.name IN $names " "RETURN d.name AS dn, c.signature AS sig", - mods=self._modules, - names=decorators, - ) - for r in rows: - result[r["dn"]].append(r["sig"]) - return result + # Decorators are not projected in graph schema 2.0.0. + raise _unprojected("get_methods_with_decorators") def get_classes_with_decorators(self, decorators: List[str]) -> Dict[str, List[str]]: - result: Dict[str, List[str]] = {d: [] for d in decorators} - rows = self._run( - "MATCH (c:Class)-[:DECORATED_BY]->(d:Decorator) " "WHERE c._module IN $mods AND d.name IN $names " "RETURN d.name AS dn, c.signature AS sig", - mods=self._modules, - names=decorators, - ) - for r in rows: - result[r["dn"]].append(r["sig"]) - return result + # Decorators are not projected in graph schema 2.0.0. + raise _unprojected("get_classes_with_decorators") diff --git a/cldk/analysis/typescript/neo4j/reconstruct.py b/cldk/analysis/typescript/neo4j/reconstruct.py index 8c687652..03f0b29d 100644 --- a/cldk/analysis/typescript/neo4j/reconstruct.py +++ b/cldk/analysis/typescript/neo4j/reconstruct.py @@ -106,7 +106,8 @@ def callsite(props: Props) -> TSCallsite: argument_types=list(props.get("argument_types", []) or []), type_arguments=list(props.get("type_arguments", []) or []), return_type=props.get("return_type"), - callee_signature=props.get("callee_signature"), + # schema 2.0.0 call-site body nodes key the resolved target as ``callee``. + callee_signature=props.get("callee_signature", props.get("callee")), is_constructor_call=props.get("is_constructor_call", False), is_optional_chain=props.get("is_optional_chain", False), start_line=props.get("start_line", -1), @@ -370,8 +371,11 @@ def namespace( def module(props: Props, **children: Any) -> TSModule: return TSModule( - file_path=props.get("file_key", props.get("file_path", "")), - module_name=props.get("module_name", ""), + # schema 2.0.0 modules carry the project-relative path as ``_module``. + file_path=props.get("_module", props.get("file_path", "")), + # VERIFY(2.0.0-e2e): assumes the 2.0.0 module node names the module via `name` (falling + # back from the pre-2.0.0 `module_name`) — validate against a live 1.0.0 graph (Task 9). + module_name=props.get("module_name", props.get("name", "")), is_tsx=props.get("is_tsx", False), is_declaration_file=props.get("is_declaration_file", False), content_hash=props.get("content_hash"), diff --git a/cldk/graph/__init__.py b/cldk/graph/__init__.py new file mode 100644 index 00000000..ce475114 --- /dev/null +++ b/cldk/graph/__init__.py @@ -0,0 +1,5 @@ +from cldk.graph.capability import CapabilityError +from cldk.graph.engine import Engine +from cldk.graph.result import FlowPath, FlowResult, SliceResult + +__all__ = ["CapabilityError", "Engine", "FlowPath", "FlowResult", "SliceResult"] diff --git a/cldk/graph/_cpg_local.py b/cldk/graph/_cpg_local.py new file mode 100644 index 00000000..8d9b6bb4 --- /dev/null +++ b/cldk/graph/_cpg_local.py @@ -0,0 +1,163 @@ +# cldk/graph/_cpg_local.py +from __future__ import annotations +from typing import Any, Dict, Iterable, List, Optional, Tuple +import networkx as nx +from cldk.models.cpg import Edge as CpgEdge + + +def _qualify(callable_id: str, local_key: str) -> str: + """A body node's dict key is local to its callable ('3:8', '@entry', '@formal_in:0', ...). + Application-level param_in/param_out already cross-reference these bodies as + '@', with the local key's own leading '@' (entry/exit/formal_*) + doubling as the join separator (never '...)@@entry'). Reproducing that exact scheme is what + lets provider-synthesized vertex ids agree with the ids the Application already uses.""" + return f"{callable_id}{local_key}" if local_key.startswith("@") else f"{callable_id}@{local_key}" + + +class CpgLocalProviderMixin: + """Implements ProgramGraphProvider's five read primitives over an in-memory cpg Application + on self.application. Language-neutral: the cpg models are shared across every analyzer, so + one mixin serves every local backend. Concrete backends supply max_level() (Task 8). + + A body node is keyed by local position in Node.body and normally carries no `.id` of its own + — only durable nodes (types/callables/functions/fields) are required to have one (see + Node's docstring and tests/models/cpg/test_node.py::test_body_node_missing_id_parses). This + mixin's canonical vertex id for a body node is `bn.id` when the analyzer did set one, else + the synthesized `_qualify(callable_id, local_key)` — matching how real param_in/param_out + already reference these nodes. + """ + + application: Any # cpg Application + + # --- internal index (built lazily, cached on the instance) --- + def _index(self) -> Dict[str, Any]: + idx = getattr(self, "_cpg_idx", None) + if idx is not None: + return idx + callables: Dict[str, Any] = {} # callable id -> (callable Node, Module, path) + canon_of: Dict[str, Dict[str, str]] = {} # callable id -> {local key: canonical vertex id} + n2c: Dict[str, str] = {} # canonical vertex id -> owning callable id + nodes: Dict[str, Any] = {} # canonical vertex id -> (body Node, Module, path) + + def _add(c, mod, path): + # bn.id: only durable body nodes (rare) carry an explicit id; the upstream + # codeanalyzer-python BodyNode has no `.id` attribute at all, so probe with + # getattr first — `bn.id` below is only ever reached once that's confirmed present. + canon = {k: (bn.id if getattr(bn, "id", None) is not None else _qualify(c.id, k)) + for k, bn in c.body.items()} + canon_of[c.id] = canon + callables[c.id] = (c, mod, path) + for k, bn in c.body.items(): + vid = canon[k] + n2c[vid] = c.id + nodes[vid] = (bn, mod, path) + # Recurse into this callable's own closures (a function/method can declare further + # nested callables, e.g. a factory's inner helper) and any classes it declares + # locally (e.g. a function defining a small helper class) — getattr-defensive since + # the slimmer upstream models may omit these containers entirely. This mirrors + # codeanalyzer-python's own recursive walk (semantic_analysis/call_graph.py's + # _walk_callable/_walk_class_callables) that the Neo4j emitter also follows + # (neo4j/project.py's _project_callable/_project_class), so every callable that owns + # a body — however deeply nested — ends up indexed here too. + for inner_c in (getattr(c, "callables", None) or {}).values(): + _add(inner_c, mod, path) + for inner_t in (getattr(c, "types", None) or {}).values(): + _add_type(inner_t, mod, path) + + def _add_type(t, mod, path): + # A class contributes no body of its own — only its methods and any nested classes + # (which may themselves nest further, arbitrarily deep) do. + for c in (getattr(t, "callables", None) or {}).values(): + _add(c, mod, path) + for inner_t in (getattr(t, "types", None) or {}).values(): + _add_type(inner_t, mod, path) + + for path, mod in self.application.symbol_table.items(): + for t in mod.types.values(): + _add_type(t, mod, path) + for f in mod.functions.values(): + _add(f, mod, path) + + idx = {"callables": callables, "canon": canon_of, "n2c": n2c, "nodes": nodes} + self._cpg_idx = idx + return idx + + def program_graph(self, callable_uri: str) -> nx.MultiDiGraph: + g = nx.MultiDiGraph() + entry = self._index()["callables"].get(callable_uri) + if entry is None: + # An unknown callable (e.g. the engine's callable_of() passed an id straight + # through because it wasn't a body vertex) is a structural non-match, not an + # error — return the empty graph rather than raising KeyError. + return g + c, _, _ = entry + canon = self._index()["canon"][callable_uri] + for k, bn in c.body.items(): + g.add_node(canon[k], kind=bn.kind, span=bn.span) + # No explicit edge key: 'family' alone does not identify a parallel edge uniquely (e.g. a + # callable can have several ddg edges between the same pair, one per var, or two cfg + # edges to the same successor with different kinds). provider.py's ABC docstring requires + # such edges to stay distinct, so let MultiDiGraph auto-assign a fresh key per edge + # instead of colliding same-family parallels onto one. + # kind/var/prov are duck-typed via getattr: the upstream codeanalyzer-python edge models + # are slimmer than the cldk cpg Edge (e.g. CdgEdge has no .kind, CfgEdge has no .var/.prov) + # — absent attributes surface as None/[] rather than raising AttributeError. + for fam, edges in (("cfg", c.cfg), ("cdg", c.cdg), ("ddg", c.ddg)): + for e in edges: + g.add_edge(canon.get(e.src, e.src), canon.get(e.dst, e.dst), + family=fam, kind=getattr(e, "kind", None), var=getattr(e, "var", None), + prov=list(getattr(e, "prov", None) or [])) + return g + + def sdg_edges(self) -> Iterable[Any]: + # param_in/param_out are already '@'-qualified at the Application + # level; summary is per-callable and LOCAL like cfg/cdg/ddg, so it needs the same + # qualification program_graph applies. All three are stamped with their own kind + # ("param_in"/"param_out"/"summary"): real edges carry kind=None in the raw analysis, and + # Engine.flows_to reports a boundary hop's bare family ("sdg") whenever kind is unset, so + # leaving these untagged would surface every interprocedural hop as opaque "sdg". + # + # Built as fresh cldk.models.cpg.Edge objects rather than model_copy: the upstream + # ParamEdge/SummaryEdge models don't declare `kind`/`var`/`prov` at all, so + # model_copy(update={...}) on them would set undeclared fields (undefined behavior). + idx = self._index() + out = [CpgEdge(src=e.src, dst=e.dst, kind="param_in", + var=getattr(e, "var", None), prov=list(getattr(e, "prov", None) or [])) + for e in self.application.param_in] + out += [CpgEdge(src=e.src, dst=e.dst, kind="param_out", + var=getattr(e, "var", None), prov=list(getattr(e, "prov", None) or [])) + for e in self.application.param_out] + for c, _, _ in idx["callables"].values(): + canon = idx["canon"][c.id] + out += [CpgEdge(src=canon.get(e.src, e.src), dst=canon.get(e.dst, e.dst), + kind="summary", var=getattr(e, "var", None), + prov=list(getattr(e, "prov", None) or [])) + for e in c.summary] + return out + + def resolve_location(self, file: str, line: int, col: Optional[int] = None) -> List[str]: + hits = [] + for vid, (bn, _mod, path) in self._index()["nodes"].items(): + if path != file and not path.endswith("/" + file): + continue + if bn.span is None or bn.span.start[0] != line: + continue + if col is not None and bn.span.start[1] != col: + continue + hits.append((bn.span.start, vid)) + # Deterministic order — a backend-agnostic tie-break the Neo4j provider must + # reproduce too — rather than whatever order the body dict happens to iterate in. + return [vid for _, vid in sorted(hits)] + + def source_slice(self, vertex_uri: str) -> Tuple[Optional[str], Optional[str]]: + node = self._index()["nodes"].get(vertex_uri) + if node is None: + return (None, None) + bn, mod, path = node + if bn.span is None: + return (path, None) + code = mod.source[bn.span.bytes[0]:bn.span.bytes[1]] if mod.source else None + return (f"{path}:{bn.span.start[0]}", code) + + def callable_of(self, vertex_uri: str) -> Optional[str]: + return self._index()["n2c"].get(vertex_uri, vertex_uri) diff --git a/cldk/graph/capability.py b/cldk/graph/capability.py new file mode 100644 index 00000000..a95f9993 --- /dev/null +++ b/cldk/graph/capability.py @@ -0,0 +1,22 @@ +from __future__ import annotations +from typing import Optional, Dict + + +class CapabilityError(Exception): + pass + + +def require(level_needed: int, provider, *, strict: bool, what: str) -> Optional[Dict]: + available = provider.max_level() + if available >= level_needed: + return None + if strict: + raise CapabilityError( + f"{what} requires analysis level {level_needed}; backend is at level {available}. " + f"Re-analyze at -a {level_needed} or drop strict=True to degrade.") + return { + "requested": level_needed, + "available": available, + "gap": f"{what} requires level {level_needed}; backend at level {available} — " + f"reduced result returned; absence of a result here is UNKNOWN, not safety.", + } diff --git a/cldk/graph/engine.py b/cldk/graph/engine.py new file mode 100644 index 00000000..673900bb --- /dev/null +++ b/cldk/graph/engine.py @@ -0,0 +1,196 @@ +# cldk/graph/engine.py +from __future__ import annotations +from typing import Iterable, List, Optional, Tuple +import networkx as nx +from cldk.graph.provider import ProgramGraphProvider, resolve_vertex +from cldk.graph.capability import require +from cldk.graph.result import SliceResult, FlowResult, FlowPath + + +def _filter_edges(g: nx.MultiDiGraph, families: Iterable[str]) -> nx.MultiDiGraph: + fam = set(families) + out = nx.MultiDiGraph() + out.add_nodes_from(g.nodes(data=True)) + for u, v, k, d in g.edges(keys=True, data=True): + if d.get("family") in fam: + out.add_edge(u, v, key=k, **d) + return out + + +_TIER_RANK = {"unresolved": 0, "structural": 1, "resolved": 2} +_RANK_TIER = {v: k for k, v in _TIER_RANK.items()} + +# Provisional witness-enumeration bounds (to be tuned against real graphs in a later +# task): flows_to explores simple paths no deeper than _PATH_CUTOFF hops and stops +# collecting witnesses at _MAX_PATHS; explain()["truncated"] reports whether the +# path cap was hit (depth-cutoff drops are not separately detectable and are folded +# into the same provisional-bounds caveat). +_MAX_PATHS = 1000 +_PATH_CUTOFF = 64 + + +def _ddg_tier(prov) -> str: + # Membership, not exact-list: prov is a provenance set in list form, and + # ["ssa", "points-to"] is STRONGER evidence than ["points-to"] alone. + prov = prov or [] + if "points-to" in prov: + return "resolved" + if "ssa" in prov: + return "structural" + return "unresolved" + + +class Engine: + def __init__(self, provider: ProgramGraphProvider): + self.p = provider + + def _evidence(self, uris, seeds, roles=None, default_role="def"): + # Seeds are always "seed"; other vertices take the verb's default_role + # (slices/flows: "def", control_deps: "control", def_use: "use"). + roles = roles or {} + ev = [] + for u in uris: + fl, code = self.p.source_slice(u) + ev.append({"uri": u, "file_line": fl, "code": code, + "role": "seed" if u in seeds else roles.get(u, default_role)}) + return ev + + def _intra(self, seed, edges, backward, strict, what, interprocedural=None, + default_role="def") -> SliceResult: + note = require(3, self.p, strict=strict, what=what) + want_inter = interprocedural if interprocedural is not None else (self.p.max_level() >= 4) + if interprocedural is True: + inter_note = require(4, self.p, strict=strict, what=f"interprocedural {what}") + if inter_note: + note = inter_note + want_inter = False + # Only dataflow (param_in/param_out/summary) crosses callable boundaries, so the + # sdg overlay is additionally gated on the ddg family being requested: a cfg- or + # cdg-only slice never crosses, even on an L4 backend. want_inter is the single + # source of truth — it both gates the overlay and feeds explain()["interprocedural"]. + want_inter = want_inter and self.p.max_level() >= 4 and "ddg" in set(edges) + seeds = resolve_vertex(self.p, seed) + g = _filter_edges(self.p.program_graph(self.p.callable_of(seeds[0])), edges) + if want_inter: + for e in self.p.sdg_edges(): + g.add_edge(e.src, e.dst, family="sdg", kind=getattr(e, "kind", None), + var=getattr(e, "var", None), prov=getattr(e, "prov", [])) + walk = g.reverse(copy=False) if backward else g + reached = set(seeds) + for s in seeds: + if s in walk: + reached |= nx.descendants(walk, s) + # seed-consistency (Task 4 fix, carried here): evidence/uris must equal subgraph nodes, + # and a seed is trivially in its own slice. + sub = g.subgraph(reached & set(g.nodes())).copy() # MultiDiGraph + for s in seeds: + if s not in sub: + sub.add_node(s, kind="seed") + ev_nodes = sorted(sub.nodes()) # deterministic; evidence set == subgraph nodes + explain = {"seed": seeds, "direction": "backward" if backward else "forward", + "edges": list(edges), "level": self.p.max_level(), + "vertices": len(sub), "interprocedural": bool(want_inter)} + if note: + explain["degraded"] = note + return SliceResult(subgraph=sub, + evidence=self._evidence(ev_nodes, set(seeds), + default_role=default_role), + _explain=explain) + + def slice_backward(self, seed, *, edges=("cfg", "cdg", "ddg"), + interprocedural: Optional[bool] = None, strict: bool = False) -> SliceResult: + return self._intra(seed, edges, True, strict, "slice_backward", interprocedural) + + def slice_forward(self, seed, *, edges=("cfg", "cdg", "ddg"), + interprocedural: Optional[bool] = None, strict: bool = False) -> SliceResult: + return self._intra(seed, edges, False, strict, "slice_forward", interprocedural) + + def control_deps(self, seed, *, strict: bool = False) -> SliceResult: + # Control dependence is intraprocedural in this model — only dataflow (param/summary) + # crosses boundaries. Force interprocedural=False so the sdg overlay is never merged + # into a pure CDG slice, even on an L4 backend. + return self._intra(seed, ("cdg",), backward=True, strict=strict, + what="control_deps", interprocedural=False, + default_role="control") + + def _dataflow_graph(self, *callable_uris) -> nx.MultiDiGraph: + # Union of the given callables' intra ddg graphs, plus the summary/param_* + # (inter) sdg overlay at L4; below L4 this is intraprocedural ddg only. + g = nx.MultiDiGraph() + for c in dict.fromkeys(callable_uris): # dedupe, keep order + cg = _filter_edges(self.p.program_graph(c), ("ddg",)) + g.add_nodes_from(cg.nodes(data=True)) + for u, v, k, d in cg.edges(keys=True, data=True): + g.add_edge(u, v, key=k, **d) + if self.p.max_level() >= 4: + for e in self.p.sdg_edges(): + g.add_edge(e.src, e.dst, family="sdg", kind=getattr(e, "kind", None), + var=getattr(e, "var", None), prov=getattr(e, "prov", [])) + return g + + def flows_to(self, source_seed, sink_seed, *, strict: bool = False) -> FlowResult: + # Full flows_to semantics are interprocedural (ddg + param_in/param_out/summary), + # which is L4. Below that, non-strict degrades honestly: the note is attached and + # the intra-only ddg witnesses that CAN be computed are still returned. + note = require(4, self.p, strict=strict, what="flows_to") + src = resolve_vertex(self.p, source_seed)[0] + dst = resolve_vertex(self.p, sink_seed)[0] + # A sink in a different callable is reachable via param_in/param_out/summary, + # so the dataflow graph must span BOTH endpoint callables. (Multi-hop flows + # through a THIRD callable's interior need the whole-program graph — deferred.) + g = self._dataflow_graph(self.p.callable_of(src), self.p.callable_of(dst)) + paths: List[FlowPath] = [] + reached = set() + truncated = False + if src in g and dst in g: + # A MultiDiGraph enumerates a route once per parallel-edge combination, yielding + # byte-identical duplicate witnesses. Enumerate over a plain-DiGraph VIEW (one path + # per distinct node route) and read per-hop parallel evidence from the MultiDiGraph g. + routes = nx.DiGraph(g) + for path in nx.all_simple_paths(routes, src, dst, cutoff=_PATH_CUTOFF): + if len(paths) >= _MAX_PATHS: + truncated = True + break + hops, tiers = [], [] + for a, b in zip(path, path[1:]): + # MultiDiGraph: get_edge_data returns {key: attrdict} over parallel edges. + # Pick the strongest-confidence parallel edge as the hop's evidence (the step + # is as strong as its best evidence; the path is as weak as its weakest step). + parallels = g.get_edge_data(a, b) + best = max(parallels.values(), + key=lambda d: _TIER_RANK[_ddg_tier(d.get("prov", []))]) + t = _ddg_tier(best.get("prov", [])) + tiers.append(t) + # Intra edges report their family (cfg/cdg/ddg have no kind); sdg + # boundary edges report the concrete kind (param_in/param_out/summary). + hops.append({"from": a, "to": b, + "kind": best.get("kind") or best.get("family"), + "var": best.get("var"), "confidence": t}) + conf = _RANK_TIER[min(_TIER_RANK[t] for t in tiers)] if tiers else "unresolved" + paths.append(FlowPath(source=src, sink=dst, hops=hops, confidence=conf)) + reached.update(path) + explain = {"source": src, "sink": dst, "level": self.p.max_level(), + "paths": len(paths), "truncated": truncated} + if note: + explain["degraded"] = note + sub = g.subgraph(reached).copy() + return FlowResult(subgraph=sub, evidence=self._evidence(sorted(sub.nodes()), {src, dst}), + _explain=explain, paths=paths) + + def def_use(self, seed, *, strict: bool = False) -> FlowResult: + note = require(3, self.p, strict=strict, what="def_use") + s = resolve_vertex(self.p, seed)[0] + # NOTE: currently scoped to the seed's callable plus sdg endpoints; uses inside + # OTHER callables' interiors arrive with the whole-program dataflow graph (deferred). + g = self._dataflow_graph(self.p.callable_of(s)) + reached = {s} | (nx.descendants(g, s) if s in g else set()) + sub = g.subgraph(reached & set(g.nodes())).copy() + if s not in sub: # a seed is trivially in its own def-use result + sub.add_node(s, kind="seed") + explain = {"seed": s, "level": self.p.max_level(), "vertices": len(sub)} + if note: + explain["degraded"] = note + return FlowResult(subgraph=sub, + evidence=self._evidence(sorted(sub.nodes()), {s}, + default_role="use"), + _explain=explain, paths=[]) diff --git a/cldk/graph/provider.py b/cldk/graph/provider.py new file mode 100644 index 00000000..eb5ea3e6 --- /dev/null +++ b/cldk/graph/provider.py @@ -0,0 +1,52 @@ +from __future__ import annotations +import re +from abc import ABC, abstractmethod +from typing import List, Tuple, Iterable, Optional, Any +import networkx as nx + +_LOC = re.compile(r"^(?P.+?):(?P\d+)(?::(?P\d+))?$") + + +class ProgramGraphProvider(ABC): + """The per-backend data seam the shared engine consumes. Implemented by local backends + (from cpg models) and Neo4j backends (from Cypher). Traversal lives in the engine, not here. + + Below the level a verb requires, the engine gates via require(...); providers should + still answer program_graph/resolve_location/callable_of structurally — none of these + ever need L3+ data to do so.""" + + @abstractmethod + def program_graph(self, callable_uri: str) -> nx.MultiDiGraph: + """Parallel cfg/cdg/ddg edges between the same vertex pair must stay distinct + edges, each carrying its own family/var/prov/kind.""" + @abstractmethod + def sdg_edges(self) -> Iterable[Any]: ... + @abstractmethod + def resolve_location(self, file: str, line: int, col: Optional[int] = None) -> List[str]: ... + @abstractmethod + def source_slice(self, vertex_uri: str) -> Tuple[Optional[str], Optional[str]]: ... + @abstractmethod + def callable_of(self, vertex_uri: str) -> Optional[str]: ... + @abstractmethod + def max_level(self) -> int: ... + + +def resolve_vertex(provider: ProgramGraphProvider, seed: Any) -> List[str]: + """Normalize a polymorphic seed to vertex ids: a BodyNode-like object (has .id), a can:// id + string, or a 'file:line[:col]' location string.""" + if hasattr(seed, "id"): + return [seed.id] + if isinstance(seed, str): + if seed.startswith("can://"): + return [seed] + m = _LOC.match(seed) + if m: + col = int(m["col"]) if m["col"] is not None else None + found = provider.resolve_location(m["file"], int(m["line"]), col) + if not found: + # An ordinary user miss (no vertex at that line) must surface as a clean + # ValueError here — every verb indexes the result, and [] would IndexError. + raise ValueError(f"no vertex at location {seed!r}") + return found + raise ValueError(f"cannot resolve seed to a vertex: {seed!r} " + f"(expected 'file:line[:col]', a can:// id, or a body node)") diff --git a/cldk/graph/result.py b/cldk/graph/result.py new file mode 100644 index 00000000..5d61be0d --- /dev/null +++ b/cldk/graph/result.py @@ -0,0 +1,53 @@ +from __future__ import annotations +import json +from dataclasses import dataclass, field, asdict +from typing import List, Dict, Literal +import networkx as nx + +Confidence = Literal["resolved", "structural", "unresolved"] + + +@dataclass(frozen=True) +class FlowPath: + source: str + sink: str + hops: List[Dict] = field(default_factory=list) + confidence: Confidence = "unresolved" + + +@dataclass +class GraphResult: + subgraph: nx.MultiDiGraph + evidence: List[Dict] + _explain: Dict + + def uris(self) -> List[str]: + return [e["uri"] for e in self.evidence] + + def explain(self) -> Dict: + return dict(self._explain) + + def to_json(self) -> str: + return json.dumps({"evidence": self.evidence, "explain": self._explain, + "vertices": list(self.subgraph.nodes)}, sort_keys=True) + + def __len__(self) -> int: + return self.subgraph.number_of_nodes() + + def __bool__(self) -> bool: + return self.subgraph.number_of_nodes() > 0 + + +@dataclass +class SliceResult(GraphResult): + pass + + +@dataclass +class FlowResult(GraphResult): + paths: List[FlowPath] = field(default_factory=list) + + def to_json(self) -> str: + base = json.loads(super().to_json()) + base["paths"] = [asdict(p) for p in self.paths] + return json.dumps(base, sort_keys=True) diff --git a/cldk/models/cpg/__init__.py b/cldk/models/cpg/__init__.py new file mode 100644 index 00000000..8efbe01f --- /dev/null +++ b/cldk/models/cpg/__init__.py @@ -0,0 +1,8 @@ +from cldk.models.cpg.base import _NullSafeBase +from cldk.models.cpg.models import ( + Span, Import, Edge, Node, Module, Application, Analyzer, AnalysisPayload, +) + +__all__ = [ + "AnalysisPayload", "Application", "Module", "Node", "Edge", "Span", "Import", "Analyzer", +] diff --git a/cldk/models/cpg/base.py b/cldk/models/cpg/base.py new file mode 100644 index 00000000..ad9df709 --- /dev/null +++ b/cldk/models/cpg/base.py @@ -0,0 +1,19 @@ +from __future__ import annotations +from pydantic import BaseModel, ConfigDict, model_validator + + +class _NullSafeBase(BaseModel): + """Shared base for every canonical (cpg) model. `extra="allow"` so language-specific fields + (TS is_tsx/exports, Python package, …) are tolerated and preserved rather than rejected — the + device that lets ONE model set parse every analyzer. The before-validator drops None-valued + keys so a collection serialized as `null` (Go/Rust/C) falls back to its field default; the one + sanctioned null (a body-node `callee`) simply resolves to its `None` default.""" + + model_config = ConfigDict(extra="allow") + + @model_validator(mode="before") + @classmethod + def _drop_nulls(cls, data): + if isinstance(data, dict): + return {k: v for k, v in data.items() if v is not None} + return data diff --git a/cldk/models/cpg/models.py b/cldk/models/cpg/models.py new file mode 100644 index 00000000..1bbae9f1 --- /dev/null +++ b/cldk/models/cpg/models.py @@ -0,0 +1,113 @@ +from __future__ import annotations +from typing import Any, Dict, List, Optional, Tuple +from pydantic import model_validator +from cldk.models.cpg.base import _NullSafeBase + + +class Span(_NullSafeBase): + start: Tuple[int, int] + end: Tuple[int, int] + bytes: Tuple[int, int] + + +class Edge(_NullSafeBase): + src: str + dst: str + kind: Optional[str] = None # cfg edge kind; absent on identity edges + var: Optional[str] = None # ddg access path + prov: List[str] = [] # ["jedi"|"pycg"|"tsc"|"jelly"] (call) / ["ssa"|"points-to"] (ddg) + weight: int = 1 + + +class Import(_NullSafeBase): + name: str + path: Optional[str] = None + alias: Optional[str] = None + span: Optional[Span] = None + + +class Node(_NullSafeBase): + # Optional because sub-callable body nodes are keyed by local position and omit id; durable + # nodes (types/callables/functions/fields) are required to carry it — enforced positionally by + # the container validators. + id: Optional[str] = None + kind: str + span: Optional[Span] = None + parent: Optional[str] = None + # type facet + base_types: List[str] = [] + interfaces: List[str] = [] + modifiers: List[str] = [] + decorators: List[Any] = [] + callables: Dict[str, "Node"] = {} + fields: Dict[str, "Node"] = {} + # callable facet + signature: Optional[str] = None + parameters: List[Any] = [] + return_type: Optional[str] = None + error_channel: List[str] = [] + metrics: Dict[str, Any] = {} + refs: Dict[str, Any] = {} + body: Dict[str, "Node"] = {} + cfg: List[Edge] = [] + cdg: List[Edge] = [] + ddg: List[Edge] = [] + summary: List[Edge] = [] + # field / body-node facet + type: Optional[str] = None + callee: Optional[str] = None + arguments: List[str] = [] + of: Optional[str] = None + # open vocab + tags: Dict[str, str] = {} + + @model_validator(mode="after") + def _durable_children_require_id(self): + for container in (self.callables, self.fields): + for key, node in container.items(): + if node.id is None: + raise ValueError(f"durable node {key!r} under {self.id or ''!r} is missing required id") + return self + + +class Module(_NullSafeBase): + id: str + kind: str = "module" + span: Optional[Span] = None + package: Optional[str] = None + source: str = "" + imports: List[Import] = [] + types: Dict[str, Node] = {} + functions: Dict[str, Node] = {} + content_hash: Optional[str] = None + + @model_validator(mode="after") + def _durable_children_require_id(self): + for container in (self.types, self.functions): + for key, node in container.items(): + if node.id is None: + raise ValueError(f"durable node {key!r} in module {self.id!r} is missing required id") + return self + + +class Application(_NullSafeBase): + id: str + kind: str = "application" + symbol_table: Dict[str, Module] = {} + call_graph: List[Edge] = [] + param_in: List[Edge] = [] + param_out: List[Edge] = [] + + +class Analyzer(_NullSafeBase): + name: str + version: Optional[str] = None + + +class AnalysisPayload(_NullSafeBase): + schema_version: str + language: str + max_level: int + k_limit: Optional[int] = None + analyzer: Optional[Analyzer] = None + application: Application diff --git a/cldk/utils/exceptions/__init__.py b/cldk/utils/exceptions/__init__.py index 7c9395c5..cc639298 100644 --- a/cldk/utils/exceptions/__init__.py +++ b/cldk/utils/exceptions/__init__.py @@ -20,10 +20,12 @@ from .exceptions import ( CldkInitializationException, + CldkSchemaMismatchException, CodeanalyzerExecutionException, ) __all__ = [ "CodeanalyzerExecutionException", "CldkInitializationException", + "CldkSchemaMismatchException", ] diff --git a/cldk/utils/exceptions/exceptions.py b/cldk/utils/exceptions/exceptions.py index 2bca21bf..b7f3162e 100644 --- a/cldk/utils/exceptions/exceptions.py +++ b/cldk/utils/exceptions/exceptions.py @@ -87,6 +87,32 @@ def __init__(self, message: str) -> None: super().__init__(self.message) +class CldkSchemaMismatchException(Exception): + """Exception raised when a persisted graph's schema version is not the one this SDK speaks. + + The Neo4j-backed TypeScript backend reads ``(:Application).schema_version`` on first use and + raises this if it differs from the graph schema the SDK was written against. Re-analyze the + project with a ``codeanalyzer-typescript`` whose Neo4j projection emits the expected schema. + + Attributes: + message (str): A descriptive error message naming the found and expected schema versions. + + See Also: + :class:`~cldk.analysis.typescript.neo4j.neo4j_backend.TSNeo4jBackend`: raises this on a + version mismatch. + """ + + def __init__(self, message: str) -> None: + """Initialize the exception with a descriptive message. + + Args: + message: A descriptive error message naming the found vs. expected schema versions + and how to resolve the mismatch. + """ + self.message = message + super().__init__(self.message) + + class CodeanalyzerUsageException(Exception): """Exception raised for incorrect CodeAnalyzer usage. diff --git a/pyproject.toml b/pyproject.toml index 86bff923..50ac6f79 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "cldk" -version = "1.4.3" +version = "2.0.0-rc.1" description = "The official Python SDK for Codellm-Devkit." readme = "README.md" license = { text = "Apache-2.0" } @@ -39,8 +39,8 @@ dependencies = [ "tree-sitter-javascript==0.23.1", "clang==17.0.6", "libclang==17.0.6", - "codeanalyzer-python==0.3.1", - "codeanalyzer-typescript==0.4.3", + "codeanalyzer-python==1.1.0", + "codeanalyzer-typescript==1.0.0", ] [project.optional-dependencies] @@ -74,6 +74,13 @@ test = [ requires = ["hatchling"] build-backend = "hatchling.build" +# The codeanalyzer-java JAR is force-included here: it lives under a `*.jar` .gitignore +# (re-included for git via a nested `!codeanalyzer-*.jar`), but hatchling honors the root +# ignore and not the nested negation, so without this it is silently dropped from every +# wheel/sdist built inside a git repo (i.e. in CI). See issue #284. +[tool.hatch.build] +artifacts = ["cldk/analysis/java/codeanalyzer/jar/*.jar"] + [tool.hatch.build.targets.wheel] packages = ["cldk"] @@ -87,8 +94,8 @@ include = [ [tool.backend-versions] codeanalyzer-java = "2.4.1" -codeanalyzer-python = "0.3.1" -codeanalyzer-typescript = "0.4.3" +codeanalyzer-python = "1.1.0" +codeanalyzer-typescript = "1.0.0" ######################################## # Tool configurations diff --git a/tests/analysis/java/test_jcodeanalyzer.py b/tests/analysis/java/test_jcodeanalyzer.py index f674c30e..bec79314 100644 --- a/tests/analysis/java/test_jcodeanalyzer.py +++ b/tests/analysis/java/test_jcodeanalyzer.py @@ -19,7 +19,6 @@ """ import os -import sys import json from typing import Dict, List, Tuple from unittest.mock import patch, MagicMock @@ -197,27 +196,6 @@ def test_init_codeanalyzer_reuses_legacy_cache_when_compatible(test_fixture, cod assert compilation_unit.import_declarations[0].is_wildcard is False -def test_get_codeanalyzer_exec(test_fixture, analysis_json, tmp_path): - """Should resolve the codeanalyzer native binary command (packaged binary only).""" - - # Patch subprocess so that it does not run codeanalyzer - with patch("cldk.analysis.java.codeanalyzer.codeanalyzer.subprocess.run") as run_mock: - run_mock.return_value = MagicMock(stdout=analysis_json, returncode=0) - - code_analyzer = JCodeanalyzer( - project_dir=test_fixture, - source_code=None, - analysis_json_path=None, - analysis_level=AnalysisLevel.symbol_table, - eager_analysis=False, - target_files=None, - ) - - # The PyPI native binary, invoked via `python -m codeanalyzer_java`. There is no longer a - # backend-path override (the binary ships with the packaged dependency). - assert code_analyzer._get_codeanalyzer_exec() == [sys.executable, "-m", "codeanalyzer_java"] - - def test_generate_call_graph(test_fixture, analysis_json): """Should generate a graph""" diff --git a/tests/analysis/python/test_python_bulk_accessors.py b/tests/analysis/python/test_python_bulk_accessors.py index fb4542a8..603fae49 100644 --- a/tests/analysis/python/test_python_bulk_accessors.py +++ b/tests/analysis/python/test_python_bulk_accessors.py @@ -23,64 +23,83 @@ ``test_python_neo4j_backend.py`` when a server is available. """ -from codeanalyzer.schema.py_schema import PyApplication, PyCallable, PyCallsite, PyClass, PyModule +from codeanalyzer.schema.py_schema import PyApplication, PyCallable, PyCallsite, PyClass, PyModule, Span from cldk.analysis.python.codeanalyzer.codeanalyzer import PyCodeanalyzer -def _callable(name, signature, *, code="", decorators=None, inner_callables=None, inner_classes=None, call_sites=None): +def _callable(name, signature, *, decorators=None, callables=None, types=None, call_sites=None): return PyCallable( name=name, path="pkg/models.py", signature=signature, - code=code, decorators=decorators or [], - inner_callables=inner_callables or {}, - inner_classes=inner_classes or {}, + callables=callables or {}, + types=types or {}, call_sites=call_sites or [], ) -def _class(name, signature, *, methods=None, inner_classes=None): - return PyClass(name=name, signature=signature, methods=methods or {}, inner_classes=inner_classes or {}) +def _class(name, signature, *, callables=None, types=None): + return PyClass(name=name, signature=signature, callables=callables or {}, types=types or {}) + + +def _stamp_source(module, snippets): + """Assemble ``module.source`` from per-callable code snippets and stamp byte-offset spans. + + Schema 2.0.0 stores source once on the module; each callable carries a ``Span`` whose + ``bytes`` slice into it (the analyzer's shape — see ``_code_of`` in the backend). One + snippet per line keeps the offsets trivial. + """ + offset = 0 + lines = [] + for lineno, (c, code) in enumerate(snippets, start=1): + c.span = Span(start=(lineno, 0), end=(lineno, len(code)), bytes=(offset, offset + len(code.encode("utf-8")))) + lines.append(code) + offset += len(code.encode("utf-8")) + 1 # + newline + module.source = "\n".join(lines) + "\n" def _backend(): """A PyCodeanalyzer wired to a hand-built application, bypassing the analyzer run.""" - decorate = _callable("_decorate", "pkg.models.greet.._decorate", code="return s.upper()") + decorate = _callable("_decorate", "pkg.models.greet.._decorate") greet = _callable( "greet", "pkg.models.greet", - code="def greet(who): ...", decorators=["app.route"], - inner_callables={"_decorate": decorate}, + callables={"_decorate": decorate}, ) - meta = _class( - "Meta", - "pkg.models.Entity.Meta", - methods={"m": _callable("m", "pkg.models.Entity.Meta.m", code="return 1")}, + meta_m = _callable("m", "pkg.models.Entity.Meta.m") + meta = _class("Meta", "pkg.models.Entity.Meta", callables={"m": meta_m}) + init = _callable("__init__", "pkg.models.Entity.__init__") + describe = _callable( + "describe", + "pkg.models.Entity.describe", + decorators=["property"], + call_sites=[PyCallsite(method_name="greet", start_line=7, start_column=4)], ) entity = _class( "Entity", "pkg.models.Entity", - methods={ - "__init__": _callable("__init__", "pkg.models.Entity.__init__", code="self.x = 1"), - "describe": _callable( - "describe", - "pkg.models.Entity.describe", - code="return self.x", - decorators=["property"], - call_sites=[PyCallsite(method_name="greet", start_line=7, start_column=4)], - ), - }, - inner_classes={"pkg.models.Entity.Meta": meta}, + callables={"__init__": init, "describe": describe}, + types={"pkg.models.Entity.Meta": meta}, ) module = PyModule( file_path="pkg/models.py", module_name="pkg.models", - classes={"pkg.models.Entity": entity}, + types={"pkg.models.Entity": entity}, functions={"greet": greet}, ) + _stamp_source( + module, + [ + (decorate, "return s.upper()"), + (greet, "def greet(who): ..."), + (meta_m, "return 1"), + (init, "self.x = 1"), + (describe, "return self.x"), + ], + ) app = PyApplication(symbol_table={"pkg/models.py": module}) backend = object.__new__(PyCodeanalyzer) diff --git a/tests/analysis/python/test_python_l34_levels.py b/tests/analysis/python/test_python_l34_levels.py new file mode 100644 index 00000000..8c691e08 --- /dev/null +++ b/tests/analysis/python/test_python_l34_levels.py @@ -0,0 +1,140 @@ +"""Level plumbing for L3/L4 (#270): the facade's AnalysisLevel maps to the analyzer's integer +``analysis_level`` option, and the backend reports the envelope's ``max_level`` — captured from +the SAME in-process run, never re-derived or sniffed. codeanalyzer-python >= 1.0.2 models are +schema-2.0-shaped (PyCallable.body/cfg/cdg/ddg/summary, PyApplication.param_in/param_out), so +``self.application`` doubles as the cpg source the graph provider mixin walks — no second parse. +""" +import json +from pathlib import Path + +import pytest + +from cldk.analysis import ANALYSIS_LEVEL_TO_INT, AnalysisLevel, to_analysis_level + +RES = Path(__file__).parent.parent.parent / "resources" / "cpg" + + +def test_analysis_level_map_is_total(): + assert ANALYSIS_LEVEL_TO_INT == { + AnalysisLevel.symbol_table: 1, + AnalysisLevel.call_graph: 2, + AnalysisLevel.program_dependency_graph: 3, + AnalysisLevel.system_dependency_graph: 4, + } + + +def test_to_analysis_level_accepts_enum(): + """to_analysis_level passes through enum values unchanged.""" + assert to_analysis_level(AnalysisLevel.call_graph) is AnalysisLevel.call_graph + + +def test_to_analysis_level_accepts_value_form(): + """to_analysis_level accepts the enum's value (space-separated).""" + assert to_analysis_level("call graph") is AnalysisLevel.call_graph + + +def test_to_analysis_level_accepts_name_form(): + """to_analysis_level accepts the enum's name (underscore form).""" + assert to_analysis_level("call_graph") is AnalysisLevel.call_graph + assert to_analysis_level("symbol_table") is AnalysisLevel.symbol_table + assert to_analysis_level("program_dependency_graph") is AnalysisLevel.program_dependency_graph + assert to_analysis_level("system_dependency_graph") is AnalysisLevel.system_dependency_graph + + +def test_to_analysis_level_rejects_garbage(): + """to_analysis_level raises on unknown values.""" + with pytest.raises(ValueError, match="unknown analysis level"): + to_analysis_level("garbage") + + +class _FakeEnvelope: + """Stands in for the analyzer's schema-v2 Analysis envelope.""" + + def __init__(self, payload_dict): + from cldk.models.python import PyApplication + + self.schema_version = payload_dict["schema_version"] + self.max_level = payload_dict["max_level"] + self.application = PyApplication(**payload_dict["application"]) + + +@pytest.fixture +def backend(monkeypatch, tmp_path): + payload = json.loads((RES / "py-a4.json").read_text()) + captured = {} + + class _FakeCodeanalyzer: + def __init__(self, options): + captured["options"] = options + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def analyze(self): + return _FakeEnvelope(payload) + + import cldk.analysis.python.codeanalyzer.codeanalyzer as mod + + monkeypatch.setattr(mod, "Codeanalyzer", _FakeCodeanalyzer) + from cldk.analysis.python.codeanalyzer.codeanalyzer import PyCodeanalyzer + + b = PyCodeanalyzer( + project_dir=tmp_path, + analysis_level=AnalysisLevel.system_dependency_graph, + analysis_json_path=None, + eager_analysis=False, + ) + return b, captured + + +def test_max_level_captured_from_same_run(backend): + b, _ = backend + assert b.max_level() == 4 + + +def test_analyzer_receives_int_level(backend): + _, captured = backend + assert captured["options"].analysis_level == 4 + + +def test_call_graph_built_at_level_ge_2(backend): + b, _ = backend + assert b.call_graph is not None + # the pyfix sample's three internal callables all appear + assert b.call_graph.number_of_edges() >= 3 + + +def test_backend_accepts_underscore_analysis_level(monkeypatch, tmp_path): + """PyCodeanalyzer accepts analysis_level as underscore form (e.g. "call_graph").""" + payload = json.loads((RES / "py-a4.json").read_text()) + + class _FakeCodeanalyzer: + def __init__(self, options): + self.captured_options = options + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def analyze(self): + return _FakeEnvelope(payload) + + import cldk.analysis.python.codeanalyzer.codeanalyzer as mod + + monkeypatch.setattr(mod, "Codeanalyzer", _FakeCodeanalyzer) + from cldk.analysis.python.codeanalyzer.codeanalyzer import PyCodeanalyzer + + # Construct with underscore form + b = PyCodeanalyzer( + project_dir=tmp_path, + analysis_level="call_graph", + analysis_json_path=None, + eager_analysis=False, + ) + assert b.max_level() == 4 + assert b.call_graph is not None diff --git a/tests/analysis/python/test_python_method_lookup.py b/tests/analysis/python/test_python_method_lookup.py index a242c1f4..d751ff6a 100644 --- a/tests/analysis/python/test_python_method_lookup.py +++ b/tests/analysis/python/test_python_method_lookup.py @@ -48,8 +48,8 @@ def _local_backend(): """A PyCodeanalyzer wired to a hand-built application, bypassing the analyzer run.""" - entry = PyCallable(name="entry", path="pkg/mod.py", signature=ENTRY_SIG, code="helper()") - helper = PyCallable(name="helper", path="pkg/mod.py", signature=HELPER_SIG, code="return 1") + entry = PyCallable(name="entry", path="pkg/mod.py", signature=ENTRY_SIG) + helper = PyCallable(name="helper", path="pkg/mod.py", signature=HELPER_SIG) module = PyModule( file_path="pkg/mod.py", module_name=MODULE_NAME, @@ -57,7 +57,7 @@ def _local_backend(): ) app = PyApplication( symbol_table={"pkg/mod.py": module}, - call_graph=[PyCallEdge(source=ENTRY_SIG, target=HELPER_SIG)], + call_graph=[PyCallEdge(src=ENTRY_SIG, dst=HELPER_SIG)], ) backend = object.__new__(PyCodeanalyzer) @@ -87,7 +87,7 @@ def run(query, **params): mod = modules.get(params["name"]) return [{"p": p} for p in mod["functions"]] if mod else [] if "PY_CALLS" in query: - return [{"src": e[0], "tgt": e[1], "p": {"weight": 1, "provenance": []}} for e in call_edges] + return [{"src": e[0], "tgt": e[1], "p": {"weight": 1, "prov": []}} for e in call_edges] return [] # attributes / inner classes / inner callables / call sites / local vars: none in this fixture return run @@ -193,9 +193,9 @@ def test_backend_parity_for_module_level_lookup(): # regression: class-scoped lookup keeps working (get_method must not become module-only) # ---------------------------------------------------------------------------------------------- def test_get_method_still_resolves_class_methods_local(): - greet = PyCallable(name="greet", path="pkg/models.py", signature="pkg.models.Entity.greet", code="...") - entity = PyClass(name="Entity", signature="pkg.models.Entity", methods={"greet": greet}) - module = PyModule(file_path="pkg/models.py", module_name="pkg.models", classes={"pkg.models.Entity": entity}) + greet = PyCallable(name="greet", path="pkg/models.py", signature="pkg.models.Entity.greet") + entity = PyClass(name="Entity", signature="pkg.models.Entity", callables={"greet": greet}) + module = PyModule(file_path="pkg/models.py", module_name="pkg.models", types={"pkg.models.Entity": entity}) app = PyApplication(symbol_table={"pkg/models.py": module}) backend = object.__new__(PyCodeanalyzer) diff --git a/tests/analysis/python/test_python_schema_contract.py b/tests/analysis/python/test_python_schema_contract.py new file mode 100644 index 00000000..7fec0f22 --- /dev/null +++ b/tests/analysis/python/test_python_schema_contract.py @@ -0,0 +1,161 @@ +################################################################################ +# Copyright IBM Corporation 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +"""Unit tests for the Python backends' schema-2.0.0 contract (no analyzer run, no live Neo4j). + +Three things are exercised here: + +* the in-process backend's fail-fast on the ``Analysis`` envelope's ``schema_version`` + (``_run_analyzer`` unwraps ``Analysis.application`` and refuses any other schema); +* the Neo4j backend's fail-fast ``schema_version`` gate (``_check_schema_version``), read from + the scoped ``:PyApplication`` node — mirroring ``TSNeo4jBackend``; +* the call-graph CanNode-id translation: schema-2.0.0 ``PyCallEdge.src``/``dst`` are ``can://`` + ids, but the SDK's public call-graph vocabulary stays dotted signatures (externals keep their + raw ``can://`` id). +""" + +import pytest +from codeanalyzer.schema.py_schema import PyApplication, PyCallable, PyCallEdge, PyClass, PyModule + +from cldk.analysis.python.codeanalyzer.codeanalyzer import PyCodeanalyzer +from cldk.analysis.python.neo4j import PyNeo4jBackend +from cldk.utils.exceptions.exceptions import CldkSchemaMismatchException + + +# -----[ Neo4j schema-version gate ]----- +def _bare_neo4j_backend() -> PyNeo4jBackend: + """A backend instance with no live driver — enough to exercise the pure guard logic.""" + backend = PyNeo4jBackend.__new__(PyNeo4jBackend) + backend.application_name = "test_app" + return backend + + +def test_neo4j_schema_version_mismatch_fails_fast(): + with pytest.raises(CldkSchemaMismatchException): + _bare_neo4j_backend()._check_schema_version(expected="2.0.0", found="1.0.0") + + +def test_neo4j_schema_version_match_passes(): + assert _bare_neo4j_backend()._check_schema_version(expected="2.0.0", found="2.0.0") is None + + +def test_neo4j_schema_version_queried_from_application_when_not_supplied(monkeypatch): + backend = _bare_neo4j_backend() + monkeypatch.setattr(backend, "_run", lambda *a, **k: [{"v": "2.0.0"}]) + assert backend._check_schema_version(expected="2.0.0") is None + + monkeypatch.setattr(backend, "_run", lambda *a, **k: [{"v": "1.0.0"}]) + with pytest.raises(CldkSchemaMismatchException): + backend._check_schema_version(expected="2.0.0") + + +def test_neo4j_schema_version_absent_fails_fast(monkeypatch): + backend = _bare_neo4j_backend() + # No :PyApplication row at all (empty/foreign DB, pre-2.0 emitter) ⇒ found is None ⇒ mismatch. + monkeypatch.setattr(backend, "_run", lambda *a, **k: []) + with pytest.raises(CldkSchemaMismatchException): + backend._check_schema_version(expected="2.0.0") + + +# -----[ in-process envelope gate ]----- +class _StubAnalysis: + def __init__(self, schema_version, application=None, max_level=1): + self.schema_version = schema_version + self.application = application + # Real schema 2.0.0+ envelopes always carry max_level (pydantic default 1) — + # _run_analyzer now captures it unconditionally, so the stub must too. + self.max_level = max_level + + +class _StubCodeanalyzer: + """Stands in for ``codeanalyzer.core.Codeanalyzer`` — returns a canned envelope.""" + + envelope: _StubAnalysis = None + + def __init__(self, options): + pass + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def analyze(self): + return self.envelope + + +def _bare_local_backend() -> PyCodeanalyzer: + """An instance with just the attributes ``_run_analyzer`` reads — no analysis at init.""" + backend = PyCodeanalyzer.__new__(PyCodeanalyzer) + backend.project_dir = "unused" + backend.analysis_json_path = None + backend.use_ray = False + backend.eager_analysis = False + backend.cache_dir = None + backend.target_files = None + return backend + + +def test_local_backend_rejects_wrong_envelope_schema(monkeypatch): + import cldk.analysis.python.codeanalyzer.codeanalyzer as mod + + _StubCodeanalyzer.envelope = _StubAnalysis(schema_version="3.0.0") + monkeypatch.setattr(mod, "Codeanalyzer", _StubCodeanalyzer) + with pytest.raises(CldkSchemaMismatchException): + _bare_local_backend()._run_analyzer() + + +def test_local_backend_unwraps_matching_envelope(monkeypatch): + import cldk.analysis.python.codeanalyzer.codeanalyzer as mod + + app = PyApplication(symbol_table={}) + _StubCodeanalyzer.envelope = _StubAnalysis(schema_version=PyCodeanalyzer.SUPPORTED_ANALYSIS_SCHEMA, application=app) + monkeypatch.setattr(mod, "Codeanalyzer", _StubCodeanalyzer) + assert _bare_local_backend()._run_analyzer() is app + + +# -----[ call-graph CanNode-id → signature translation ]----- +def test_call_graph_translates_can_ids_to_signatures(): + f = PyCallable(name="f", path="pkg/mod.py", signature="pkg.mod.f", id="can://python/proj/pkg/mod.py/f()") + m = PyCallable(name="m", path="pkg/mod.py", signature="pkg.mod.A.m", id="can://python/proj/pkg/mod.py/A/m(self)") + module = PyModule( + file_path="pkg/mod.py", + module_name="mod", + types={"pkg.mod.A": PyClass(name="A", signature="pkg.mod.A", callables={"m": m})}, + functions={"f": f}, + ) + external_id = "can://python/proj/@external/os/getcwd" + app = PyApplication( + symbol_table={"pkg/mod.py": module}, + call_graph=[ + PyCallEdge(src=f.id, dst=m.id, weight=1, prov=["jedi"]), + PyCallEdge(src=f.id, dst=external_id, weight=1, prov=["jedi"]), + ], + ) + + backend = PyCodeanalyzer.__new__(PyCodeanalyzer) + backend.application = app + backend.call_graph = None + + graph = backend.get_call_graph() + # Symbol-table callables appear under their dotted signatures; the external keeps its can:// id. + assert set(graph.nodes) == {"pkg.mod.f", "pkg.mod.A.m", external_id} + assert graph.has_edge("pkg.mod.f", "pkg.mod.A.m") + assert graph.has_edge("pkg.mod.f", external_id) + + callers = backend.get_all_callers("pkg.mod.A", "m") + assert [c["caller_signature"] for c in callers["caller_details"]] == ["pkg.mod.f"] diff --git a/tests/analysis/typescript/test_typescript_get_method_functions.py b/tests/analysis/typescript/test_typescript_get_method_functions.py index 962f62fa..bf1fba5e 100644 --- a/tests/analysis/typescript/test_typescript_get_method_functions.py +++ b/tests/analysis/typescript/test_typescript_get_method_functions.py @@ -218,15 +218,15 @@ def stub_neo4j_backend(): baz_props = _callable_props(_baz()) qux_props = _callable_props(_qux()) - has_method_query = "MATCH (o:Symbol {signature: $sig})-[:HAS_METHOD]->(m:Callable {name: $name}) RETURN properties(m) AS p LIMIT 1" + has_method_query = "MATCH (o:CanNode {signature: $sig})-[:TS_HAS_METHOD]->(m:TSCallable {name: $name}) RETURN properties(m) AS p LIMIT 1" exact_sig_query = ( - "MATCH (parent)-[:DECLARES]->(c:Callable {signature: $sig}) " - "WHERE (parent:Module OR parent:Namespace) AND c._module IN $mods " + "MATCH (parent)-[:TS_DECLARES]->(c:TSCallable {signature: $sig}) " + "WHERE (parent:TSModule OR parent:TSNamespace) AND c._module IN $mods " "RETURN properties(c) AS p LIMIT 1" ) short_name_query = ( - "MATCH (parent)-[:DECLARES]->(c:Callable {name: $name}) " - "WHERE (parent:Module OR parent:Namespace) AND c._module IN $mods AND c.signature STARTS WITH $prefix " + "MATCH (parent)-[:TS_DECLARES]->(c:TSCallable {name: $name}) " + "WHERE (parent:TSModule OR parent:TSNamespace) AND c._module IN $mods AND c.signature STARTS WITH $prefix " "RETURN properties(c) AS p LIMIT 1" ) diff --git a/tests/analysis/typescript/test_typescript_neo4j_backend.py b/tests/analysis/typescript/test_typescript_neo4j_backend.py index 8013c55b..87d70eff 100644 --- a/tests/analysis/typescript/test_typescript_neo4j_backend.py +++ b/tests/analysis/typescript/test_typescript_neo4j_backend.py @@ -68,10 +68,20 @@ def _neo4j_reachable() -> bool: return False -pytestmark = pytest.mark.skipif( - not _neo4j_reachable(), - reason=f"no Neo4j reachable at {NEO4J_URI} (set CLDK_TEST_NEO4J_URI / _USER / _PASSWORD)", -) +# The read-only backend now speaks graph schema 2.0.0 (TS-prefixed vocabulary, CanNode keys), but +# the pinned ``codeanalyzer-typescript`` is still 0.4.3, whose ``--emit neo4j`` projection predates +# that schema — so this out-of-band-populated integration suite cannot produce a conformant graph +# yet. It is unconditionally skipped until the analyzer pin moves to >=1.0.0 (release-train Task 9); +# the ``_neo4j_reachable`` guard is retained for when it is re-enabled. +pytestmark = [ + pytest.mark.skipif( + not _neo4j_reachable(), + reason=f"no Neo4j reachable at {NEO4J_URI} (set CLDK_TEST_NEO4J_URI / _USER / _PASSWORD)", + ), + pytest.mark.skip( + reason="needs a graph schema 2.0.0 database (codeanalyzer-typescript>=1.0.0); the pin is still 0.4.3 until Task 9 moves it", + ), +] def _codeanalyzer_ts_exec() -> list[str]: diff --git a/tests/analysis/typescript/test_typescript_neo4j_schema.py b/tests/analysis/typescript/test_typescript_neo4j_schema.py new file mode 100644 index 00000000..aec8b952 --- /dev/null +++ b/tests/analysis/typescript/test_typescript_neo4j_schema.py @@ -0,0 +1,193 @@ +################################################################################ +# Copyright IBM Corporation 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +"""Unit tests for the TS Neo4j backend's graph-schema contract (no live Neo4j required). + +Four things are exercised here without ever opening a Bolt connection: + +* the fail-fast ``schema_version`` gate (``_check_schema_version``) the backend runs on first use; +* the application-id resolution guard (``_resolve_application_id``) — the suffix match must bind + exactly one ``:Application`` or raise, never silently merge two apps' module scopes; +* the accessors whose vocabulary is *not projected* into graph schema 2.0.0 (decorators, + attributes/fields, module imports/exports, variables) — these must raise a clear + ``NotImplementedError`` rather than silently returning wrong data; +* the constructed Cypher for the two riskiest 2.0.0 rewrites (the call-site body-node path and + module `_module` scoping) — behavioral verification against a live graph is Task 9's e2e run. + +All are tested against a bare ``__new__`` instance (no ``__init__``, so no driver), because the +logic under test never needs a real session. +""" + +import pytest + +from cldk.analysis.typescript.neo4j import TSNeo4jBackend +from cldk.utils.exceptions.exceptions import CldkSchemaMismatchException, CodeanalyzerUsageException + + +def _bare_backend() -> TSNeo4jBackend: + """A backend instance with no live driver — enough to exercise pure query/guard logic.""" + return TSNeo4jBackend.__new__(TSNeo4jBackend) + + +# -----[ schema-version gate ]----- +def test_schema_version_mismatch_fails_fast(): + backend = _bare_backend() + with pytest.raises(CldkSchemaMismatchException): + backend._check_schema_version(expected="2.0.0", found="1.0.0") + + +def test_schema_version_match_passes(): + backend = _bare_backend() + # An exact match is a no-op (returns None, raises nothing). + assert backend._check_schema_version(expected="2.0.0", found="2.0.0") is None + + +def test_schema_version_queried_from_application_when_not_supplied(monkeypatch): + backend = _bare_backend() + monkeypatch.setattr(backend, "_run", lambda *a, **k: [{"v": "2.0.0"}]) + # Reads (:Application).schema_version and finds the supported version ⇒ passes. + assert backend._check_schema_version(expected="2.0.0") is None + + monkeypatch.setattr(backend, "_run", lambda *a, **k: [{"v": "1.0.0"}]) + with pytest.raises(CldkSchemaMismatchException): + backend._check_schema_version(expected="2.0.0") + + +def test_schema_version_absent_fails_fast(monkeypatch): + backend = _bare_backend() + # No Application row at all (empty/foreign DB) ⇒ found is None ⇒ mismatch. + monkeypatch.setattr(backend, "_run", lambda *a, **k: []) + with pytest.raises(CldkSchemaMismatchException): + backend._check_schema_version(expected="2.0.0") + + +# -----[ application-id resolution guard ]----- +def test_resolve_application_id_unique_match(monkeypatch): + backend = _bare_backend() + backend.application_name = "frontend" + monkeypatch.setattr(backend, "_run", lambda *a, **k: [{"id": "can://repo-a/frontend"}]) + assert backend._resolve_application_id() == "can://repo-a/frontend" + + +def test_resolve_application_id_no_match_raises(monkeypatch): + backend = _bare_backend() + backend.application_name = "frontend" + monkeypatch.setattr(backend, "_run", lambda *a, **k: []) + with pytest.raises(CodeanalyzerUsageException, match="no :Application found"): + backend._resolve_application_id() + + +def test_resolve_application_id_ambiguous_raises_naming_candidates(monkeypatch): + backend = _bare_backend() + backend.application_name = "frontend" + monkeypatch.setattr( + backend, + "_run", + lambda *a, **k: [{"id": "can://repo-a/frontend"}, {"id": "can://repo-b/frontend"}], + ) + with pytest.raises(CodeanalyzerUsageException) as exc_info: + backend._resolve_application_id() + message = str(exc_info.value) + assert "ambiguous" in message + assert "can://repo-a/frontend" in message + assert "can://repo-b/frontend" in message + + +# -----[ constructed-Cypher shape for the riskiest 2.0.0 rewrites ]----- +def _recording_backend(monkeypatch): + """A bare backend whose ``_run`` records every (query, params) call and returns no rows.""" + backend = _bare_backend() + backend._modules = ["src/mod.ts"] + calls = [] + + def _run(query, **params): + calls.append((query, params)) + return [] + + monkeypatch.setattr(backend, "_run", _run) + return backend, calls + + +def test_call_site_query_uses_body_node_path(monkeypatch): + backend, calls = _recording_backend(monkeypatch) + backend.get_call_sites("src/mod.Foo.bar") + (query, params), = calls + assert "-[:TS_HAS_BODY_NODE]->" in query + assert "TSBodyNode {kind: 'call'}" in query + assert params == {"sig": "src/mod.Foo.bar"} + + +def test_calling_lines_query_reads_callee_property(monkeypatch): + backend, calls = _recording_backend(monkeypatch) + backend.get_calling_lines("src/mod.Foo.bar") + (query, _), = calls + assert "TSBodyNode {kind: 'call'}" in query + assert "cs.callee = $sig" in query + + +def test_call_targets_query_reads_callee_property(monkeypatch): + backend, calls = _recording_backend(monkeypatch) + backend.get_call_targets("src/mod.Foo.bar") + (query, _), = calls + assert "-[:TS_HAS_BODY_NODE]->" in query + assert "cs.callee AS" in query + + +def test_external_symbols_query_resolves_via_body_nodes(monkeypatch): + backend, calls = _recording_backend(monkeypatch) + backend.get_external_symbols() + (query, _), = calls + assert "-[:TS_CALLS]->(e:TSExternal)" in query + assert "(cs:TSBodyNode {kind: 'call'})-[:TS_RESOLVES_TO]->(e:TSExternal)" in query + + +def test_module_lookup_query_keys_on_module_property(monkeypatch): + backend, calls = _recording_backend(monkeypatch) + assert backend.get_typescript_module("src/mod.ts") is None # no rows stubbed + (query, params), = calls + assert "TSModule {_module: $key}" in query + assert params == {"key": "src/mod.ts"} + + +def test_module_keys_query_scoped_to_resolved_application_id(monkeypatch): + backend, calls = _recording_backend(monkeypatch) + backend._app_id = "can://repo-a/frontend" + backend._load_module_keys() + (query, params), = calls + assert "Application {id: $app_id}" in query + assert "-[:TS_HAS_MODULE]->" in query + assert params == {"app_id": "can://repo-a/frontend"} + + +# -----[ accessors with no graph support in schema 2.0.0 ]----- +_FALLBACK_CALLS = [ + ("get_decorators", ("src/x.f",)), + ("get_class_decorators", ("src/x.C",)), + ("get_methods_with_decorators", (["Get"],)), + ("get_classes_with_decorators", (["Controller"],)), + ("get_all_fields", ("src/x.C",)), + ("get_interface_properties", ("src/x.I",)), + ("get_imports", ()), + ("get_all_exports", ()), + ("get_all_variables", ()), +] + + +@pytest.mark.parametrize("method_name,args", _FALLBACK_CALLS) +def test_unprojected_accessor_raises_not_implemented(method_name, args): + backend = _bare_backend() + with pytest.raises(NotImplementedError, match="graph schema 2.0.0"): + getattr(backend, method_name)(*args) diff --git a/tests/conftest.py b/tests/conftest.py index 5f3c0f1f..18837128 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -66,8 +66,8 @@ def analysis_json(analysis_json_fixture) -> str: def codeanalyzer_backend_path(): """Backend-path override for the Java analyzer in tests. - Returns None so the analyzer uses its default: the JVM-free native binary shipped in the - ``codeanalyzer-java`` PyPI package (``python -m codeanalyzer_java``). + Returns None so the analyzer uses its default: the ``codeanalyzer-*.jar`` bundled under + ``cldk/analysis/java/codeanalyzer/jar/``, run on a cached JDK (``[java, -jar, ]``). """ return None diff --git a/tests/graph/__init__.py b/tests/graph/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/graph/test_capability.py b/tests/graph/test_capability.py new file mode 100644 index 00000000..19253061 --- /dev/null +++ b/tests/graph/test_capability.py @@ -0,0 +1,23 @@ +import pytest +from cldk.graph.capability import require, CapabilityError + + +class P: + def __init__(self, lvl): self._l = lvl + def max_level(self): return self._l + + +def test_satisfied_returns_none(): + assert require(3, P(4), strict=False, what="slice_backward") is None + + +def test_degrade_returns_note(): + note = require(4, P(3), strict=False, what="interprocedural flows_to") + assert note["requested"] == 4 and note["available"] == 3 + assert "UNKNOWN, not safety" in note["gap"] + + +def test_strict_raises(): + with pytest.raises(CapabilityError) as e: + require(4, P(3), strict=True, what="flows_to") + assert "level 4" in str(e.value) diff --git a/tests/graph/test_cpg_local.py b/tests/graph/test_cpg_local.py new file mode 100644 index 00000000..1c26ef7a --- /dev/null +++ b/tests/graph/test_cpg_local.py @@ -0,0 +1,412 @@ +"""CpgLocalProviderMixin: Step 1's hand-built Application (kept verbatim) plus the same five +primitives exercised end-to-end against a REAL committed L4 golden sample +(tests/resources/cpg/py-a4.json) — including driving the merged Engine on real data.""" +import json +from pathlib import Path +import networkx as nx +from cldk.graph._cpg_local import CpgLocalProviderMixin +from cldk.graph.engine import Engine +from cldk.models.cpg.models import Application, Module, Node, Edge, Span +from cldk.models.cpg import AnalysisPayload +from cldk.models.cpg import Edge as CpgEdge + + +def _app(): + call = Node(id="can://x/m.py/f", kind="function", signature="f", + span=Span(start=(1, 0), end=(4, 0), bytes=(0, 40)), + body={"f@1:0": Node(id="can://x/m.py/f@1:0", kind="statement", + span=Span(start=(1, 0), end=(1, 8), bytes=(0, 8))), + "f@2:0": Node(id="can://x/m.py/f@2:0", kind="statement", + span=Span(start=(2, 0), end=(2, 8), bytes=(9, 17)))}, + cfg=[Edge(src="can://x/m.py/f@1:0", dst="can://x/m.py/f@2:0", kind="fallthrough")], + ddg=[Edge(src="can://x/m.py/f@1:0", dst="can://x/m.py/f@2:0", var="a", prov=["ssa"])], + cdg=[], summary=[]) + mod = Module(id="can://x/m.py", source="a = 1\nb = a\n", functions={"f": call}, types={}) + return Application(id="can://x", symbol_table={"m.py": mod}, call_graph=[], param_in=[], param_out=[]) + + +class Backend(CpgLocalProviderMixin): + def __init__(self): self.application = _app(); self._level = 3 + def max_level(self): return self._level + + +def test_program_graph_has_body_and_edges(): + g = Backend().program_graph("can://x/m.py/f") + assert set(g.nodes) == {"can://x/m.py/f@1:0", "can://x/m.py/f@2:0"} + fams = {d["family"] for _, _, d in g.edges(data=True)} + assert fams == {"cfg", "ddg"} + + +def test_resolve_location_hits_line(): + assert Backend().resolve_location("m.py", 2) == ["can://x/m.py/f@2:0"] + + +def test_source_slice_reads_module_source(): + fl, code = Backend().source_slice("can://x/m.py/f@1:0") + assert fl == "m.py:1" and code == "a = 1\nb " + + +# --- Golden fixture: a REAL, conformant L4 codeanalyzer-python sample ------------------------ +# Unlike the hand-built Application above (whose body nodes carry an explicit .id matching the +# brief's own convention), a real analyzer leaves body-node .id unset — they're keyed by LOCAL +# position ("3:8", "@entry", ...) per Node's own docstring and tests/models/cpg/test_node.py's +# test_body_node_missing_id_parses. Application-level param_in/param_out already reference these +# as "@" (verified against the raw JSON below), which is the scheme the +# mixin must reproduce so provider-synthesized ids agree with the application's own. +RES = Path(__file__).parent.parent / "resources" / "cpg" +GOLDEN = AnalysisPayload(**json.loads((RES / "py-a4.json").read_text())).application + +MOD_PATH = "pkg/mod.py" +ENTRY = "can://python/pyfix/pkg/mod.py/entry()" +RESET_PW = "can://python/pyfix/pkg/mod.py/ResUsers/reset_password(self,login)" +ACTION = "can://python/pyfix/pkg/mod.py/ResUsers/_action_reset_password(self,ids)" + +# reset_password's own body keys, straight from py-a4.json (11 total: 2 spanned source +# positions, entry/exit, 2 formal_in/2 formal_out ports, 3 actual_in/out ports for its one +# call site). +RESET_PW_BODY_KEYS = ["3:15", "@entry", "3:8", "@exit", "@formal_in:0", "@formal_in:1", + "@formal_out:0", "@formal_out:1", "3:8/actual_in:0", "3:8/actual_in:1", + "3:8/actual_out"] + + +def _qid(callable_id: str, local_key: str) -> str: + # Independent re-derivation of the qualification scheme (not imported from the mixin under + # test) so a bug in the mixin's own _qualify can't silently validate itself. + return callable_id + local_key if local_key.startswith("@") else f"{callable_id}@{local_key}" + + +class GoldenBackend(CpgLocalProviderMixin): + application = GOLDEN + def max_level(self): return 4 + + +def test_golden_program_graph_preserves_parallel_edges(): + # reset_password has THREE ddg edges between the same pair (@entry -> 3:8, one per var) and + # TWO cfg edges between another same pair (3:8 -> @exit, kinds 'exception' and 'return'). + # provider.py's ABC docstring requires these to stay distinct parallel edges, each with its + # own family/var/prov/kind — a naive MultiDiGraph key of just the family name would silently + # collapse each trio/pair into one edge. + g = GoldenBackend().program_graph(RESET_PW) + assert isinstance(g, nx.MultiDiGraph) + assert set(g.nodes) == {_qid(RESET_PW, k) for k in RESET_PW_BODY_KEYS} + assert g.number_of_edges() == 7 # 3 cfg + 1 cdg + 3 ddg (summary excluded) + + fams = [d["family"] for _, _, d in g.edges(data=True)] + assert sorted(fams) == sorted(["cfg", "cfg", "cfg", "cdg", "ddg", "ddg", "ddg"]) + + ddg_vars = {d["var"] for _, _, d in g.edges(data=True) if d["family"] == "ddg"} + assert ddg_vars == {"login", "self", "self._action_reset_password"} # none collapsed away + + exit_kinds = {d["kind"] for _, _, d in g.edges(data=True) + if d["family"] == "cfg" and d["kind"] in ("exception", "return")} + assert exit_kinds == {"exception", "return"} # both 3:8 -> @exit edges survive + + assert g.has_edge(_qid(RESET_PW, "@entry"), _qid(RESET_PW, "3:8")) + + +def test_golden_resolve_location_hits_real_lines(): + b = GoldenBackend() + # line 3 has TWO spanned body nodes in reset_password: the call (col 15) and the return + # (col 8) — col=None must return both; a col narrows to exactly one. + assert set(b.resolve_location(MOD_PATH, 3)) == {_qid(RESET_PW, "3:15"), _qid(RESET_PW, "3:8")} + assert b.resolve_location(MOD_PATH, 3, 8) == [_qid(RESET_PW, "3:8")] + assert b.resolve_location(MOD_PATH, 3, 15) == [_qid(RESET_PW, "3:15")] + assert b.resolve_location("mod.py", 3, 8) == [_qid(RESET_PW, "3:8")] # basename suffix match + assert b.resolve_location(MOD_PATH, 2) == [] # the `def` line has no body node + + +def test_golden_source_slice_reads_real_module_source(): + b = GoldenBackend() + fl, code = b.source_slice(_qid(RESET_PW, "3:8")) + assert fl == "pkg/mod.py:3" and code == "return self._action_reset_password([login])" + fl2, code2 = b.source_slice(_qid(RESET_PW, "3:15")) + assert fl2 == "pkg/mod.py:3" and code2 == "self._action_reset_password([login])" + # @entry has no span (it's a synthetic CFG node, not a source position) — degrades to + # (path, None) rather than raising. + fl3, code3 = b.source_slice(_qid(RESET_PW, "@entry")) + assert fl3 == "pkg/mod.py" and code3 is None + + +def test_golden_callable_of_maps_body_node_and_is_identity_on_callable(): + b = GoldenBackend() + assert b.callable_of(_qid(RESET_PW, "3:8")) == RESET_PW + assert b.callable_of(_qid(RESET_PW, "@entry")) == RESET_PW + assert b.callable_of(_qid(ACTION, "5:8")) == ACTION + assert b.callable_of(RESET_PW) == RESET_PW # passthrough: not a body vertex + assert b.callable_of("can://nonexistent") == "can://nonexistent" + + +def test_golden_sdg_edges_include_param_in_out_and_summary_tagged_by_kind(): + # param_in/param_out are already @ qualified at the Application level; + # summary is per-callable and LOCAL like cfg/cdg/ddg, so it needs the same qualification the + # mixin applies to program_graph. All three must be tagged with their OWN kind + # ("param_in"/"param_out"/"summary") — real param_in/param_out/summary edges carry kind=None + # in the raw JSON, and engine.flows_to reports a boundary hop's family ("sdg") only when kind + # is unset (test_engine_interproc.py::test_flow_boundary_hop_reports_sdg_kind, already + # merged), so an untagged edge here would surface as the opaque "sdg" instead. + got = {(e.src, e.dst, e.kind) for e in GoldenBackend().sdg_edges()} + assert got == { + (_qid(RESET_PW, "3:8/actual_in:0"), _qid(ACTION, "formal_in:0"), "param_in"), + (_qid(RESET_PW, "3:8/actual_in:1"), _qid(ACTION, "formal_in:1"), "param_in"), + (_qid(ENTRY, "7:4/actual_in:0"), _qid(RESET_PW, "formal_in:1"), "param_in"), + (_qid(ACTION, "formal_out:0"), _qid(RESET_PW, "3:8/actual_out"), "param_out"), + (_qid(RESET_PW, "formal_out:0"), _qid(ENTRY, "7:4/actual_out"), "param_out"), + (_qid(RESET_PW, "3:8/actual_in:1"), _qid(RESET_PW, "3:8/actual_out"), "summary"), + (_qid(ENTRY, "7:4/actual_in:0"), _qid(ENTRY, "7:4/actual_out"), "summary"), + } + + +def test_golden_engine_slice_backward_on_real_line(): + # Feeds the mixin to the real merged Engine and drives it with a plain 'file:line[:col]' + # seed string, exactly as an SDK caller would — proving the mixin's five primitives compose + # correctly through resolve_vertex -> callable_of -> program_graph -> sdg overlay -> subgraph. + r = Engine(GoldenBackend()).slice_backward(f"{MOD_PATH}:3:8") + assert bool(r) and len(r) > 0 + # only @entry precedes 3:8 (via cfg/cdg/ddg); the sdg overlay is engaged (level 4, ddg + # requested) but nothing points INTO the bare '3:8' return node from another callable in + # this sample — the interprocedural wiring attaches at the unspanned actual/formal ports. + assert set(r.uris()) == {_qid(RESET_PW, "3:8"), _qid(RESET_PW, "@entry")} + assert r.explain()["interprocedural"] is True + assert r.explain()["level"] == 4 + + ev = {e["uri"]: e for e in r.evidence} + assert ev[_qid(RESET_PW, "3:8")]["code"] == "return self._action_reset_password([login])" + assert ev[_qid(RESET_PW, "3:8")]["role"] == "seed" + assert ev[_qid(RESET_PW, "@entry")]["code"] is None + assert ev[_qid(RESET_PW, "@entry")]["role"] == "def" + + +def test_golden_engine_flows_to_crosses_callable_boundary(): + # A real cross-callable dataflow hop, driven end to end through the merged Engine: the value + # passed as reset_password's 2nd call argument (3:8/actual_in:1) flows via param_in into + # _action_reset_password's 2nd formal parameter — proving the mixin's sdg_edges() actually + # drives Engine.flows_to's interprocedural path on real data, not just synthetic fixtures. + src = _qid(RESET_PW, "3:8/actual_in:1") + dst = _qid(ACTION, "formal_in:1") + r = Engine(GoldenBackend()).flows_to(src, dst) + assert len(r.paths) == 1 + hop = r.paths[0].hops[0] + assert hop["from"] == src and hop["to"] == dst + assert hop["kind"] == "param_in" # the specific sdg kind, not the opaque family "sdg" + # this param_in edge carries no ssa/points-to provenance in the raw sample, so the honest + # confidence is 'unresolved' rather than a stronger tier it can't back up. + assert r.paths[0].confidence == "unresolved" + + +# --- Slim-model hardening: the mixin must duck-type BOTH the cldk cpg models AND the slimmer +# upstream codeanalyzer-python 1.0.2 models. These stand-ins carry ONLY the fields the upstream +# BodyNode/CfgEdge/CdgEdge/DdgEdge/ParamEdge/SummaryEdge models declare (verified against +# codeanalyzer.schema.py_schema): no .id on body nodes, no .var/.prov on cfg edges, no .kind at +# all on cdg/param/summary edges. Accessing a field the real upstream model lacks must raise +# AttributeError here too, or this test would validate nothing. +class _SlimSpan: + def __init__(self, start): + self.start = start + + +class _SlimBodyNode: + """Mirrors upstream BodyNode: {kind, span, callee, of, parent} — no `.id`.""" + + def __init__(self, kind, span=None, callee=None, of=None, parent=None): + self.kind = kind + self.span = span + self.callee = callee + self.of = of + self.parent = parent + + +class _SlimCfgEdge: + """Mirrors upstream CfgEdge: {src, dst, kind} — no `.var`/`.prov`.""" + + def __init__(self, src, dst, kind="fallthrough"): + self.src = src + self.dst = dst + self.kind = kind + + +class _SlimCdgEdge: + """Mirrors upstream CdgEdge: {src, dst} — no `.kind`/`.var`/`.prov`.""" + + def __init__(self, src, dst): + self.src = src + self.dst = dst + + +class _SlimDdgEdge: + """Mirrors upstream DdgEdge: {src, dst, var, prov} — no `.kind`.""" + + def __init__(self, src, dst, var=None, prov=None): + self.src = src + self.dst = dst + self.var = var + self.prov = prov or [] + + +class _SlimParamEdge: + """Mirrors upstream ParamEdge: {src, dst} only.""" + + def __init__(self, src, dst): + self.src = src + self.dst = dst + + +class _SlimSummaryEdge: + """Mirrors upstream SummaryEdge: {src, dst} only.""" + + def __init__(self, src, dst): + self.src = src + self.dst = dst + + +class _SlimCallable: + def __init__(self, id, body, cfg=None, cdg=None, ddg=None, summary=None, callables=None, types=None): + self.id = id + self.body = body + self.cfg = cfg or [] + self.cdg = cdg or [] + self.ddg = ddg or [] + self.summary = summary or [] + # A callable can declare further nested callables (closures) and/or locally-defined + # classes — both absent by default (upstream PyCallable.callables/.types, empty dicts). + self.callables = callables or {} + self.types = types or {} + + +class _SlimClass: + """Mirrors upstream PyClass's type-facet containers: {callables, types} — methods and any + classes nested inside this one. No `.body`/`.cfg`/etc — a class contributes no body of its + own, only its members do.""" + + def __init__(self, callables=None, types=None): + self.callables = callables or {} + self.types = types or {} + + +class _SlimModule: + def __init__(self, functions, types=None): + self.types = types or {} + self.functions = functions + + +class _SlimApplication: + def __init__(self, symbol_table, param_in=None, param_out=None): + self.symbol_table = symbol_table + self.param_in = param_in or [] + self.param_out = param_out or [] + + +SLIM_CID = "can://slim/m.py/f" + + +# Ids for the nested-recursion fixture (#270 final review Finding 1): a closure declared +# inside the top-level function `f`, and a method reachable only through a class nested inside +# another class — both must be discoverable by _index()'s recursive walk, mirroring +# codeanalyzer-python's own _walk_callable/_walk_class_callables and the Neo4j emitter's +# _project_callable/_project_class. +CLOSURE_CID = f"{SLIM_CID}/g" +OUTER_METHOD_CID = "can://slim/m.py/Outer.m" +INNER_METHOD_CID = "can://slim/m.py/Outer.Inner.n" + + +def _slim_app(): + # Body dict deliberately inserts the col-15 key BEFORE the col-8 key, so a resolve_location + # that merely preserved dict insertion order would return them in the wrong order. + body = { + "3:15": _SlimBodyNode(kind="call", span=_SlimSpan(start=(3, 15))), + "3:8": _SlimBodyNode(kind="statement", span=_SlimSpan(start=(3, 8))), + "@entry": _SlimBodyNode(kind="entry", span=None), + } + # A closure nested inside `f` — its own callable, with its own body, reachable only via + # f.callables (never listed at module/class top level). + g = _SlimCallable(id=CLOSURE_CID, body={"5:0": _SlimBodyNode(kind="statement", span=_SlimSpan(start=(5, 0)))}) + f = _SlimCallable( + id=SLIM_CID, + body=body, + cfg=[_SlimCfgEdge(src="@entry", dst="3:8", kind="fallthrough")], + cdg=[_SlimCdgEdge(src="@entry", dst="3:15")], + ddg=[_SlimDdgEdge(src="@entry", dst="3:8", var="x", prov=["ssa"])], + summary=[_SlimSummaryEdge(src="3:8", dst="3:15")], + callables={"g": g}, + ) + # Outer.m is an ordinary method; Outer.Inner.n is a method on a class nested TWO levels deep + # (a class inside a class) — both must surface through mod.types' recursive walk, not just + # a single top-level pass over each class's own .callables. + inner_method = _SlimCallable(id=INNER_METHOD_CID, + body={"30:0": _SlimBodyNode(kind="statement", span=_SlimSpan(start=(30, 0)))}) + inner_cls = _SlimClass(callables={"n": inner_method}) + outer_method = _SlimCallable(id=OUTER_METHOD_CID, + body={"20:0": _SlimBodyNode(kind="statement", span=_SlimSpan(start=(20, 0)))}) + outer_cls = _SlimClass(callables={"m": outer_method}, types={"Inner": inner_cls}) + mod = _SlimModule(functions={"f": f}, types={"Outer": outer_cls}) + return _SlimApplication( + symbol_table={"m.py": mod}, + param_in=[_SlimParamEdge(src="can://slim/other@formal_in:0", dst=f"{SLIM_CID}@3:8")], + param_out=[_SlimParamEdge(src=f"{SLIM_CID}@3:8", dst="can://slim/other@formal_out:0")], + ) + + +class SlimBackend(CpgLocalProviderMixin): + application = _slim_app() + + def max_level(self): + return 4 + + +def test_slim_program_graph_duck_types_absent_fields_to_none_or_empty(): + g = SlimBackend().program_graph(SLIM_CID) + assert isinstance(g, nx.MultiDiGraph) + by_family = {d["family"]: d for _, _, d in g.edges(data=True)} + # every edge datum has family/kind/var/prov regardless of what the source model declared + assert by_family["cfg"]["kind"] == "fallthrough" + assert by_family["cfg"]["var"] is None and by_family["cfg"]["prov"] == [] + assert by_family["cdg"]["kind"] is None # CdgEdge has no .kind at all + assert by_family["cdg"]["var"] is None and by_family["cdg"]["prov"] == [] + assert by_family["ddg"]["kind"] is None # DdgEdge has no .kind at all + assert by_family["ddg"]["var"] == "x" and by_family["ddg"]["prov"] == ["ssa"] + + +def test_slim_sdg_edges_are_cpg_edge_instances_with_stamped_kind(): + edges = SlimBackend().sdg_edges() + got = {(e.src, e.dst, e.kind) for e in edges} + assert got == { + ("can://slim/other@formal_in:0", f"{SLIM_CID}@3:8", "param_in"), + (f"{SLIM_CID}@3:8", "can://slim/other@formal_out:0", "param_out"), + (f"{SLIM_CID}@3:8", f"{SLIM_CID}@3:15", "summary"), # summary re-qualified like cfg/cdg/ddg + } + assert all(isinstance(e, CpgEdge) for e in edges) + + +def test_slim_resolve_location_orders_by_start_line_and_col_not_insertion(): + hits = SlimBackend().resolve_location("m.py", 3) + assert hits == [f"{SLIM_CID}@3:8", f"{SLIM_CID}@3:15"] # col 8 first, despite 3:15 inserted first + + +def test_slim_program_graph_unknown_callable_returns_empty_graph_not_keyerror(): + g = SlimBackend().program_graph("can://nowhere") + assert isinstance(g, nx.MultiDiGraph) + assert g.number_of_nodes() == 0 and g.number_of_edges() == 0 + + +def test_index_recurses_into_closures_and_doubly_nested_class_methods(): + # #270 final review Finding 1 (Critical): _index() used to walk only mod.functions and + # mod.types[*].callables, one level — a closure declared inside a function, or a method on a + # class nested inside another class, was silently invisible (empty program_graph, no owning + # callable, no resolve_location hit) even though the Neo4j emitter walks these recursively. + b = SlimBackend() + + g = b.program_graph(CLOSURE_CID) + assert set(g.nodes) == {f"{CLOSURE_CID}@5:0"} + + m = b.program_graph(OUTER_METHOD_CID) + assert set(m.nodes) == {f"{OUTER_METHOD_CID}@20:0"} + + n = b.program_graph(INNER_METHOD_CID) + assert set(n.nodes) == {f"{INNER_METHOD_CID}@30:0"} + + assert b.callable_of(f"{CLOSURE_CID}@5:0") == CLOSURE_CID + assert b.callable_of(f"{OUTER_METHOD_CID}@20:0") == OUTER_METHOD_CID + assert b.callable_of(f"{INNER_METHOD_CID}@30:0") == INNER_METHOD_CID + + assert b.resolve_location("m.py", 5) == [f"{CLOSURE_CID}@5:0"] + assert b.resolve_location("m.py", 20) == [f"{OUTER_METHOD_CID}@20:0"] + assert b.resolve_location("m.py", 30) == [f"{INNER_METHOD_CID}@30:0"] diff --git a/tests/graph/test_engine_flows.py b/tests/graph/test_engine_flows.py new file mode 100644 index 00000000..3b6b59ae --- /dev/null +++ b/tests/graph/test_engine_flows.py @@ -0,0 +1,156 @@ +# tests/graph/test_engine_flows.py +import networkx as nx +import pytest +from cldk.graph.engine import Engine, _ddg_tier +from cldk.graph.provider import ProgramGraphProvider +from cldk.graph.capability import CapabilityError +from tests.graph.test_engine_slice import OneCallableProvider + + +def test_ddg_tier_uses_membership_not_exact_list(): + # I8: prov is a provenance SET in list form. ["ssa", "points-to"] carries strictly + # MORE evidence than ["points-to"] alone and must rank "resolved" — an exact-list + # comparison would let the stronger provenance fall through to "unresolved". + assert _ddg_tier(["ssa", "points-to"]) == "resolved" + assert _ddg_tier(["points-to"]) == "resolved" + assert _ddg_tier(["ssa"]) == "structural" + assert _ddg_tier([]) == "unresolved" + + +class ParallelEdgeProvider(ProgramGraphProvider): + # ONE node route s1 -> s2 -> s3, but the s1->s2 hop carries TWO parallel ddg edges + # (var x via ssa, var y via points-to). A MultiDiGraph enumerates the route once per + # parallel edge; the engine must collapse to one witness per distinct node route. + def program_graph(self, callable_uri): + g = nx.MultiDiGraph() + for n in ["c@1:0", "c@2:0", "c@3:0"]: + g.add_node(n, kind="statement", span=None) + g.add_edge("c@1:0", "c@2:0", key="ddg:x", family="ddg", var="x", prov=["ssa"]) + g.add_edge("c@1:0", "c@2:0", key="ddg:y", family="ddg", var="y", prov=["points-to"]) + g.add_edge("c@2:0", "c@3:0", key="ddg", family="ddg", var="z", prov=["ssa"]) + return g + + def sdg_edges(self): return [] + def resolve_location(self, file, line, col=None): return [f"c@{line}:{col or 0}"] + def source_slice(self, vertex_uri): return (f"m:{vertex_uri}", vertex_uri) + def callable_of(self, vertex_uri): return "c" + def max_level(self): return 3 + + +def test_flows_to_finds_witness_with_min_confidence(): + e = Engine(OneCallableProvider()) + r = e.flows_to("m:1", "m:3") # s1 -> s2 (ssa) -> s3 (points-to); min = structural + assert bool(r) is True + assert len(r.paths) == 1 + hops = r.paths[0].hops + assert [h["from"] for h in hops] == ["c@1:0", "c@2:0"] + assert [h["to"] for h in hops] == ["c@2:0", "c@3:0"] + assert [h["kind"] for h in hops] == ["ddg", "ddg"] + assert [h["var"] for h in hops] == ["x", "y"] + assert [h["confidence"] for h in hops] == ["structural", "resolved"] + assert r.paths[0].confidence == "structural" + + +def test_flows_to_no_path_is_falsy(): + e = Engine(OneCallableProvider()) + r = e.flows_to("m:3", "m:1") # no forward ddg path s3 -> s1 + assert not r.paths + assert bool(r) is False + + +def test_flows_to_dedups_parallel_edge_routes(): + # one node route, two parallel ddg edges on the first hop -> exactly one witness. + e = Engine(ParallelEdgeProvider()) + r = e.flows_to("m:1", "m:3") + assert len(r.paths) == 1 + p = r.paths[0] + assert [h["from"] for h in p.hops] == ["c@1:0", "c@2:0"] + assert [h["to"] for h in p.hops] == ["c@2:0", "c@3:0"] + # per-hop confidence uses the BEST (max-tier) parallel: s1->s2 resolved (points-to + # beats ssa), s2->s3 structural. Path confidence is the min over hops. + assert [h["confidence"] for h in p.hops] == ["resolved", "structural"] + assert p.confidence == "structural" + + +def test_flows_to_on_l3_degrades_but_still_returns_intra_paths(): + # C1: full flows_to semantics are interprocedural (ddg + param_in/param_out/summary), + # which is L4. On an L3 backend the non-strict call must attach a degraded note AND + # still return the intraprocedural ddg witnesses it can compute — honest degrade, + # not silent completeness and not a refusal. + e = Engine(OneCallableProvider()) # L3 backend + r = e.flows_to("m:1", "m:3") + assert "degraded" in r.explain() + assert r.explain()["degraded"]["requested"] == 4 + assert len(r.paths) == 1 # intra ddg witness still computed + + +def test_flows_to_strict_on_l3_raises(): + e = Engine(OneCallableProvider()) # L3 backend + with pytest.raises(CapabilityError): + e.flows_to("m:1", "m:3", strict=True) + + +class DiamondProvider(ProgramGraphProvider): + # TWO distinct node routes: c@s -> c@a -> c@t and c@s -> c@b -> c@t. + def program_graph(self, callable_uri): + g = nx.MultiDiGraph() + g.add_edge("c@s", "c@a", key="ddg", family="ddg", var="x", prov=["ssa"]) + g.add_edge("c@a", "c@t", key="ddg", family="ddg", var="x", prov=["ssa"]) + g.add_edge("c@s", "c@b", key="ddg", family="ddg", var="x", prov=["ssa"]) + g.add_edge("c@b", "c@t", key="ddg", family="ddg", var="x", prov=["ssa"]) + return g + def sdg_edges(self): return [] + def resolve_location(self, file, line, col=None): return [f"c@{line}"] + def source_slice(self, vertex_uri): return (vertex_uri, vertex_uri) + def callable_of(self, vertex_uri): return "c" + def max_level(self): return 4 + + +def test_flows_to_sets_truncated_when_path_cap_hit(monkeypatch): + # I5: witness enumeration is bounded; when the cap drops paths the result must SAY + # so via explain()["truncated"], instead of silently presenting a partial set as + # complete. Shrink the cap to 1 so the diamond's second route is dropped. + import cldk.graph.engine as eng + monkeypatch.setattr(eng, "_MAX_PATHS", 1) + e = Engine(DiamondProvider()) + class S: id = "c@s" + class T: id = "c@t" + r = e.flows_to(S(), T()) + assert len(r.paths) == 1 # capped at _MAX_PATHS + assert r.explain()["truncated"] is True + + +def test_flows_to_not_truncated_within_bounds(): + e = Engine(DiamondProvider()) + class S: id = "c@s" + class T: id = "c@t" + r = e.flows_to(S(), T()) + assert len(r.paths) == 2 # both diamond routes enumerated + assert r.explain()["truncated"] is False + + +def test_def_use_returns_downstream_uses(): + e = Engine(OneCallableProvider()) + r = e.def_use("m:1") # def at s1 flows to s2, s3 + assert set(r.uris()) == {"c@1:0", "c@2:0", "c@3:0"} + + +def test_def_use_evidence_role_is_use(): + # I4: downstream vertices in a def_use result are USES of the seed's definition. + e = Engine(OneCallableProvider()) + r = e.def_use("m:1") + roles = {ev["uri"]: ev["role"] for ev in r.evidence} + assert roles["c@1:0"] == "seed" + assert roles["c@2:0"] == "use" + assert roles["c@3:0"] == "use" + + +def test_def_use_seed_absent_is_consistent(): + # seed resolving to a vertex not in the dataflow graph is still in its own result; + # uris()/evidence must equal the subgraph node set (no uris/bool contradiction). + e = Engine(OneCallableProvider()) + r = e.def_use("m:99") # c@99:0 is not a node in the ddg graph + assert set(r.uris()) == set(r.subgraph.nodes()) + assert "c@99:0" in set(r.uris()) + assert bool(r) is True + assert len(r) == r.subgraph.number_of_nodes() diff --git a/tests/graph/test_engine_interproc.py b/tests/graph/test_engine_interproc.py new file mode 100644 index 00000000..3f4f1f72 --- /dev/null +++ b/tests/graph/test_engine_interproc.py @@ -0,0 +1,156 @@ +# tests/graph/test_engine_interproc.py +import networkx as nx +from cldk.graph.engine import Engine +from cldk.graph.provider import ProgramGraphProvider +from cldk.graph.capability import CapabilityError +import pytest + + +class _Edge: + def __init__(self, src, dst): self.src, self.dst, self.var, self.prov = src, dst, "a", ["points-to"] + + +class TwoCallableProvider(ProgramGraphProvider): + # caller c: c@call --param_in--> callee d; d@ret --param_out--> c@after + def program_graph(self, callable_uri): + g = nx.MultiDiGraph() # engine's _filter_edges iterates edges(keys=True) + if callable_uri == "c": + g.add_edge("c@call", "c@after", key="ddg", family="ddg", var="a", prov=["points-to"]) + else: + g.add_node("d@ret", kind="statement", span=None) + return g + def sdg_edges(self): return [_Edge("c@call", "d@in"), _Edge("d@ret", "c@after")] + def resolve_location(self, file, line, col=None): return [f"c@{line}"] + def source_slice(self, vertex_uri): return (vertex_uri, vertex_uri) + def callable_of(self, vertex_uri): return vertex_uri.split("@")[0] + def max_level(self): return 4 + + +def test_interproc_none_crosses_at_l4(): + e = Engine(TwoCallableProvider()) + # resolve_vertex only accepts a .id-bearing object, a can:// id, or a file:line[:col] + # string (see test_provider.py::test_resolve_node_object_uses_id for the same pattern) — + # "c@call" is a raw vertex id, so it must go through the .id-object path. + class Seed: id = "c@call" + r = e.slice_forward(Seed(), edges=("ddg",), interprocedural=None) + assert "d@in" in set(r.uris()) # crossed the param_in boundary + + +def test_explicit_interproc_on_l3_strict_raises(): + class L3(TwoCallableProvider): + def max_level(self): return 3 + with pytest.raises(CapabilityError): + Engine(L3()).slice_forward("c@call", interprocedural=True, strict=True) + + +class _SDGEdge: + def __init__(self, src, dst, kind): + self.src, self.dst, self.kind = src, dst, kind + self.var, self.prov = "a", ["points-to"] + + +class CrossCallableFlowProvider(ProgramGraphProvider): + # caller c: c@src --ddg--> c@call; sdg: c@call --param_in--> d@in; + # callee d: d@in --ddg--> d@sink. The flow c@src -> d@sink exists only if the + # dataflow graph spans BOTH endpoint callables plus the sdg overlay. + def program_graph(self, callable_uri): + g = nx.MultiDiGraph() + if callable_uri == "c": + g.add_edge("c@src", "c@call", key="ddg", family="ddg", var="a", prov=["ssa"]) + else: + g.add_edge("d@in", "d@sink", key="ddg", family="ddg", var="a", prov=["ssa"]) + return g + def sdg_edges(self): return [_SDGEdge("c@call", "d@in", "param_in")] + def resolve_location(self, file, line, col=None): return [f"c@{line}"] + def source_slice(self, vertex_uri): return (vertex_uri, vertex_uri) + def callable_of(self, vertex_uri): return vertex_uri.split("@")[0] + def max_level(self): return 4 + + +def test_flows_to_crosses_callable_boundary(): + # C2: a sink in a DIFFERENT callable (reachable via param_in into the callee's + # interior) must be found. Building the dataflow graph from the source's callable + # alone loses the callee's intra ddg edges and yields a false "no flow". + e = Engine(CrossCallableFlowProvider()) + class Src: id = "c@src" + class Snk: id = "d@sink" + r = e.flows_to(Src(), Snk()) + assert len(r.paths) >= 1 # a real cross-callable flow, not empty + p = r.paths[0] + assert [h["from"] for h in p.hops] == ["c@src", "c@call", "d@in"] + assert [h["to"] for h in p.hops] == ["c@call", "d@in", "d@sink"] + + +def test_flow_boundary_hop_reports_sdg_kind(): + # I6: a hop crossing the callable boundary must report WHICH sdg edge carried the + # flow (param_in/param_out/summary), not the opaque family name "sdg". Intra hops + # keep reporting their family ("ddg"). + e = Engine(CrossCallableFlowProvider()) + class Src: id = "c@src" + class Snk: id = "d@sink" + p = e.flows_to(Src(), Snk()).paths[0] + kinds = [h["kind"] for h in p.hops] + assert kinds[0] == "ddg" # intra hop: family + assert kinds[1] in {"param_in", "param_out", "summary"} # boundary hop: sdg kind + assert kinds[1] == "param_in" + assert kinds[2] == "ddg" + + +def test_family_scoped_slice_has_no_sdg_overlay_at_l4(): + # C3: the sdg (dataflow: param_in/param_out/summary) overlay must be gated on the + # ddg family being REQUESTED, not just on level/interprocedural intent. A cfg-only + # backward slice on an L4 backend must not pull in dataflow vertices from other + # callables — only dataflow crosses boundaries, and no dataflow family was asked for. + class CFGProvider(TwoCallableProvider): + def program_graph(self, callable_uri): + g = nx.MultiDiGraph() + g.add_edge("c@1", "c@2", key="cfg", family="cfg") + g.add_edge("c@2", "c@3", key="cfg", family="cfg") + return g + def sdg_edges(self): return [_Edge("d@in", "c@2")] # foreign DATAFLOW vertex + e = Engine(CFGProvider()) + class Seed: id = "c@3" + r = e.slice_backward(Seed(), edges=("cfg",)) + assert "d@in" not in set(r.uris()) # no dataflow contamination + assert set(r.uris()) == {"c@1", "c@2", "c@3"} + assert r.explain()["interprocedural"] is False # no dataflow family => no crossing + + +def test_control_deps_stays_intraprocedural_at_l4(): + # Control dependence has NO interprocedural notion in this model — only dataflow + # (param_in/param_out/summary) crosses callable boundaries. control_deps must force + # interprocedural=False; otherwise, on an L4 backend, _intra defaults to want_inter=True and + # merges the sdg dataflow overlay into a pure CDG slice, and the backward walk pulls in + # dataflow-reachable vertices from other callables with no control-dependence relation. + # + # control_deps is a BACKWARD slice, so a leaking sdg edge must be a forward-ANCESTOR edge of + # the seed: d@in -> c@body means d@in reaches c@body, so a backward slice from c@body WOULD + # pull in d@in if the overlay were applied (verified: it leaks against the unfixed code). + class CDGProvider(TwoCallableProvider): + def program_graph(self, callable_uri): + g = nx.MultiDiGraph() # only a control-dependence edge + g.add_edge("c@guard", "c@body", key="cdg", family="cdg") + return g + def sdg_edges(self): return [_Edge("d@in", "c@body")] # cross-callable DATAFLOW + e = Engine(CDGProvider()) + class Seed: id = "c@body" + r = e.control_deps(Seed()) + assert set(r.uris()) == {"c@guard", "c@body"} # only intra cdg reachability, no d@in + assert "d@in" not in set(r.uris()) # sdg dataflow did NOT cross the boundary + assert r.explain()["interprocedural"] is False # control_deps is always intraprocedural + + +def test_control_deps_evidence_role_is_control(): + # I4: a non-seed vertex in a control_deps result is there as a controlling guard, + # not as a definition — its evidence role must say so. + class CDGProvider(TwoCallableProvider): + def program_graph(self, callable_uri): + g = nx.MultiDiGraph() + g.add_edge("c@guard", "c@body", key="cdg", family="cdg") + return g + e = Engine(CDGProvider()) + class Seed: id = "c@body" + r = e.control_deps(Seed()) + roles = {ev["uri"]: ev["role"] for ev in r.evidence} + assert roles["c@body"] == "seed" + assert roles["c@guard"] == "control" diff --git a/tests/graph/test_engine_slice.py b/tests/graph/test_engine_slice.py new file mode 100644 index 00000000..d183995f --- /dev/null +++ b/tests/graph/test_engine_slice.py @@ -0,0 +1,79 @@ +# tests/graph/test_engine_slice.py +import networkx as nx +from cldk.graph.engine import Engine +from cldk.graph.provider import ProgramGraphProvider + + +def _callable_graph(): + # entry -> s1(x=1) -> s2(y=x) -> s3(return y); ddg x:s1->s2, y:s2->s3. + # MultiDiGraph so the cfg fallthrough (s1->s2, s2->s3) and the ddg edges on the + # same statement pairs stay as DISTINCT parallel edges, each keeping its attrs. + g = nx.MultiDiGraph() + for n in ["c@entry", "c@1:0", "c@2:0", "c@3:0"]: + g.add_node(n, kind="statement", span=None) + g.add_edge("c@entry", "c@1:0", key="cfg", family="cfg") + g.add_edge("c@1:0", "c@2:0", key="cfg", family="cfg") + g.add_edge("c@2:0", "c@3:0", key="cfg", family="cfg") + g.add_edge("c@1:0", "c@2:0", key="ddg", family="ddg", var="x", prov=["ssa"]) + g.add_edge("c@2:0", "c@3:0", key="ddg", family="ddg", var="y", prov=["points-to"]) + assert g.number_of_edges() == 5 # genuine parallel edges, not overwrites + return g + + +class OneCallableProvider(ProgramGraphProvider): + def program_graph(self, callable_uri): return _callable_graph() + def sdg_edges(self): return [] + def resolve_location(self, file, line, col=None): return [f"c@{line}:{col or 0}"] + def source_slice(self, vertex_uri): return (f"m:{vertex_uri}", vertex_uri) + def callable_of(self, vertex_uri): return "c" + def max_level(self): return 3 + + +def test_backward_slice_exact_set(): + e = Engine(OneCallableProvider()) + r = e.slice_backward("m:3", edges=("cfg", "ddg")) # seed s3 + assert set(r.uris()) == {"c@3:0", "c@2:0", "c@1:0", "c@entry"} + + +def test_forward_slice_exact_set(): + e = Engine(OneCallableProvider()) + r = e.slice_forward("m:1", edges=("ddg",)) # seed s1, ddg only + assert set(r.uris()) == {"c@1:0", "c@2:0", "c@3:0"} + + +def test_ddg_only_backward_from_s3(): + e = Engine(OneCallableProvider()) + r = e.slice_backward("m:3", edges=("ddg",)) + assert set(r.uris()) == {"c@3:0", "c@2:0", "c@1:0"} # follows ddg chain, not entry + + +def test_family_scoped_slices_differ(): + # cfg and ddg share the endpoint pairs s1->s2 and s2->s3; a MultiDiGraph keeps + # them as distinct parallel edges, so family-scoped slices must NOT collapse. + e = Engine(OneCallableProvider()) + ddg_set = set(e.slice_backward("m:3", edges=("ddg",)).uris()) + cfg_set = set(e.slice_backward("m:3", edges=("cfg",)).uris()) + assert "c@entry" not in ddg_set # entry reachable only via the cfg chain + assert "c@entry" in cfg_set # cfg fallthrough reaches entry + assert ddg_set != cfg_set # families are distinct, not merged + + +def test_slice_evidence_default_role_is_def(): + # I4 guard: slices keep the "def" role for non-seed vertices (only control_deps + # and def_use re-role their evidence). + e = Engine(OneCallableProvider()) + r = e.slice_backward("m:3", edges=("ddg",)) + roles = {ev["uri"]: ev["role"] for ev in r.evidence} + assert roles["c@3:0"] == "seed" + assert roles["c@1:0"] == "def" and roles["c@2:0"] == "def" + + +def test_seed_absent_from_graph_is_consistent(): + # a seed resolving to a vertex not in the callable graph is still in its own + # slice; uris()/evidence must equal the subgraph's node set (no contradiction). + e = Engine(OneCallableProvider()) + r = e.slice_backward("m:99", edges=("cfg", "ddg")) # c@99:0 is not a node + assert set(r.uris()) == set(r.subgraph.nodes()) + assert len(r) == r.subgraph.number_of_nodes() + assert "c@99:0" in set(r.uris()) # seed present in its own slice + assert bool(r) is True # non-empty; uris() and bool() agree diff --git a/tests/graph/test_facade_delegates.py b/tests/graph/test_facade_delegates.py new file mode 100644 index 00000000..8000c993 --- /dev/null +++ b/tests/graph/test_facade_delegates.py @@ -0,0 +1,49 @@ +from unittest.mock import MagicMock + +import networkx as nx + +from cldk.analysis.python.python_analysis import PythonAnalysis +from cldk.graph import SliceResult, FlowResult + + +def _facade_with_backend(backend): + pa = PythonAnalysis.__new__(PythonAnalysis) + pa.backend = backend + return pa + + +def _mock_backend(): + b = MagicMock() + b.max_level.return_value = 3 + b.callable_of.side_effect = lambda u: "c" + g = nx.MultiDiGraph() + # Edge direction: def→use; the definition at c@1:0 flows into the use at c@2:0. + # A backward slice from the use (c@2:0) walks incoming edges to reach the def (c@1:0). + g.add_edge("c@1:0", "c@2:0", family="ddg", prov=["ssa"], var="x", kind=None) + b.program_graph.return_value = g + b.sdg_edges.return_value = [] + b.resolve_location.return_value = ["c@2:0"] + b.source_slice.side_effect = lambda u: (u, u) + return b + + +def test_slice_backward_delegates(): + r = _facade_with_backend(_mock_backend()).slice_backward("m.py:2", edges=("ddg",)) + assert isinstance(r, SliceResult) + assert set(r.uris()) == {"c@2:0", "c@1:0"} + + +def test_flows_to_delegates(): + r = _facade_with_backend(_mock_backend()).flows_to("c@1:0", "c@2:0") + assert isinstance(r, FlowResult) + + +def test_all_five_verbs_exist(): + for verb in ("slice_backward", "slice_forward", "flows_to", "def_use", "control_deps"): + assert callable(getattr(PythonAnalysis, verb, None)), verb + + +def test_public_exports(): + import cldk.graph as gr + for name in ("Engine", "SliceResult", "FlowResult", "FlowPath", "CapabilityError"): + assert hasattr(gr, name), name diff --git a/tests/graph/test_golden_pyfix.py b/tests/graph/test_golden_pyfix.py new file mode 100644 index 00000000..58634480 --- /dev/null +++ b/tests/graph/test_golden_pyfix.py @@ -0,0 +1,106 @@ +"""Engine goldens over the pyfix L4 sample + the REAL duck-typed backend end-to-end: +PyCodeanalyzer's own upstream application (codeanalyzer-python 1.0.2 models) drives the +shared Engine through the mixin — the exact object graph production uses.""" +import json +from pathlib import Path + +import pytest + +from cldk.analysis import AnalysisLevel +from cldk.graph import CapabilityError +from cldk.graph._cpg_local import CpgLocalProviderMixin +from cldk.graph.engine import Engine +from cldk.models.cpg import AnalysisPayload + +RES = Path(__file__).parent.parent / "resources" / "cpg" +MOD = "pkg/mod.py" +ENTRY = "can://python/pyfix/pkg/mod.py/entry()" +C2 = "can://python/pyfix/pkg/mod.py/ResUsers/reset_password(self,login)" +C3 = "can://python/pyfix/pkg/mod.py/ResUsers/_action_reset_password(self,ids)" + + +class _Local(CpgLocalProviderMixin): + def __init__(self, application, level=4): + self.application = application + self._level = level + + def max_level(self): + return self._level + + +@pytest.fixture(scope="module") +def eng(): + app = AnalysisPayload(**json.loads((RES / "py-a4.json").read_text())).application + return Engine(_Local(app)) + + +def test_backward_slice_from_location_multi_seed(eng): + r = eng.slice_backward(f"{MOD}:5") + assert set(r.uris()) == {f"{C3}@entry", f"{C3}@5:8", f"{C3}@5:15"} + assert r.explain()["level"] == 4 and "degraded" not in r.explain() + + +def test_control_deps_exact(eng): + r = eng.control_deps(f"{C2}@3:8") + assert set(r.uris()) == {f"{C2}@entry", f"{C2}@3:8"} + + +def test_def_use_exact(eng): + r = eng.def_use(f"{C2}@entry") + assert set(r.uris()) == {f"{C2}@entry", f"{C2}@3:8"} + + +def test_flows_to_via_summary(eng): + r = eng.flows_to(f"{C2}@3:8/actual_in:1", f"{C2}@3:8/actual_out") + assert len(r.paths) == 1 + assert [h["kind"] for h in r.paths[0].hops] == ["summary"] + + +def test_flows_to_no_route_is_empty_not_error(eng): + r = eng.flows_to(f"{C3}@5:8", f"{ENTRY}@7:4") + assert len(r.paths) == 0 and not r + + +def test_capability_degrade_below_l4(eng): + # A level-2 provider over the SAME application object as `eng` (no re-analysis, no + # subclassing): flows_to needs L4 for its sdg (param_in/param_out/summary) overlay, so a + # cross-callable flow degrades to "no route" rather than crashing or silently completing. + low = Engine(_Local(eng.p.application, level=2)) + r = low.flows_to(f"{ENTRY}@7:4/actual_in:0", f"{C2}@formal_in:1") + assert len(r.paths) == 0 # no sdg overlay below L4 + assert r.explain()["degraded"]["requested"] == 4 + with pytest.raises(CapabilityError): + low.flows_to(f"{ENTRY}@7:4/actual_in:0", f"{C2}@formal_in:1", strict=True) + + +def test_real_backend_is_a_provider_end_to_end(monkeypatch, tmp_path): + # Build a REAL PyCodeanalyzer (upstream models, not cldk cpg models) via the fake-analyzer + # pattern from tests/analysis/python/test_python_l34_levels.py, then drive the Engine + # through it — this is the production object graph, and it is what catches upstream-model + # slimness (BodyNode without id, CfgEdge without var/prov). + import cldk.analysis.python.codeanalyzer.codeanalyzer as mod + from cldk.models.python import PyApplication + + payload = json.loads((RES / "py-a4.json").read_text()) + + class _Env: + schema_version = payload["schema_version"] + max_level = payload["max_level"] + application = PyApplication(**payload["application"]) + + class _Fake: + def __init__(self, options): ... + def __enter__(self): return self + def __exit__(self, *a): return False + def analyze(self): return _Env() + + monkeypatch.setattr(mod, "Codeanalyzer", _Fake) + b = mod.PyCodeanalyzer(project_dir=tmp_path, + analysis_level=AnalysisLevel.system_dependency_graph, + analysis_json_path=None, eager_analysis=False) + assert isinstance(b, CpgLocalProviderMixin) and b.max_level() == 4 + # exact same golden as the cpg-model path: + r = Engine(b).slice_backward(f"{MOD}:5") + assert set(r.uris()) == {f"{C3}@entry", f"{C3}@5:8", f"{C3}@5:15"} + flows = Engine(b).flows_to(f"{C2}@3:8/actual_in:1", f"{C3}@formal_in:1") + assert len(flows.paths) == 1 and flows.paths[0].hops[0]["kind"] == "param_in" diff --git a/tests/graph/test_provider.py b/tests/graph/test_provider.py new file mode 100644 index 00000000..39923aa3 --- /dev/null +++ b/tests/graph/test_provider.py @@ -0,0 +1,45 @@ +import pytest +from cldk.graph.provider import resolve_vertex, ProgramGraphProvider + + +class FakeProvider(ProgramGraphProvider): + def program_graph(self, callable_uri): ... + def sdg_edges(self): return [] + def resolve_location(self, file, line, col=None): + return [f"can://x/{file}/f@{line}:{col or 0}"] + def source_slice(self, vertex_uri): return ("m.py:1", "code") + def callable_of(self, vertex_uri): return "can://x/f" + def max_level(self): return 4 + + +def test_resolve_location_string(): + p = FakeProvider() + assert resolve_vertex(p, "src/m.py:42") == ["can://x/src/m.py/f@42:0"] + assert resolve_vertex(p, "src/m.py:42:5") == ["can://x/src/m.py/f@42:5"] + + +def test_resolve_can_id_passthrough(): + p = FakeProvider() + assert resolve_vertex(p, "can://x/src/m.py/f@42:5") == ["can://x/src/m.py/f@42:5"] + + +def test_resolve_node_object_uses_id(): + p = FakeProvider() + class N: id = "can://x/src/m.py/f@42:5" + assert resolve_vertex(p, N()) == ["can://x/src/m.py/f@42:5"] + + +def test_resolve_rejects_garbage(): + p = FakeProvider() + with pytest.raises(ValueError): + resolve_vertex(p, 12345) + + +def test_resolve_location_with_no_vertex_raises(): + # I1: resolve_location legitimately returns [] when no vertex sits at that line. + # Every engine verb indexes resolve_vertex(...)[0], so [] must surface as a clean + # ValueError here — not an IndexError at the call site. + class EmptyProvider(FakeProvider): + def resolve_location(self, file, line, col=None): return [] + with pytest.raises(ValueError, match="no vertex at location"): + resolve_vertex(EmptyProvider(), "m.py:99") diff --git a/tests/graph/test_py_neo4j_provider.py b/tests/graph/test_py_neo4j_provider.py new file mode 100644 index 00000000..b2992abe --- /dev/null +++ b/tests/graph/test_py_neo4j_provider.py @@ -0,0 +1,133 @@ +"""ProgramGraphProvider conformance for the read-only Neo4j Python backend (#270). + +Stub-based: fakes ``_run`` so no live Neo4j server is required. Verifies the identity +translation and that every query stays scoped to ``application_name`` the same way the file's +existing accessors do. + +``PyCFGNode.id`` rows below mirror the REAL codeanalyzer-python 1.0.2 emitter, confirmed against +a live Neo4j instance in #270 Task 6: ``n.id`` is already the fully-qualified ``can://...@key`` +vertex id, with no ``#`` separator anywhere (see issue #295, which this stub previously masked by +hard-coding the dotted-sig ``"#"``-separated form the analyzer repo's ``schema.py`` comment +describes but that has never actually been observed on the wire). ``test_to_uri_...hash_form`` +below is the one deliberately-kept case exercising ``_to_uri``'s defensive ``#`` fallback branch. +""" + +import pytest + +from cldk.analysis.python.neo4j.neo4j_backend import PyNeo4jBackend +from cldk.graph.provider import ProgramGraphProvider + +SIG2 = "pkg.mod.ResUsers.reset_password" +C2 = "can://python/pyfix/pkg/mod.py/ResUsers/reset_password(self,login)" +SIG3 = "pkg.mod.ResUsers._action_reset_password" +C3 = "can://python/pyfix/pkg/mod.py/ResUsers/_action_reset_password(self,ids)" + + +@pytest.fixture +def backend(monkeypatch): + b = PyNeo4jBackend.__new__(PyNeo4jBackend) + b.application_name = "pyfix" # real attribute name (neo4j_backend.py:134) + + def fake_run(query, **params): + q = " ".join(query.split()) + if "PY_PARAM_IN|PY_PARAM_OUT|PY_SUMMARY" in q and "LIMIT 1" in q: + return [{"one": 1}] + if "MATCH (c:PyCallable)" in q and "RETURN c.signature" in q: + # Both endpoints resolved — a param_in/out edge's dst (a @formal_in/@formal_out + # vertex on the CALLEE) must not silently fall through the sig_is_None guard in + # source_slice/callable_of; that would mask a parse bug on the synthetic-key path. + return [{"sig": SIG2, "id": C2}, {"sig": SIG3, "id": C3}] + if "PY_HAS_CFG_NODE" in q and "n._module AS mod" in q: + # source_slice's per-callable node scan (checked BEFORE the generic + # program_graph node query below, since both match "PY_HAS_CFG_NODE"/"RETURN n.id"). + if params.get("sig") == SIG2: + return [{"id": f"{C2}@entry", "sl": None, "mod": "pkg/mod.py"}, + {"id": f"{C2}@3:8", "sl": 3, "mod": "pkg/mod.py"}] + if params.get("sig") == SIG3: + return [{"id": f"{C3}@formal_in:1", "sl": None, "mod": "pkg/mod.py"}] + return [] + if "PY_HAS_CFG_NODE" in q and "RETURN n.id" in q: + # Real emitter form: n.id is already the minted can:// URI, no "#". + return [{"id": f"{C2}@entry", "kind": "entry", "sl": None, "el": None}, + {"id": f"{C2}@3:8", "kind": "return", "sl": 3, "el": 3}] + if "r:PY_CFG_NEXT" in q: + return [{"src": f"{C2}@entry", "dst": f"{C2}@3:8", "kind": "fallthrough", + "var": None, "prov": None}] + if "r:PY_CDG" in q or "r:PY_DDG" in q: + return [] + if "PY_PARAM_IN" in q: + return [{"src": f"{C2}@3:8/actual_in:1", + "dst": f"{C3}@formal_in:1", "var": None}] + if "PY_PARAM_OUT" in q or "PY_SUMMARY" in q: + return [] + if "n.start_line = $line" in q: + return [{"id": f"{C2}@3:8", "mod": "pkg/mod.py", "sl": 3}] + raise AssertionError(f"unstubbed query: {q}") + + monkeypatch.setattr(b, "_run", fake_run) + return b + + +def test_is_a_provider(backend): + assert isinstance(backend, ProgramGraphProvider) + + +def test_max_level_derived_from_overlay(backend): + assert backend.max_level() == 4 + + +def test_program_graph_translates_to_can_uris(backend): + g = backend.program_graph(C2) + assert set(g.nodes) == {f"{C2}@entry", f"{C2}@3:8"} + (d,) = g.get_edge_data(f"{C2}@entry", f"{C2}@3:8").values() + assert d["family"] == "cfg" and d["kind"] == "fallthrough" + + +def test_sdg_edges_translated_and_kinded(backend): + edges = list(backend.sdg_edges()) + assert edges and edges[0].kind == "param_in" + assert edges[0].src == f"{C2}@3:8/actual_in:1" + + +def test_resolve_location_orders_by_parsed_col(backend): + assert backend.resolve_location("pkg/mod.py", 3) == [f"{C2}@3:8"] + + +def test_resolve_location_basename_suffix_match(backend): + # A basename/suffix seed must behave identically to the fully-qualified module path — the + # same latitude the local backend's mixin already gives (#270 final review Finding 2). + assert backend.resolve_location("mod.py", 3) == [f"{C2}@3:8"] + + +def test_source_slice_lossy_code_none(backend): + fl, code = backend.source_slice(f"{C2}@3:8") + assert fl == "pkg/mod.py:3" and code is None + + +def test_source_slice_degrades_to_module_path_on_synthetic_formal_in_vertex(backend): + # A resolved callable (sig_is_None guard doesn't hide it) whose vertex is a synthetic + # @formal_in:N body node — it EXISTS but carries no start_line of its own, so per the + # adjudicated contract (#270 final review Finding 3) this degrades to (module_path, None), + # matching the local backend's mixin exactly — never (None, None) and never a fabricated + # line parsed out of the "formal_in" key shape. + assert backend.source_slice(f"{C3}@formal_in:1") == ("pkg/mod.py", None) + + +def test_source_slice_unknown_vertex_is_none_none(backend): + # A vertex whose callable can't be resolved at all (not a can:// id this app knows about) + # is a structural non-match, not a synthetic-vertex degrade — (None, None), never fabricated. + assert backend.source_slice("can://nonexistent@0:0") == (None, None) + + +def test_callable_of_partitions_at_first_at(backend): + assert backend.callable_of(f"{C2}@3:8/actual_in:1") == C2 + + +def test_to_uri_translates_dotted_sig_hash_form_defensively(backend): + # The analyzer repo's schema.py comment (and this file's original brief) documents a + # "#"-separated dotted-sig PyCFGNode.id form; never observed on a real graph (issue #295 — + # the real emitter stores the can:// id directly, exercised by every other test above), but + # _to_uri keeps translating it correctly as a defensive fallback should some emitter version + # ever actually produce it. + assert backend._to_uri(f"{SIG2}#3:8") == f"{C2}@3:8" + assert backend._to_uri(f"{SIG2}#@entry") == f"{C2}@entry" diff --git a/tests/graph/test_py_parity_live.py b/tests/graph/test_py_parity_live.py new file mode 100644 index 00000000..b4ca7292 --- /dev/null +++ b/tests/graph/test_py_parity_live.py @@ -0,0 +1,203 @@ +################################################################################ +# Copyright IBM Corporation 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +"""Live dual-backend parity for the five verbs' data seam (env-gated, mirrors +tests/analysis/python/test_python_neo4j_backend.py). Writes the pyfix sources from the +py-a4.json fixture to a temp project, runs the real analyzer at level 4, emits to Neo4j +in-process from that SAME analysis, then asserts each provider primitive agrees modulo +documented lossiness (Neo4j source_slice code is None). + +The whole module is skipped unless CLDK_TEST_NEO4J_URI is set. Point it at a server with: + + CLDK_TEST_NEO4J_URI=bolt://localhost:7687 \ + CLDK_TEST_NEO4J_USER=neo4j \ + CLDK_TEST_NEO4J_PASSWORD=testpassword \ + uv run pytest tests/graph/test_py_parity_live.py -v + +(e.g. `podman run -d -p 7687:7687 -e NEO4J_AUTH=neo4j/testpassword neo4j:5` — a DEDICATED +container is still advised, even though the analyzer's bolt writer scopes its orphan-module +prune to THIS application's own app_name (codeanalyzer/neo4j/bolt.py's full-run prune reads +`WHERE ... app=app_name`, never touching other applications' data) — a fresh container just +keeps this suite free of any stale state left over from a previous run.) +""" + +import json +import os +from pathlib import Path + +import pytest + +RES = Path(__file__).parent.parent / "resources" / "cpg" +NEO4J_URI = os.environ.get("CLDK_TEST_NEO4J_URI") +pytestmark = pytest.mark.skipif(not NEO4J_URI, reason="set CLDK_TEST_NEO4J_URI to run") + + +@pytest.fixture(scope="module") +def both_backends(tmp_path_factory): + """(local, remote): the real in-process PyCodeanalyzer and a PyNeo4jBackend over the + SAME analysis, projected to Neo4j out of band.""" + payload = json.loads((RES / "py-a4.json").read_text()) + proj = tmp_path_factory.mktemp("pyfix") + for path, mod in payload["application"]["symbol_table"].items(): + f = proj / path + f.parent.mkdir(parents=True, exist_ok=True) + f.write_text(mod["source"] or "") + + from cldk.analysis import AnalysisLevel + from cldk.analysis.python.codeanalyzer.codeanalyzer import PyCodeanalyzer + + local = PyCodeanalyzer( + project_dir=proj, + analysis_level=AnalysisLevel.system_dependency_graph, + analysis_json_path=None, + eager_analysis=True, + ) + + # Populate Neo4j from the SAME analysis `local` already ran — never a second analyze(): + # jedi call-resolution is nondeterministic across runs and a re-analyze would masquerade + # as a backend diff. codeanalyzer-python 1.0.2's emit_neo4j takes the v2 `Analysis` + # envelope (schema_version/max_level/analyzer/application), not a bare PyApplication — + # confirmed against its real signature and tests/analysis/python/test_python_neo4j_backend.py + # (the emit-API sketch in the task brief predates that envelope). `analyzer` lives only on + # the envelope Codeanalyzer.analyze() built (not on PyApplication itself), and `local` + # doesn't keep that envelope around, so it defaults here — pure metadata, not read by + # emit_neo4j's projection logic. Wrapping `local`'s own already-analyzed application in a + # fresh envelope reuses the identical analysis result (zero re-analysis). + from codeanalyzer.neo4j.emit import emit_neo4j + from codeanalyzer.options import AnalysisOptions + from codeanalyzer.schema.py_schema import Analysis + + neo4j_user = os.environ.get("CLDK_TEST_NEO4J_USER", "neo4j") + neo4j_password = os.environ.get("CLDK_TEST_NEO4J_PASSWORD", "neo4j") + + # app_name MUST equal proj.name (the same value PyCodeanalyzer's own AnalysisOptions + # defaulted to at analysis time — cldk never sets app_name, so codeanalyzer-python + # falls back to `self.project_dir.name`). emit_neo4j's assign_ids() is only idempotent + # when re-run with the SAME app_name: it re-stamps every module/class/callable `.id` in + # place on `local.application`, but does NOT touch the already-baked-in cfg/ddg/param_in/ + # param_out edge src/dst strings from the original analysis. A different app_name here + # would rename callable ids out from under those edges, corrupting `local`'s own identity + # consistency as a side effect of populating Neo4j (a real footgun in this two-actor + # in-place-mutation harness, hand-verified against codeanalyzer-python 1.0.2's + # `Codeanalyzer.analyze()` / `codeanalyzer.schema.assign_ids.assign_ids`). + app_name = proj.name + analysis = Analysis( + max_level=local.max_level(), + application=local.get_application_view(), + ) + emit_neo4j( + analysis, + AnalysisOptions( + input=proj, + app_name=app_name, + neo4j_uri=NEO4J_URI, + neo4j_user=neo4j_user, + neo4j_password=neo4j_password, + ), + ) + + from cldk.analysis.python.neo4j.neo4j_backend import PyNeo4jBackend + + remote = PyNeo4jBackend( + neo4j_uri=NEO4J_URI, + neo4j_username=neo4j_user, + neo4j_password=neo4j_password, + neo4j_database=None, + application_name=app_name, + ) + yield local, remote + remote.close() + + +def test_max_level_parity(both_backends): + local, remote = both_backends + assert local.max_level() == remote.max_level() == 4 + + +def test_program_graph_parity_per_callable(both_backends): + local, remote = both_backends + for cid in local._index()["callables"]: + lg, rg = local.program_graph(cid), remote.program_graph(cid) + assert set(lg.nodes) == set(rg.nodes), cid + # prov included (as a tuple, so it hashes into the set key) — #270 final review + # Finding 4(a): a backend that dropped/mismatched ddg provenance would previously pass + # this parity check silently since prov wasn't compared at all. + lset = {(u, v, d["family"], d.get("kind"), d.get("var"), tuple(d.get("prov") or [])) + for u, v, d in lg.edges(data=True)} + rset = {(u, v, d["family"], d.get("kind"), d.get("var"), tuple(d.get("prov") or [])) + for u, v, d in rg.edges(data=True)} + assert lset == rset, cid + + +def test_sdg_parity(both_backends): + local, remote = both_backends + key = lambda es: {(e.src, e.dst, e.kind) for e in es} # noqa: E731 + assert key(local.sdg_edges()) == key(remote.sdg_edges()) + + +def test_source_slice_and_callable_of_parity_per_vertex(both_backends): + # #270 final review Finding 4(b): every vertex of every callable must agree on callable_of + # (both backends), and on source_slice's location half (Neo4j's `code` is documented-lossy — + # always None there — so only the (module[:line] | None) half is compared). + local, remote = both_backends + checked = 0 + for cid in local._index()["callables"]: + for v in local.program_graph(cid).nodes(): + assert local.callable_of(v) == remote.callable_of(v), v + l_fl, _ = local.source_slice(v) + r_fl, r_code = remote.source_slice(v) + assert l_fl == r_fl, v + assert r_code is None + checked += 1 + assert checked > 0 # sanity: the fixture actually has vertices to compare + + +def test_resolve_location_parity_for_every_spanned_vertex(both_backends): + # #270 final review Finding 4(b): every (file, line) that owns at least one real vertex must + # return the SAME full hit-list on both backends, not just agree on a single hand-picked + # location (the existing verb-parity tests only ever probe one seed per verb). + local, remote = both_backends + locations = set() + for cid in local._index()["callables"]: + for v in local.program_graph(cid).nodes(): + fl, _ = local.source_slice(v) + if not fl or ":" not in fl: + continue # synthetic vertex (@entry/@exit/formal_*/actual_*) — no line to probe + file, _, line = fl.rpartition(":") + locations.add((file, int(line))) + assert locations # sanity: the fixture actually has spanned (real source line) vertices + for file, line in locations: + assert set(local.resolve_location(file, line)) == set(remote.resolve_location(file, line)), (file, line) + + +def test_verb_parity(both_backends): + from cldk.graph import Engine + + local, remote = both_backends + # A real param_in pair, discovered from the live application rather than hardcoded ids: the + # fixture's app_name is a random tmp-dir name (see both_backends), so can:// ids aren't + # stable across runs and can't be pinned as string literals here. + param_in_edge = next(e for e in local.sdg_edges() if e.kind == "param_in") + for verb, args in ( + ("slice_backward", ("pkg/mod.py:5",)), + ("slice_forward", ("pkg/mod.py:3",)), + ("def_use", ("pkg/mod.py:3",)), + ("control_deps", ("pkg/mod.py:3",)), + ("flows_to", (param_in_edge.src, param_in_edge.dst)), + ): + l = getattr(Engine(local), verb)(*args) # noqa: E741 + r = getattr(Engine(remote), verb)(*args) + assert set(l.uris()) == set(r.uris()), verb diff --git a/tests/graph/test_result.py b/tests/graph/test_result.py new file mode 100644 index 00000000..c8080163 --- /dev/null +++ b/tests/graph/test_result.py @@ -0,0 +1,45 @@ +import networkx as nx +from cldk.graph.result import GraphResult, SliceResult, FlowResult, FlowPath + + +def _graph(*nodes): + g = nx.DiGraph() + g.add_nodes_from(nodes) + return g + + +def test_graphresult_len_bool_uris(): + g = _graph("a", "b") + r = SliceResult(subgraph=g, evidence=[{"uri": "a"}, {"uri": "b"}], _explain={"level": 3}) + assert len(r) == 2 + assert bool(r) is True + assert r.uris() == ["a", "b"] + assert r.explain() == {"level": 3} + + +def test_empty_result_is_falsy(): + r = SliceResult(subgraph=_graph(), evidence=[], _explain={}) + assert not r + assert len(r) == 0 + + +def test_flowresult_carries_paths_and_serializes(): + p = FlowPath(source="a", sink="c", + hops=[{"from": "a", "to": "b", "kind": "ddg", "var": "x", "confidence": "structural"}], + confidence="structural") + r = FlowResult(subgraph=_graph("a", "b", "c"), + evidence=[{"uri": "a", "file_line": "m.py:1", "code": "x = 1", "role": "seed"}], + _explain={"level": 4}, paths=[p]) + assert r.paths[0].confidence == "structural" + assert '"file_line": "m.py:1"' in r.to_json() + + +def test_flowresult_to_json_includes_paths(): + p = FlowPath(source="a", sink="c", + hops=[{"from": "a", "to": "b", "kind": "ddg", "var": "x", "confidence": "structural"}], + confidence="structural") + r = FlowResult(subgraph=_graph("a", "b", "c"), + evidence=[{"uri": "a"}], _explain={"level": 4}, paths=[p]) + dumped = r.to_json() + assert '"confidence": "structural"' in dumped + assert '"var": "x"' in dumped diff --git a/tests/models/cpg/__init__.py b/tests/models/cpg/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/models/cpg/test_accessor_contract.py b/tests/models/cpg/test_accessor_contract.py new file mode 100644 index 00000000..269ebb23 --- /dev/null +++ b/tests/models/cpg/test_accessor_contract.py @@ -0,0 +1,66 @@ +"""Pin the accessors F7 (Task-7 provider) and F3 (views) depend on, against a real L4 sample.""" +import json +from pathlib import Path +from cldk.models.cpg import AnalysisPayload + +RES = Path(__file__).parent.parent.parent / "resources" / "cpg" + + +def _app(name): + return AnalysisPayload(**json.loads((RES / name).read_text())).application + + +def test_symbol_table_module_source_and_containment(): + app = _app("py-a4.json") + mod = app.symbol_table["pkg/mod.py"] + assert isinstance(mod.source, str) and mod.source # module.source (byte-slice base) + assert isinstance(mod.types, dict) and isinstance(mod.functions, dict) # both accessors pinned + assert mod.types or mod.functions # at least one populated + + +def test_callable_body_and_dataflow_edges_present_at_l4(): + app = _app("py-a4.json") + mod = app.symbol_table["pkg/mod.py"] + cls = next(iter(mod.types.values())) + call = next(iter(cls.callables.values())) + assert call.signature # callable.signature + assert call.body # body{} populated at L4 + assert call.cfg and call.ddg # cfg/ddg edge lists + # cdg/summary are unpinned elsewhere and are the sole extra="allow" guard for these two + # fields (both in F7's cfg/cdg/ddg/summary read set) — dereference an element attribute so a + # deleted field (which would fall back to a raw dict under extra="allow") fails loudly. + assert isinstance(call.cdg, list) and isinstance(call.summary, list) + assert call.cdg[0].src and call.summary[0].src + # span.bytes present for slicing a body node + some = next(iter(call.body.values())) + assert some.span is None or (some.span.bytes and len(some.span.bytes) == 2) + + +def test_envelope_k_limit_at_l4(): + payload = AnalysisPayload(**json.loads((RES / "py-a4.json").read_text())) + assert payload.k_limit == 3 + # a plain value check alone would still pass via the extra="allow" passthrough even if + # k_limit were deleted from the model — assert it's a declared field, not an extras leak. + assert "k_limit" not in (payload.model_extra or {}) + + +def test_application_interprocedural_edges_at_l4(): + app = _app("py-a4.json") + assert app.call_graph # L2 call graph + assert app.param_in and app.param_out # L4 SDG param edges + for e in app.call_graph: + assert e.src.startswith("can://") and e.dst.startswith("can://") # can:// identity + + +def test_typescript_sample_same_accessors(): + app = _app("ts-a4.json") + mod = next(iter(app.symbol_table.values())) + assert isinstance(mod.source, str) and mod.source + # a TS type node with callables + typ = next((t for t in mod.types.values() if t.callables), None) + assert typ is not None and next(iter(typ.callables.values())).signature is not None + # resetPassword under type Users: body/cfg must resolve to parsed Node/Edge, not raw dicts, + # so the TS analyzer path isn't pinned on source/signature alone. + call = typ.callables["resetPassword"] + assert isinstance(call.body, dict) and next(iter(call.body.values())).kind + assert isinstance(call.cfg, list) and call.cfg[0].kind diff --git a/tests/models/cpg/test_base_span.py b/tests/models/cpg/test_base_span.py new file mode 100644 index 00000000..ad9df5a1 --- /dev/null +++ b/tests/models/cpg/test_base_span.py @@ -0,0 +1,26 @@ +from cldk.models.cpg.models import Span +from cldk.models.cpg.base import _NullSafeBase +from pydantic import ConfigDict +from typing import Dict, List + + +class _M(_NullSafeBase): + xs: List[int] = [] + d: Dict[str, int] = {} + opt: int | None = None + + +def test_null_collections_coerce_to_defaults(): + m = _M(**{"xs": None, "d": None, "opt": None}) + assert m.xs == [] and m.d == {} and m.opt is None + + +def test_extra_fields_are_allowed_and_preserved(): + m = _M(**{"xs": [1], "is_tsx": True}) # language-specific extra + assert m.xs == [1] + assert m.model_extra.get("is_tsx") is True + + +def test_span_parses_byte_offsets(): + s = Span(**{"start": [1, 0], "end": [4, 2], "bytes": [0, 40]}) + assert s.start == (1, 0) and s.end == (4, 2) and s.bytes == (0, 40) diff --git a/tests/models/cpg/test_containers.py b/tests/models/cpg/test_containers.py new file mode 100644 index 00000000..f61d10a5 --- /dev/null +++ b/tests/models/cpg/test_containers.py @@ -0,0 +1,59 @@ +import pytest +from pydantic import ValidationError + +from cldk.models.cpg import AnalysisPayload, Application, Module, Node, Edge, Span, Import, Analyzer + + +def test_envelope_reads_authoritative_level(): + p = AnalysisPayload(**{ + "schema_version": "2.0.0", "language": "python", "max_level": 4, "k_limit": 3, + "analyzer": {"name": "codeanalyzer-python", "version": "0.4.0"}, + "application": {"id": "can://python/app", "kind": "application", "symbol_table": {}}, + }) + assert p.schema_version == "2.0.0" and p.max_level == 4 and p.k_limit == 3 + assert p.analyzer.name == "codeanalyzer-python" + assert p.application.id == "can://python/app" + + +def test_module_holds_source_and_containment(): + m = Module(**{"id": "can://python/app/m.py", "kind": "module", "source": "x = 1\n", + "types": {"C": {"id": "can://python/app/m.py/C", "kind": "class"}}, + "functions": {"f()": {"id": "can://python/app/m.py/f()", "kind": "function"}}}) + assert m.source == "x = 1\n" + assert m.types["C"].kind == "class" and m.functions["f()"].kind == "function" + + +def test_module_durable_node_missing_id_raises(): + # a type reached through the durable-containment dict MUST carry the join key id + with pytest.raises(ValidationError): + Module(**{"id": "m", "types": {"C": {"kind": "class"}}}) # no id on the type + + +def test_application_edge_lists(): + a = Application(**{"id": "can://python/app", "kind": "application", "symbol_table": {}, + "call_graph": [{"src": "a", "dst": "b", "prov": ["jedi"], "weight": 1}], + "param_in": [{"src": "c@in", "dst": "d@in"}]}) + assert a.call_graph[0].dst == "b" and a.param_in[0].src == "c@in" and a.param_out == [] + + +def test_symbol_table_deep_composition(): + from cldk.models.cpg import Application, Module, Node + a = Application(**{ + "id": "can://python/app", "kind": "application", + "symbol_table": { + "pkg/m.py": { + "id": "can://python/app/pkg/m.py", "kind": "module", "source": "x = 1\n", + "types": {"C": {"id": "can://python/app/pkg/m.py/C", "kind": "class", + "callables": {"C.f()": {"id": "can://python/app/pkg/m.py/C/f()", + "kind": "method", "signature": "f"}}}}, + "functions": {"g()": {"id": "can://python/app/pkg/m.py/g()", "kind": "function"}}, + } + }, + }) + mod = a.symbol_table["pkg/m.py"] + assert isinstance(mod, Module) and mod.source == "x = 1\n" + cls = mod.types["C"] + assert isinstance(cls, Node) and cls.kind == "class" + method = cls.callables["C.f()"] + assert isinstance(method, Node) and method.kind == "method" and method.signature == "f" + assert isinstance(mod.functions["g()"], Node) diff --git a/tests/models/cpg/test_edge_import.py b/tests/models/cpg/test_edge_import.py new file mode 100644 index 00000000..42067bbf --- /dev/null +++ b/tests/models/cpg/test_edge_import.py @@ -0,0 +1,22 @@ +from cldk.models.cpg.models import Edge, Import + + +def test_call_edge_shape(): + e = Edge(**{"src": "can://p/a#f", "dst": "can://p/a#g", "prov": ["jedi", "pycg"], "weight": 2}) + assert e.src.endswith("#f") and e.dst.endswith("#g") + assert e.prov == ["jedi", "pycg"] and e.weight == 2 and e.kind is None and e.var is None + + +def test_ddg_edge_carries_var_and_prov(): + e = Edge(**{"src": "a@1:0", "dst": "a@2:0", "var": "x", "prov": ["ssa"]}) + assert e.var == "x" and e.prov == ["ssa"] and e.weight == 1 + + +def test_edge_empty_prov_and_weight_defaults(): + e = Edge(src="a", dst="b") + assert e.prov == [] and e.weight == 1 + + +def test_import_optional_fields(): + i = Import(**{"name": "os"}) + assert i.name == "os" and i.path is None and i.alias is None and i.span is None diff --git a/tests/models/cpg/test_node.py b/tests/models/cpg/test_node.py new file mode 100644 index 00000000..bb3f22e7 --- /dev/null +++ b/tests/models/cpg/test_node.py @@ -0,0 +1,44 @@ +import pytest +from pydantic import ValidationError + +from cldk.models.cpg.models import Node + + +def test_class_node_facet(): + n = Node(**{"id": "can://p/m.py/C", "kind": "class", + "callables": {"C.f()": {"id": "can://p/m.py/C/f()", "kind": "method", "signature": "f"}}}) + assert n.kind == "class" + assert n.callables["C.f()"].kind == "method" and n.callables["C.f()"].signature == "f" + + +def test_callable_node_carries_body_and_edges(): + n = Node(**{"id": "can://p/m.py/f()", "kind": "function", "signature": "f()", + "body": {"f@1:0": {"id": "can://p/m.py/f()@1:0", "kind": "statement"}}, + "cfg": [{"src": "can://p/m.py/f()@1:0", "dst": "can://p/m.py/f()@2:0", "kind": "fallthrough"}], + "ddg": [{"src": "can://p/m.py/f()@1:0", "dst": "can://p/m.py/f()@2:0", "var": "x", "prov": ["ssa"]}]}) + assert set(n.body) == {"f@1:0"} + assert n.cfg[0].kind == "fallthrough" and n.ddg[0].var == "x" and n.ddg[0].prov == ["ssa"] + + +def test_call_body_node_callee_refines_from_null(): + n = Node(**{"id": "a@2:0", "kind": "call", "callee": None, "arguments": ["a@2:0/arg0"]}) + assert n.kind == "call" and n.callee is None and n.arguments == ["a@2:0/arg0"] + + +def test_language_extra_field_preserved(): + n = Node(**{"id": "x", "kind": "class", "is_abstract": True}) # a language-specific flag + assert n.model_extra.get("is_abstract") is True + + +def test_durable_callable_missing_id_raises(): + # a callable reached through the durable-containment dict MUST carry the join key id + with pytest.raises(ValidationError): + Node(**{"id": "can://p/m.py/C", "kind": "class", + "callables": {"C.f()": {"kind": "method"}}}) # no id on the callable + + +def test_body_node_missing_id_parses(): + # body nodes are keyed by local position and legitimately omit id — must NOT raise + n = Node(**{"id": "can://p/m.py/f()", "kind": "function", + "body": {"1:0": {"kind": "statement"}}}) + assert n.body["1:0"].id is None and n.body["1:0"].kind == "statement" diff --git a/tests/models/cpg/test_real_samples.py b/tests/models/cpg/test_real_samples.py new file mode 100644 index 00000000..2eaeeda8 --- /dev/null +++ b/tests/models/cpg/test_real_samples.py @@ -0,0 +1,53 @@ +"""The models must parse REAL, conformant analysis.json from BOTH analyzers at L1 and L4, and the +L1 tree must be a subset of the L4 tree (additive-levels invariant).""" +import json +from pathlib import Path +import pytest +from cldk.models.cpg import AnalysisPayload, Span + +RES = Path(__file__).parent.parent.parent / "resources" / "cpg" + + +def _load(name): + return AnalysisPayload(**json.loads((RES / name).read_text())) + + +@pytest.mark.parametrize("name,lang,level", [ + ("py-a1.json", "python", 1), ("py-a4.json", "python", 4), + ("ts-a1.json", "typescript", 1), ("ts-a4.json", "typescript", 4), +]) +def test_real_sample_parses(name, lang, level): + p = _load(name) + assert p.schema_version == "2.0.0" and p.language == lang and p.max_level == level + assert p.application.symbol_table # non-empty tree + # every call-graph edge is identity-only src/dst (no dangling shape) + for e in p.application.call_graph: + assert e.src and e.dst + + +def _keys(obj, prefix=""): + out = set() + if isinstance(obj, dict): + for k, v in obj.items(): + out.add(prefix + str(k)); out |= _keys(v, prefix + str(k) + "/") + elif isinstance(obj, list): + for v in obj: + out |= _keys(v, prefix + "[]/") + return out + + +@pytest.mark.parametrize("lo,hi", [("py-a1.json", "py-a4.json"), ("ts-a1.json", "ts-a4.json")]) +def test_l1_subset_of_l4(lo, hi): + lo_t = json.loads((RES / lo).read_text())["application"]["symbol_table"] + hi_t = json.loads((RES / hi).read_text())["application"]["symbol_table"] + assert not (_keys(lo_t) - _keys(hi_t)), "L1 tree keys must be a subset of L4" + + +def test_module_span_parses_on_typescript_sample(): + # span is a common field per the keystone (Part II), module included; ts-a4 emits it on the + # module node — it must parse into Span, not fall through to model_extra as a raw dict. + p = _load("ts-a4.json") + mod = next(iter(p.application.symbol_table.values())) + assert isinstance(mod.span, Span) + assert mod.span.bytes == (0, 254) + assert len(mod.span.bytes) == 2 and all(isinstance(b, int) for b in mod.span.bytes) diff --git a/tests/resources/cpg/py-a1.json b/tests/resources/cpg/py-a1.json new file mode 100644 index 00000000..8890155a --- /dev/null +++ b/tests/resources/cpg/py-a1.json @@ -0,0 +1 @@ +{"schema_version":"2.0.0","language":"python","max_level":1,"analyzer":{"name":"codeanalyzer-python","version":"1.0.0","config":{"analysis_level":1}},"application":{"symbol_table":{"pkg/__init__.py":{"file_path":"/private/tmp/claude-501/-Users-rkrsn-workspace-codellm-devkit-python-sdk/d8e0db27-8f7f-482f-9d10-132b01e7abe7/scratchpad/v2-confirm/pyfix/pkg/__init__.py","module_name":"__init__","id":"can://python/pyfix/pkg/__init__.py","kind":"module","source":"","imports":[],"comments":[],"types":{},"functions":{},"variables":[],"content_hash":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","last_modified":1784053428.8736746,"file_size":0},"pkg/mod.py":{"file_path":"/private/tmp/claude-501/-Users-rkrsn-workspace-codellm-devkit-python-sdk/d8e0db27-8f7f-482f-9d10-132b01e7abe7/scratchpad/v2-confirm/pyfix/pkg/mod.py","module_name":"mod","id":"can://python/pyfix/pkg/mod.py","kind":"module","source":"class ResUsers:\n def reset_password(self, login):\n return self._action_reset_password([login])\n def _action_reset_password(self, ids):\n return list(ids)\ndef entry():\n return ResUsers().reset_password(\"x\")\n","imports":[],"comments":[],"types":{"pkg.mod.ResUsers":{"name":"ResUsers","signature":"pkg.mod.ResUsers","id":"can://python/pyfix/pkg/mod.py/ResUsers","kind":"class","span":{"start":[1,0],"end":[5,24],"bytes":[0,172]},"comments":[],"base_classes":[],"callables":{"reset_password":{"name":"reset_password","path":"/private/tmp/claude-501/-Users-rkrsn-workspace-codellm-devkit-python-sdk/d8e0db27-8f7f-482f-9d10-132b01e7abe7/scratchpad/v2-confirm/pyfix/pkg/mod.py","signature":"pkg.mod.ResUsers.reset_password","id":"can://python/pyfix/pkg/mod.py/ResUsers/reset_password(self,login)","kind":"function","span":{"start":[2,4],"end":[3,51],"bytes":[20,104]},"comments":[],"decorators":[],"parameters":[{"name":"self","type":"ResUsers","start_line":2,"end_line":2,"start_column":23,"end_column":27},{"name":"login","type":"str","start_line":2,"end_line":2,"start_column":29,"end_column":34}],"start_line":2,"end_line":3,"code_start_line":3,"accessed_symbols":[{"name":"self","scope":"local","kind":"variable","type":"ResUsers","qualified_name":"pkg.mod.ResUsers","is_builtin":false,"lineno":3,"col_offset":15},{"name":"login","scope":"local","kind":"variable","type":"list","qualified_name":"builtins.list","is_builtin":false,"lineno":3,"col_offset":44}],"call_sites":[{"method_name":"_action_reset_password","receiver_expr":"self","receiver_type":"ResUsers","argument_types":["list"],"arguments":[{"ast_kind":"List","inferred_type":"list"}],"return_type":"list","callee_signature":"pkg.mod.ResUsers._action_reset_password","is_constructor_call":false,"start_line":3,"start_column":15,"end_line":3,"end_column":51}],"callables":{},"types":{},"local_variables":[],"cyclomatic_complexity":2,"body":{"3:15":{"kind":"call","span":{"start":[3,15],"end":[3,51],"bytes":[68,104]}}},"cfg":[],"cdg":[],"ddg":[],"summary":[]},"_action_reset_password":{"name":"_action_reset_password","path":"/private/tmp/claude-501/-Users-rkrsn-workspace-codellm-devkit-python-sdk/d8e0db27-8f7f-482f-9d10-132b01e7abe7/scratchpad/v2-confirm/pyfix/pkg/mod.py","signature":"pkg.mod.ResUsers._action_reset_password","id":"can://python/pyfix/pkg/mod.py/ResUsers/_action_reset_password(self,ids)","kind":"function","span":{"start":[4,4],"end":[5,24],"bytes":[109,172]},"comments":[],"decorators":[],"parameters":[{"name":"self","type":"ResUsers","start_line":4,"end_line":4,"start_column":31,"end_column":35},{"name":"ids","type":"list","start_line":4,"end_line":4,"start_column":37,"end_column":40}],"start_line":4,"end_line":5,"code_start_line":5,"accessed_symbols":[{"name":"list","scope":"local","kind":"class","type":"list","qualified_name":"builtins.list","is_builtin":false,"lineno":5,"col_offset":15},{"name":"ids","scope":"local","kind":"variable","type":"list","qualified_name":"builtins.list","is_builtin":false,"lineno":5,"col_offset":20}],"call_sites":[{"method_name":"list","argument_types":["list"],"arguments":[{"ast_kind":"Name","inferred_type":"list"}],"return_type":"list","callee_signature":"builtins.list.__init__","is_constructor_call":true,"start_line":5,"start_column":15,"end_line":5,"end_column":24}],"callables":{},"types":{},"local_variables":[],"cyclomatic_complexity":2,"body":{"5:15":{"kind":"call","span":{"start":[5,15],"end":[5,24],"bytes":[163,172]}}},"cfg":[],"cdg":[],"ddg":[],"summary":[]}},"attributes":{},"types":{},"start_line":1,"end_line":5}},"functions":{"entry":{"name":"entry","path":"/private/tmp/claude-501/-Users-rkrsn-workspace-codellm-devkit-python-sdk/d8e0db27-8f7f-482f-9d10-132b01e7abe7/scratchpad/v2-confirm/pyfix/pkg/mod.py","signature":"pkg.mod.entry","id":"can://python/pyfix/pkg/mod.py/entry()","kind":"function","span":{"start":[6,0],"end":[7,41],"bytes":[173,227]},"comments":[],"decorators":[],"parameters":[],"start_line":6,"end_line":7,"code_start_line":7,"accessed_symbols":[{"name":"ResUsers","scope":"local","kind":"class","type":"ResUsers","qualified_name":"pkg.mod.ResUsers","is_builtin":false,"lineno":7,"col_offset":11}],"call_sites":[{"method_name":"reset_password","receiver_expr":"ResUsers()","receiver_type":"ResUsers","argument_types":["str"],"arguments":[{"ast_kind":"Constant","inferred_type":"str"}],"return_type":"list","callee_signature":"pkg.mod.ResUsers.reset_password","is_constructor_call":false,"start_line":7,"start_column":11,"end_line":7,"end_column":41},{"method_name":"ResUsers","argument_types":[],"arguments":[],"return_type":"ResUsers","callee_signature":"pkg.mod.ResUsers.__init__","is_constructor_call":true,"start_line":7,"start_column":11,"end_line":7,"end_column":21}],"callables":{},"types":{},"local_variables":[],"cyclomatic_complexity":2,"body":{"7:11":{"kind":"call","span":{"start":[7,11],"end":[7,21],"bytes":[197,207]}}},"cfg":[],"cdg":[],"ddg":[],"summary":[]}},"variables":[],"content_hash":"52fd7728f588c48291e131b8819c7e48fd7956dfd236ca93ccd366a57a82fc29","last_modified":1784053428.887157,"file_size":228}},"id":"can://python/pyfix","kind":"application","call_graph":[{"src":"can://python/pyfix/pkg/mod.py/entry()","dst":"can://python/pyfix/pkg/mod.py/ResUsers/reset_password(self,login)","weight":1,"prov":["jedi"]},{"src":"can://python/pyfix/pkg/mod.py/entry()","dst":"can://python/pyfix/@external/pkg.mod.ResUsers/__init__","weight":1,"prov":["jedi"]},{"src":"can://python/pyfix/pkg/mod.py/ResUsers/reset_password(self,login)","dst":"can://python/pyfix/pkg/mod.py/ResUsers/_action_reset_password(self,ids)","weight":1,"prov":["jedi"]},{"src":"can://python/pyfix/pkg/mod.py/ResUsers/_action_reset_password(self,ids)","dst":"can://python/pyfix/@external/builtins.list/__init__","weight":1,"prov":["jedi"]}],"external_symbols":{"can://python/pyfix/@external/pkg.mod.ResUsers/__init__":{"id":"can://python/pyfix/@external/pkg.mod.ResUsers/__init__","kind":"external","name":"__init__","module":"pkg.mod.ResUsers"},"can://python/pyfix/@external/builtins.list/__init__":{"id":"can://python/pyfix/@external/builtins.list/__init__","kind":"external","name":"__init__","module":"builtins.list"}},"param_in":[],"param_out":[]}} \ No newline at end of file diff --git a/tests/resources/cpg/py-a4.json b/tests/resources/cpg/py-a4.json new file mode 100644 index 00000000..408a0ab1 --- /dev/null +++ b/tests/resources/cpg/py-a4.json @@ -0,0 +1 @@ +{"schema_version":"2.0.0","language":"python","max_level":4,"k_limit":3,"analyzer":{"name":"codeanalyzer-python","version":"1.0.0","config":{"analysis_level":4}},"application":{"symbol_table":{"pkg/__init__.py":{"file_path":"/private/tmp/claude-501/-Users-rkrsn-workspace-codellm-devkit-python-sdk/d8e0db27-8f7f-482f-9d10-132b01e7abe7/scratchpad/v2-confirm/pyfix/pkg/__init__.py","module_name":"__init__","id":"can://python/pyfix/pkg/__init__.py","kind":"module","source":"","imports":[],"comments":[],"types":{},"functions":{},"variables":[],"content_hash":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","last_modified":1784053428.8736746,"file_size":0},"pkg/mod.py":{"file_path":"/private/tmp/claude-501/-Users-rkrsn-workspace-codellm-devkit-python-sdk/d8e0db27-8f7f-482f-9d10-132b01e7abe7/scratchpad/v2-confirm/pyfix/pkg/mod.py","module_name":"mod","id":"can://python/pyfix/pkg/mod.py","kind":"module","source":"class ResUsers:\n def reset_password(self, login):\n return self._action_reset_password([login])\n def _action_reset_password(self, ids):\n return list(ids)\ndef entry():\n return ResUsers().reset_password(\"x\")\n","imports":[],"comments":[],"types":{"pkg.mod.ResUsers":{"name":"ResUsers","signature":"pkg.mod.ResUsers","id":"can://python/pyfix/pkg/mod.py/ResUsers","kind":"class","span":{"start":[1,0],"end":[5,24],"bytes":[0,172]},"comments":[],"base_classes":[],"callables":{"reset_password":{"name":"reset_password","path":"/private/tmp/claude-501/-Users-rkrsn-workspace-codellm-devkit-python-sdk/d8e0db27-8f7f-482f-9d10-132b01e7abe7/scratchpad/v2-confirm/pyfix/pkg/mod.py","signature":"pkg.mod.ResUsers.reset_password","id":"can://python/pyfix/pkg/mod.py/ResUsers/reset_password(self,login)","kind":"function","span":{"start":[2,4],"end":[3,51],"bytes":[20,104]},"comments":[],"decorators":[],"parameters":[{"name":"self","type":"ResUsers","start_line":2,"end_line":2,"start_column":23,"end_column":27},{"name":"login","type":"str","start_line":2,"end_line":2,"start_column":29,"end_column":34}],"start_line":2,"end_line":3,"code_start_line":3,"accessed_symbols":[{"name":"self","scope":"local","kind":"variable","type":"ResUsers","qualified_name":"pkg.mod.ResUsers","is_builtin":false,"lineno":3,"col_offset":15},{"name":"login","scope":"local","kind":"variable","type":"list","qualified_name":"builtins.list","is_builtin":false,"lineno":3,"col_offset":44}],"call_sites":[{"method_name":"_action_reset_password","receiver_expr":"self","receiver_type":"ResUsers","argument_types":["list"],"arguments":[{"ast_kind":"List","inferred_type":"list"}],"return_type":"list","callee_signature":"pkg.mod.ResUsers._action_reset_password","is_constructor_call":false,"start_line":3,"start_column":15,"end_line":3,"end_column":51}],"callables":{},"types":{},"local_variables":[],"cyclomatic_complexity":2,"body":{"3:15":{"kind":"call","span":{"start":[3,15],"end":[3,51],"bytes":[68,104]},"callee":"can://python/pyfix/pkg/mod.py/ResUsers/_action_reset_password(self,ids)"},"@entry":{"kind":"entry"},"3:8":{"kind":"return","span":{"start":[3,8],"end":[3,51],"bytes":[61,104]}},"@exit":{"kind":"exit"},"@formal_in:0":{"kind":"formal_in","of":"self"},"@formal_in:1":{"kind":"formal_in","of":"login"},"@formal_out:0":{"kind":"formal_out","of":""},"@formal_out:1":{"kind":"formal_out","of":"self"},"3:8/actual_in:0":{"kind":"actual_in","of":"self","parent":"3:8"},"3:8/actual_in:1":{"kind":"actual_in","of":"ids","parent":"3:8"},"3:8/actual_out":{"kind":"actual_out","of":"","parent":"3:8"}},"cfg":[{"src":"@entry","dst":"3:8","kind":"fallthrough"},{"src":"3:8","dst":"@exit","kind":"exception"},{"src":"3:8","dst":"@exit","kind":"return"}],"cdg":[{"src":"@entry","dst":"3:8"}],"ddg":[{"src":"@entry","dst":"3:8","var":"login","prov":["ssa"]},{"src":"@entry","dst":"3:8","var":"self","prov":["ssa"]},{"src":"@entry","dst":"3:8","var":"self._action_reset_password","prov":["ssa"]}],"summary":[{"src":"3:8/actual_in:1","dst":"3:8/actual_out"}]},"_action_reset_password":{"name":"_action_reset_password","path":"/private/tmp/claude-501/-Users-rkrsn-workspace-codellm-devkit-python-sdk/d8e0db27-8f7f-482f-9d10-132b01e7abe7/scratchpad/v2-confirm/pyfix/pkg/mod.py","signature":"pkg.mod.ResUsers._action_reset_password","id":"can://python/pyfix/pkg/mod.py/ResUsers/_action_reset_password(self,ids)","kind":"function","span":{"start":[4,4],"end":[5,24],"bytes":[109,172]},"comments":[],"decorators":[],"parameters":[{"name":"self","type":"ResUsers","start_line":4,"end_line":4,"start_column":31,"end_column":35},{"name":"ids","type":"list","start_line":4,"end_line":4,"start_column":37,"end_column":40}],"start_line":4,"end_line":5,"code_start_line":5,"accessed_symbols":[{"name":"list","scope":"local","kind":"class","type":"list","qualified_name":"builtins.list","is_builtin":false,"lineno":5,"col_offset":15},{"name":"ids","scope":"local","kind":"variable","type":"list","qualified_name":"builtins.list","is_builtin":false,"lineno":5,"col_offset":20}],"call_sites":[{"method_name":"list","argument_types":["list"],"arguments":[{"ast_kind":"Name","inferred_type":"list"}],"return_type":"list","callee_signature":"builtins.list.__init__","is_constructor_call":true,"start_line":5,"start_column":15,"end_line":5,"end_column":24}],"callables":{},"types":{},"local_variables":[],"cyclomatic_complexity":2,"body":{"5:15":{"kind":"call","span":{"start":[5,15],"end":[5,24],"bytes":[163,172]},"callee":"can://python/pyfix/@external/builtins.list/__init__"},"@entry":{"kind":"entry"},"5:8":{"kind":"return","span":{"start":[5,8],"end":[5,24],"bytes":[156,172]}},"@exit":{"kind":"exit"},"@formal_in:0":{"kind":"formal_in","of":"self"},"@formal_in:1":{"kind":"formal_in","of":"ids"},"@formal_out:0":{"kind":"formal_out","of":""},"@formal_out:1":{"kind":"formal_out","of":"ids"}},"cfg":[{"src":"@entry","dst":"5:8","kind":"fallthrough"},{"src":"5:8","dst":"@exit","kind":"exception"},{"src":"5:8","dst":"@exit","kind":"return"}],"cdg":[{"src":"@entry","dst":"5:8"}],"ddg":[{"src":"@entry","dst":"5:8","var":"ids","prov":["ssa"]},{"src":"@entry","dst":"5:8","var":"list","prov":["ssa"]}],"summary":[]}},"attributes":{},"types":{},"start_line":1,"end_line":5}},"functions":{"entry":{"name":"entry","path":"/private/tmp/claude-501/-Users-rkrsn-workspace-codellm-devkit-python-sdk/d8e0db27-8f7f-482f-9d10-132b01e7abe7/scratchpad/v2-confirm/pyfix/pkg/mod.py","signature":"pkg.mod.entry","id":"can://python/pyfix/pkg/mod.py/entry()","kind":"function","span":{"start":[6,0],"end":[7,41],"bytes":[173,227]},"comments":[],"decorators":[],"parameters":[],"start_line":6,"end_line":7,"code_start_line":7,"accessed_symbols":[{"name":"ResUsers","scope":"local","kind":"class","type":"ResUsers","qualified_name":"pkg.mod.ResUsers","is_builtin":false,"lineno":7,"col_offset":11}],"call_sites":[{"method_name":"reset_password","receiver_expr":"ResUsers()","receiver_type":"ResUsers","argument_types":["str"],"arguments":[{"ast_kind":"Constant","inferred_type":"str"}],"return_type":"list","callee_signature":"pkg.mod.ResUsers.reset_password","is_constructor_call":false,"start_line":7,"start_column":11,"end_line":7,"end_column":41},{"method_name":"ResUsers","argument_types":[],"arguments":[],"return_type":"ResUsers","callee_signature":"pkg.mod.ResUsers.__init__","is_constructor_call":true,"start_line":7,"start_column":11,"end_line":7,"end_column":21}],"callables":{},"types":{},"local_variables":[],"cyclomatic_complexity":2,"body":{"7:11":{"kind":"call","span":{"start":[7,11],"end":[7,21],"bytes":[197,207]},"callee":"can://python/pyfix/@external/pkg.mod.ResUsers/__init__"},"@entry":{"kind":"entry"},"7:4":{"kind":"return","span":{"start":[7,4],"end":[7,41],"bytes":[190,227]}},"@exit":{"kind":"exit"},"@formal_in:0":{"kind":"formal_in","of":":mod::ResUsers"},"@formal_out":{"kind":"formal_out","of":""},"7:4/actual_in:0":{"kind":"actual_in","of":"login","parent":"7:4"},"7:4/actual_out":{"kind":"actual_out","of":"","parent":"7:4"}},"cfg":[{"src":"@entry","dst":"7:4","kind":"fallthrough"},{"src":"7:4","dst":"@exit","kind":"exception"},{"src":"7:4","dst":"@exit","kind":"return"}],"cdg":[{"src":"@entry","dst":"7:4"}],"ddg":[{"src":"@entry","dst":"7:4","var":"mod::ResUsers","prov":["ssa"]}],"summary":[{"src":"7:4/actual_in:0","dst":"7:4/actual_out"}]}},"variables":[],"content_hash":"52fd7728f588c48291e131b8819c7e48fd7956dfd236ca93ccd366a57a82fc29","last_modified":1784053428.887157,"file_size":228}},"id":"can://python/pyfix","kind":"application","call_graph":[{"src":"can://python/pyfix/pkg/mod.py/entry()","dst":"can://python/pyfix/pkg/mod.py/ResUsers/reset_password(self,login)","weight":2,"prov":["jedi","pycg"]},{"src":"can://python/pyfix/pkg/mod.py/entry()","dst":"can://python/pyfix/@external/pkg.mod.ResUsers/__init__","weight":1,"prov":["jedi"]},{"src":"can://python/pyfix/pkg/mod.py/ResUsers/reset_password(self,login)","dst":"can://python/pyfix/pkg/mod.py/ResUsers/_action_reset_password(self,ids)","weight":2,"prov":["jedi","pycg"]},{"src":"can://python/pyfix/pkg/mod.py/ResUsers/_action_reset_password(self,ids)","dst":"can://python/pyfix/@external/builtins.list/__init__","weight":1,"prov":["jedi"]},{"src":"can://python/pyfix/pkg/mod.py/ResUsers/_action_reset_password(self,ids)","dst":"can://python/pyfix/@external//list","weight":1,"prov":["pycg"]}],"external_symbols":{"can://python/pyfix/@external/pkg.mod.ResUsers/__init__":{"id":"can://python/pyfix/@external/pkg.mod.ResUsers/__init__","kind":"external","name":"__init__","module":"pkg.mod.ResUsers"},"can://python/pyfix/@external/builtins.list/__init__":{"id":"can://python/pyfix/@external/builtins.list/__init__","kind":"external","name":"__init__","module":"builtins.list"},"can://python/pyfix/@external//list":{"id":"can://python/pyfix/@external//list","kind":"external","name":"list","module":""}},"param_in":[{"src":"can://python/pyfix/pkg/mod.py/ResUsers/reset_password(self,login)@3:8/actual_in:0","dst":"can://python/pyfix/pkg/mod.py/ResUsers/_action_reset_password(self,ids)@formal_in:0"},{"src":"can://python/pyfix/pkg/mod.py/ResUsers/reset_password(self,login)@3:8/actual_in:1","dst":"can://python/pyfix/pkg/mod.py/ResUsers/_action_reset_password(self,ids)@formal_in:1"},{"src":"can://python/pyfix/pkg/mod.py/entry()@7:4/actual_in:0","dst":"can://python/pyfix/pkg/mod.py/ResUsers/reset_password(self,login)@formal_in:1"}],"param_out":[{"src":"can://python/pyfix/pkg/mod.py/ResUsers/_action_reset_password(self,ids)@formal_out:0","dst":"can://python/pyfix/pkg/mod.py/ResUsers/reset_password(self,login)@3:8/actual_out"},{"src":"can://python/pyfix/pkg/mod.py/ResUsers/reset_password(self,login)@formal_out:0","dst":"can://python/pyfix/pkg/mod.py/entry()@7:4/actual_out"}]}} \ No newline at end of file diff --git a/tests/resources/cpg/ts-a1.json b/tests/resources/cpg/ts-a1.json new file mode 100644 index 00000000..1a432bb3 --- /dev/null +++ b/tests/resources/cpg/ts-a1.json @@ -0,0 +1 @@ +{"schema_version":"2.0.0","language":"typescript","max_level":1,"analyzer":{"name":"codeanalyzer-typescript","version":"0.5.0"},"application":{"id":"can://typescript/tsfix","kind":"application","symbol_table":{"src/index.ts":{"source":"export class Users {\n resetPassword(login: string): string[] { return this.actionReset([login]); }\n private actionReset(ids: string[]): string[] { return ids.map(i => i); }\n}\nexport function entry(): string[] { return new Users().resetPassword(\"x\"); }\n","imports":[],"exports":[],"comments":[],"is_tsx":false,"is_declaration_file":false,"id":"can://typescript/tsfix/src/index.ts","kind":"module","span":{"start":[1,1],"end":[6,1],"bytes":[0,254]},"types":{"Users":{"name":"Users","signature":"src/index.Users","comments":[],"decorators":[],"base_classes":[],"implements_types":[],"type_parameters":[],"entrypoints":[],"is_abstract":false,"is_exported":true,"is_ambient":false,"id":"can://typescript/tsfix/src/index.ts/Users","kind":"class","span":{"start":[1,1],"end":[4,2],"bytes":[0,176]},"callables":{"resetPassword":{"name":"resetPassword","signature":"src/index.Users.resetPassword","comments":[],"decorators":[],"parameters":[{"name":"login","type":"string","is_optional":false,"is_rest":false,"is_readonly":false,"decorators":[],"start_line":2,"end_line":2,"start_column":17,"end_column":30}],"type_parameters":[],"return_type":"string[]","accessed_symbols":[],"cyclomatic_complexity":1,"entrypoints":[],"kind":"method","is_static":false,"is_abstract":false,"is_async":false,"is_generator":false,"is_optional":false,"is_readonly":false,"is_exported":false,"is_ambient":false,"is_implicit":false,"overload_signatures":[],"id":"can://typescript/tsfix/src/index.ts/Users/resetPassword","span":{"start":[2,3],"end":[2,79],"bytes":[23,99]},"body":{"2:51":{"method_name":"actionReset","receiver_expr":"this","receiver_type":"this","argument_types":["string[]"],"type_arguments":[],"return_type":"string[]","is_constructor_call":false,"is_optional_chain":false,"kind":"call","span":{"start":[2,51],"end":[2,76],"bytes":[71,96]},"callee":null}}},"actionReset":{"name":"actionReset","signature":"src/index.Users.actionReset","comments":[],"decorators":[],"parameters":[{"name":"ids","type":"string[]","is_optional":false,"is_rest":false,"is_readonly":false,"decorators":[],"start_line":3,"end_line":3,"start_column":23,"end_column":36}],"type_parameters":[],"return_type":"string[]","accessed_symbols":[],"cyclomatic_complexity":1,"entrypoints":[],"kind":"method","accessibility":"private","is_static":false,"is_abstract":false,"is_async":false,"is_generator":false,"is_optional":false,"is_readonly":false,"is_exported":false,"is_ambient":false,"is_implicit":false,"overload_signatures":[],"id":"can://typescript/tsfix/src/index.ts/Users/actionReset","span":{"start":[3,3],"end":[3,75],"bytes":[102,174]},"body":{"3:57":{"method_name":"map","receiver_expr":"ids","receiver_type":"string[]","argument_types":["(i: string) => string"],"type_arguments":[],"return_type":"string[]","is_constructor_call":false,"is_optional_chain":false,"kind":"call","span":{"start":[3,57],"end":[3,72],"bytes":[156,171]},"callee":null}}},"constructor":{"name":"constructor","signature":"src/index.Users.constructor","comments":[],"decorators":[],"parameters":[],"type_parameters":[],"accessed_symbols":[],"cyclomatic_complexity":0,"entrypoints":[],"kind":"constructor","is_static":false,"is_abstract":false,"is_async":false,"is_generator":false,"is_optional":false,"is_readonly":false,"is_exported":false,"is_ambient":false,"is_implicit":true,"overload_signatures":[],"id":"can://typescript/tsfix/src/index.ts/Users/constructor","span":{"start":[0,0],"end":[0,0],"bytes":[0,0]},"body":{}}},"fields":{}}},"functions":{"entry":{"name":"entry","signature":"src/index.entry","comments":[],"decorators":[],"parameters":[],"type_parameters":[],"return_type":"string[]","accessed_symbols":[],"cyclomatic_complexity":1,"entrypoints":[],"kind":"function","is_static":false,"is_abstract":false,"is_async":false,"is_generator":false,"is_optional":false,"is_readonly":false,"is_exported":true,"is_ambient":false,"is_implicit":false,"overload_signatures":[],"id":"can://typescript/tsfix/src/index.ts/entry","span":{"start":[5,1],"end":[5,77],"bytes":[177,253]},"body":{"5:44":{"method_name":"resetPassword","receiver_expr":"new Users()","receiver_type":"Users","argument_types":["\"x\""],"type_arguments":[],"return_type":"string[]","is_constructor_call":false,"is_optional_chain":false,"kind":"call","span":{"start":[5,44],"end":[5,74],"bytes":[220,250]},"callee":null},"5:44/2":{"method_name":"Users","argument_types":[],"type_arguments":[],"return_type":"Users","is_constructor_call":true,"is_optional_chain":false,"kind":"call","span":{"start":[5,44],"end":[5,55],"bytes":[220,231]},"callee":null}}}},"fields":{}}},"call_graph":[],"param_in":[],"param_out":[]}} \ No newline at end of file diff --git a/tests/resources/cpg/ts-a4.json b/tests/resources/cpg/ts-a4.json new file mode 100644 index 00000000..b9a7f4f3 --- /dev/null +++ b/tests/resources/cpg/ts-a4.json @@ -0,0 +1 @@ +{"schema_version":"2.0.0","language":"typescript","max_level":4,"k_limit":3,"analyzer":{"name":"codeanalyzer-typescript","version":"0.5.0"},"application":{"id":"can://typescript/tsfix","kind":"application","symbol_table":{"src/index.ts":{"source":"export class Users {\n resetPassword(login: string): string[] { return this.actionReset([login]); }\n private actionReset(ids: string[]): string[] { return ids.map(i => i); }\n}\nexport function entry(): string[] { return new Users().resetPassword(\"x\"); }\n","imports":[],"exports":[],"comments":[],"is_tsx":false,"is_declaration_file":false,"id":"can://typescript/tsfix/src/index.ts","kind":"module","span":{"start":[1,1],"end":[6,1],"bytes":[0,254]},"types":{"Users":{"name":"Users","signature":"src/index.Users","comments":[],"decorators":[],"base_classes":[],"implements_types":[],"type_parameters":[],"entrypoints":[],"is_abstract":false,"is_exported":true,"is_ambient":false,"id":"can://typescript/tsfix/src/index.ts/Users","kind":"class","span":{"start":[1,1],"end":[4,2],"bytes":[0,176]},"callables":{"resetPassword":{"name":"resetPassword","signature":"src/index.Users.resetPassword","comments":[],"decorators":[],"parameters":[{"name":"login","type":"string","is_optional":false,"is_rest":false,"is_readonly":false,"decorators":[],"start_line":2,"end_line":2,"start_column":17,"end_column":30}],"type_parameters":[],"return_type":"string[]","accessed_symbols":[],"cyclomatic_complexity":1,"entrypoints":[],"kind":"method","is_static":false,"is_abstract":false,"is_async":false,"is_generator":false,"is_optional":false,"is_readonly":false,"is_exported":false,"is_ambient":false,"is_implicit":false,"overload_signatures":[],"id":"can://typescript/tsfix/src/index.ts/Users/resetPassword","span":{"start":[2,3],"end":[2,79],"bytes":[23,99]},"body":{"2:51":{"method_name":"actionReset","receiver_expr":"this","receiver_type":"this","argument_types":["string[]"],"type_arguments":[],"return_type":"string[]","is_constructor_call":false,"is_optional_chain":false,"kind":"call","span":{"start":[2,51],"end":[2,76],"bytes":[71,96]},"callee":"can://typescript/tsfix/src/index.ts/Users/actionReset"},"@entry":{"kind":"entry","span":{"start":[2,3],"end":[2,79],"bytes":[23,99]}},"2:44":{"kind":"statement","span":{"start":[2,44],"end":[2,77],"bytes":[64,97]}},"@exit":{"kind":"exit","span":{"start":[2,3],"end":[2,79],"bytes":[23,99]}},"@formal_in:0":{"kind":"formal_in","of":"login"},"@formal_out":{"kind":"formal_out","of":"$ret"},"2:44/actual_out":{"kind":"actual_out","of":"$ret","parent":"2:44"},"2:44/actual_in:0":{"kind":"actual_in","of":"arg0","parent":"2:44"}},"cfg":[{"src":"@entry","dst":"2:44","kind":"fallthrough"},{"src":"2:44","dst":"@exit","kind":"exception"},{"src":"2:44","dst":"@exit","kind":"return"}],"cdg":[{"src":"@entry","dst":"2:44"}],"ddg":[{"src":"@entry","dst":"2:44","var":"login","prov":["reaching-defs"]},{"src":"@entry","dst":"2:44","var":"this.actionReset","prov":["reaching-defs"]},{"src":"2:44","dst":"@formal_out","var":"return","prov":["reaching-defs"]}],"summary":[{"src":"2:44/actual_in:0","dst":"2:44/actual_out"}]},"actionReset":{"name":"actionReset","signature":"src/index.Users.actionReset","comments":[],"decorators":[],"parameters":[{"name":"ids","type":"string[]","is_optional":false,"is_rest":false,"is_readonly":false,"decorators":[],"start_line":3,"end_line":3,"start_column":23,"end_column":36}],"type_parameters":[],"return_type":"string[]","accessed_symbols":[],"cyclomatic_complexity":1,"entrypoints":[],"kind":"method","accessibility":"private","is_static":false,"is_abstract":false,"is_async":false,"is_generator":false,"is_optional":false,"is_readonly":false,"is_exported":false,"is_ambient":false,"is_implicit":false,"overload_signatures":[],"id":"can://typescript/tsfix/src/index.ts/Users/actionReset","span":{"start":[3,3],"end":[3,75],"bytes":[102,174]},"body":{"3:57":{"method_name":"map","receiver_expr":"ids","receiver_type":"string[]","argument_types":["(i: string) => string"],"type_arguments":[],"return_type":"string[]","is_constructor_call":false,"is_optional_chain":false,"kind":"call","span":{"start":[3,57],"end":[3,72],"bytes":[156,171]},"callee":null},"@entry":{"kind":"entry","span":{"start":[3,3],"end":[3,75],"bytes":[102,174]}},"3:50":{"kind":"statement","span":{"start":[3,50],"end":[3,73],"bytes":[149,172]}},"@exit":{"kind":"exit","span":{"start":[3,3],"end":[3,75],"bytes":[102,174]}},"@formal_in:0":{"kind":"formal_in","of":"ids"},"@formal_out":{"kind":"formal_out","of":"$ret"},"3:50/actual_in:0":{"kind":"actual_in","of":"arg0","parent":"3:50"},"3:50/actual_out":{"kind":"actual_out","of":"$ret","parent":"3:50"}},"cfg":[{"src":"@entry","dst":"3:50","kind":"fallthrough"},{"src":"3:50","dst":"@exit","kind":"exception"},{"src":"3:50","dst":"@exit","kind":"return"}],"cdg":[{"src":"@entry","dst":"3:50"}],"ddg":[{"src":"@entry","dst":"3:50","var":"ids.map","prov":["reaching-defs"]},{"src":"3:50","dst":"@formal_out","var":"return","prov":["reaching-defs"]}],"summary":[{"src":"3:50/actual_in:0","dst":"3:50/actual_out"}]},"constructor":{"name":"constructor","signature":"src/index.Users.constructor","comments":[],"decorators":[],"parameters":[],"type_parameters":[],"accessed_symbols":[],"cyclomatic_complexity":0,"entrypoints":[],"kind":"constructor","is_static":false,"is_abstract":false,"is_async":false,"is_generator":false,"is_optional":false,"is_readonly":false,"is_exported":false,"is_ambient":false,"is_implicit":true,"overload_signatures":[],"id":"can://typescript/tsfix/src/index.ts/Users/constructor","span":{"start":[0,0],"end":[0,0],"bytes":[0,0]},"body":{}}},"fields":{}}},"functions":{"entry":{"name":"entry","signature":"src/index.entry","comments":[],"decorators":[],"parameters":[],"type_parameters":[],"return_type":"string[]","accessed_symbols":[],"cyclomatic_complexity":1,"entrypoints":[],"kind":"function","is_static":false,"is_abstract":false,"is_async":false,"is_generator":false,"is_optional":false,"is_readonly":false,"is_exported":true,"is_ambient":false,"is_implicit":false,"overload_signatures":[],"id":"can://typescript/tsfix/src/index.ts/entry","span":{"start":[5,1],"end":[5,77],"bytes":[177,253]},"body":{"5:44":{"method_name":"resetPassword","receiver_expr":"new Users()","receiver_type":"Users","argument_types":["\"x\""],"type_arguments":[],"return_type":"string[]","is_constructor_call":false,"is_optional_chain":false,"kind":"call","span":{"start":[5,44],"end":[5,74],"bytes":[220,250]},"callee":"can://typescript/tsfix/src/index.ts/Users/resetPassword"},"5:44/2":{"method_name":"Users","argument_types":[],"type_arguments":[],"return_type":"Users","is_constructor_call":true,"is_optional_chain":false,"kind":"call","span":{"start":[5,44],"end":[5,55],"bytes":[220,231]},"callee":"can://typescript/tsfix/src/index.ts/Users/constructor"},"@entry":{"kind":"entry","span":{"start":[5,1],"end":[5,77],"bytes":[177,253]}},"5:37":{"kind":"statement","span":{"start":[5,37],"end":[5,75],"bytes":[213,251]}},"@exit":{"kind":"exit","span":{"start":[5,1],"end":[5,77],"bytes":[177,253]}},"@formal_out":{"kind":"formal_out","of":"$ret"},"5:37/actual_in:0":{"kind":"actual_in","of":"arg0","parent":"5:37"},"5:37/actual_out":{"kind":"actual_out","of":"$ret","parent":"5:37"}},"cfg":[{"src":"@entry","dst":"5:37","kind":"fallthrough"},{"src":"5:37","dst":"@exit","kind":"exception"},{"src":"5:37","dst":"@exit","kind":"return"}],"cdg":[{"src":"@entry","dst":"5:37"}],"ddg":[{"src":"5:37","dst":"@formal_out","var":"return","prov":["reaching-defs"]}],"summary":[{"src":"5:37/actual_in:0","dst":"5:37/actual_out"}]}},"fields":{}}},"call_graph":[{"src":"can://typescript/tsfix/src/index.ts/entry","dst":"can://typescript/tsfix/src/index.ts/Users/resetPassword","prov":["tsc","jelly"],"weight":2},{"src":"can://typescript/tsfix/src/index.ts/entry","dst":"can://typescript/tsfix/src/index.ts/Users/constructor","prov":["tsc"],"weight":1},{"src":"can://typescript/tsfix/src/index.ts/Users/resetPassword","dst":"can://typescript/tsfix/src/index.ts/Users/actionReset","prov":["tsc","jelly"],"weight":2},{"src":"can://typescript/tsfix/src/index.ts/Users/actionReset","dst":"can://typescript/tsfix/src/index.ts/Users/actionReset@3:65","prov":["jelly"],"weight":1}],"param_in":[{"src":"can://typescript/tsfix/src/index.ts/entry@5:37/actual_in:0","dst":"can://typescript/tsfix/src/index.ts/Users/resetPassword@formal_in:0"},{"src":"can://typescript/tsfix/src/index.ts/Users/resetPassword@2:44/actual_in:0","dst":"can://typescript/tsfix/src/index.ts/Users/actionReset@formal_in:0"}],"param_out":[{"src":"can://typescript/tsfix/src/index.ts/Users/actionReset@formal_out","dst":"can://typescript/tsfix/src/index.ts/Users/resetPassword@2:44/actual_out"},{"src":"can://typescript/tsfix/src/index.ts/Users/resetPassword@formal_out","dst":"can://typescript/tsfix/src/index.ts/entry@5:37/actual_out"}],"external_symbols":{},"synthesized_callables":{"can://typescript/tsfix/src/index.ts/Users/actionReset@3:65":{"id":"can://typescript/tsfix/src/index.ts/Users/actionReset@3:65","kind":"callable","name":"","path":"src/index.ts","span":{"start":[3,65],"end":[3,65],"bytes":[0,0]}}}}} \ No newline at end of file diff --git a/uv.lock b/uv.lock index 4f68f2a6..af64b189 100644 --- a/uv.lock +++ b/uv.lock @@ -84,6 +84,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f8/ed/e97229a566617f2ae958a6b13e7cc0f585470eac730a73e9e82c32a3cdd2/arrow-1.3.0-py3-none-any.whl", hash = "sha256:c728b120ebc00eb84e01882a6f5e7927a53960aa990ce7dd2b10f39005a67f80", size = 66419, upload-time = "2023-09-30T22:11:16.072Z" }, ] +[[package]] +name = "astor" +version = "0.8.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/21/75b771132fee241dfe601d39ade629548a9626d1d39f333fde31bc46febe/astor-0.8.1.tar.gz", hash = "sha256:6a6effda93f4e1ce9f618779b2dd1d9d84f1e32812c23a29b3fff6fd7f63fa5e", size = 35090, upload-time = "2019-12-10T01:50:35.51Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/88/97eef84f48fa04fbd6750e62dcceafba6c63c81b7ac1420856c8dcc0a3f9/astor-0.8.1-py2.py3-none-any.whl", hash = "sha256:070a54e890cefb5b3739d19f30f5a5ec840ffc9c50ffa7d23cc9fc1a38ebbfc5", size = 27488, upload-time = "2019-12-10T01:50:33.628Z" }, +] + [[package]] name = "astroid" version = "3.3.11" @@ -300,7 +309,7 @@ wheels = [ [[package]] name = "cldk" -version = "1.4.3" +version = "2.0.0rc1" source = { editable = "." } dependencies = [ { name = "clang" }, @@ -345,8 +354,8 @@ test = [ [package.metadata] requires-dist = [ { name = "clang", specifier = "==17.0.6" }, - { name = "codeanalyzer-python", specifier = "==0.3.1" }, - { name = "codeanalyzer-typescript", specifier = "==0.4.3" }, + { name = "codeanalyzer-python", specifier = "==1.1.0" }, + { name = "codeanalyzer-typescript", specifier = "==1.0.0" }, { name = "libclang", specifier = "==17.0.6" }, { name = "neo4j", marker = "extra == 'neo4j'", specifier = ">=5.14,<7" }, { name = "networkx", specifier = ">=3.4.2,<4" }, @@ -394,15 +403,17 @@ wheels = [ [[package]] name = "codeanalyzer-python" -version = "0.3.1" +version = "1.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "astor" }, { name = "jedi" }, { name = "msgpack" }, { name = "networkx" }, { name = "numpy" }, { name = "packaging" }, { name = "pandas" }, + { name = "parso" }, { name = "pycg" }, { name = "pydantic" }, { name = "ray" }, @@ -412,21 +423,21 @@ dependencies = [ { name = "typing-extensions" }, { name = "uv" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bb/65/f3b0563def7cd4d846147e7b7b29d1ae6c4acf900ed217dc3011ec6efdda/codeanalyzer_python-0.3.1.tar.gz", hash = "sha256:e4a3a17df4c694e128a2ba121779fc755c505774b8e66ed27162f23d1f598d89", size = 96974, upload-time = "2026-07-14T16:36:12.325Z" } +sdist = { url = "https://files.pythonhosted.org/packages/2b/cb/c223115d8866f4436066d3cc24e68ad5585c07dc408d3b688f915392989a/codeanalyzer_python-1.1.0.tar.gz", hash = "sha256:e706763fb8d690ecb91466060cef44098caab583cb2d2fc8b95f77c1fdaf9d77", size = 178430, upload-time = "2026-07-27T15:39:40.072Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e9/49/222a823db55ea2423a58eff426bcf71877d296ff2df1f893619e5d39eadb/codeanalyzer_python-0.3.1-py3-none-any.whl", hash = "sha256:7a24a643edfea2d079a6dd9558187c7a2ee0a2ce7c877d20cd8cd3b05789c305", size = 89547, upload-time = "2026-07-14T16:36:11.175Z" }, + { url = "https://files.pythonhosted.org/packages/dd/8a/88f924b4d1ccbb0c04fae651085f43b0b558647c5b129e16c420c5c3ec1a/codeanalyzer_python-1.1.0-py3-none-any.whl", hash = "sha256:8bd302d927bdfbe79f26fdbf75a4f7e14f91058483e09c66981b294e7dbd2127", size = 180702, upload-time = "2026-07-27T15:39:38.693Z" }, ] [[package]] name = "codeanalyzer-typescript" -version = "0.4.3" +version = "1.0.0" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f9/ff/be99765bd13613eb518184df08141a44b5d55a96306c9c177dba725b9310/codeanalyzer_typescript-0.4.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:f0d60a457c30b94ac52140701f89d20cb552118fc63146d7d69199ae226ca853", size = 31040212, upload-time = "2026-06-27T18:50:27.121Z" }, - { url = "https://files.pythonhosted.org/packages/ed/df/3c2eaf131bc1c8c1333fc33c0157d5230e347390d982013f71d87e010177/codeanalyzer_typescript-0.4.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:48098a7cc38a8acc74a10a64c19f79a549e919503e6d89b25200935d910424db", size = 28680981, upload-time = "2026-06-27T18:50:30.005Z" }, - { url = "https://files.pythonhosted.org/packages/f5/98/19e27a40be65b76853a8e21713cb9ffa3126956481c65d748e3727918ac0/codeanalyzer_typescript-0.4.3-py3-none-manylinux2014_aarch64.whl", hash = "sha256:5cd3b76040719914463d9aa0bf87bcb346e7f4c568e7d62085bb88c03f555ab8", size = 40234190, upload-time = "2026-06-27T18:50:32.951Z" }, - { url = "https://files.pythonhosted.org/packages/14/61/fa7e886b66c67dd19fc16c8da032a304ff6ad8b06d1232b716fcc55e1f16/codeanalyzer_typescript-0.4.3-py3-none-manylinux2014_x86_64.whl", hash = "sha256:6f87e9e2fdc2b6d926c536c5517661c70aee3720a5803c91fcf4af54d52320e6", size = 40503499, upload-time = "2026-06-27T18:50:36.031Z" }, - { url = "https://files.pythonhosted.org/packages/de/bf/06f1a820ec8ded7721ce61057d67c2758079f5c31f0e70933c3c29dd78c4/codeanalyzer_typescript-0.4.3-py3-none-win_amd64.whl", hash = "sha256:1af81d1ec28c503790d7d3dc745bb5c5418f87b69c427535b76f7c112a297359", size = 42890644, upload-time = "2026-06-27T18:50:38.974Z" }, + { url = "https://files.pythonhosted.org/packages/d8/58/c4a56c8d27c5f6eca1236357fd454034ec49c1bb7d31d34ad8f6d0f66025/codeanalyzer_typescript-1.0.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:6fff6e7c36cd68c66eb281e8e4cdb98a16d8bc1ad9f4b508cb5bded9faec87d4", size = 31071250, upload-time = "2026-07-15T22:06:01.922Z" }, + { url = "https://files.pythonhosted.org/packages/f4/6c/e53b5bcc5211df3a0f22f104601ea23a0cada14ec8a067ef0d02fa6a45b6/codeanalyzer_typescript-1.0.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:f868bd71a9f3971ac887a11c2ca262b2a3180f3c7d67e2a247859843142ebbd7", size = 28710753, upload-time = "2026-07-15T22:06:04.791Z" }, + { url = "https://files.pythonhosted.org/packages/5b/32/e71c426cef7b38780acd0c44b5d981e28e03e09c3db068e5c4dfc6b4718e/codeanalyzer_typescript-1.0.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:a357b0dca83866f963a03214f933fa884792e3587a2e3cbaabdaad519cc59d8b", size = 40264977, upload-time = "2026-07-15T22:06:07.747Z" }, + { url = "https://files.pythonhosted.org/packages/72/d7/36e9fd3cae0bff0354032b807889431481af32812acc097ba303aeb9b893/codeanalyzer_typescript-1.0.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a54cacf33d3f174e010f32f3732f29f97475a38ddc02cfdb215ff78c85c561af", size = 40533525, upload-time = "2026-07-15T22:06:10.95Z" }, + { url = "https://files.pythonhosted.org/packages/57/2e/26272454344628dd9bc4b0246ce11933b771de1619c9446307cfafe1e723/codeanalyzer_typescript-1.0.0-py3-none-win_amd64.whl", hash = "sha256:8f4d627fbe116ea2047196eccf5e2c0d18a6b183eb1b8d3f7702ee0d38ce89eb", size = 42918784, upload-time = "2026-07-15T22:06:13.564Z" }, ] [[package]] @@ -1417,11 +1428,11 @@ wheels = [ [[package]] name = "parso" -version = "0.8.4" +version = "0.8.7" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/66/94/68e2e17afaa9169cf6412ab0f28623903be73d1b32e208d9e8e541bb086d/parso-0.8.4.tar.gz", hash = "sha256:eb3a7b58240fb99099a345571deecc0f9540ea5f4dd2fe14c2a99d6b281ab92d", size = 400609, upload-time = "2024-04-05T09:43:55.897Z" } +sdist = { url = "https://files.pythonhosted.org/packages/30/4b/90c937815137d43ce71ba043cd3566221e9df6b9c805f24b5d138c9d40a7/parso-0.8.7.tar.gz", hash = "sha256:eaaac4c9fdd5e9e8852dc778d2d7405897ec510f2a298071453e5e3a07914bb1", size = 401824, upload-time = "2026-05-01T23:13:02.138Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c6/ac/dac4a63f978e4dcb3c6d3a78c4d8e0192a113d288502a1216950c41b1027/parso-0.8.4-py2.py3-none-any.whl", hash = "sha256:a418670a20291dacd2dddc80c377c5c3791378ee1e8d12bffc35420643d43f18", size = 103650, upload-time = "2024-04-05T09:43:53.299Z" }, + { url = "https://files.pythonhosted.org/packages/99/5d/8268b644392ee874ee82a635cd0df1773de230bde356c38de28e298392cc/parso-0.8.7-py2.py3-none-any.whl", hash = "sha256:a8926eb2a1b915486941fdbd31e86a4baf88fe8c210f25f2f35ecec5b574ca1c", size = 107025, upload-time = "2026-05-01T23:12:58.867Z" }, ] [[package]]