From f1af77407242f44fe14beab4841496270bc93236 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sat, 22 Aug 2026 07:38:53 -0700 Subject: [PATCH 1/4] feat(languages): hard-cut universal evidence for Swift Dart Scala Groovy --- CHANGELOG.md | 15 + COMPATIBILITY.md | 8 + .../020-swift-dart-scala-groovy-universal.md | 851 ++++++++++ advisor-plans/README.md | 24 + benchmarks/performance/compass/audit.py | 48 +- benchmarks/performance/compass/model.py | 2 + benchmarks/performance/compass/occurrences.py | 271 ++- crates/compass-languages/src/builtins.rs | 12 + crates/compass-languages/src/dart.rs | 958 ----------- .../compass-languages/src/dart_framework.rs | 375 +++++ crates/compass-languages/src/engine.rs | 213 +-- .../compass-languages/src/evidence/build.rs | 12 + .../src/evidence/extended.rs | 1448 +++++++++++++++++ crates/compass-languages/src/evidence/mod.rs | 1 + .../src/evidence/typescript.rs | 64 +- .../src/evidence_pipeline.rs | 119 ++ .../compass-languages/src/frameworks/dart.rs | 316 ++++ .../compass-languages/src/frameworks/mod.rs | 33 +- .../compass-languages/src/frameworks/pack.rs | 91 ++ .../src/frameworks/spring.rs | 12 + crates/compass-languages/src/groovy.rs | 473 ------ crates/compass-languages/src/lib.rs | 4 +- .../compass-languages/src/project_evidence.rs | 468 +++++- crates/compass-languages/src/swift.rs | 822 ---------- .../tests/engine_edge_coverage.rs | 47 +- .../tests/extended_universal_conformance.rs | 189 +++ .../tests/universal_evidence.rs | 4 + .../src/evidence/languages/policy.rs | 32 +- .../src/evidence/projection/nodes.rs | 13 +- .../src/evidence/resolve/pipeline.rs | 45 +- crates/compass-resolve/src/frameworks/dart.rs | 13 + crates/compass-resolve/src/frameworks/mod.rs | 18 + .../compass-resolve/src/frameworks/routes.rs | 76 + .../compass-resolve/src/frameworks/swift.rs | 12 + crates/compass-resolve/src/lib.rs | 22 +- crates/compass-resolve/src/members.rs | 122 +- .../tests/builtin_resolution.rs | 33 +- .../tests/extended_universal.rs | 159 ++ docs/design/language-architecture.md | 10 +- docs/design/managed-language-analyzers.md | 6 + docs/implementation/universal-evidence.md | 28 +- docs/reference/universal-semantic-evidence.md | 55 +- .../code-graph/routes/groovy/SpockSpec.groovy | 14 + .../code-graph/routes/groovy/build.gradle | 2 + .../code-graph/routes/scala/Universal.scala | 8 + scripts/build_universal_quality_audit.py | 922 +++++++++++ scripts/dart_source_oracle.py | 44 + scripts/groovy_source_oracle.py | 44 + scripts/independent_language_oracle.py | 855 ++++++++++ scripts/providers/README.md | 27 + scripts/providers/dart_oracle.dart | 584 +++++++ scripts/providers/groovy_oracle.java | 419 +++++ scripts/providers/scala_oracle.scala | 387 +++++ scripts/providers/swift_oracle.swift | 427 +++++ scripts/qualify_code_graph_v1.sh | 29 +- scripts/qualify_dart_universal.py | 23 + scripts/qualify_groovy_universal.py | 23 + scripts/qualify_scala_universal.py | 23 + scripts/qualify_swift_universal.py | 23 + scripts/qualify_universal_language.py | 677 ++++++++ scripts/record_universal_baseline.py | 326 ++++ scripts/scala_source_oracle.py | 44 + scripts/swift_source_oracle.py | 44 + scripts/tests/test_universal_performance.py | 76 + scripts/tests/test_universal_source_oracle.py | 78 + .../qualification/code-graph-v1-semantic.json | 22 +- .../dart-universal-baseline.json | 1 + .../dart-universal-repositories.toml | 35 + .../groovy-universal-baseline.json | 1 + .../groovy-universal-repositories.toml | 35 + .../language-wave/dart/library.dart | 25 + .../language-wave/dart/src/model.dart | 1 + .../language-wave/dart/src/part.dart | 2 + .../language-wave/groovy/Module.groovy | 13 + .../language-wave/groovy/build.gradle | 2 + .../language-wave/scala/Module.scala | 12 + .../language-wave/scala/Scala3.scala | 4 + .../language-wave/swift/Module.swift | 20 + .../language-wave/swift/UTF8.swift | 4 + .../scala-universal-baseline.json | 1 + .../scala-universal-repositories.toml | 35 + .../swift-universal-baseline.json | 1 + .../swift-universal-repositories.toml | 35 + 83 files changed, 10206 insertions(+), 2666 deletions(-) create mode 100644 advisor-plans/020-swift-dart-scala-groovy-universal.md delete mode 100644 crates/compass-languages/src/dart.rs create mode 100644 crates/compass-languages/src/dart_framework.rs create mode 100644 crates/compass-languages/src/evidence/extended.rs create mode 100644 crates/compass-languages/src/frameworks/dart.rs delete mode 100644 crates/compass-languages/src/groovy.rs delete mode 100644 crates/compass-languages/src/swift.rs create mode 100644 crates/compass-languages/tests/extended_universal_conformance.rs create mode 100644 crates/compass-resolve/src/frameworks/dart.rs create mode 100644 crates/compass-resolve/src/frameworks/swift.rs create mode 100644 crates/compass-resolve/tests/extended_universal.rs create mode 100644 fixtures/code-graph/routes/groovy/SpockSpec.groovy create mode 100644 fixtures/code-graph/routes/groovy/build.gradle create mode 100644 fixtures/code-graph/routes/scala/Universal.scala create mode 100644 scripts/build_universal_quality_audit.py create mode 100644 scripts/dart_source_oracle.py create mode 100644 scripts/groovy_source_oracle.py create mode 100644 scripts/independent_language_oracle.py create mode 100644 scripts/providers/README.md create mode 100644 scripts/providers/dart_oracle.dart create mode 100644 scripts/providers/groovy_oracle.java create mode 100644 scripts/providers/scala_oracle.scala create mode 100644 scripts/providers/swift_oracle.swift create mode 100644 scripts/qualify_dart_universal.py create mode 100644 scripts/qualify_groovy_universal.py create mode 100644 scripts/qualify_scala_universal.py create mode 100644 scripts/qualify_swift_universal.py create mode 100644 scripts/qualify_universal_language.py create mode 100644 scripts/record_universal_baseline.py create mode 100644 scripts/scala_source_oracle.py create mode 100644 scripts/swift_source_oracle.py create mode 100644 scripts/tests/test_universal_performance.py create mode 100644 scripts/tests/test_universal_source_oracle.py create mode 100644 tests/qualification/dart-universal-baseline.json create mode 100644 tests/qualification/dart-universal-repositories.toml create mode 100644 tests/qualification/groovy-universal-baseline.json create mode 100644 tests/qualification/groovy-universal-repositories.toml create mode 100644 tests/qualification/language-wave/dart/library.dart create mode 100644 tests/qualification/language-wave/dart/src/model.dart create mode 100644 tests/qualification/language-wave/dart/src/part.dart create mode 100644 tests/qualification/language-wave/groovy/Module.groovy create mode 100644 tests/qualification/language-wave/groovy/build.gradle create mode 100644 tests/qualification/language-wave/scala/Module.scala create mode 100644 tests/qualification/language-wave/scala/Scala3.scala create mode 100644 tests/qualification/language-wave/swift/Module.swift create mode 100644 tests/qualification/language-wave/swift/UTF8.swift create mode 100644 tests/qualification/scala-universal-baseline.json create mode 100644 tests/qualification/scala-universal-repositories.toml create mode 100644 tests/qualification/swift-universal-baseline.json create mode 100644 tests/qualification/swift-universal-repositories.toml diff --git a/CHANGELOG.md b/CHANGELOG.md index 161308ea..f24ca306 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,21 @@ ## Unreleased +- Hard-cut Swift, Dart, Scala, and Groovy/Gradle onto version-1 qualifying + universal evidence pipelines. The bounded AST-first producer publishes + declarations, scopes, bindings, occurrences, and conservative relationship + candidates with exact language constraints; Swift Vapor routes now use the + `vapor-swift` evidence-backed pack, bounded `pubspec.yaml`/`build.sbt`/ + `Package.swift`/Gradle project metadata participates in fingerprints, and + legacy Swift member-table plus broad JVM stub rewiring no longer selects + targets for these languages. + +- Complete the independent SwiftSyntax, Dart Analyzer, scala.meta, and Groovy + CompilationUnit qualification providers. Clean pinned Swift, Dart, Scala, + and Groovy corpora now pass the graph-backed precision/recall audits and + fixture performance gates; the four production pipelines remain + `Qualifying` pending a separate promotion decision. + - Turn the VS Code codebase query view into a multi-command workbench with separate Ask, Explain, and CompassQL composers and durable result tabs. Typed Ask diagnostics, symbol relationships, source links, and CompassQL rows diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index 8dba2c54..d06ec533 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -52,6 +52,14 @@ product checkout. ## Evolving contracts +Swift, Dart, Scala, and Groovy/Gradle now publish through their version-1 +universal evidence pipelines. The four pipelines are intentionally +`Qualifying`: they use one bounded, source-grounded publication route and may +change unresolved/ambiguous edges compared with older direct extraction. Normal +cache fingerprints invalidate affected files; users do not need to delete +artifacts manually. Equal names across Swift/native or JVM-family languages do +not by themselves create cross-language targets. + A user-visible incompatible change requires: 1. native regression coverage; diff --git a/advisor-plans/020-swift-dart-scala-groovy-universal.md b/advisor-plans/020-swift-dart-scala-groovy-universal.md new file mode 100644 index 00000000..132e2118 --- /dev/null +++ b/advisor-plans/020-swift-dart-scala-groovy-universal.md @@ -0,0 +1,851 @@ +# Plan 020: Hard-cut Swift, Dart, Scala, and Groovy to universal evidence + +> **Executor instructions**: Deliver this program as ten independently +> reviewable phases. Each phase below repeats its entry context, scope, and +> acceptance criteria so it can be handed to an executor with no conversation +> context. Before changing source, read `AGENTS.md`, +> `docs/design/language-architecture.md`, +> `docs/implementation/universal-evidence.md`, +> `docs/implementation/evidence-resolution-framework-technical-design.md`, +> and `docs/reference/universal-semantic-evidence.md`. Never dual-publish a +> candidate in production: candidate emitters remain test/qualification-only, +> and each language switches its registry, extractor, resolver, cache +> requirements, and framework integration in one atomic hard-cut phase. +> +> **Drift check (run before every phase)**: +> `git diff --stat 88abe4c0..HEAD -- crates/compass-files crates/compass-languages crates/compass-resolve crates/compass-model crates/compass-graph crates/compass-core vendor/compass-tree-sitter-language-pack fixtures/code-graph tests/qualification scripts benchmarks/performance docs COMPATIBILITY.md MIGRATION.md CHANGELOG.md advisor-plans` +> Reconcile changed producer versions, evidence fields, framework pack IDs, +> cache contracts, qualification thresholds, and language-specific paths before +> implementation. If current code contradicts the inventory in this plan, stop +> and update the plan rather than preserving an obsolete path. + +## Status + +- **Execution status (2026-08-22)**: DONE. The production hard cuts, universal + registry entries, framework boundaries, deterministic fixture baselines, + pinned read-only manifests, parser-backed source providers, and fail-closed + audit harnesses are implemented. SwiftSyntax 603.0.0, Dart Analyzer 8.4.0, + scala.meta 4.13.10, and Groovy 4.0.27 providers are provisioned outside the + checkout under the mounted qualification target. The Swift, Dart, Scala, + and Groovy three-corpus graph audits all pass precision, recall, coverage, + and diversity gates. Registry state remains version-1 `Qualifying`; a later + promotion decision is intentionally separate from this delivery. +- **Priority**: P1 +- **Effort**: XXL, delivered as ten phases and at least ten PRs +- **Risk**: HIGH +- **Depends on**: no code prerequisite; release automation should consume plan + 005 or an equivalent exact-commit qualification gate +- **Category**: language architecture, correctness, resolution, frameworks, + performance, tests, documentation +- **Planned at**: commit `88abe4c0`, 2026-08-21 + +## Why this matters + +Compass already recognizes, parses, and publishes some facts for Swift, Dart, +Scala, and Groovy. That is useful established support, but it is not the same +support contract as Python, Go, Rust, Java, Kotlin, Ruby, TypeScript, +JavaScript, PHP, and C#: the four requested languages are absent from +`UniversalEvidenceRegistry`, lack independent quality audits, and still depend +on direct extractors or generic resolver behavior. + +This program makes all four languages first-class hard-cut universal evidence +pipelines. “Supported” for this program means one production route per +language, exact and bounded source evidence, conservative project-wide +resolution, deterministic cold/warm/incremental publication, and the full +quality-audit thresholds in `docs/reference/universal-semantic-evidence.md`. +The initial production state is version 1 `Qualifying`, matching Compass's +current language-transition policy. Promotion to `Qualified` is a separate +product decision and must not be inferred merely from registration or a green +fixture suite. + +## Extension contract + +Files are classified by `compass-files` and the static language registry; the +vendored package supplies pinned parsers only; `compass-languages` emits one +validated, bounded `SemanticEvidenceBatch` per source; `compass-resolve` +performs exact-language, project-aware, fail-closed resolution; framework packs +consume exact normalized evidence; and `compass-graph`/`compass-core` publish a +coherent deterministic graph. Inputs are untrusted source and bounded project +manifests. Outputs preserve declaration identity, direction, multiplicity, +source anchors, ambiguity, diagnostics, and producer provenance. A missing +grammar or invalid evidence is fatal; parser recovery or a spent budget is +explicitly incomplete; competing targets remain unresolved. + +## Current state at the planned commit + +| Surface | Swift | Dart | Scala | Groovy | +| --- | --- | --- | --- | --- | +| Discovery | `.swift` recognized | `.dart` recognized | `.scala` recognized | `.groovy` and `.gradle` recognized | +| Grammar | pinned static grammar | pinned static grammar, but production extractor does not use it | pinned static grammar | pinned static grammar, but production extractor does not use it | +| Extraction | dedicated AST walker in `src/swift.rs` | source/regex walker in `src/dart.rs` | branches inside generic `engine.rs` | line/regex walker plus Spock branch in `src/groovy.rs` | +| Calls | local edges plus `RawCall`; typed member table | framework-specific edges only; `raw_calls: None` | generic `RawCall` behavior | regex member calls plus `RawCall` | +| Project facts | bounded `Package.swift` dependency scan | no `pubspec.yaml` project evidence | no `build.sbt` project evidence | Gradle dependency scan only | +| Resolution | legacy Swift type table and compatibility pass | no Dart-specific resolver | JVM-family stub rewiring and generic resolution | JVM-family stub rewiring and generic resolution | +| Frameworks | established `vapor-routes` source pack | Flutter/BLoC/Riverpod/navigation rules are embedded in the direct extractor | `play-routes-config` can target Scala handlers | Spock recognition is embedded in the direct extractor | +| Universal registry | absent | absent | absent | absent | +| Qualification corpus | one matrix file plus focused Swift/Vapor and Dart range tests | one matrix file plus focused Dart range tests | one matrix file, type-shape smoke case, and Play handler | one matrix file; no dedicated conformance suite | + +Important current boundaries: + +- `crates/compass-languages/src/evidence_pipeline.rs` registers C#, Go, Java, + JavaScript, Kotlin, PHP, Python, Ruby, Rust, and TypeScript only. +- `crates/compass-resolve/src/evidence/languages/policy.rs` has no Swift, Dart, + Scala, or Groovy policy. +- `crates/compass-resolve/src/members.rs` contains `swift_type_table` handling + and an observable cross-language Swift compatibility pass. The hard cut must + remove that behavior; it must not migrate it into universal resolution. +- `crates/compass-resolve/src/lib.rs` groups Scala and Groovy with JVM sources + for direct stub rewiring. Shared JVM membership is not evidence for a + cross-language call target. +- `tests/qualification/code-graph-v1-corpus.json` proves recognition for all + four languages, not semantic completeness. +- The four grammars already exist in + `vendor/compass-tree-sitter-language-pack/language_definitions.json` and its + static build set. Parser availability must be reverified, not reimplemented. + +## Required semantic decisions + +Freeze these decisions in tests and documentation before a hard cut. + +### Swift v1 + +- Canonical identity is module-qualified, not file-stem-qualified. Nested + types retain lexical ownership. Overload identity includes the Swift base + name and argument-label sequence; `init`, subscripts, and operators do not + collapse by terminal spelling. +- `class`, `struct`, `enum`, `actor`, and `protocol` are distinct declaration + kinds. Protocol conformance is not class inheritance. Extensions attach only + after exact module/type resolution; competing same-named extension targets + remain unresolved. +- Imports bind modules. `typealias` is an alias. Attributes and property + wrappers remain anchored evidence but advertise `Decorators` only if the + producer emits and audits a truthful relationship family. +- Optional chaining, trailing closures, async/await, generic specialization, + and call-result chains may emit occurrences; they resolve only from exact + owner, type, or result evidence. +- Objective-C/C/C++ interoperability never uses native-family terminal-name + matching. Cross-language endpoints require exact fresh compiler/SCIP facts. + +### Dart v1 + +- Canonical identity uses the Dart library URI and lexical owner. Relative + libraries, `package:` URIs, `part`, and `part of` must converge on one + contained library identity without consulting files outside the project. +- Classes, mixins, extensions, extension types, enums, typedefs, top-level + functions, methods, getters, setters, operators, unnamed constructors, + named constructors, and factory constructors remain distinct where Dart + dispatch distinguishes them. +- Imports, deferred imports, prefixes, `show`/`hide`, exports, and aliases are + explicit bounded bindings. No first matching import wins. +- Named arguments, cascades, null-aware access, generic invocation, and + extension methods preserve exact occurrences. A `dynamic` receiver or an + ambiguous extension remains unresolved. +- Flutter, BLoC, Riverpod, and navigation meaning moves out of the language + extractor into statically registered evidence-backed framework packs. + +### Scala v1 + +- One semantic language identity, `scala`, covers Scala 2 and Scala 3. A + dialect field is emitted only when bounded project/toolchain evidence proves + it; syntax guessing must not change declaration identity. +- Packages, package objects, classes, traits, objects, companion pairs, enums, + case classes/objects, methods, values, variables, type aliases, givens, and + extension declarations receive distinct source identities. A class and its + companion object are linked but never collapsed. +- Import selectors, aliases, exclusions, wildcards, Scala 3 exports, overloads, + multiple parameter lists, named/default arguments, and inheritance/mixins + are represented explicitly and resolved within bounded candidate sets. +- Implicit conversions, implicit search, contextual `given` selection, + compiler-synthesized members, macros, and quoted code remain unresolved in + the structural v1 tier unless exact fresh compiler evidence names both ends. +- Java, Kotlin, and Groovy declarations are never selected from shared JVM + packages or terminal spelling. Cross-language calls require exact anchored + compiler/SCIP endpoints. + +### Groovy v1 + +- Canonical identity uses package plus lexical owner. Scripts have a + source-scoped script owner; their top-level declarations do not become one + repository-global namespace. +- Classes, interfaces, traits, enums, records, annotations, methods, + constructors, fields/properties, closures, aliases, and quoted Spock feature + methods retain exact ranges and distinct identities. +- Static imports, aliases, safe navigation, spread access, closures, named + arguments, and constructor calls may produce evidence. Runtime metaclass + mutation, `methodMissing`, `propertyMissing`, dynamic GStrings, categories, + and nonliteral DSL dispatch never create convenient exact targets. +- `@CompileStatic` or `@TypeChecked` may strengthen source-proven local type + evidence; it does not authorize using a compiler result that is absent or + stale. +- Gradle DSL calls remain qualified external/unresolved unless a separately + versioned framework pack proves their meaning. Spock feature methods may be + test declarations, but `Tests` is advertised only after relationships to the + subject under test are independently proven and audited. +- Java/Kotlin/Scala interop follows the same exact-endpoint rule as Scala. + +## Qualification truth and corpora + +Phase 0 must create immutable manifests with full commit SHAs and inventory +digests for these proposed three-corpus sets. The executor may replace a corpus +only if it is unavailable, unbuildable by its documented pinned toolchain, or +fails the diversity rule; record the replacement and reason in the plan and +qualification documentation before continuing. + +| Language | Proposed corpora | Independent qualification-only oracle | +| --- | --- | --- | +| Swift | `apple/swift-nio`, `vapor/vapor`, `apple/swift-collections` | pinned SwiftSyntax parser; optional SourceKit/IndexStore compiler endpoints stay a separate provider profile | +| Dart | `dart-lang/sdk` library subset, `flutter/flutter` packages subset, `rrousselGit/riverpod` | pinned Dart SDK `analyzer` AST helper | +| Scala | `scala/scala3`, `akka/akka`, `playframework/playframework` | pinned scala.meta source parser for constructs plus separately identified SemanticDB compiler endpoints | +| Groovy | `apache/groovy`, `gradle/gradle`, `spockframework/spock` | pinned Groovy compiler `CompilationUnit` AST helper that never executes repository build scripts | + +All external repositories live under +`/Volumes/Workspace/Github//`, are treated as read-only, +and are never reset, cleaned, updated, or executed by qualification. Oracle +helpers may parse source and checked-in manifests only. They must record +provider/toolchain versions, complete file inventories, partial files, exact +UTF-8 ranges, canonical inventory digests, and deterministic output. No oracle +may reuse Tree-sitter, Compass graph output, Graphify, or another language's +terminal-name index as truth. + +## Commands executors will need + +Every compiling Cargo command must use a unique mounted target directory for +the implementation checkout or worktree. Replace `` with the phase +number and never fall back to a local `target/` directory. + +| Purpose | Command | Expected on success | +| --- | --- | --- | +| Target preflight | `test -d /Volumes/Workspace && mkdir -p /Volumes/Workspace/crabbuild-target/compass-language-wave- && test -w /Volumes/Workspace/crabbuild-target/compass-language-wave-` | exit 0 | +| Language crate | `CARGO_TARGET_DIR=/Volumes/Workspace/crabbuild-target/compass-language-wave- cargo test -p compass-languages --locked` | exit 0 | +| Resolver crate | `CARGO_TARGET_DIR=/Volumes/Workspace/crabbuild-target/compass-language-wave- cargo test -p compass-resolve --locked` | exit 0 | +| Core incremental contract | `CARGO_TARGET_DIR=/Volumes/Workspace/crabbuild-target/compass-language-wave- cargo test -p compass-core --test code_graph_v1_determinism --locked` | exit 0 | +| Product boundary | `sh scripts/check_product_boundary.sh` | exit 0; no Graphify/runtime boundary violations | +| Fixture qualification | `CARGO_TARGET_DIR=/Volumes/Workspace/crabbuild-target/compass-language-wave- ./scripts/qualify_code_graph_v1.sh --fixtures-only` | all manifest, determinism, incremental, and graph assertions pass | +| Format | `cargo fmt --all -- --check` | exit 0, no diff | +| Baseline Clippy | `CARGO_TARGET_DIR=/Volumes/Workspace/crabbuild-target/compass-language-wave- cargo clippy --workspace --lib --bins --locked -- -D warnings` | exit 0 | +| Baseline tests | `CARGO_TARGET_DIR=/Volumes/Workspace/crabbuild-target/compass-language-wave- cargo test --workspace --lib --bins --locked` | exit 0 | + +Each language adds `scripts/qualify__universal.py` with `fixture`, +`pinned`, `quality-audit`, and `performance` modes, modeled on +`scripts/qualify_ruby_universal.py`. Each mode must use caller-provided clean +checkouts and temporary outputs; it must not clone, mutate, compile, or execute +qualification repository code. + +## Scope + +**In scope across the program**: + +- `crates/compass-files/src/detect.rs` and tests only if recognition rules need + a documented correction; +- `crates/compass-languages/src/{registry.rs,evidence_pipeline.rs,engine.rs}`; +- new `crates/compass-languages/src/evidence/{swift,dart,scala,groovy}.rs`; +- deletion or narrowing of the replaced direct files + `src/{swift,dart,groovy}.rs` and Scala-only generic-engine branches; +- bounded project evidence for `Package.swift`, `pubspec.yaml`, `build.sbt`, + and Gradle/Groovy project inputs; +- `crates/compass-languages/src/frameworks/` for Vapor and Dart framework + migration; +- `crates/compass-resolve/src/evidence/` and focused closed language policies; +- removal of replaced Swift member-table and Scala/Groovy JVM stub behavior; +- framework expansion/targeting required by the migrated packs; +- focused fixtures, conformance/resolver tests, qualification harnesses, + manifests, performance evidence, compatibility notes, changelog, and current + support documentation. + +**Out of scope**: + +- runtime parser downloads, dynamic grammar loading, Graphify dependencies, or + executing untrusted project builds; +- compiler-grade completeness, IDE/LSP transport work, or mandatory Swift, + Dart, Scala, Groovy, Gradle, sbt, Flutter, or Xcode installations at normal + Compass runtime; +- speculative cross-language calls based on JVM/native family, package, + import, or terminal-name similarity; +- changing `compass.graph/1`, the universal evidence schema, public lookup + budgets, or `LanguageCapability` solely to accommodate one language; +- broad framework expansion beyond preserving existing Vapor, Dart heuristic, + Play, Spock, and Gradle behavior; +- vendored grammar changes. If a pinned grammar cannot represent a required + stable construct, stop and propose a separate attributed vendor update with + parser conformance evidence. + +## Git and delivery workflow + +- Branches: `advisor/020-p-`. +- Use conventional commits matching current history, for example + `feat(languages): add Swift universal evidence candidate`. +- One phase per PR. Keep candidate and production hard-cut changes separate. +- Do not push, merge, or open a PR unless the operator explicitly requests it. +- Every PR description records compatibility effect, exact checks, pinned + corpus/toolchain identities, performance comparison, and checks not run. + +## Phase map + +| Phase | Deliverable | Depends on | Production behavior changes? | +| --- | --- | --- | --- | +| 0 | Freeze contracts, direct baselines, corpora, and independent oracles | none | No | +| 1 | Build a qualification-only Swift v1 candidate | 0 | No | +| 2 | Qualify, migrate Vapor, and atomically hard-cut Swift | 1 | Yes, Swift only | +| 3 | Build a qualification-only Dart v1 candidate | 0 | No | +| 4 | Qualify, migrate Dart frameworks, and atomically hard-cut Dart | 3 | Yes, Dart only | +| 5 | Build a qualification-only Scala v1 candidate | 0 | No | +| 6 | Qualify Play targeting and atomically hard-cut Scala | 5 | Yes, Scala only | +| 7 | Build a qualification-only Groovy v1 candidate | 0; reuse JVM decisions from 5 where landed | No | +| 8 | Qualify Gradle/Spock behavior and atomically hard-cut Groovy | 7 | Yes, Groovy only | +| 9 | Run mixed-language release qualification and close public support docs | 2, 4, 6, 8 | Documentation/status only unless a gate exposes a defect | + +Phases 1, 3, and 5 can run in parallel after Phase 0. Phase 7 may also run in +parallel, but it must adopt—not duplicate—the exact-language JVM boundary from +Phase 5 if that boundary has landed. + +## Phase 0: Freeze contracts, baselines, corpora, and independent truth + +### Phase context + +This phase changes no production extraction. At commit `88abe4c0`, all four +languages use established direct paths and none is registered in +`UniversalEvidenceRegistry`. The purpose is to prevent implementation from +silently redefining existing behavior or measuring itself as its own oracle. + +### Phase scope and work + +- Create one versioned repository manifest per language under + `tests/qualification/`, using full immutable SHAs, purpose, dialect/profile, + required source globs, exclusion policy, and clean-checkout requirements. +- Create qualification-only source-oracle helpers and canonical output schemas. + Pin toolchain/package versions and verify byte-deterministic repeated output. +- Add established-path baselines covering relation counts, graph/evidence + digests, diagnostics, omitted facts, identity collisions, cold/warm/neutral + edit/semantic edit/restore timing, and peak RSS. +- Add focused fixtures for every semantic decision listed above, including + positive, negative, ambiguity, malformed, UTF-8, multiline, repeated-site, + and limit cases. Baseline corrections are recorded as intended changes; they + are not forced into parity. +- Define per-language target capability lists. A capability is excluded unless + the source oracle can inventory it and the audit can satisfy at least 100 + accepted records for that capability identity. +- Record performance gates in each baseline: cold median no worse than the + larger of 110% of established or established plus one second; warm and + fact-neutral median no worse than the larger of 115% or established plus + 100 ms/250 ms respectively; peak RSS no worse than the larger of 115% or + established plus 32 MiB. A stricter existing repository gate wins. + +### Phase acceptance criteria + +The phase checklists below are historical gates: a criterion describing the +pre-cutover registry records the state verified before the following atomic +hard cut. The final registry state is recorded in Phase 9 and the program done +criteria. + +- [x] Four manifests validate, use full SHAs, and point only to clean mounted + read-only checkouts. +- [x] Each oracle output is byte-identical across two runs and includes exact + toolchain/provider identity, complete inventory digest, partial-file count, + and exact source ranges. +- [x] Each established baseline is reproduced twice with identical graph + bytes on cold rebuild and warm cache reuse; edit/restore returns to the exact + baseline hash. +- [x] The candidate capability list has positive, negative, ambiguity, limit, + and independent-oracle strata for every advertised capability. +- [x] No production registry, extractor dispatch, resolver, or framework pack + changes appear in the diff. +- [x] `git diff --check` and all new harness unit tests pass. + +## Phase 1: Build a qualification-only Swift v1 candidate + +### Phase context + +Swift currently uses `crates/compass-languages/src/swift.rs`, file-stem IDs, +`RawCall`, `swift_type_table`, and legacy member resolution. The existing +Vapor source pack remains production-active. This phase adds a parallel +candidate API callable only by tests and qualification; it must not register +`compass.swift` or alter production graphs. + +### Phase scope and work + +- Add `evidence/swift.rs` as a bounded AST consumer using the already prepared + Swift Tree-sitter root. Emit module-qualified declarations/scopes/bindings, + protocol and extension evidence, exact imports/typealiases, calls, + construction, type references, ownership, receiver evidence, and truthful + diagnostics. +- Encode overload identity with argument labels and distinguish type kinds, + constructors, deinit, subscript, operators, nested declarations, enum cases, + properties, local bindings, actors, and extensions. +- Emit complete direct-base evidence only when parser recovery does not overlap + the declaration. Preserve ambiguous class/protocol classification rather + than inheriting the direct extractor's “first base” heuristic. +- Add `swift_universal_conformance.rs` and a qualification-only resolver module + covering module imports, nested scopes, overloads, extensions, protocol + dispatch, optional/result chains, ambiguity, builtins, UTF-8, malformed + source, repeated calls, and evidence limits. +- Compare the candidate against Phase 0 baselines and the SwiftSyntax oracle. + Record intentional ID/relation corrections separately from regressions. + +### Phase acceptance criteria + +- [x] `UniversalEvidenceRegistry::pipeline("swift")` remains `None` and normal + `Engine::extract` output is byte-identical to the Phase 0 Swift baseline. +- [x] The candidate batch validates, contains no direct graph nodes/edges or + `RawCall`, is deterministic, and advertises only audited capabilities. +- [x] Every emitted occurrence slices the original UTF-8 bytes exactly; + repeated same-line occurrences remain distinct. +- [x] Duplicate module/type/method names, competing extensions, unknown + receivers, and native-family near matches remain unresolved. +- [x] `cargo test -p compass-languages --test swift_universal_conformance + --locked` and `cargo test -p compass-resolve --test universal_resolution + --locked swift` pass with the required external target directory. + +## Phase 2: Qualify, migrate Vapor, and atomically hard-cut Swift + +### Phase context + +Phase 1 produced a non-production Swift evidence candidate. Production still +uses the direct Swift publisher, `swift_type_table`, legacy member resolution, +and the established `vapor-routes` source pack. This phase may begin only when +the candidate passes fixture conformance and a reproducible pinned-corpus +comparison; this phase owns the blocking quality audit and performance gate. + +### Phase scope and work + +- Complete SwiftPM project evidence: package/module name, bounded target source + roots, test targets, dependencies, and contained module imports. Do not + evaluate `Package.swift` or invoke SwiftPM. +- Prefer generic resolver stages; add `LanguagePolicyKind::Swift` only for + extension/protocol/overload rules that cannot be expressed generically. +- Convert Vapor to one `vapor-swift` universal descriptor and matching resolver + adapter keyed by the same pack ID. Consume exact call/import/ownership facts; + preserve grouped literal routes and explicit handlers; keep opaque closures + unresolved. +- Run the independent three-corpus audit. Meet every threshold in the reference + contract: 2,000 accepted records, 400 per corpus, 100 per relation and + advertised capability, cluster diversity, precision/Wilson/recall gates, and + zero critical violations. +- Atomically register `compass.swift` version 1 as `Qualifying`, switch Engine + publication to universal evidence, require valid cached evidence, remove the + direct publisher/`swift_type_table`/legacy compatibility resolver, replace + `vapor-routes`, and update cache fingerprints without changing unrelated + language versions. +- Update Swift qualification, framework-route, compatibility, changelog, and + language-status documentation. + +### Phase acceptance criteria + +- [x] No production Swift file publishes through `src/swift.rs`, `RawCall`, + `swift_type_table`, or `resolve_swift_registry_compatibility`; searches find + no active references to the replaced route. +- [x] `UniversalEvidenceRegistry::pipeline("swift")` returns + `compass.swift`, version 1, `Qualifying`, with the exact audited capability + list. +- [x] Cold, warm, forced rebuild, alternate checkout, fact-neutral edit, + semantic edit, delete, rename, and restore cases are deterministic and meet + Phase 0 performance/RSS gates. +- [x] The Swift quality-audit report meets all numerical and zero-tolerance + gates and binds the exact source-oracle inventory and graph digest. +- [x] Vapor activation/negative/ambiguity/handler/range/limit tests pass and + descriptor/expansion registry sets match exactly. +- [x] Swift conformance, resolver, framework, crate, core determinism, full + fixture qualification, product boundary, baseline Clippy, and baseline tests + all pass. + +## Phase 3: Build a qualification-only Dart v1 candidate + +### Phase context + +Dart currently bypasses its linked grammar and uses `src/dart.rs`, a regex +extractor with `raw_calls: None`. Flutter, BLoC, Riverpod, and navigation +relations are mixed into language extraction. There is no `pubspec.yaml` +project evidence. This phase adds a test-only AST candidate and leaves normal +Dart output unchanged. + +### Phase scope and work + +- Add `evidence/dart.rs` using the prepared Dart Tree-sitter AST. Emit library- + qualified declarations, lexical scopes, parts, imports/exports/prefixes, + `show`/`hide`, typedefs, classes/mixins/extensions/extension types, members, + functions, constructors, annotations, calls, type references, receivers, and + exact diagnostics. +- Model named/factory constructors, getters, setters, operators, named/default + arguments, cascades, null-aware access, generics, and local/result types + without resolving `dynamic` or ambiguous extension dispatch. +- Add a bounded `pubspec.yaml` parser to qualification-only project context for + package name, dependencies, SDK constraints, and project-contained roots. + Never run `pub`, Flutter, builders, or generated code. +- Split current Dart framework facts into candidate evidence-backed adapters; + they remain unregistered until Phase 4. +- Add Dart conformance and resolver tests plus differential reports against the + Dart analyzer oracle and Phase 0 direct baseline. + +### Phase acceptance criteria + +- [x] `UniversalEvidenceRegistry::pipeline("dart")` remains `None`; normal Dart + extraction and current framework output match the Phase 0 baseline. +- [x] Candidate output validates, is byte deterministic, has exact ranges, and + contains no direct graph records or `RawCall`. +- [x] `part`/`part of`, package/relative imports, prefixes, filters, exports, + duplicate names, named constructors, and ambiguous extensions have positive + and fail-closed tests. +- [x] `dynamic`, malformed syntax, nonliteral navigation, and out-of-project + package paths never create invented local targets. +- [x] Dart conformance and `universal_resolution dart` targeted tests pass. + +## Phase 4: Qualify, migrate Dart frameworks, and atomically hard-cut Dart + +### Phase context + +Phase 3 supplied a qualification-only Dart AST candidate and candidate project +context. Production still uses the regex direct extractor and embeds framework +rules. This phase performs the sole production switch after independent audit +and performance gates pass. + +### Phase scope and work + +- Promote bounded `pubspec.yaml` project facts into normal project evidence, + fingerprinting package name, contained roots, dependencies, and diagnostics. +- Add Dart-specific resolver policy only for language rules such as extension + selection or named constructors that generic stages cannot express. +- Register focused universal packs for the established behavior: use stable + separate IDs for Flutter navigation, BLoC, and Riverpod when their activation + and accepted evidence differ. Exact pack IDs and dependencies must be frozen + in tests; one catch-all detector is not acceptable. +- Complete the three-corpus source-oracle audit and Phase 0 performance gates. +- Atomically register `compass.dart` version 1 `Qualifying`, use universal + publication, reject stale cache entries lacking current evidence, delete the + regex direct path, and remove embedded framework branches. +- Update Dart qualification, compatibility, changelog, framework, and status + documentation. + +### Phase acceptance criteria + +- [x] No production Dart path calls `src/dart.rs` or emits framework relations + from the language producer. +- [x] The production registry exposes only `compass.dart` version 1 + `Qualifying`; project/package changes invalidate only relevant extraction. +- [x] The full audit satisfies all record-count, diversity, precision, Wilson, + recall, and zero-critical-violation gates. +- [x] Framework packs require positive manifest/source activation, reject wrong + frameworks, preserve exact anchors and repeated occurrences, and remain + bounded/deterministic. +- [x] Cold/warm/rebuild/incremental/delete/rename/restore graphs and evidence are + deterministic and within Phase 0 performance/RSS limits. +- [x] Dart targeted suites, full language/resolver crates, core determinism, + fixture qualification, product boundary, Clippy, and baseline tests pass. + +## Phase 5: Build a qualification-only Scala v1 candidate + +### Phase context + +Scala currently uses generic Tree-sitter extraction with Scala-only helper +branches in `engine.rs`; collection resolution may rewire Scala stubs through +the broad JVM family. Play config routes can target Scala handlers. This phase +adds test-only evidence for Scala 2 and Scala 3 and does not register it. + +### Phase scope and work + +- Add `evidence/scala.rs` over the prepared tree. Emit packages/scopes, + classes/traits/objects/companions/enums/case declarations, methods, + constructors, vals/vars, type aliases, imports/selectors/aliases/exclusions, + Scala 3 exports, annotations, calls, construction, ownership, receivers, + type parameters/bounds, and inheritance/mixins. +- Preserve overload signatures, multiple parameter lists, named/default + arguments, by-name/varargs types, nested packages, and exact companion + identities. Emit givens and extension declarations as declarations, but do + not claim implicit search or compiler-synthesized dispatch. +- Add bounded qualification-only `build.sbt`/project metadata sufficient to + identify source roots, Scala version/dialect when explicit, and dependency + coordinates without evaluating sbt. +- Establish one explicit cross-JVM invariant test matrix: equal package/name + Java, Kotlin, Groovy, and Scala declarations never cross-resolve without + exact fresh compiler evidence. +- Add conformance/resolver tests and differential reports against scala.meta, + SemanticDB where available, and the established baseline. + +### Phase acceptance criteria + +- [x] The production registry still returns no Scala pipeline and normal Scala + graphs remain at the Phase 0 baseline. +- [x] Candidate batches validate, sort deterministically, preserve exact Scala + 2/3 anchors, and advertise no implicit/macro/compiler-only capability. +- [x] Companions, overloads, imports, exclusions, wildcards, exports, givens, + extensions, inheritance, ambiguity, malformed syntax, and limits have direct + tests. +- [x] Cross-JVM collision tests publish no guessed calls or type edges. +- [x] Scala conformance and `universal_resolution scala` tests pass. + +## Phase 6: Qualify Play targeting and atomically hard-cut Scala + +### Phase context + +Phase 5 produced a non-production Scala candidate and exact-language JVM +tests. Production still uses generic Scala branches and JVM-family stub +rewiring. The existing `play-routes-config` pack is a configuration producer, +not a reason to infer Scala/Java target identity. + +### Phase scope and work + +- Promote bounded sbt/Scala project evidence needed for module/package/source- + root resolution. Keep build evaluation and compiler invocation optional and + outside normal runtime. +- Prefer generic resolution. Add `LanguagePolicyKind::Scala` only for + companion, extension, or overload rules backed by explicit source facts. +- Update Play target resolution so an exact Scala handler resolves from Scala + package/owner/signature evidence. Java or injected handlers remain separate; + equal terminal spelling is explicit ambiguity. +- Complete the three-corpus quality audit and performance gates. +- Atomically register `compass.scala` version 1 `Qualifying`, remove Scala-only + generic-engine helpers and Scala participation in broad JVM stub rewiring, + enforce current evidence on cache reuse, and update docs/contracts. + +### Phase acceptance criteria + +- [x] Searches show no active Scala-specific branches in the direct generic + walker and no JVM-family terminal-name fallback for Scala. +- [x] The registry returns `compass.scala` version 1 `Qualifying` with the exact + audited capability list. +- [x] Scala 2 and Scala 3 each appear in fixture and pinned-corpus evidence; an + unknown dialect does not fabricate a dialect or change identity. +- [x] The full independent audit and Phase 0 performance/RSS gates pass. +- [x] Play route tests cover exact Scala, exact Java, injected, duplicate, + ambiguous, malformed, and unresolved handlers with correct direction and + anchors. +- [x] Scala targeted/full suites and all repository gates listed in Phase 2 + pass. + +## Phase 7: Build a qualification-only Groovy v1 candidate + +### Phase context + +Groovy currently bypasses its grammar and parses lines with regular +expressions. It has a separate Spock feature branch, raw member calls, limited +class/method identity, and broad JVM stub compatibility. `.gradle` is treated +as Groovy source. This phase replaces none of that in production; it builds a +test-only AST candidate. + +### Phase scope and work + +- Add `evidence/groovy.rs` over the prepared Groovy tree. Emit packages, + scripts, classes/interfaces/traits/enums/records/annotations, methods, + constructors, fields/properties, closures, imports/static imports/aliases, + annotations, calls, construction, ownership, type references, inheritance, + receivers, and quoted Spock feature declarations. +- Give top-level scripts source-scoped owners. Preserve exact closure and + feature ranges. Do not interpret Gradle DSL or dynamic metaprogramming as + exact local calls. +- Add bounded qualification-only Gradle/Groovy project facts from checked-in + settings/build/property/version-catalog files without executing Gradle or + Groovy. Reuse the exact-language JVM boundary from Scala instead of adding a + second family heuristic. +- Add conformance/resolver tests covering static/dynamic calls, aliases, + traits, scripts, closures, Spock, Gradle, parser recovery, ambiguity, UTF-8, + repeated sites, and limits. +- Compare against the pinned Groovy compiler AST oracle and direct baseline. + +### Phase acceptance criteria + +- [x] Production Groovy/Gradle graphs and registry state remain unchanged. +- [x] Candidate batches validate and are deterministic; the candidate never + executes repositories, Gradle, Groovy scripts, transforms, or AST macros. +- [x] `methodMissing`, metaclass mutation, dynamic GStrings, DSL calls, and + cross-JVM near matches remain unresolved. +- [x] Spock quoted features are exact declarations without advertising an + unaudited `Tests` relationship capability. +- [x] Groovy conformance and `universal_resolution groovy` tests pass. + +## Phase 8: Qualify Gradle/Spock behavior and atomically hard-cut Groovy + +### Phase context + +Phase 7 produced a qualification-only Groovy candidate. Production still uses +the regex/line extractor, Spock branch, `RawCall`, and JVM stub rewiring. This +phase switches only after the Groovy compiler oracle, three corpora, and +performance evidence pass. + +### Phase scope and work + +- Promote safe Gradle/Groovy project evidence required for contained modules, + dependencies, source roots, and framework activation. Preserve dynamic DSL + calls as external/unresolved facts. +- Add `LanguagePolicyKind::Groovy` only for source-proven language behavior not + covered by generic stages. Never use a “dynamic language” broad fallback. +- Keep Spock declaration/test classification in the language producer or a + narrowly scoped universal pack based on evidence ownership; freeze the choice + and pack ID before implementation. Do not create subject-under-test edges + without independent proof. +- Complete the three-corpus audit and performance gates. +- Atomically register `compass.groovy` version 1 `Qualifying`, delete the regex + and Spock direct branches, remove Groovy JVM stub rewiring, enforce evidence- + aware cache reuse, and update documentation. + +### Phase acceptance criteria + +- [x] No production call reaches `src/groovy.rs`, regex class/method/call + extractors, or JVM terminal-name rewiring for Groovy. +- [x] The registry returns `compass.groovy` version 1 `Qualifying` with only + audited capabilities. +- [x] Apache Groovy, Gradle, and Spock corpora meet the full audit and diversity + gates with zero fabricated/dynamic/cross-language targets. +- [x] `.groovy` application sources, `.gradle` scripts, and Spock features each + have dedicated deterministic incremental coverage and meet performance/RSS + limits. +- [x] Groovy targeted/full suites and all repository gates listed in Phase 2 + pass. + +## Phase 9: Qualify the mixed-language release and close support claims + +### Phase context + +Swift, Dart, Scala, and Groovy are now independently hard-cut version-1 +`Qualifying` pipelines. This phase proves they coexist with existing languages, +frameworks, caches, and output contracts. It does not weaken a failing +language gate or promote a pipeline to `Qualified` automatically. + +### Phase scope and work + +- Extend the Code Graph v1 fixture corpus from recognition-only files to a + reviewable multilingual slice containing imports, declarations, calls, + construction, members, inheritance/traits, ambiguity, malformed files, + framework facts, and exact negative cross-language collisions. +- Run clean, warm, forced, alternate-checkout, edit, delete, rename, and restore + qualification over the combined corpus. Assert byte-identical graphs and + evidence where the input state is equivalent. +- Verify universal registry order/uniqueness, framework descriptor/adapter + parity, cache invalidation isolation, diagnostics, limits, publication + omission accounting, stable IDs, direction, multiplicity, anchors, and + provenance. +- Update `docs/design/language-architecture.md`, + `docs/implementation/universal-evidence.md`, + `docs/reference/universal-semantic-evidence.md`, user-facing language/support + docs, `COMPATIBILITY.md`, and `CHANGELOG.md`. Update `MIGRATION.md` only if a + user must discard or rebuild artifacts manually rather than through normal + fingerprint invalidation. +- Record each language's exact `Qualifying` evidence and open promotion work. + Change a language to `Qualified` only in a separate approved decision backed + by its complete audit artifact. + +### Phase acceptance criteria + +- [x] All four languages appear once in the sorted universal registry, each at + version 1 `Qualifying`, with no production direct fallback. +- [x] The mixed fixture proves exact-language boundaries: Swift/native and + Scala/Groovy/Java/Kotlin name collisions never create cross-language edges; + Dart package names never bind unrelated filesystem stubs. +- [x] All four independent audit artifacts still meet their thresholds against + the exact release-candidate graph and source inventories. +- [x] `cargo fmt`, workspace lib/bin Clippy, workspace lib/bin tests, product + boundary, CLI product contract, and `qualify_code_graph_v1.sh --fixtures-only` + pass with the required external target directory. +- [x] `git diff --check` passes; the implementation status contains only + intended source, test, fixture, qualification, and documentation changes + plus pre-existing user paths preserved untouched; no generated graph, + `.compass/`, `compass-out/`, credentials, or external-repository changes are + present. + +## Cross-phase test plan + +Every language needs tests at four layers: + +1. `compass-languages`: parser/evidence conformance, identity, exact UTF-8 + ranges, capabilities, deterministic ordering, malformed syntax, and limits. +2. `compass-resolve`: local, lexical, import/package, member, hierarchy, + overload/argument, ambiguity, external, and negative cross-language cases. +3. `compass-core`/`compass-graph`: cold/warm/incremental/delete/rename/restore, + cache-version enforcement, stable normalized graph, omissions, diagnostics, + direction, multiplicity, and provenance. +4. Qualification: three independent corpora, source-oracle recall, accepted- + edge precision, framework behavior, target-cluster diversity, performance, + RSS, and exact release-candidate binding. + +Tests must model existing conformance suites such as +`kotlin_universal_conformance.rs`, the language modules under +`crates/compass-resolve/tests/universal_resolution/`, Ruby's independent +qualification harness, and `code_graph_v1_determinism.rs`. They must not use +Graphify, real credentials, network services, runtime grammar downloads, or +untrusted project execution. + +## Program done criteria + +All items must hold; a green registration test alone is insufficient. + +- [x] Swift, Dart, Scala, and Groovy are each registered once as version-1 + `Qualifying` universal evidence pipelines. +- [x] Each production extractor emits validated typed evidence directly and + has no dual direct graph publisher, raw-call fallback, or replaced resolver. +- [x] Each language has exact project/module/package identity, conservative + imports/calls/members/hierarchy, explicit ambiguity, and no guessed + cross-language endpoints. +- [x] Existing Vapor, Dart application-framework, Play, Spock, and Gradle + behavior is preserved or intentionally corrected through evidence-backed, + bounded, independently tested contracts. +- [x] Each language passes its three-corpus independent quality audit and Phase + 0 performance/RSS gates. +- [x] Equivalent clean, warm, forced, alternate-checkout, incremental, and + restored inputs publish byte-identical artifacts. +- [x] Public compatibility, support, framework, implementation, and changelog + documentation accurately distinguishes recognition, established support, + `Qualifying`, and `Qualified`. +- [x] All targeted and repository-wide verification gates pass on the final + release-candidate commit. + +## STOP conditions + +Stop and report rather than improvising if any of these occurs: + +- The mounted workspace or the phase-specific external Cargo target directory + is unavailable or unwritable. +- A selected corpus is not at its pinned SHA, is dirty, requires executing + repository code to inventory source, or cannot be parsed with the pinned + qualification-only oracle. +- The vendored grammar lacks a required stable construct or exceeds parser + recovery limits on a corpus. Do not add regex fallback or modify `vendor/` + inside this plan. +- A proposed evidence fact requires changing `compass.graph/1`, the universal + evidence schema, a public limit, or central publisher language branching. +- Candidate and established paths would both publish in production, even + behind an environment variable. +- An audit misses a numerical, diversity, recall, precision, or zero-tolerance + gate. Keep the direct production path active until corrected. +- Resolution requires choosing among multiple source-valid candidates, + selecting the first filesystem/hash iteration result, or using JVM/native + family or terminal-name similarity. +- A framework migration cannot prove exact activation or handler ownership. + Preserve the established pack until a separate bounded design is approved. +- A phase's verification fails twice after a reasonable correction, or the + implementation needs files outside that phase's stated scope. + +## Maintenance notes + +- Producer versions are per-language cache identities. Increment only the + changed language for later semantic changes; do not bump the universal + evidence schema for producer-local evolution. +- Project evidence and framework pack descriptors are fingerprint inputs. + Review bounded scans, symlink containment, deterministic ordering, and stale + cache rejection whenever new manifest fields are added. +- Reviewers should scrutinize exact range slicing, overload/argument identity, + incomplete evidence, ambiguity, cross-language boundaries, and removal of + replaced paths more than total node/edge growth. +- Optional compiler/SCIP enrichment is a separate fresh, bounded provider + profile. It may strengthen exact endpoints but never replaces structural + evidence or becomes a normal runtime dependency. +- Future Swift macros, Dart code generation, Scala implicit/compiler synthesis, + and Groovy metaprogramming require separate capability and provider audits; + this plan deliberately leaves unsupported dynamic meaning unresolved. + +## Findings considered and rejected + +- **Treat parser availability as support**: rejected because all four parsers + are already linked while semantic and qualification maturity differ sharply. +- **Keep the direct publisher as fallback after registration**: rejected + because Compass's hard-cut contract permits one production path only. +- **Migrate all four languages in one code change**: rejected because one + language's audit or grammar gap must not block, weaken, or destabilize the + others. +- **Share one JVM resolver across Java, Kotlin, Scala, and Groovy by terminal + name**: rejected because package/family proximity is not an exact endpoint + and creates fabricated cross-language edges. +- **Retain Dart/Groovy regex extraction beside AST evidence**: rejected because + it duplicates identities and lets line heuristics bypass evidence validation. +- **Run SwiftPM, pub, sbt, Gradle, Flutter, or repository tests during normal + extraction**: rejected because Compass is native, local-first, bounded, and + must not execute untrusted project code. +- **Claim compiler-grade or dynamic-language completeness**: rejected. The v1 + structural tier publishes only independently auditable source evidence and + explicit qualified externals. diff --git a/advisor-plans/README.md b/advisor-plans/README.md index d8996cab..076af837 100644 --- a/advisor-plans/README.md +++ b/advisor-plans/README.md @@ -43,6 +43,23 @@ quality gates. The pinned three-corpus audit now passes (89,981 accepted relationships, 100% observed precision, 98.5567% recall); Ruby remains `Qualifying` until a separate promotion decision. +Plan 020 is the Swift, Dart, Scala, and Groovy universal-evidence program. It +was planned at Compass commit `88abe4c0` on 2026-08-21. All four languages are +already recognized and have established extraction, so the program freezes +that behavior, builds independent source oracles and qualification-only +candidates, performs one atomic hard cut per language, preserves existing +Vapor/Dart/Play/Spock/Gradle behavior through evidence-backed boundaries, and +finishes with a mixed-language release gate. Swift, Dart, and Scala candidates +can proceed independently after the shared baseline; Groovy reuses Scala's +exact-language JVM boundary. +The production hard cut, deterministic fixture baselines, pinned manifests, +parser-backed source-oracle providers, audit builder, mixed fixture gate, and +three-corpus quality audits are implemented. The plan is `DONE`; all four +registry entries intentionally remain version-1 `Qualifying` until a separate +promotion decision. The mounted qualification target records the pinned +SwiftSyntax, Dart Analyzer, scala.meta, and Groovy CompilationUnit toolchains +and the immutable audit results. + ## Execution order and status | Plan | Title | Priority | Effort | Depends on | Status | @@ -66,6 +83,7 @@ relationships, 100% observed precision, 98.5567% recall); Ruby remains | 017 | Derive bounded, ranked execution flows from entry points | P2 | L | Existing universal call graph | TODO | | 018 | Expose five native MCP workflow prompts | P2 | M | — | TODO | | 019 | Hard-cut Ruby to a qualifying universal evidence pipeline | P1 | XL | —; final gate should consume 005 or equivalent | IN PROGRESS | +| 020 | Hard-cut Swift, Dart, Scala, and Groovy to universal evidence | P1 | XXL | —; final gate should consume 005 or equivalent | DONE | Status values: `TODO`, `IN PROGRESS`, `DONE`, `BLOCKED`, or `REJECTED`. @@ -106,6 +124,12 @@ Status values: `TODO`, `IN PROGRESS`, `DONE`, `BLOCKED`, or `REJECTED`. Rails pack stay qualification-only until one atomic production hard cut; optimization follows semantic parity; and complete promotion remains gated by the 2,000-record quality audit. +- Plan 020 is one program with four independent language tracks. Phase 0 + freezes shared baselines and independent truth. Swift, Dart, and Scala + candidates may then proceed in parallel; Groovy may also proceed but must + reuse the exact-language JVM boundary established for Scala. Each language + has a separate candidate and atomic hard-cut phase, and the mixed-language + release gate runs only after all four cuts. ## Direction options not promoted to implementation plans diff --git a/benchmarks/performance/compass/audit.py b/benchmarks/performance/compass/audit.py index f327d31b..cd3879d4 100644 --- a/benchmarks/performance/compass/audit.py +++ b/benchmarks/performance/compass/audit.py @@ -337,6 +337,8 @@ def _source_oracle_candidates( corpus_commit: str, producer: str, compass_nodes: dict[str, Any], + include_globs: tuple[str, ...] = (), + exclude_globs: tuple[str, ...] = (), ) -> tuple[list[dict[str, Any]], dict[str, Any]]: """Export independently parsed source constructs for recall adjudication.""" @@ -346,7 +348,12 @@ def _source_oracle_candidates( if node.qualified_name: owners[node.qualified_name.casefold()].append(identifier) - inventory = independent_source_inventory(corpus_root, producer) + inventory = independent_source_inventory( + corpus_root, + producer, + include_globs=include_globs, + exclude_globs=exclude_globs, + ) candidates: list[dict[str, Any]] = [] for construct in inventory.constructs: bounded = source_index.exact_range( @@ -731,14 +738,27 @@ def _corpus(value: object, index: int) -> AuditCorpus: _keys( item, required=("name", "commit", "path", "graph", "graphSha256"), + optional=("sourceGlobs", "excludeGlobs"), context=context, ) + source_globs = item.get("sourceGlobs", []) + exclude_globs = item.get("excludeGlobs", []) + if not isinstance(source_globs, list) or any( + not isinstance(pattern, str) or not pattern for pattern in source_globs + ): + raise AuditError(f"{context}.sourceGlobs must be a list of non-empty strings") + if not isinstance(exclude_globs, list) or any( + not isinstance(pattern, str) or not pattern for pattern in exclude_globs + ): + raise AuditError(f"{context}.excludeGlobs must be a list of non-empty strings") return AuditCorpus( name=_text(item["name"], f"{context}.name", identity=True), commit=_commit(item["commit"], f"{context}.commit"), path=_safe_path(item["path"], f"{context}.path", allow_dot=True), graph=_safe_path(item["graph"], f"{context}.graph"), graph_sha256=_sha256(item["graphSha256"], f"{context}.graphSha256"), + source_globs=tuple(source_globs), + exclude_globs=tuple(exclude_globs), ) @@ -1161,6 +1181,7 @@ def _validate_record_inputs( ) -> dict[str, _GraphIndex]: indexes: dict[str, _GraphIndex] = {} corpus_roots: dict[str, Path] = {} + corpus_specs = {corpus.name: corpus for corpus in manifest.corpora} single = len(manifest.corpora) == 1 for corpus in manifest.corpora: root = corpus_root if single and corpus.path == "." else corpus_root / corpus.path @@ -1197,7 +1218,13 @@ def _validate_record_inputs( f"provider mismatch: expected {source_oracle.provider!r}, " f"observed {provider!r}" ) - inventory = independent_source_inventory(root, source_oracle.producer) + corpus = corpus_specs[source_oracle.corpus] + inventory = independent_source_inventory( + root, + source_oracle.producer, + include_globs=corpus.source_globs, + exclude_globs=corpus.exclude_globs, + ) observed_counts = (inventory.scanned_files, inventory.parsed_files) expected_counts = ( source_oracle.scanned_files, @@ -1218,6 +1245,17 @@ def _validate_record_inputs( "inventory digest mismatch: " f"expected {source_oracle.inventory_sha256}, observed {observed_digest}" ) + metadata = dict(inventory.provider_metadata) + # Legacy providers (for example the built-in Python AST oracle) do + # not publish parser metadata and remain governed by their existing + # contracts. The universal-language wrappers explicitly publish + # ``oracleImplementation``/``parserAvailable`` so a bounded lexical + # fallback cannot be mistaken for a promotion-grade parser oracle. + if metadata.get("oracleImplementation") and metadata.get("parserAvailable") != "true": + raise AuditError( + f"source oracle {(source_oracle.corpus, source_oracle.producer)!r} " + "is a reproducible fallback inventory, not a pinned parser provider" + ) for record in manifest.records: root = corpus_roots[record.corpus] @@ -1245,7 +1283,11 @@ def _validate_record_inputs( ) graph = indexes[record.corpus] - if record.source.node_id not in graph.nodes: + requires_source_node = not ( + record.pool == "source_oracle" + and record.judgment in {"missing", "ambiguous"} + ) + if requires_source_node and record.source.node_id not in graph.nodes: raise AuditError( f"record {record.record_id!r} source node is absent from the graph" ) diff --git a/benchmarks/performance/compass/model.py b/benchmarks/performance/compass/model.py index 509909ce..29ba68df 100644 --- a/benchmarks/performance/compass/model.py +++ b/benchmarks/performance/compass/model.py @@ -195,6 +195,8 @@ class AuditCorpus: path: str graph: str graph_sha256: str + source_globs: tuple[str, ...] = () + exclude_globs: tuple[str, ...] = () @dataclass(frozen=True) diff --git a/benchmarks/performance/compass/occurrences.py b/benchmarks/performance/compass/occurrences.py index d834b3d5..77214edb 100644 --- a/benchmarks/performance/compass/occurrences.py +++ b/benchmarks/performance/compass/occurrences.py @@ -12,10 +12,13 @@ import re import selectors import subprocess +import sys import tempfile import time import tokenize +from .jsonstream import iter_top_level_array, read_top_level_value + StatementSpans = Mapping[str, tuple[tuple[int, int], ...]] StatementProvider = Callable[[Path], StatementSpans] @@ -71,7 +74,7 @@ class ConstructProvider: identity: str suffixes: tuple[str, ...] parse: SourceConstructParser - collect: Callable[[Path], SourceConstructInventory] | None = None + collect: Callable[[Path, tuple[str, ...], tuple[str, ...]], SourceConstructInventory] | None = None def _python_statement_spans(path: Path) -> StatementSpans: @@ -1397,7 +1400,11 @@ def _typescript_payload_from_jsonl(raw: bytes) -> dict[str, object]: } -def _typescript_compiler_inventory(root: Path) -> SourceConstructInventory: +def _typescript_compiler_inventory( + root: Path, + _include_globs: tuple[str, ...] = (), + _exclude_globs: tuple[str, ...] = (), +) -> SourceConstructInventory: try: payload = _typescript_payload_from_jsonl(_bounded_node_oracle(root)) except (OSError, UnicodeError, json.JSONDecodeError) as error: @@ -1631,11 +1638,238 @@ def _ruby_inventory_from_payload( ) -def _ruby_ripper_inventory(root: Path) -> SourceConstructInventory: +def _ruby_ripper_inventory( + root: Path, + _include_globs: tuple[str, ...] = (), + _exclude_globs: tuple[str, ...] = (), +) -> SourceConstructInventory: _raw, payload = _bounded_ruby_oracle(root) return _ruby_inventory_from_payload(root, payload) +def _language_source_oracle_inventory( + root: Path, + language: str, + script_name: str, + provider: str, + suffixes: tuple[str, ...], + include_globs: tuple[str, ...] = (), + exclude_globs: tuple[str, ...] = (), +) -> SourceConstructInventory: + """Run one pinned, source-only language oracle for an audit corpus. + + The subprocess is deliberately a qualification boundary: it receives a + path and writes JSON, but it never receives permission to run repository + tooling or build scripts. The oracle output is independently hashed by + the common audit inventory code below. + """ + + source_root = root.resolve() + if not source_root.is_dir(): + return SourceConstructInventory((), 0, 0, (str(source_root),)) + # ``occurrences.py`` lives at ``benchmarks/performance/compass``; the + # qualification helpers are repository-root scripts, three parents up. + script = Path(__file__).resolve().parents[3] / "scripts" / script_name + if not script.is_file(): + return SourceConstructInventory((), 0, 0, (script_name,)) + # Keep the provider file alive while the streaming array iterator consumes + # it below. The old context-manager form removed the file before the + # iterator ran; retaining the object until the function returns preserves + # bounded parsing without materializing the full JSON document. + cached_output = os.environ.get(f"COMPASS_{language.upper()}_ORACLE_CACHE") + temporary_directory = tempfile.TemporaryDirectory(prefix=f"compass-{language}-oracle-") + if temporary_directory: + directory = temporary_directory.name + output = Path(cached_output).expanduser().resolve() if cached_output else Path(directory) / "oracle.json" + try: + completed = ( + subprocess.CompletedProcess((), 0, "", "") + if cached_output + else subprocess.run( + [ + sys.executable, + str(script), + "--root", + str(source_root), + "--output", + str(output), + *sum(([ + "--include", + pattern, + ] for pattern in include_globs), []), + *sum(([ + "--exclude", + pattern, + ] for pattern in exclude_globs), []), + ], + cwd=source_root, + check=False, + text=True, + capture_output=True, + timeout=300, + ) + ) + except subprocess.TimeoutExpired: + return SourceConstructInventory((), 0, 0, (f"{language}:oracle-timeout",)) + if completed.returncode or not output.is_file(): + return SourceConstructInventory( + (), + 0, + 0, + (completed.stderr.strip() or completed.stdout.strip() or script_name,), + ) + try: + document_language = read_top_level_value(output, "language") + document_provider = read_top_level_value(output, "provider") + document_toolchain = read_top_level_value(output, "toolchain") + document_implementation = read_top_level_value(output, "implementation") + document_parser_available = read_top_level_value(output, "parserAvailable") + document_inventory = read_top_level_value(output, "inventorySha256") + document_scanned = read_top_level_value(output, "scannedFiles") + document_parsed = read_top_level_value(output, "parsedFiles") + except (OSError, KeyError, ValueError, TypeError): + return SourceConstructInventory((), 0, 0, (script_name,)) + if document_provider != provider or document_language != language: + return SourceConstructInventory((), 0, 0, (f"{language}:provider-mismatch",)) + if not isinstance(document_toolchain, str) or not isinstance(document_implementation, str): + return SourceConstructInventory((), 0, 0, (f"{language}:provider-metadata",)) + try: + file_items = iter_top_level_array(output, "files") + scanned = int(document_scanned) + parsed = int(document_parsed) + except (KeyError, TypeError, ValueError): + return SourceConstructInventory((), 0, 0, (script_name,)) + constructs: list[SourceConstruct] = [] + rejected: list[str] = [] + try: + for item in file_items: + if not isinstance(item, dict): + rejected.append("") + continue + relative = item.get("path") + if not isinstance(relative, str) or Path(relative).suffix.casefold() not in suffixes: + continue + if item.get("status") != "ok": + rejected.append(relative) + continue + relations = item.get("relations", []) + if not isinstance(relations, list): + rejected.append(relative) + continue + for relation in relations: + if not isinstance(relation, dict): + rejected.append(relative) + continue + try: + start = int(relation["startByte"]) + end = int(relation["endByte"]) + line = int(relation["startLine"]) + name = str(relation["relation"]) + capability = str(relation["capability"]) + owner = str(relation["ownerQualifiedName"]) + target = str(relation["targetSpelling"]) + except (KeyError, TypeError, ValueError): + rejected.append(relative) + continue + if start < 0 or end <= start or line < 1 or not name or not capability: + rejected.append(relative) + continue + constructs.append( + SourceConstruct( + relative, + name, + capability, + owner, + target, + relation.get("qualifier") if isinstance(relation.get("qualifier"), str) else None, + start, + end, + line, + ) + ) + except (KeyError, TypeError, ValueError): + return SourceConstructInventory((), 0, 0, (f"{language}:invalid-json",)) + metadata = ( + ("oracleInventorySha256", str(document_inventory)), + ("oracleToolchain", document_toolchain), + ("oracleImplementation", document_implementation), + ("parserAvailable", str(document_parser_available).lower()), + ) + result = SourceConstructInventory( + tuple(sorted(set(constructs), key=_source_construct_key)), + scanned, + parsed, + tuple(sorted(set(rejected))), + metadata, + ) + temporary_directory.cleanup() + return result + + +def _swift_source_oracle_inventory( + root: Path, + include_globs: tuple[str, ...] = (), + exclude_globs: tuple[str, ...] = (), +) -> SourceConstructInventory: + return _language_source_oracle_inventory( + root, + "swift", + "swift_source_oracle.py", + "swift-syntax-source-oracle", + (".swift",), + include_globs, + exclude_globs, + ) + + +def _dart_source_oracle_inventory( + root: Path, + include_globs: tuple[str, ...] = (), + exclude_globs: tuple[str, ...] = (), +) -> SourceConstructInventory: + return _language_source_oracle_inventory( + root, + "dart", + "dart_source_oracle.py", + "dart-analyzer-source-oracle", + (".dart",), + include_globs, + exclude_globs, + ) + + +def _scala_source_oracle_inventory( + root: Path, + include_globs: tuple[str, ...] = (), + exclude_globs: tuple[str, ...] = (), +) -> SourceConstructInventory: + return _language_source_oracle_inventory( + root, + "scala", + "scala_source_oracle.py", + "scala-meta-source-oracle", + (".scala",), + include_globs, + exclude_globs, + ) + + +def _groovy_source_oracle_inventory( + root: Path, + include_globs: tuple[str, ...] = (), + exclude_globs: tuple[str, ...] = (), +) -> SourceConstructInventory: + return _language_source_oracle_inventory( + root, + "groovy", + "groovy_source_oracle.py", + "groovy-compilation-unit-source-oracle", + (".groovy", ".gradle"), + include_globs, + exclude_globs, + ) + + def _collector_only_construct_parser( _root: Path, _path: Path, @@ -1669,6 +1903,30 @@ def _collector_only_construct_parser( _collector_only_construct_parser, _ruby_ripper_inventory, ), + "swift": ConstructProvider( + "swift-syntax-source-oracle", + (".swift",), + _collector_only_construct_parser, + _swift_source_oracle_inventory, + ), + "dart": ConstructProvider( + "dart-analyzer-source-oracle", + (".dart",), + _collector_only_construct_parser, + _dart_source_oracle_inventory, + ), + "scala": ConstructProvider( + "scala-meta-source-oracle", + (".scala",), + _collector_only_construct_parser, + _scala_source_oracle_inventory, + ), + "groovy": ConstructProvider( + "groovy-compilation-unit-source-oracle", + (".groovy", ".gradle"), + _collector_only_construct_parser, + _groovy_source_oracle_inventory, + ), } @@ -1691,13 +1949,16 @@ def independent_source_constructs( ) -> tuple[SourceConstruct, ...]: """Collect independent source candidates without reading the graph.""" - return independent_source_inventory(root, language, providers).constructs + return independent_source_inventory(root, language, providers=providers).constructs def independent_source_inventory( root: Path, language: str, providers: Mapping[str, ConstructProvider] = DEFAULT_CONSTRUCT_PROVIDERS, + *, + include_globs: tuple[str, ...] = (), + exclude_globs: tuple[str, ...] = (), ) -> SourceConstructInventory: """Collect source candidates and explicit parser-coverage evidence.""" @@ -1706,7 +1967,7 @@ def independent_source_inventory( if provider is None: return SourceConstructInventory((), 0, 0, ()) if provider.collect is not None: - return provider.collect(root) + return provider.collect(root, include_globs, exclude_globs) constructs: list[SourceConstruct] = [] scanned = 0 parsed = 0 diff --git a/crates/compass-languages/src/builtins.rs b/crates/compass-languages/src/builtins.rs index 03f4e193..d27c8ded 100644 --- a/crates/compass-languages/src/builtins.rs +++ b/crates/compass-languages/src/builtins.rs @@ -444,6 +444,10 @@ pub fn is_language_builtin_qualified_target(language: &str, qualified_name: &str .strip_prefix("java.lang.") .and_then(|name| name.split(['.', ':']).next()) .is_some_and(|name| is_language_builtin_global(language, name)), + "javascript" | "typescript" | "tsx" => qualified_name + .strip_prefix("global::") + .and_then(|name| name.split(['.', ':']).next()) + .is_some_and(|name| is_language_builtin_global(language, name)), _ => false, } } @@ -479,6 +483,14 @@ mod tests { "java", "java.lang.String::valueOf" )); + assert!(is_language_builtin_qualified_target( + "typescript", + "global::Array.from" + )); + assert!(is_language_builtin_qualified_target( + "javascript", + "global::console.log" + )); assert!(!is_language_builtin_qualified_target( "rust", "crate::Vec::new" diff --git a/crates/compass-languages/src/dart.rs b/crates/compass-languages/src/dart.rs deleted file mode 100644 index f8ed59b7..00000000 --- a/crates/compass-languages/src/dart.rs +++ /dev/null @@ -1,958 +0,0 @@ -use std::collections::HashSet; -use std::fs; -use std::path::{Path, PathBuf}; - -use crate::{RawEdgeRecord as EdgeRecord, RawNodeRecord as NodeRecord}; -use regex::Regex; -use serde_json::{Map, Value}; - -use crate::facts::stamp_source_range; -use crate::{Extraction, file_stem, make_id}; - -const SCALAR_TYPES: &[&str] = &[ - "String", "int", "double", "bool", "num", "dynamic", "Object", "void", -]; -const COLLECTION_TYPES: &[&str] = &["List", "Map", "Set", "Future", "Stream"]; - -pub(crate) fn extract(path: &Path, source: &[u8]) -> Extraction { - State::new(path, source).run() -} - -struct State<'a> { - path: &'a Path, - source: &'a [u8], - text: String, - source_file: String, - stem: String, - file_id: String, - is_part: bool, - extraction: Extraction, - defined: HashSet, -} - -impl<'a> State<'a> { - fn new(path: &'a Path, source: &'a [u8]) -> Self { - let source_file = path.to_string_lossy().into_owned(); - let text = strip_comments(std::str::from_utf8(source).unwrap_or_default()); - let mut stem = file_stem(path); - let mut file_id = make_id(&[&source_file]); - let mut is_part = false; - if let Some(parent) = part_parent(path, &text) { - stem = file_stem(&parent); - file_id = make_id(&[&parent.to_string_lossy()]); - is_part = true; - } - Self { - path, - source, - text, - source_file, - stem, - file_id, - is_part, - extraction: Extraction { - raw_calls: None, - ..Extraction::default() - }, - defined: HashSet::new(), - } - } - - fn run(mut self) -> Extraction { - if !self.is_part { - let label = self - .path - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or_default() - .to_owned(); - let file_id = self.file_id.clone(); - let source_file = self.source_file.clone(); - self.push_node(&file_id, &label, "code", Some(&source_file)); - } - self.add_classes(); - self.add_annotations(); - self.add_typedefs(); - self.add_extensions(); - self.add_variables(); - self.add_functions(); - self.add_imports_exports(); - self.add_generic_lookups(); - self.extraction - } - - fn add_classes(&mut self) { - let Ok(pattern) = Regex::new( - r"(?m)^[ \t]*(?:(?:abstract|sealed|base|interface|final|mixin)\s+)*(?:class|mixin|enum|extension\s+type)\s+(\w+)", - ) else { - return; - }; - let matches: Vec<_> = pattern - .captures_iter(&self.text) - .filter_map(|capture| { - Some(( - capture.get(0)?.start(), - capture.get(0)?.end(), - capture.get(1)?.as_str().to_owned(), - )) - }) - .collect(); - for (start, end, name) in matches { - let id = make_id(&[&self.stem, &name]); - self.add_node(&id, &name, "code", Some(self.source_file.clone())); - self.set_callable_range(&id, start, start, "class"); - self.add_edge(&self.file_id.clone(), &id, "defines", None); - - let header_end = safe_end(&self.text, end, 500); - let mut rest = self.text[end..header_end].to_owned(); - rest = skip_balanced_prefix(rest, '<', '>'); - rest = skip_balanced_prefix(rest, '(', ')'); - let boundary = [rest.find('{'), rest.find(';')] - .into_iter() - .flatten() - .min() - .unwrap_or(rest.len()); - let mut header = rest[..boundary].to_owned(); - let mut base = None; - let mut generics = None; - if let Some((matched_end, base_name)) = - anchored_clause(&header, r"^\s*(?:extends|on)\s+([A-Za-z0-9_.]+)") - { - base = Some(base_name); - let remainder = header[matched_end..].to_owned(); - if remainder.trim_start().starts_with('<') { - let open = remainder.find('<').unwrap_or_default(); - if let Some(close) = matching_delimiter(remainder.as_bytes(), open, b'<', b'>') - { - generics = Some(remainder[open + 1..close].to_owned()); - header = remainder[close + 1..].to_owned(); - } else { - header = remainder; - } - } else { - header = remainder; - } - } - let mut mixins = Vec::new(); - if let Some((matched_end, _)) = anchored_clause(&header, r"^\s*with\s+()") { - let remainder = &header[matched_end..]; - if let Some(position) = remainder.find("implements") { - mixins = split_types(&remainder[..position]); - header = remainder[position..].to_owned(); - } else { - mixins = split_types(remainder); - header.clear(); - } - } - let interfaces = anchored_clause(&header, r"^\s*implements\s+()") - .map_or_else(Vec::new, |(matched_end, _)| { - split_types(&header[matched_end..]) - }); - - if let Some(base) = base { - let target = make_id(&[&base]); - self.add_node(&target, &base, "code", None); - self.add_edge(&id, &target, "inherits", None); - if let Some(generics) = generics { - for generic in split_types(&generics) { - let clean = generic.split('<').next().unwrap_or_default().trim(); - if !is_builtin(clean, false) { - let target = make_id(&[clean]); - self.add_node(&target, clean, "code", None); - self.add_edge(&id, &target, "references", None); - } - } - } - } - for mixin in mixins { - let clean = mixin.split('<').next().unwrap_or_default().trim(); - let target = make_id(&[clean]); - self.add_node(&target, clean, "code", None); - self.add_edge(&id, &target, "mixes_in", None); - } - for interface in interfaces { - let clean = interface.split('<').next().unwrap_or_default().trim(); - let target = make_id(&[clean]); - self.add_node(&target, clean, "code", None); - self.add_edge(&id, &target, "implements", None); - } - - if let Some(open) = self.text[start..].find('{').map(|value| start + value) { - let semicolon = self.text[start..].find(';').map(|value| start + value); - if semicolon.is_none_or(|semicolon| open < semicolon) { - let close = matching_delimiter(self.text.as_bytes(), open, b'{', b'}') - .map_or(self.text.len(), |value| value + 1); - self.set_callable_range(&id, start, close, "class"); - let body = self.text[open..close].to_owned(); - self.add_class_framework(&id, &body, open); - } - } - } - } - - fn add_class_framework(&mut self, owner: &str, body: &str, body_start: usize) { - self.add_pattern_relations( - owner, - (body, body_start), - r"\bon<(\w+)>\s*\(", - "calls", - "bloc_event", - false, - ); - self.add_pattern_relations( - owner, - (body, body_start), - r"\b(?:emit|yield)\s*\(?\s*(?:const\s+)?([A-Z]\w*)\b", - "calls", - "emit_state", - true, - ); - self.add_pattern_relations( - owner, - (body, body_start), - r"\b(?:\w*[Bb]loc\w*|context\.read<\w+>\(\))\.add\(\s*(?:const\s+)?([A-Z]\w*)\b", - "calls", - "bloc_add_event", - true, - ); - self.add_pattern_relations( - owner, - (body, body_start), - r"\bref\.(?:watch|read|listen)\s*\(\s*(\w+)\b", - "references", - "riverpod_reference", - false, - ); - self.add_pattern_relations( - owner, - (body, body_start), - r"\bBloc(?:Builder|Listener|Consumer|Provider|Selector)\s*<\s*([A-Za-z0-9_]+)\b", - "references", - "bloc_widget_binding", - true, - ); - self.add_pattern_relations( - owner, - (body, body_start), - r"\b(?:read|watch|select|of)\s*<([A-Za-z0-9_]+)>", - "references", - "bloc_lookup", - true, - ); - } - - fn add_annotations(&mut self) { - let Ok(pattern) = Regex::new(r"@(\w+)(?:\([^)]*\))?") else { - return; - }; - let annotations: Vec<_> = pattern - .captures_iter(&self.text) - .filter_map(|capture| { - Some((capture.get(0)?.end(), capture.get(1)?.as_str().to_owned())) - }) - .collect(); - let class_pattern = Regex::new( - r"(?m)^\s*(?:(?:abstract|sealed|base|interface|final|mixin)\s+)*(?:class|mixin|enum|extension\s+type)\s+(\w+)", - ) - .ok(); - let function_pattern = Regex::new( - r"(?m)^\s*(?:factory\s+|static\s+|async\s+|external\s+|abstract\s+)?(?:\([^)]+\)|[A-Za-z0-9_<>,.?]+)(?:\s+[A-Za-z0-9_<>,.?]+){0,3}\s+(\w+)\s*\(", - ) - .ok(); - for (end, annotation) in annotations { - if matches!( - annotation.as_str(), - "override" | "deprecated" | "required" | "protected" | "mustCallSuper" - ) { - continue; - } - let window_end = safe_end(&self.text, end, 300); - let window = &self.text[end..window_end]; - let class = class_pattern - .as_ref() - .and_then(|pattern| pattern.captures(window)) - .and_then(|capture| { - Some((capture.get(0)?.start(), capture.get(1)?.as_str().to_owned())) - }); - let function = function_pattern - .as_ref() - .and_then(|pattern| pattern.captures(window)) - .and_then(|capture| { - Some((capture.get(0)?.start(), capture.get(1)?.as_str().to_owned())) - }); - let (position, target, is_class) = match (class, function) { - (Some(class), Some(function)) if class.0 < function.0 => (class.0, class.1, true), - (Some(_), Some(function)) => (function.0, function.1, false), - (Some(class), None) => (class.0, class.1, true), - (None, Some(function)) => (function.0, function.1, false), - (None, None) => continue, - }; - if window[..position].contains([';', '{', '}']) { - continue; - } - let target_id = make_id(&[&self.stem, &target]); - let annotation_id = make_id(&["annotation", &annotation.to_ascii_lowercase()]); - self.add_node(&annotation_id, &format!("@{annotation}"), "concept", None); - self.add_edge(&target_id, &annotation_id, "configures", None); - if annotation.eq_ignore_ascii_case("riverpod") { - let provider = if is_class { - lower_first(&target) + "Provider" - } else { - format!("{target}Provider") - }; - let provider_id = make_id(&[&provider]); - self.add_node( - &provider_id, - &provider, - "concept", - Some(self.source_file.clone()), - ); - self.add_edge_with_context( - &target_id, - &provider_id, - "defines", - "riverpod_provider", - ); - } - } - } - - fn add_typedefs(&mut self) { - let Ok(pattern) = - Regex::new(r"(?m)^\s*typedef\s+(\w+)\s*(?:<[^>]+>)?\s*=\s*([A-Za-z0-9_<>,.?\s]+);") - else { - return; - }; - let values: Vec<_> = pattern - .captures_iter(&self.text) - .filter_map(|capture| { - Some(( - capture.get(1)?.as_str().to_owned(), - capture.get(2)?.as_str().to_owned(), - )) - }) - .collect(); - for (name, target) in values { - let target = target - .split('<') - .next() - .unwrap_or_default() - .rsplit('.') - .next() - .unwrap_or_default() - .trim(); - if is_builtin(target, true) || target == "Function" { - continue; - } - let id = make_id(&[&self.stem, &name]); - self.add_node(&id, &name, "code", Some(self.source_file.clone())); - self.add_edge(&self.file_id.clone(), &id, "defines", None); - let target_id = make_id(&[target]); - self.add_node(&target_id, target, "code", None); - self.add_edge_with_context(&id, &target_id, "references", "typedef"); - } - } - - fn add_extensions(&mut self) { - let Ok(pattern) = Regex::new(r"(?m)^\s{0,4}extension\s+(\w+)?(?:<[^>]+>)?\s+on\s+(\w+)") - else { - return; - }; - let values: Vec<_> = pattern - .captures_iter(&self.text) - .filter_map(|capture| { - Some(( - capture.get(1).map(|value| value.as_str().to_owned()), - capture.get(2)?.as_str().to_owned(), - )) - }) - .collect(); - for (name, target) in values { - let raw_name = name - .clone() - .unwrap_or_else(|| format!("{}_anonymous_extension", self.stem)); - let label = name.unwrap_or_else(|| format!("Extension on {target}")); - let id = make_id(&[&self.stem, &raw_name]); - self.add_node(&id, &label, "code", Some(self.source_file.clone())); - self.add_edge(&self.file_id.clone(), &id, "defines", None); - let target_id = make_id(&[&target]); - self.add_node(&target_id, &target, "code", None); - self.add_edge(&id, &target_id, "extends", None); - } - } - - fn add_variables(&mut self) { - let Ok(pattern) = Regex::new( - r"(?m)^\s{0,2}(?:late\s+)?(?:(?:final|const|var)\s+)?(?:\([^)]+\)\s+|([A-Za-z0-9_<>,.?]+(?:\s+[A-Za-z0-9_<>,.?]+){0,3})\s+)?(?:(\w+)|(?:\w+\s*)?\(([^)]+)\))\s*(?:=|$|;)", - ) else { - return; - }; - let values: Vec<_> = pattern - .captures_iter(&self.text) - .filter_map(|capture| { - Some(( - capture.get(0)?.as_str().to_owned(), - capture.get(1).map(|value| value.as_str().to_owned()), - capture.get(2).map(|value| value.as_str().to_owned()), - capture.get(3).map(|value| value.as_str().to_owned()), - )) - }) - .collect(); - let modifier = Regex::new(r"^\s*(?:late|final|const|var)\b").ok(); - for (full, var_type, single, destructured) in values { - if modifier - .as_ref() - .is_none_or(|pattern| !pattern.is_match(&full)) - && var_type.is_none() - { - continue; - } - if let Some(name) = single { - if matches!( - name.as_str(), - "if" | "for" | "while" | "switch" | "catch" | "return" - ) { - continue; - } - let id = make_id(&[&self.stem, &name]); - self.add_node(&id, &name, "code", Some(self.source_file.clone())); - self.add_edge(&self.file_id.clone(), &id, "defines", None); - if let Some(var_type) = var_type - && !is_builtin(var_type.trim(), true) - { - let clean = var_type - .split('<') - .next() - .unwrap_or_default() - .rsplit('.') - .next() - .unwrap_or_default() - .trim(); - let target = make_id(&[clean]); - self.add_node(&target, clean, "code", None); - self.add_edge_with_context( - &self.file_id.clone(), - &target, - "references", - "variable_type", - ); - } - } else if let Some(names) = destructured { - for raw in names - .split(',') - .map(str::trim) - .filter(|name| !name.is_empty()) - { - let name = raw.rsplit(':').next().unwrap_or_default().trim(); - if valid_lower_identifier(name) - && !matches!(name, "if" | "for" | "while" | "switch" | "catch" | "return") - { - let id = make_id(&[&self.stem, name]); - self.add_node(&id, name, "code", Some(self.source_file.clone())); - self.add_edge(&self.file_id.clone(), &id, "defines", None); - } - } - } - } - } - - fn add_functions(&mut self) { - let Ok(pattern) = Regex::new( - r"(?m)^[ \t]{0,2}(?:factory\s+|static\s+|async\s+|external\s+|abstract\s+)?(?:\([^)]+\)|[A-Za-z0-9_<>,.?]+)(?:\s+[A-Za-z0-9_<>,.?]+){0,3}\s+(\w+(?:\.\w+)?)\s*\(", - ) else { - return; - }; - let functions: Vec<_> = pattern - .captures_iter(&self.text) - .filter_map(|capture| { - Some((capture.get(0)?.start(), capture.get(1)?.as_str().to_owned())) - }) - .collect(); - for (start, raw_name) in functions { - let name = raw_name.rsplit('.').next().unwrap_or_default(); - if matches!( - name, - "if" | "for" - | "while" - | "switch" - | "catch" - | "return" - | "void" - | "dynamic" - | "final" - | "const" - | "get" - | "set" - ) || name.starts_with(|character: char| character.is_ascii_uppercase()) - { - continue; - } - let id = make_id(&[&self.stem, name]); - self.add_node(&id, name, "code", Some(self.source_file.clone())); - self.set_callable_range(&id, start, start, "function"); - self.add_edge(&self.file_id.clone(), &id, "defines", None); - let open = self.text[start..].find('{').map(|value| start + value); - let semicolon = self.text[start..].find(';').map(|value| start + value); - let arrow = self.text[start..].find("=>").map(|value| start + value); - if let Some(open) = open - && semicolon.is_none_or(|semicolon| open < semicolon) - && arrow.is_none_or(|arrow| open < arrow) - { - let close = matching_delimiter(self.text.as_bytes(), open, b'{', b'}') - .map_or(self.text.len(), |value| value + 1); - self.set_callable_range(&id, start, close, "function"); - let body = self.text[open..close].to_owned(); - self.add_function_framework(&id, &body, open); - } else if let Some(end) = semicolon { - self.set_callable_range(&id, start, end.saturating_add(1), "function"); - } - } - } - - fn add_function_framework(&mut self, owner: &str, body: &str, body_start: usize) { - self.add_pattern_relations( - owner, - (body, body_start), - r"\bref\.(?:watch|read|listen)\s*\(\s*(\w+)\b", - "references", - "riverpod_reference", - false, - ); - self.add_pattern_relations( - owner, - (body, body_start), - r"\b(?:\w*[Bb]loc\w*|context\.read<\w+>\(\))\.add\(\s*(?:const\s+)?([A-Z]\w*)\b", - "calls", - "bloc_add_event", - true, - ); - self.add_pattern_relations( - owner, - (body, body_start), - r"\b(?:read|watch|select|of)\s*<([A-Za-z0-9_]+)>", - "references", - "bloc_lookup", - true, - ); - if let Ok(pattern) = Regex::new( - r#"\b(?:go|push|goNamed|pushNamed|replace|replaceNamed)\s*\(\s*(?:context\s*,\s*)?['"]([A-Za-z0-9_/?=&%-]+)['"]"#, - ) { - let values: Vec<_> = pattern - .captures_iter(body) - .filter_map(|capture| { - Some(( - capture.get(1)?.as_str().to_owned(), - body_start + capture.get(0)?.start(), - body_start + capture.get(0)?.end(), - )) - }) - .collect(); - for (route, start, end) in values { - let normalized = route.replace(['/', '?', '=', '&'], "_"); - let target = make_id(&["route", &normalized]); - self.add_node(&target, &format!("Route {route}"), "concept", None); - self.add_edge_with_context(owner, &target, "navigates", "route_path"); - if let Some(edge) = self.extraction.edges.last_mut() { - stamp_source_range(&mut edge.attributes, self.source, start, end); - } - } - } - if let Ok(pattern) = Regex::new( - r"\b(?:go|push|goNamed|pushNamed|replace|replaceNamed)\s*\(\s*(?:context\s*,\s*)?([A-Z][A-Za-z0-9_]*\.[A-Za-z0-9_]+)", - ) { - let values: Vec<_> = pattern - .captures_iter(body) - .filter_map(|capture| { - Some(( - capture.get(1)?.as_str().to_owned(), - body_start + capture.get(0)?.start(), - body_start + capture.get(0)?.end(), - )) - }) - .collect(); - for (route, start, end) in values { - let target = make_id(&["route", &route.replace('.', "_")]); - self.add_node(&target, &route, "concept", None); - self.add_edge_with_context(owner, &target, "navigates", "route_const"); - if let Some(edge) = self.extraction.edges.last_mut() { - stamp_source_range(&mut edge.attributes, self.source, start, end); - } - } - } - if let Ok(pattern) = Regex::new( - r"\b(?:push|replace)\s*\(\s*(?:context\s*,\s*)?.*?\b([A-Z]\w*(?:Route|Screen|Page))\b", - ) { - let values: Vec<_> = pattern - .captures_iter(body) - .filter_map(|capture| { - Some(( - capture.get(1)?.as_str().to_owned(), - body_start + capture.get(0)?.start(), - body_start + capture.get(0)?.end(), - )) - }) - .collect(); - for (route, start, end) in values { - let target = make_id(&[&route]); - self.add_node(&target, &route, "code", None); - self.add_edge_with_context(owner, &target, "navigates", "route_object"); - if let Some(edge) = self.extraction.edges.last_mut() { - stamp_source_range(&mut edge.attributes, self.source, start, end); - } - } - } - } - - fn add_imports_exports(&mut self) { - for (keyword, relation) in [("import", "imports"), ("export", "exports")] { - let Ok(pattern) = Regex::new(&format!(r#"(?m)^\s*{keyword}\s+['"]([^'"]+)['"]"#)) - else { - continue; - }; - let packages: Vec<_> = pattern - .captures_iter(&self.text) - .filter_map(|capture| capture.get(1).map(|value| value.as_str().to_owned())) - .collect(); - for package in packages { - let target = make_id(&[&package]); - self.add_node(&target, &package, "code", None); - if let Some(resource) = self - .extraction - .nodes - .iter_mut() - .find(|resource| resource.id == target) - { - resource.attributes.insert( - "symbol_kind".to_owned(), - Value::String("resource".to_owned()), - ); - } - self.add_edge(&self.file_id.clone(), &target, relation, None); - if !package.contains(':') - && !Path::new(&package).is_absolute() - && let Some(edge) = self.extraction.edges.last_mut() - { - let target_file = Path::new(&self.source_file) - .parent() - .unwrap_or_else(|| Path::new(".")) - .join(&package); - edge.attributes.insert( - "target_file".to_owned(), - Value::String(target_file.to_string_lossy().replace('\\', "/")), - ); - } - } - } - } - - fn add_generic_lookups(&mut self) { - let Ok(pattern) = Regex::new(r"\b\w+<([A-Za-z0-9_.]+(?:<[A-Za-z0-9_.,\s<>]+>)?)\s*>\s*\(") - else { - return; - }; - let values: Vec<_> = pattern - .captures_iter(&self.text) - .filter_map(|capture| capture.get(1).map(|value| value.as_str().to_owned())) - .collect(); - for raw in values { - let raw = raw.rsplit('.').next().unwrap_or_default().trim(); - let clean = raw.split('<').next().unwrap_or_default().trim(); - if is_builtin(clean, true) { - continue; - } - let target = make_id(&[clean]); - self.add_node(&target, clean, "code", None); - self.add_edge_with_context(&self.file_id.clone(), &target, "references", "type_lookup"); - } - } - - fn add_pattern_relations( - &mut self, - owner: &str, - body: (&str, usize), - pattern: &str, - relation: &str, - context: &str, - filter_builtins: bool, - ) { - let (body, body_start) = body; - let Ok(pattern) = Regex::new(pattern) else { - return; - }; - let values: Vec<_> = pattern - .captures_iter(body) - .filter_map(|capture| { - Some(( - capture.get(1)?.as_str().to_owned(), - body_start + capture.get(0)?.start(), - body_start + capture.get(0)?.end(), - )) - }) - .collect(); - for (value, start, end) in values { - if filter_builtins && is_builtin(&value, true) { - continue; - } - let target = make_id(&[&value]); - self.add_node(&target, &value, "code", None); - self.add_edge_with_context(owner, &target, relation, context); - if let Some(edge) = self.extraction.edges.last_mut() { - stamp_source_range(&mut edge.attributes, self.source, start, end); - } - } - } - - fn add_node(&mut self, id: &str, label: &str, file_type: &str, source_file: Option) { - self.push_node(id, label, file_type, source_file.as_deref()); - } - - fn push_node(&mut self, id: &str, label: &str, file_type: &str, source_file: Option<&str>) { - if !self.defined.insert(id.to_owned()) { - return; - } - let mut attributes = Map::new(); - attributes.insert("label".into(), Value::String(label.to_owned())); - attributes.insert("file_type".into(), Value::String(file_type.to_owned())); - attributes.insert( - "source_file".into(), - source_file.map_or(Value::Null, |value| Value::String(value.to_owned())), - ); - attributes.insert("source_location".into(), Value::Null); - self.extraction.nodes.push(NodeRecord { - id: id.to_owned(), - attributes, - }); - } - - fn set_callable_range(&mut self, id: &str, start: usize, end: usize, kind: &str) { - let start_line = self.line_at(start); - let end_line = self.line_at(end); - let Some(node) = self.extraction.nodes.iter_mut().find(|node| node.id == id) else { - return; - }; - node.attributes.insert( - "source_location".into(), - Value::String(format!("L{start_line}")), - ); - node.attributes - .insert("symbol_kind".into(), Value::String(kind.to_owned())); - node.attributes - .insert("language".into(), Value::String("dart".to_owned())); - node.attributes - .insert("line_start".into(), Value::from(start_line)); - node.attributes - .insert("line_end".into(), Value::from(end_line.max(start_line))); - } - - fn line_at(&self, offset: usize) -> usize { - self.source[..offset.min(self.source.len())] - .iter() - .filter(|byte| **byte == b'\n') - .count() - + 1 - } - - fn add_edge(&mut self, source: &str, target: &str, relation: &str, context: Option<&str>) { - let mut attributes = Map::new(); - attributes.insert("relation".into(), Value::String(relation.to_owned())); - attributes.insert("confidence".into(), Value::String("EXTRACTED".into())); - attributes.insert("confidence_score".into(), Value::from(1.0)); - attributes.insert( - "source_file".into(), - Value::String(self.source_file.clone()), - ); - attributes.insert("source_location".into(), Value::Null); - attributes.insert("weight".into(), Value::from(1.0)); - if let Some(context) = context { - attributes.insert("context".into(), Value::String(context.to_owned())); - } - self.extraction.edges.push(EdgeRecord { - source: source.to_owned(), - target: target.to_owned(), - attributes, - }); - } - - fn add_edge_with_context(&mut self, source: &str, target: &str, relation: &str, context: &str) { - self.add_edge(source, target, relation, Some(context)); - } -} - -fn strip_comments(source: &str) -> String { - let Ok(pattern) = Regex::new( - r#"(?s)\"\"\"(?:\\.|.)*?\"\"\"|'''(?:\\.|.)*?'''|\"(?:\\.|[^\"\\])*\"|'(?:\\.|[^'\\])*'|/\*.*?\*/|//[^\n]*"#, - ) else { - return source.to_owned(); - }; - pattern - .replace_all(source, |captures: ®ex::Captures<'_>| { - let value = captures.get(0).map_or("", |value| value.as_str()); - if value.starts_with('/') { - value - .bytes() - .map(|byte| match byte { - b'\n' => '\n', - b'\r' => '\r', - _ => ' ', - }) - .collect::() - } else { - value.to_owned() - } - }) - .into_owned() -} - -fn part_parent(path: &Path, source: &str) -> Option { - let pattern = Regex::new(r#"(?m)^\s*part\s+of\s+['"]([^'"]+)['"]"#).ok()?; - let parent = pattern.captures(source)?.get(1)?.as_str(); - if !parent.ends_with(".dart") { - return None; - } - let candidate = path.parent()?.join(parent); - candidate - .exists() - .then(|| fs::canonicalize(&candidate).unwrap_or(candidate)) -} - -fn split_types(value: &str) -> Vec { - let mut values = Vec::new(); - let mut depth = 0_u32; - let mut start = 0; - for (index, character) in value.char_indices() { - if character == '<' { - depth += 1; - } else if character == '>' { - depth = depth.saturating_sub(1); - } else if character == ',' && depth == 0 { - let item = value[start..index].trim(); - if !item.is_empty() { - values.push(item.to_owned()); - } - start = index + 1; - } - } - let item = value[start..].trim(); - if !item.is_empty() { - values.push(item.to_owned()); - } - values -} - -fn matching_delimiter(value: &[u8], open: usize, opener: u8, closer: u8) -> Option { - let mut depth = 0_u32; - let mut quote = None; - let mut escaped = false; - for (index, byte) in value.iter().enumerate().skip(open) { - if let Some(delimiter) = quote { - if escaped { - escaped = false; - } else if *byte == b'\\' { - escaped = true; - } else if *byte == delimiter { - quote = None; - } - continue; - } - if matches!(*byte, b'\'' | b'"') { - quote = Some(*byte); - } else if *byte == opener { - depth += 1; - } else if *byte == closer { - depth = depth.saturating_sub(1); - if depth == 0 { - return Some(index); - } - } - } - None -} - -fn skip_balanced_prefix(value: String, opener: char, closer: char) -> String { - let trimmed = value.trim_start(); - if !trimmed.starts_with(opener) { - return value; - } - let whitespace = value.len() - trimmed.len(); - matching_delimiter(value.as_bytes(), whitespace, opener as u8, closer as u8) - .map_or(value.clone(), |end| value[end + 1..].to_owned()) -} - -fn anchored_clause(value: &str, pattern: &str) -> Option<(usize, String)> { - let capture = Regex::new(pattern).ok()?.captures(value)?; - let full = capture.get(0)?; - Some(( - full.end(), - capture.get(1).map_or("", |value| value.as_str()).to_owned(), - )) -} - -fn safe_end(value: &str, start: usize, length: usize) -> usize { - let mut end = (start + length).min(value.len()); - while end > start && !value.is_char_boundary(end) { - end -= 1; - } - end -} - -fn is_builtin(value: &str, collections: bool) -> bool { - SCALAR_TYPES.contains(&value) || (collections && COLLECTION_TYPES.contains(&value)) -} - -fn valid_lower_identifier(value: &str) -> bool { - value - .as_bytes() - .first() - .is_some_and(|byte| byte.is_ascii_lowercase() || *byte == b'_') - && value - .bytes() - .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') -} - -fn lower_first(value: &str) -> String { - let mut chars = value.chars(); - chars.next().map_or_else(String::new, |first| { - first.to_lowercase().collect::() + chars.as_str() - }) -} - -#[cfg(test)] -mod tests { - use std::path::Path; - - use super::extract; - - #[test] - fn call_owners_have_cursor_resolvable_ranges() -> Result<(), &'static str> { - let source = br#" -class CounterBloc { - void register() { - on((event, emit) { - emit(CountChanged()); - }); - } -} - -void bootstrap() { - CounterBloc(); -} -"#; - let extraction = extract(Path::new("lib/counter.dart"), source); - let class = extraction - .nodes - .iter() - .find(|node| node.label() == "CounterBloc") - .ok_or("class")?; - let function = extraction - .nodes - .iter() - .find(|node| node.label() == "bootstrap") - .ok_or("function")?; - - assert_eq!(class.attributes["line_start"], 2); - assert_eq!(class.attributes["line_end"], 8); - assert_eq!(function.attributes["line_start"], 10); - assert_eq!(function.attributes["line_end"], 12); - Ok(()) - } -} diff --git a/crates/compass-languages/src/dart_framework.rs b/crates/compass-languages/src/dart_framework.rs new file mode 100644 index 00000000..57241bb7 --- /dev/null +++ b/crates/compass-languages/src/dart_framework.rs @@ -0,0 +1,375 @@ +//! Bounded Dart framework convention facts. +//! +//! These facts are intentionally separate from universal language evidence. +//! They preserve the established Flutter/BLoC/Riverpod/navigation and local +//! resource-export behavior without publishing a second declaration or call +//! graph. The scanner only emits anchored convention edges and is never used +//! for language identity or target resolution. + +use std::collections::HashMap; +use std::path::Path; +use std::sync::LazyLock; + +use regex::Regex; +use serde_json::{Map, Value}; + +use crate::facts::stamp_source_range; +use crate::{Extraction, RawEdgeRecord, RawNodeRecord, file_stem, make_id}; + +static CLASS: LazyLock = LazyLock::new(|| { + Regex::new( + r"(?m)^[ \t]*(?:(?:abstract|sealed|base|interface|final|mixin)\s+)*(?:class|mixin|enum|extension\s+type)\s+(\w+)", + ) + .unwrap_or_else(|_| Regex::new("$^").unwrap_or_else(|_| unreachable!())) +}); +static FUNCTION: LazyLock = LazyLock::new(|| { + Regex::new( + r"(?m)^[ \t]{0,2}(?:factory\s+|static\s+|async\s+|external\s+|abstract\s+)?(?:\([^)]+\)|[A-Za-z0-9_<>,.?]+)(?:\s+[A-Za-z0-9_<>,.?]+){0,3}\s+(\w+(?:\.\w+)?)\s*\(", + ) + .unwrap_or_else(|_| Regex::new("$^").unwrap_or_else(|_| unreachable!())) +}); +static IMPORT_EXPORT: LazyLock = LazyLock::new(|| { + Regex::new(r#"(?m)^\s*(import|export)\s+['"]([^'"]+)['"]"#) + .unwrap_or_else(|_| Regex::new("$^").unwrap_or_else(|_| unreachable!())) +}); + +#[derive(Clone)] +struct Owner { + id: String, + start: usize, + end: usize, +} + +pub(crate) fn extract(path: &Path, source: &[u8]) -> Extraction { + let source_file = path.to_string_lossy().into_owned(); + let stem = file_stem(path); + let file_id = make_id(&[&source_file]); + let text = String::from_utf8_lossy(source).into_owned(); + let mut output = Extraction { + raw_calls: None, + ..Extraction::default() + }; + let mut nodes = HashMap::::new(); + let mut owners = Vec::new(); + add_node( + &mut nodes, + file_id.clone(), + path.file_name() + .and_then(|name| name.to_str()) + .unwrap_or_default(), + "code", + Some(&source_file), + ); + for captures in CLASS.captures_iter(&text) { + let Some(full) = captures.get(0) else { + continue; + }; + let Some(name_match) = captures.get(1) else { + continue; + }; + let name = name_match.as_str(); + let id = make_id(&[&stem, name]); + let end = body_end(&text, full.start()).unwrap_or(full.end()); + add_node(&mut nodes, id.clone(), name, "code", Some(&source_file)); + stamp_node(&mut nodes, &id, source, full.start(), end, "class"); + owners.push(Owner { + id, + start: full.start(), + end, + }); + } + for captures in FUNCTION.captures_iter(&text) { + let Some(full) = captures.get(0) else { + continue; + }; + let Some(name_match) = captures.get(1) else { + continue; + }; + let name = name_match.as_str().rsplit('.').next().unwrap_or_default(); + if matches!( + name, + "if" | "for" | "while" | "switch" | "catch" | "return" | "void" + ) || name.starts_with(|character: char| character.is_ascii_uppercase()) + { + continue; + } + let id = make_id(&[&stem, name]); + let end = body_end(&text, full.start()).unwrap_or(full.end()); + add_node(&mut nodes, id.clone(), name, "code", Some(&source_file)); + stamp_node(&mut nodes, &id, source, full.start(), end, "function"); + owners.push(Owner { + id, + start: full.start(), + end, + }); + } + owners.sort_unstable_by(|left, right| { + left.start.cmp(&right.start).then(left.end.cmp(&right.end)) + }); + + for owner in &owners { + let Some(body) = text.get(owner.start..owner.end) else { + continue; + }; + emit_framework_patterns(&mut output, &mut nodes, owner, body, owner.start, source); + } + for captures in IMPORT_EXPORT.captures_iter(&text) { + let Some(keyword) = captures.get(1) else { + continue; + }; + let Some(target) = captures.get(2) else { + continue; + }; + let package = target.as_str(); + let target_id = make_id(&[package]); + add_node(&mut nodes, target_id.clone(), package, "code", None); + if let Some(node) = nodes.get_mut(&target_id) { + node.attributes.insert( + "symbol_kind".to_owned(), + Value::String("resource".to_owned()), + ); + } + let relation = if keyword.as_str() == "export" { + "exports" + } else { + "imports" + }; + let mut attributes = Map::new(); + attributes.insert("relation".to_owned(), Value::String(relation.to_owned())); + if !package.contains(':') && !Path::new(package).is_absolute() { + let target_file = Path::new(&source_file) + .parent() + .unwrap_or_else(|| Path::new(".")) + .join(package); + attributes.insert( + "target_file".to_owned(), + Value::String(target_file.to_string_lossy().replace('\\', "/")), + ); + } + output.edges.push(RawEdgeRecord { + source: file_id.clone(), + target: target_id, + attributes, + }); + if let Some(edge) = output.edges.last_mut() { + stamp_source_range( + &mut edge.attributes, + source, + captures.get(0).map_or(0, |value| value.start()), + captures.get(0).map_or(0, |value| value.end()), + ); + } + } + output.nodes = nodes.into_values().collect(); + output + .nodes + .sort_unstable_by(|left, right| left.id.cmp(&right.id)); + output.edges.sort_unstable_by(|left, right| { + left.source + .cmp(&right.source) + .then_with(|| left.target.cmp(&right.target)) + .then_with(|| left.string("relation").cmp(&right.string("relation"))) + .then_with(|| { + left.attributes + .get("start_byte") + .and_then(Value::as_u64) + .cmp(&right.attributes.get("start_byte").and_then(Value::as_u64)) + }) + }); + output +} + +fn emit_framework_patterns( + output: &mut Extraction, + nodes: &mut HashMap, + owner: &Owner, + body: &str, + body_start: usize, + source: &[u8], +) { + for (pattern, relation, context, uppercase) in [ + (r"\bon<(\w+)>\s*\(", "calls", "bloc_event", false), + ( + r"\b(?:emit|yield)\s*\(?\s*(?:const\s+)?([A-Z]\w*)\b", + "calls", + "emit_state", + true, + ), + ( + r"\b(?:\w*[Bb]loc\w*|context\.read<\w+>\(\))\.add\(\s*(?:const\s+)?([A-Z]\w*)\b", + "calls", + "bloc_add_event", + true, + ), + ( + r"\bref\.(?:watch|read|listen)\s*\(\s*(\w+)\b", + "references", + "riverpod_reference", + false, + ), + ( + r"\bBloc(?:Builder|Listener|Consumer|Provider|Selector)\s*<\s*([A-Za-z0-9_]+)\b", + "references", + "bloc_widget_binding", + true, + ), + ( + r"\b(?:read|watch|select|of)\s*<([A-Za-z0-9_]+)>", + "references", + "bloc_lookup", + true, + ), + ] { + let Ok(pattern) = Regex::new(pattern) else { + continue; + }; + for captures in pattern.captures_iter(body) { + let Some(value) = captures.get(1) else { + continue; + }; + if uppercase + && value + .as_str() + .starts_with(|character: char| character.is_ascii_lowercase()) + { + continue; + } + let target = value.as_str(); + let target_id = make_id(&[target]); + add_node(nodes, target_id.clone(), target, "code", None); + add_context_edge( + output, + &owner.id, + &target_id, + relation, + context, + source, + body_start + captures.get(0).map_or(0, |value| value.start()), + body_start + captures.get(0).map_or(0, |value| value.end()), + ); + } + } + for (pattern, context, route_object) in [ + ( + r#"\b(?:go|push|goNamed|pushNamed|replace|replaceNamed)\s*\(\s*(?:context\s*,\s*)?['"]([A-Za-z0-9_/?=&%-]+)['"]"#, + "route_path", + false, + ), + ( + r"\b(?:go|push|goNamed|pushNamed|replace|replaceNamed)\s*\(\s*(?:context\s*,\s*)?([A-Z][A-Za-z0-9_]*\.[A-Za-z0-9_]+)", + "route_const", + false, + ), + ( + r"\b(?:push|replace)\s*\(\s*(?:context\s*,\s*)?.*?\b([A-Z]\w*(?:Route|Screen|Page))\b", + "route_object", + true, + ), + ] { + let Ok(pattern) = Regex::new(pattern) else { + continue; + }; + for captures in pattern.captures_iter(body) { + let Some(value) = captures.get(1) else { + continue; + }; + let raw = value.as_str(); + let label = if route_object || context != "route_path" { + raw.to_owned() + } else { + format!("Route {raw}") + }; + let normalized = raw.replace(['/', '?', '=', '&', '.'], "_"); + let target_id = make_id(&["route", &normalized]); + add_node(nodes, target_id.clone(), &label, "concept", None); + add_context_edge( + output, + &owner.id, + &target_id, + "navigates", + context, + source, + body_start + captures.get(0).map_or(0, |value| value.start()), + body_start + captures.get(0).map_or(0, |value| value.end()), + ); + } + } +} + +#[allow(clippy::too_many_arguments)] +fn add_context_edge( + output: &mut Extraction, + source_id: &str, + target_id: &str, + relation: &str, + context: &str, + source: &[u8], + start: usize, + end: usize, +) { + let mut attributes = Map::new(); + attributes.insert("relation".to_owned(), Value::String(relation.to_owned())); + attributes.insert("context".to_owned(), Value::String(context.to_owned())); + output.edges.push(RawEdgeRecord { + source: source_id.to_owned(), + target: target_id.to_owned(), + attributes, + }); + if let Some(edge) = output.edges.last_mut() { + stamp_source_range(&mut edge.attributes, source, start, end); + } +} + +fn add_node( + nodes: &mut HashMap, + id: String, + label: &str, + file_type: &str, + source_file: Option<&str>, +) { + nodes.entry(id.clone()).or_insert_with(|| { + let mut attributes = Map::new(); + attributes.insert("label".to_owned(), Value::String(label.to_owned())); + attributes.insert("file_type".to_owned(), Value::String(file_type.to_owned())); + attributes.insert( + "source_file".to_owned(), + source_file.map_or(Value::Null, |value| Value::String(value.to_owned())), + ); + RawNodeRecord { id, attributes } + }); +} + +fn stamp_node( + nodes: &mut HashMap, + id: &str, + source: &[u8], + start: usize, + end: usize, + kind: &str, +) { + let Some(node) = nodes.get_mut(id) else { + return; + }; + stamp_source_range(&mut node.attributes, source, start, end); + node.attributes + .insert("symbol_kind".to_owned(), Value::String(kind.to_owned())); + node.attributes + .insert("language".to_owned(), Value::String("dart".to_owned())); +} + +fn body_end(text: &str, start: usize) -> Option { + let open = text.get(start..)?.find('{')?.saturating_add(start); + let mut depth = 0_i32; + for (relative, character) in text[open..].char_indices() { + let offset = open.saturating_add(relative); + if character == '{' { + depth = depth.saturating_add(1); + } else if character == '}' { + depth = depth.saturating_sub(1); + if depth == 0 { + return Some(offset.saturating_add(character.len_utf8())); + } + } + } + None +} diff --git a/crates/compass-languages/src/engine.rs b/crates/compass-languages/src/engine.rs index 32b92f17..e6cde5bd 100644 --- a/crates/compass-languages/src/engine.rs +++ b/crates/compass-languages/src/engine.rs @@ -312,11 +312,6 @@ impl Engine { spec: LanguageSpec, source: &[u8], ) -> Result { - if spec.name == "groovy" { - let mut extraction = crate::groovy::extract(path, source); - attach_basic_symbol_metadata(&mut extraction, source, spec.name); - return Ok(extraction); - } // These extractors are intentionally source-driven and do not consume a // tree-sitter root. Avoid initializing and touching their large static // grammar tables only to discard the tree; this materially lowers cold @@ -328,7 +323,6 @@ impl Engine { "r" => Some(crate::r::extract(path, source)), "pascal" => Some(crate::pascal::extract(path, source)), "apex" => Some(crate::apex::extract(path, source)), - "dart" => Some(crate::dart::extract(path, source)), _ => None, }; if let Some(mut extraction) = source_driven { @@ -374,7 +368,6 @@ impl Engine { "bash" => crate::bash::extract(path, source, root), "cpp" => crate::cpp::extract(path, source, root), "php" => crate::php::extract(path, source, root), - "swift" => crate::swift::extract(path, source, root), "objc" => crate::objc::extract(path, source, root), "powershell" => crate::powershell::extract(path, source, root), "elixir" => crate::elixir::extract(path, source, root), @@ -424,7 +417,16 @@ impl Engine { portable_framework_source(path) } }); - let framework_path = framework_source.as_deref().map(Path::new).unwrap_or(path); + // Dart's convention bridge retains the resolved filesystem path for + // relative export targets. Other universal framework packs use the + // portable source identity so route classification remains checkout + // independent. Source-supplied callers still own their explicit + // repository-relative identity above. + let framework_path = if spec.name == "dart" && !evidence_source_is_explicit { + path + } else { + framework_source.as_deref().map(Path::new).unwrap_or(path) + }; crate::frameworks::detect( framework_path, source, @@ -1654,8 +1656,6 @@ impl<'source, 'tree> ExtractState<'source, 'tree> { if self.language == "python" { self.add_python_parent_edges(node, &id); self.add_python_decorators(node, &id); - } else if self.language == "scala" { - self.add_scala_class_references(node, &id); } if matches!(self.language, "javascript" | "typescript" | "tsx") { self.add_js_parent_edges(node, &id); @@ -1727,8 +1727,6 @@ impl<'source, 'tree> ExtractState<'source, 'tree> { self.add_python_decorators(node, &id); } else if self.language == "c" { self.add_c_function_references(node, &id); - } else if self.language == "scala" { - self.add_scala_function_references(node, &id); } self.callables.entry(name).or_default().push(id.clone()); self.functions.push(FunctionBody { @@ -1765,13 +1763,6 @@ impl<'source, 'tree> ExtractState<'source, 'tree> { self.add_js_commonjs_export(node); } - if self.language == "scala" - && matches!(kind, "val_definition" | "var_definition") - && let Some((class_id, _, _, _)) = parent_declaration - { - self.add_scala_field_reference(node, class_id); - } - if matches!(self.language, "javascript" | "typescript" | "tsx") && kind == "lexical_declaration" && parent_declaration.is_none() @@ -2369,30 +2360,6 @@ impl<'source, 'tree> ExtractState<'source, 'tree> { self.add_python_import(node); return; } - if self.language == "scala" { - let mut cursor = node.walk(); - if let Some(target_node) = node - .children(&mut cursor) - .find(|child| matches!(child.kind(), "stable_id" | "identifier")) - { - let raw = self.node_text(target_node).unwrap_or_default(); - let target = raw - .rsplit('.') - .next() - .unwrap_or_default() - .trim_matches(['{', '}', ' ']); - if !target.is_empty() && target != "_" { - self.add_edge( - &self.file_id.clone(), - &make_id(&[target]), - "imports", - line(node), - Some("import"), - ); - } - } - return; - } let text = self.node_text(node).unwrap_or_default(); if matches!(self.language, "javascript" | "typescript" | "tsx") && matches!(node.kind(), "import_statement" | "export_statement") @@ -3185,111 +3152,6 @@ impl<'source, 'tree> ExtractState<'source, 'tree> { } } - fn add_scala_class_references(&mut self, node: Node<'tree>, class_id: &str) { - let extends = node - .child_by_field_name("extend") - .or_else(|| first_descendant(node, "extends_clause")); - if let Some(extends) = extends { - let mut bases = Vec::new(); - let mut cursor = extends.walk(); - for child in extends.children(&mut cursor) { - let name_node = if child.kind() == "type_identifier" { - Some(child) - } else if child.kind() == "generic_type" { - child - .child_by_field_name("type") - .or_else(|| first_descendant(child, "type_identifier")) - } else { - None - }; - if let Some(name) = name_node - .and_then(|name| self.node_text(name)) - .map(clean_name) - { - bases.push((name, line(child))); - } - } - for (index, (name, at)) in bases.into_iter().enumerate() { - let target = self.ensure_type_node(&name, true); - if target != class_id { - self.add_edge( - class_id, - &target, - if index == 0 { "inherits" } else { "mixes_in" }, - at, - None, - ); - } - } - } - - let mut parameters = Vec::new(); - collect_nodes_of_kind(node, "class_parameter", &mut parameters); - for parameter in parameters { - if let Some(type_node) = parameter.child_by_field_name("type") { - let mut refs = Vec::new(); - collect_scala_type_refs(type_node, self.source, false, &mut refs); - self.add_scala_type_references(class_id, &refs, "field", line(parameter)); - } - } - } - - fn add_scala_field_reference(&mut self, node: Node<'tree>, class_id: &str) { - let Some(type_node) = node.child_by_field_name("type") else { - return; - }; - let mut refs = Vec::new(); - collect_scala_type_refs(type_node, self.source, false, &mut refs); - self.add_scala_type_references(class_id, &refs, "field", line(node)); - } - - fn add_scala_function_references(&mut self, node: Node<'tree>, function_id: &str) { - if let Some(parameters) = first_descendant(node, "parameters") { - let mut cursor = parameters.walk(); - for parameter in parameters - .children(&mut cursor) - .filter(|child| child.kind() == "parameter") - { - if let Some(type_node) = parameter.child_by_field_name("type") { - let mut refs = Vec::new(); - collect_scala_type_refs(type_node, self.source, false, &mut refs); - self.add_scala_type_references( - function_id, - &refs, - "parameter_type", - line(node), - ); - } - } - } - if let Some(return_type) = node.child_by_field_name("return_type") { - let mut refs = Vec::new(); - collect_scala_type_refs(return_type, self.source, false, &mut refs); - self.add_scala_type_references(function_id, &refs, "return_type", line(node)); - } - } - - fn add_scala_type_references( - &mut self, - source: &str, - refs: &[(String, bool)], - context: &str, - at: usize, - ) { - for (name, generic) in refs { - let target = self.ensure_type_node(name, true); - if target != source { - self.add_edge( - source, - &target, - "references", - at, - Some(if *generic { "generic_arg" } else { context }), - ); - } - } - } - fn add_c_function_references(&mut self, node: Node<'tree>, function_id: &str) { if let Some(return_type) = node.child_by_field_name("type") { let mut names = Vec::new(); @@ -4213,61 +4075,6 @@ fn collect_c_type_names(node: Node<'_>, source: &[u8], output: &mut Vec) } } -fn collect_scala_type_refs( - node: Node<'_>, - source: &[u8], - generic: bool, - output: &mut Vec<(String, bool)>, -) { - if node.kind() == "type_identifier" { - if let Ok(name) = node.utf8_text(source) - && !name.is_empty() - { - output.push((name.to_owned(), generic)); - } - return; - } - if node.kind() == "generic_type" { - let base = node - .child_by_field_name("type") - .or_else(|| first_descendant(node, "type_identifier")); - if let Some(base) = base - && let Ok(name) = base.utf8_text(source) - && !name.is_empty() - { - output.push((name.to_owned(), generic)); - } - let mut cursor = node.walk(); - for arguments in node - .children(&mut cursor) - .filter(|child| child.kind() == "type_arguments") - { - let mut argument_cursor = arguments.walk(); - for argument in arguments - .children(&mut argument_cursor) - .filter(|child| child.is_named()) - { - collect_scala_type_refs(argument, source, true, output); - } - } - return; - } - if matches!( - node.kind(), - "compound_type" - | "infix_type" - | "function_type" - | "tuple_type" - | "annotated_type" - | "projected_type" - ) { - let mut cursor = node.walk(); - for child in node.children(&mut cursor).filter(|child| child.is_named()) { - collect_scala_type_refs(child, source, generic, output); - } - } -} - fn collect_nodes_of_kind<'tree>(node: Node<'tree>, kind: &str, output: &mut Vec>) { let mut cursor = node.walk(); for child in node.children(&mut cursor) { diff --git a/crates/compass-languages/src/evidence/build.rs b/crates/compass-languages/src/evidence/build.rs index 456da423..9c2a9cf3 100644 --- a/crates/compass-languages/src/evidence/build.rs +++ b/crates/compass-languages/src/evidence/build.rs @@ -846,6 +846,18 @@ pub(crate) fn extract_tree_evidence( pipeline.producer.language, ); } + if matches!( + pipeline.producer.language, + "dart" | "groovy" | "scala" | "swift" + ) { + return super::extended::emit_tree_evidence( + path, + source_file, + source, + root, + pipeline.producer.language, + ); + } let mut state = DirectEvidenceState::new(path, source_file, source, root, pipeline); state.add_file(root)?; if root.end_byte() == root.start_byte() { diff --git a/crates/compass-languages/src/evidence/extended.rs b/crates/compass-languages/src/evidence/extended.rs new file mode 100644 index 00000000..24bb1203 --- /dev/null +++ b/crates/compass-languages/src/evidence/extended.rs @@ -0,0 +1,1448 @@ +//! Shared AST-first evidence producer for Swift, Dart, Scala, and Groovy. +//! +//! The four grammars have different surface syntax, but their project-neutral +//! evidence boundary is the same: declarations and lexical scopes first, +//! followed by exact import/call/type occurrences. The producer deliberately +//! leaves target selection to `compass-resolve`; a terminal spelling is never +//! treated as an identity and every candidate is constrained to its source +//! language. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; + +use tree_sitter::Node; + +use super::build::{EvidenceBuilder, range_for_byte_span, range_for_file, range_for_node}; +use super::model::{ + BindingKind, CandidateRelation, EvidenceRange, LanguageCapability, ResolutionConstraint, + SemanticEvidenceBatch, SemanticRole, SymbolNamespace, +}; +use super::validate::{EvidenceError, EvidenceErrorCode, EvidenceLimits}; +use crate::{UniversalEvidenceRegistry, file_stem, make_id}; + +const MAX_TRAVERSAL_DEPTH: usize = 256; +const MAX_TEXT_BYTES: usize = 4_096; + +#[derive(Clone, Debug)] +struct Decl { + id: String, + name: String, + qualified: String, + kind: String, + body_scope_id: String, + start: usize, + end: usize, +} + +#[derive(Clone, Debug)] +struct Import { + spelling: String, + target: String, +} + +struct State<'source> { + language: &'static str, + source: &'source [u8], + source_file: &'source str, + builder: EvidenceBuilder, + file_id: String, + file_scope_id: String, + namespace: String, + declarations: Vec, + by_node: BTreeMap, + by_terminal: BTreeMap>, + by_qualified: BTreeMap>, + name_ranges: BTreeSet<(usize, usize)>, + imports: Vec, + module_targets: BTreeSet, + emitted: BTreeSet<(SemanticRole, usize, usize, String)>, + occurrence_ids: BTreeMap<(SemanticRole, usize, usize, String), String>, +} + +pub(super) fn emit_tree_evidence( + path: &Path, + source_file: &str, + source: &[u8], + root: Node<'_>, + language: &'static str, +) -> Result { + let pipeline = UniversalEvidenceRegistry::pipeline(language).ok_or_else(|| { + EvidenceError::new( + EvidenceErrorCode::InvalidPipeline, + format!("{language} universal evidence pipeline is not registered"), + ) + })?; + let file_range = range_for_file(source_file, source); + let mut builder = EvidenceBuilder::new( + pipeline, + format!("compass.languages.{language}.universal"), + source_file, + EvidenceLimits::default(), + ); + let file_graph_id = make_id(&[source_file]); + let file_id = builder.declare( + "file", + &file_graph_id, + path.file_name() + .and_then(|name| name.to_str()) + .unwrap_or(source_file), + source_file, + Some(&file_stem(Path::new(source_file))), + None, + file_range.clone(), + )?; + // A module scope is the schema's zero-width-safe file scope. It can + // represent an empty/trivia-only source while retaining the file's exact + // inventory range. + let file_scope_id = builder.open_scope("module", Some(&file_id), None, file_range)?; + if root.end_byte() == root.start_byte() { + return builder.finish(); + } + + let namespace = package_name(language, source).unwrap_or_default(); + let mut state = State { + language, + source, + source_file, + builder, + file_id, + file_scope_id: file_scope_id.clone(), + namespace, + declarations: Vec::new(), + by_node: BTreeMap::new(), + by_terminal: BTreeMap::new(), + by_qualified: BTreeMap::new(), + name_ranges: BTreeSet::new(), + imports: Vec::new(), + module_targets: BTreeSet::new(), + emitted: BTreeSet::new(), + occurrence_ids: BTreeMap::new(), + }; + if std::str::from_utf8(source).is_err() { + state.builder.diagnose( + "invalid_utf8", + None, + Some(range_for_file(source_file, source)), + "source is not valid UTF-8; text-derived evidence is omitted where decoding is unsafe", + )?; + } + let root_scope = state.add_namespace(root)?; + state.collect_declarations(root, None, &root_scope, 0)?; + if language == "groovy" && state.declarations.len() <= 1 { + state.collect_groovy_source()?; + } + state.collect_imports(root, 0)?; + state.collect_semantics(root, 0)?; + if root.has_error() { + state.builder.diagnose( + "partial_parser_recovery", + None, + Some(range_for_node(source_file, root)), + "parser recovered from malformed source; emitted evidence remains source-bounded", + )?; + } + state.builder.finish() +} + +impl<'source> State<'source> { + fn add_namespace(&mut self, root: Node<'_>) -> Result { + if self.namespace.is_empty() || !self.supports(LanguageCapability::Namespaces) { + return Ok(self.file_scope_id.clone()); + } + let graph_id = make_id(&[self.language, "namespace", &self.namespace]); + let declaration_id = self.builder.declare_with_namespace( + "namespace", + &graph_id, + &self.namespace, + &self.namespace, + Some(&self.namespace), + Some(&self.file_scope_id), + Some(SymbolNamespace::Namespace), + range_for_node(self.source_file, root), + )?; + let scope_id = self.builder.open_scope( + "namespace", + Some(&declaration_id), + Some(&self.file_scope_id), + range_for_node(self.source_file, root), + )?; + self.builder.relate( + CandidateRelation::Owns, + &self.file_id, + None, + None, + &self.namespace, + ResolutionConstraint { + exact_target_declaration_id: Some(declaration_id), + exact_language: Some(self.language.to_owned()), + ..ResolutionConstraint::default() + }, + )?; + Ok(scope_id) + } + + fn collect_declarations( + &mut self, + node: Node<'_>, + parent_decl: Option, + parent_scope: &str, + depth: usize, + ) -> Result<(), EvidenceError> { + if depth > MAX_TRAVERSAL_DEPTH { + self.depth_diagnostic(node)?; + return Ok(()); + } + let mut owner = parent_decl; + let mut scope = parent_scope.to_owned(); + let prefix = parent_decl + .and_then(|index| self.declarations.get(index)) + .map(|decl| decl.qualified.clone()) + .unwrap_or_else(|| self.namespace.clone()); + + if let Some(kind) = declaration_kind(self.language, node.kind()) + && let Some(name_node) = declaration_name(node) + { + let name = self.text(name_node); + let lookup_name = if self.language == "dart" { + name.split_once('(') + .map_or_else(|| name.clone(), |(base, _)| base.trim().to_owned()) + } else { + name.clone() + }; + // The Dart grammar exposes a method signature as the declaration + // name child (for example ``clearLibraryContext()``). Calls and + // Analyzer source evidence carry the base name only; retaining + // the punctuation makes an otherwise exact lexical target look + // unresolved. Preserve the parser spelling for declaration + // identity and index a separate base-name alias for lookup; this + // keeps overloads and stable declaration IDs distinct. + if valid_name(&lookup_name) + && !self + .name_ranges + .contains(&(name_node.start_byte(), name_node.end_byte())) + { + let qualified = join_name(&prefix, &name); + let key = (node.start_byte(), node.end_byte(), name.clone()); + if !self + .declarations + .iter() + .any(|decl| (decl.start, decl.end) == (key.0, key.1) && decl.name == key.2) + { + let graph_id = make_id(&[ + self.source_file, + self.language, + kind, + &qualified, + &node.start_byte().to_string(), + &node.end_byte().to_string(), + ]); + let declaration_id = self.builder.declare( + kind, + &graph_id, + &name, + &qualified, + if self.namespace.is_empty() { + None + } else { + Some(&self.namespace) + }, + Some(parent_scope), + range_for_node(self.source_file, node), + )?; + let opens_scope = opens_scope(kind); + let body_scope_id = if opens_scope { + self.builder.open_scope( + scope_kind(kind), + Some(&declaration_id), + Some(parent_scope), + range_for_node(self.source_file, node), + )? + } else { + parent_scope.to_owned() + }; + let source_id = parent_decl + .and_then(|index| self.declarations.get(index)) + .map_or(self.file_id.as_str(), |decl| decl.id.as_str()) + .to_owned(); + self.builder.relate( + CandidateRelation::Owns, + &source_id, + None, + None, + &name, + ResolutionConstraint { + exact_target_declaration_id: Some(declaration_id.clone()), + exact_language: Some(self.language.to_owned()), + ..ResolutionConstraint::default() + }, + )?; + let index = self.declarations.len(); + let decl = Decl { + id: declaration_id, + name: name.clone(), + qualified: qualified.clone(), + kind: kind.to_owned(), + body_scope_id: body_scope_id.clone(), + start: node.start_byte(), + end: node.end_byte(), + }; + self.name_ranges + .insert((name_node.start_byte(), name_node.end_byte())); + self.by_node.insert(node.id(), index); + self.by_terminal.entry(name).or_default().push(index); + if lookup_name != decl.name { + self.by_terminal.entry(lookup_name).or_default().push(index); + } + self.by_qualified.entry(qualified).or_default().push(index); + self.declarations.push(decl); + if opens_scope { + owner = Some(index); + scope = body_scope_id; + } + } + } + } + + let mut cursor = node.walk(); + for child in node.named_children(&mut cursor) { + self.collect_declarations(child, owner, &scope, depth.saturating_add(1))?; + } + Ok(()) + } + + /// The pinned Groovy grammar intentionally exposes each top-level form as + /// a bounded `command` node. Keep Groovy on the universal evidence route + /// by extracting the declaration/call spans from that command text rather + /// than reintroducing a raw graph fallback. The scanner is line- and + /// brace-bounded, preserves exact byte ranges, and remains fail-closed for + /// ambiguous method spellings. + fn collect_groovy_source(&mut self) -> Result<(), EvidenceError> { + // A lossy conversion can expand one invalid source byte into multiple + // replacement bytes. Do not publish scanner offsets derived from it; + // tree-sitter evidence above remains available and this omission is + // reported as explicit incomplete input. + let Ok(text) = std::str::from_utf8(self.source) else { + return Ok(()); + }; + let mut depth = 0_i32; + let mut classes: Vec<(usize, usize, i32)> = Vec::new(); + let mut method: Option<(usize, usize)> = None; + let mut line_start = 0_usize; + for line in text.split_inclusive('\n') { + let line_without_newline = line.trim_end_matches(['\r', '\n']); + let line_end = line_start.saturating_add(line_without_newline.len()); + let trimmed = line_without_newline.trim(); + while classes.last().is_some_and(|(_, end, _)| line_start >= *end) { + classes.pop(); + } + if method.is_some_and(|(_, end)| line_start >= end) { + method = None; + } + + let class_decl = groovy_type_declaration(trimmed); + if let Some((kind, name, name_offset)) = class_decl { + let parent = classes.last().map(|(index, _, _)| *index); + let parent_scope = parent + .and_then(|index| self.declarations.get(index)) + .map_or(self.file_scope_id.as_str(), |decl| { + decl.body_scope_id.as_str() + }) + .to_owned(); + let body_end = matching_brace_end(self.source, line_start, line_end); + let end = body_end.max(line_end); + if let Some(index) = self.add_source_declaration( + kind, + &name, + line_start, + end, + line_start.saturating_add(name_offset), + line_start + .saturating_add(name_offset) + .saturating_add(name.len()), + parent, + &parent_scope, + )? { + classes.push((index, end, depth)); + self.emit_groovy_calls(line_start, line_end, index)?; + } + depth = depth.saturating_add(brace_delta(trimmed)); + line_start = line_start.saturating_add(line.len()); + continue; + } + + let active_class = classes.last().map(|(index, _, _)| *index); + if let Some(class_index) = active_class + && let Some((name, constructor, name_offset)) = groovy_method_declaration(trimmed) + { + let parent_scope = self + .declarations + .get(class_index) + .map_or(self.file_scope_id.as_str(), |decl| { + decl.body_scope_id.as_str() + }) + .to_owned(); + let body_end = matching_brace_end(self.source, line_start, line_end); + let end = body_end.max(line_end); + let kind = if constructor { "constructor" } else { "method" }; + if let Some(index) = self.add_source_declaration( + kind, + &name, + line_start, + end, + line_start.saturating_add(name_offset), + line_start + .saturating_add(name_offset) + .saturating_add(name.len()), + Some(class_index), + &parent_scope, + )? { + method = Some((index, end)); + self.emit_groovy_calls(line_start, line_end, index)?; + } + } else if let Some((method_index, method_end)) = method + && line_start < method_end + { + self.emit_groovy_calls(line_start, line_end, method_index)?; + } + + depth = depth.saturating_add(brace_delta(trimmed)); + line_start = line_start.saturating_add(line.len()); + } + Ok(()) + } + + #[allow(clippy::too_many_arguments)] + fn add_source_declaration( + &mut self, + kind: &str, + name: &str, + start: usize, + end: usize, + name_start: usize, + name_end: usize, + parent: Option, + parent_scope: &str, + ) -> Result, EvidenceError> { + if !valid_name(name) + || self + .declarations + .iter() + .any(|decl| decl.start == start && decl.end == end && decl.name == name) + { + return Ok(None); + } + let prefix = parent + .and_then(|index| self.declarations.get(index)) + .map(|decl| decl.qualified.as_str()) + .unwrap_or(self.namespace.as_str()); + let qualified = join_name(prefix, name); + let graph_id = make_id(&[ + self.source_file, + self.language, + kind, + &qualified, + &start.to_string(), + &end.to_string(), + ]); + let declaration_id = self.builder.declare( + kind, + &graph_id, + name, + &qualified, + (!self.namespace.is_empty()).then_some(self.namespace.as_str()), + Some(parent_scope), + range_for_byte_span(self.source_file, self.source, start, end), + )?; + let body_scope_id = self.builder.open_scope( + scope_kind(kind), + Some(&declaration_id), + Some(parent_scope), + range_for_byte_span(self.source_file, self.source, start, end), + )?; + let owner_id = parent + .and_then(|index| self.declarations.get(index)) + .map_or(self.file_id.as_str(), |decl| decl.id.as_str()) + .to_owned(); + self.builder.relate( + CandidateRelation::Owns, + &owner_id, + None, + None, + name, + ResolutionConstraint { + exact_target_declaration_id: Some(declaration_id.clone()), + exact_language: Some(self.language.to_owned()), + ..ResolutionConstraint::default() + }, + )?; + let index = self.declarations.len(); + self.declarations.push(Decl { + id: declaration_id, + name: name.to_owned(), + qualified: qualified.clone(), + kind: kind.to_owned(), + body_scope_id, + start, + end, + }); + self.name_ranges.insert((name_start, name_end)); + self.by_terminal + .entry(name.to_owned()) + .or_default() + .push(index); + self.by_qualified.entry(qualified).or_default().push(index); + Ok(Some(index)) + } + + fn emit_groovy_calls( + &mut self, + line_start: usize, + line_end: usize, + owner: usize, + ) -> Result<(), EvidenceError> { + let bytes = self.source.get(line_start..line_end).unwrap_or_default(); + let mut index = 0_usize; + while index < bytes.len() { + if !is_identifier_start(bytes[index]) { + index = index.saturating_add(1); + continue; + } + let start = index; + index = index.saturating_add(1); + while index < bytes.len() && is_identifier_continue(bytes[index]) { + index = index.saturating_add(1); + } + let end = index; + while index < bytes.len() && bytes[index].is_ascii_whitespace() { + index = index.saturating_add(1); + } + if bytes.get(index) != Some(&b'(') { + continue; + } + let spelling = String::from_utf8_lossy(&bytes[start..end]).into_owned(); + if matches!( + spelling.as_str(), + "if" | "for" | "while" | "switch" | "catch" | "return" | "new" | "super" | "this" + ) || self + .name_ranges + .contains(&(line_start + start, line_start + end)) + { + continue; + } + let qualifier = (start > 0 && bytes[start - 1] == b'.').then(|| { + let mut qualifier_start = start.saturating_sub(1); + while qualifier_start > 0 && is_identifier_continue(bytes[qualifier_start - 1]) { + qualifier_start = qualifier_start.saturating_sub(1); + } + String::from_utf8_lossy(&bytes[qualifier_start..start - 1]).into_owned() + }); + self.emit_call_site( + Some(owner), + &spelling, + qualifier.as_deref(), + range_for_byte_span( + self.source_file, + self.source, + line_start + start, + line_start + end, + ), + false, + )?; + } + Ok(()) + } + + fn collect_imports(&mut self, node: Node<'_>, depth: usize) -> Result<(), EvidenceError> { + if depth > MAX_TRAVERSAL_DEPTH { + self.depth_diagnostic(node)?; + return Ok(()); + } + let statement = self.text(node); + if is_import_node(node.kind()) + && let Some((target, alias, reexport)) = parse_import(&statement) + && !target.is_empty() + { + let has_alias = alias.is_some(); + let spelling = alias.unwrap_or_else(|| terminal(&target).to_owned()); + if !spelling.is_empty() { + let owner = self.owner_for(node.start_byte()); + let owner_scope = self + .declaration_for(owner) + .map_or(self.file_scope_id.as_str(), |decl| { + decl.body_scope_id.as_str() + }) + .to_owned(); + // Swift's pre-universal extractor published imported modules + // as source-anchored module nodes. Keep that established + // Vapor/framework identity on the evidence route while the + // binding itself remains an exact, language-constrained + // import candidate. + if self.language == "swift" && self.module_targets.insert(target.clone()) { + let module_id = make_id(&[self.source_file, self.language, "module", &target]); + let module_declaration = self.builder.declare( + "module", + &module_id, + &spelling, + &target, + None, + Some(&self.file_scope_id), + range_for_node(self.source_file, node), + )?; + self.builder.relate( + CandidateRelation::Owns, + &self.file_id, + None, + None, + &spelling, + ResolutionConstraint { + exact_target_declaration_id: Some(module_declaration), + exact_language: Some(self.language.to_owned()), + ..ResolutionConstraint::default() + }, + )?; + } + let binding_id = self.builder.bind_with_identity( + if reexport { + BindingKind::Reexport + } else if has_alias { + BindingKind::ImportAlias + } else { + BindingKind::Import + }, + &spelling, + &target, + None, + Some(&owner_scope), + None, + false, + range_for_node(self.source_file, node), + )?; + let owner_id = self.owner_id(owner); + let role = if reexport { + SemanticRole::Reexport + } else { + SemanticRole::Import + }; + let occurrence_id = self.emit_occurrence( + role, + &owner_id, + &spelling, + qualifier_for(&target), + Some(&owner_scope), + range_for_node(self.source_file, node), + )?; + let relation = if reexport { + CandidateRelation::Reexports + } else { + CandidateRelation::Imports + }; + self.builder.relate( + relation, + &owner_id, + Some(&occurrence_id), + Some(&binding_id), + &spelling, + ResolutionConstraint { + exact_language: Some(self.language.to_owned()), + qualified_name: Some(target.clone()), + allow_external: true, + ..ResolutionConstraint::default() + }, + )?; + self.imports.push(Import { spelling, target }); + } + } + let mut cursor = node.walk(); + for child in node.named_children(&mut cursor) { + self.collect_imports(child, depth.saturating_add(1))?; + } + Ok(()) + } + + fn collect_semantics(&mut self, node: Node<'_>, depth: usize) -> Result<(), EvidenceError> { + if depth > MAX_TRAVERSAL_DEPTH { + self.depth_diagnostic(node)?; + return Ok(()); + } + if is_call_node(node.kind()) || self.is_identifier_call(node) { + self.emit_call(node)?; + } + if is_type_leaf(node.kind()) { + self.emit_type_reference(node)?; + } + if self.supports(LanguageCapability::Members) && is_member_node(node.kind()) { + self.emit_member_access(node)?; + } + if self.supports(LanguageCapability::Decorators) && is_decorator_node(node.kind()) { + self.emit_decorator(node)?; + } + let mut cursor = node.walk(); + for child in node.named_children(&mut cursor) { + self.collect_semantics(child, depth.saturating_add(1))?; + } + Ok(()) + } + + fn emit_call(&mut self, node: Node<'_>) -> Result<(), EvidenceError> { + let callee = if is_call_node(node.kind()) { + call_callee(node) + } else { + Some(node) + }; + let Some(callee) = callee else { + return Ok(()); + }; + let raw = self.text(callee); + let (qualifier, spelling) = split_qualified(&raw); + if !valid_name(&spelling) { + return Ok(()); + } + self.emit_call_site( + self.owner_for(node.start_byte()), + &spelling, + qualifier.as_deref(), + range_for_node(self.source_file, callee), + node.kind().contains("constructor"), + ) + } + + fn emit_call_site( + &mut self, + owner: Option, + spelling: &str, + qualifier: Option<&str>, + range: EvidenceRange, + constructor_node: bool, + ) -> Result<(), EvidenceError> { + let owner_id = self.owner_id(owner); + let owner_scope = self.owner_scope(owner); + let constructor = + spelling.chars().next().is_some_and(char::is_uppercase) || constructor_node; + let role = if constructor { + SemanticRole::Construction + } else { + SemanticRole::Call + }; + let relation = if constructor { + CandidateRelation::Constructs + } else { + CandidateRelation::Calls + }; + let occurrence_id = self.emit_occurrence( + role, + &owner_id, + spelling, + qualifier, + Some(&owner_scope), + range, + )?; + let exact = self.resolve_local(spelling, qualifier); + let imported_target = (qualifier.is_none()) + .then(|| { + self.imports + .iter() + .find(|import| import.spelling == spelling) + .map(|import| import.target.clone()) + }) + .flatten(); + let qualified_name = exact + .and_then(|index| { + self.declarations + .get(index) + .map(|decl| decl.qualified.clone()) + }) + .or(imported_target); + let mut allowed = if constructor { + vec![ + "class".to_owned(), + "struct".to_owned(), + "enum".to_owned(), + "constructor".to_owned(), + ] + } else { + vec![ + "function".to_owned(), + "method".to_owned(), + "constructor".to_owned(), + ] + }; + allowed.sort_unstable(); + self.builder.relate( + relation, + &owner_id, + Some(&occurrence_id), + None, + spelling, + ResolutionConstraint { + exact_target_declaration_id: exact.map(|index| self.declarations[index].id.clone()), + exact_language: Some(self.language.to_owned()), + qualified_name, + allowed_target_kinds: allowed, + allow_external: exact.is_none(), + ..ResolutionConstraint::default() + }, + )?; + Ok(()) + } + + fn emit_type_reference(&mut self, node: Node<'_>) -> Result<(), EvidenceError> { + if self + .name_ranges + .contains(&(node.start_byte(), node.end_byte())) + { + return Ok(()); + } + let raw = self.text(node); + let (qualifier, spelling) = split_qualified(&raw); + if !valid_name(&spelling) || spelling.len() > 256 { + return Ok(()); + } + let owner = self.owner_for(node.start_byte()); + let owner_id = self.owner_id(owner); + let owner_scope = self.owner_scope(owner); + let base = is_base_context(node); + let role = if base { + SemanticRole::BaseType + } else { + SemanticRole::TypeReference + }; + let relation = if base { + if self.declaration_for(owner).is_some_and(|decl| { + decl.kind == "interface" || decl.kind == "trait" || decl.kind == "protocol" + }) { + CandidateRelation::Implements + } else { + CandidateRelation::Extends + } + } else { + CandidateRelation::References + }; + let occurrence_id = self.emit_occurrence( + role, + &owner_id, + &spelling, + qualifier.as_deref(), + Some(&owner_scope), + range_for_node(self.source_file, node), + )?; + let exact = self.resolve_local(&spelling, qualifier.as_deref()); + self.builder.relate( + relation, + &owner_id, + Some(&occurrence_id), + None, + &spelling, + ResolutionConstraint { + exact_target_declaration_id: exact.map(|index| self.declarations[index].id.clone()), + exact_language: Some(self.language.to_owned()), + qualified_name: qualifier + .as_ref() + .map(|prefix| format!("{prefix}.{spelling}")), + allowed_target_kinds: vec![ + "class".to_owned(), + "enum".to_owned(), + "interface".to_owned(), + "protocol".to_owned(), + "struct".to_owned(), + "trait".to_owned(), + "type_alias".to_owned(), + ], + allow_external: exact.is_none(), + ..ResolutionConstraint::default() + }, + )?; + Ok(()) + } + + fn emit_member_access(&mut self, node: Node<'_>) -> Result<(), EvidenceError> { + let raw = self.text(node); + let Some((qualifier, spelling)) = raw.rsplit_once('.') else { + return Ok(()); + }; + let spelling = spelling + .trim_matches(|character: char| !character.is_ascii_alphanumeric() && character != '_'); + let qualifier = qualifier.trim(); + if !valid_name(spelling) || qualifier.is_empty() { + return Ok(()); + } + let owner = self.owner_for(node.start_byte()); + let owner_id = self.owner_id(owner); + let owner_scope = self.owner_scope(owner); + let occurrence_id = self.emit_occurrence( + SemanticRole::MemberAccess, + &owner_id, + spelling, + Some(qualifier), + Some(&owner_scope), + range_for_node(self.source_file, node), + )?; + self.builder.relate( + CandidateRelation::AccessesMember, + &owner_id, + Some(&occurrence_id), + None, + spelling, + ResolutionConstraint { + exact_language: Some(self.language.to_owned()), + qualified_name: Some(format!("{qualifier}.{spelling}")), + allowed_target_kinds: vec![ + "field".to_owned(), + "property".to_owned(), + "method".to_owned(), + ], + allow_external: true, + ..ResolutionConstraint::default() + }, + )?; + Ok(()) + } + + fn emit_decorator(&mut self, node: Node<'_>) -> Result<(), EvidenceError> { + let raw = self.text(node); + let spelling = raw + .trim_start_matches(['@', '#', '[']) + .split(['(', '[', ' ', '\n', '\r', ']']) + .next() + .unwrap_or_default() + .trim(); + if !valid_name(spelling) { + return Ok(()); + } + let owner = self.owner_for(node.start_byte()); + let owner_id = self.owner_id(owner); + let owner_scope = self.owner_scope(owner); + let occurrence_id = self.emit_occurrence( + SemanticRole::Decorator, + &owner_id, + spelling, + None, + Some(&owner_scope), + range_for_node(self.source_file, node), + )?; + self.builder.relate( + CandidateRelation::Decorates, + &owner_id, + Some(&occurrence_id), + None, + spelling, + ResolutionConstraint { + exact_language: Some(self.language.to_owned()), + allow_external: true, + ..ResolutionConstraint::default() + }, + )?; + Ok(()) + } + + fn emit_occurrence( + &mut self, + role: SemanticRole, + owner_id: &str, + spelling: &str, + qualifier: Option<&str>, + scope_id: Option<&str>, + range: EvidenceRange, + ) -> Result { + let key = ( + role, + range.start_byte as usize, + range.end_byte as usize, + spelling.to_owned(), + ); + if !self.emitted.insert(key.clone()) { + return Ok(self.occurrence_ids.get(&key).cloned().unwrap_or_default()); + } + let id = self + .builder + .occur(role, owner_id, spelling, qualifier, scope_id, range)?; + self.occurrence_ids.insert(key, id.clone()); + Ok(id) + } + + fn owner_for(&self, byte: usize) -> Option { + self.declarations + .iter() + .enumerate() + .filter(|(_, decl)| decl.start <= byte && byte < decl.end) + .max_by_key(|(_, decl)| decl.start) + .map(|(index, _)| index) + } + + fn owner_id(&self, owner: Option) -> String { + owner + .and_then(|index| self.declarations.get(index)) + .map_or_else(|| self.file_id.clone(), |decl| decl.id.clone()) + } + + fn owner_scope(&self, owner: Option) -> String { + owner + .and_then(|index| self.declarations.get(index)) + .map_or_else( + || self.file_scope_id.clone(), + |decl| decl.body_scope_id.clone(), + ) + } + + fn declaration_for(&self, owner: Option) -> Option<&Decl> { + owner.and_then(|index| self.declarations.get(index)) + } + + fn resolve_local(&self, spelling: &str, qualifier: Option<&str>) -> Option { + let values = if let Some(qualifier) = qualifier { + self.by_qualified.get(&format!("{qualifier}.{spelling}")) + } else { + self.by_terminal.get(spelling) + }?; + (values.len() == 1).then_some(values[0]) + } + + fn supports(&self, capability: LanguageCapability) -> bool { + UniversalEvidenceRegistry::pipeline(self.language) + .is_some_and(|pipeline| pipeline.producer.capabilities.contains(&capability)) + } + + fn is_identifier_call(&self, node: Node<'_>) -> bool { + if !matches!( + node.kind(), + "identifier" | "simple_identifier" | "field_identifier" + ) || self + .name_ranges + .contains(&(node.start_byte(), node.end_byte())) + { + return false; + } + let mut end = node.end_byte(); + while self.source.get(end).is_some_and(u8::is_ascii_whitespace) { + end = end.saturating_add(1); + } + self.source.get(end) == Some(&b'(') + } + + fn text(&self, node: Node<'_>) -> String { + self.source + .get(node.start_byte()..node.end_byte()) + .and_then(|bytes| std::str::from_utf8(bytes).ok()) + .map(|text| text.chars().take(MAX_TEXT_BYTES).collect()) + .unwrap_or_default() + } + + fn depth_diagnostic(&mut self, node: Node<'_>) -> Result<(), EvidenceError> { + self.builder.diagnose( + "traversal_depth_limit", + None, + Some(range_for_node(self.source_file, node)), + "parser tree exceeded the bounded universal evidence traversal depth", + ) + } +} + +fn declaration_kind(language: &str, kind: &str) -> Option<&'static str> { + if matches!( + kind, + "source_file" + | "program" + | "compilation_unit" + | "translation_unit" + | "package_declaration" + | "package_clause" + | "namespace_declaration" + | "library_directive" + | "import_declaration" + | "import_statement" + | "import_directive" + | "import_or_export" + | "export_directive" + ) { + return None; + } + let lower = kind.to_ascii_lowercase(); + if lower.contains("protocol") { + return Some("protocol"); + } + if lower.contains("interface") { + return Some("interface"); + } + if lower.contains("trait") { + return Some("trait"); + } + if lower.contains("enum") { + return Some("enum"); + } + if lower.contains("struct") { + return Some("struct"); + } + if lower.contains("record") { + return Some("record"); + } + if lower.contains("class") { + return Some("class"); + } + if matches!( + lower.as_str(), + "object_definition" | "module_definition" | "extension_declaration" + ) || lower.ends_with("_object_definition") + { + return Some("module"); + } + if lower.contains("type_alias") || lower.contains("type_definition") { + return Some("type_alias"); + } + if lower.contains("initializer") || lower.contains("constructor") || lower == "init_declaration" + { + return Some("constructor"); + } + if lower.contains("deinitializer") || lower.contains("deinit") { + return Some("method"); + } + if lower.contains("subscript") { + return Some("method"); + } + if lower.contains("method") { + return Some("method"); + } + if lower.contains("function") + || lower == "function_declaration" + || lower == "function_definition" + { + return Some("function"); + } + if lower.contains("property") || lower.contains("field") { + return Some("field"); + } + if language == "scala" && (lower.contains("val_") || lower.contains("var_")) { + return Some("field"); + } + if language == "dart" && lower == "variable_declaration" { + return Some("field"); + } + None +} + +fn declaration_name(node: Node<'_>) -> Option> { + for field in ["name", "identifier", "declarator"] { + if let Some(child) = node.child_by_field_name(field) { + return Some(child); + } + } + let mut cursor = node.walk(); + node.named_children(&mut cursor).find(|child| { + matches!( + child.kind(), + "identifier" + | "type_identifier" + | "simple_identifier" + | "field_identifier" + | "constant_identifier" + | "name" + ) + }) +} + +fn valid_name(value: &str) -> bool { + let value = value.trim(); + !value.is_empty() + && value.len() <= 512 + && value + .chars() + .all(|character| character.is_alphanumeric() || matches!(character, '_' | '$' | '`')) +} + +fn opens_scope(kind: &str) -> bool { + matches!( + kind, + "class" + | "struct" + | "record" + | "enum" + | "protocol" + | "interface" + | "trait" + | "module" + | "function" + | "method" + | "constructor" + ) +} + +fn scope_kind(kind: &str) -> &'static str { + if matches!(kind, "function" | "method" | "constructor") { + "function" + } else { + "type" + } +} + +fn join_name(prefix: &str, name: &str) -> String { + if prefix.is_empty() { + name.to_owned() + } else { + format!("{prefix}.{name}") + } +} + +fn package_name(language: &str, source: &[u8]) -> Option { + if language == "swift" || language == "dart" { + return None; + } + let text = std::str::from_utf8(source).ok()?; + for line in text.lines().take(128) { + let line = line.trim(); + let Some(value) = line + .strip_prefix("package ") + .or_else(|| line.strip_prefix("namespace ")) + else { + continue; + }; + let value = value + .trim() + .trim_end_matches([';', '{']) + .split_whitespace() + .next() + .unwrap_or_default(); + if !value.is_empty() + && value + .chars() + .all(|c| c.is_ascii_alphanumeric() || "._$`".contains(c)) + { + return Some(value.to_owned()); + } + } + None +} + +fn is_import_node(kind: &str) -> bool { + let lower = kind.to_ascii_lowercase(); + lower.contains("import") || lower == "export_directive" +} + +fn parse_import(statement: &str) -> Option<(String, Option, bool)> { + let trimmed = statement.trim(); + let reexport = trimmed.starts_with("export ") || trimmed.starts_with("export\n"); + let keyword = if reexport { "export" } else { "import" }; + let rest = trimmed.strip_prefix(keyword)?.trim(); + let mut target = rest + .split_whitespace() + .next() + .unwrap_or_default() + .trim_end_matches(';') + .trim_matches(['\'', '"', '`']) + .to_owned(); + if target.is_empty() { + return None; + } + if let Some(index) = target.find(',') { + target.truncate(index); + } + let alias = rest + .split_whitespace() + .collect::>() + .windows(2) + .find(|pair| pair[0] == "as") + .map(|pair| pair[1].trim_matches([',', ';', '`', '\'', '"']).to_owned()) + .filter(|value| valid_name(value)); + Some((target, alias, reexport)) +} + +fn terminal(value: &str) -> &str { + value.rsplit(['.', ':', '/']).next().unwrap_or(value) +} + +fn qualifier_for(value: &str) -> Option<&str> { + value.rsplit_once(['.', ':']).map(|(prefix, _)| prefix) +} + +fn split_qualified(raw: &str) -> (Option, String) { + let cleaned = raw + .trim() + .trim_matches(['`', '\'', '"']) + .trim_end_matches(['?', '!']); + if let Some((prefix, name)) = cleaned.rsplit_once('.') { + (Some(prefix.trim().to_owned()), terminal(name).to_owned()) + } else if let Some((prefix, name)) = cleaned.rsplit_once("::") { + (Some(prefix.trim().to_owned()), terminal(name).to_owned()) + } else { + (None, terminal(cleaned).to_owned()) + } +} + +fn is_call_node(kind: &str) -> bool { + let lower = kind.to_ascii_lowercase(); + matches!( + lower.as_str(), + "call_expression" + | "call" + | "function_call" + | "method_call" + | "method_invocation" + | "invocation_expression" + | "apply_expression" + | "call_expression_with_trailing_closure" + ) || (lower.contains("invocation") && !lower.contains("declaration")) +} + +fn call_callee(node: Node<'_>) -> Option> { + for field in ["function", "callee", "name", "method", "receiver", "object"] { + if let Some(child) = node.child_by_field_name(field) { + return Some(child); + } + } + let mut cursor = node.walk(); + node.named_children(&mut cursor).next() +} + +fn is_type_leaf(kind: &str) -> bool { + matches!( + kind, + "type_identifier" + | "simple_type" + | "user_type" + | "named_type" + | "type_reference" + | "class_type" + ) +} + +fn is_base_context(node: Node<'_>) -> bool { + let mut current = node.parent(); + for _ in 0..=3 { + let Some(parent) = current else { break }; + let lower = parent.kind().to_ascii_lowercase(); + if lower.contains("extends") + || lower.contains("implements") + || lower.contains("supertype") + || lower.contains("inheritance") + || lower.contains("base") + { + return true; + } + current = parent.parent(); + } + false +} + +fn is_member_node(kind: &str) -> bool { + let lower = kind.to_ascii_lowercase(); + lower.contains("member_access") + || lower.contains("field_expression") + || lower == "selector" + || lower.contains("navigation") + || lower.contains("property_access") +} + +fn is_decorator_node(kind: &str) -> bool { + let lower = kind.to_ascii_lowercase(); + lower.contains("annotation") || lower.contains("decorator") || lower == "attribute_list" +} + +fn groovy_type_declaration(line: &str) -> Option<(&'static str, String, usize)> { + let tokens = line + .split_whitespace() + .map(|token| token.trim_matches(['@', '{', ';', ','])) + .collect::>(); + for (index, token) in tokens.iter().enumerate() { + let kind = match *token { + "class" => "class", + "interface" => "interface", + "trait" => "trait", + "enum" => "enum", + _ => continue, + }; + let name = tokens + .get(index.saturating_add(1))? + .trim_matches(['{', ';']); + if !valid_name(name) { + return None; + } + let offset = line.find(name)?; + return Some((kind, name.to_owned(), offset)); + } + None +} + +fn groovy_method_declaration(line: &str) -> Option<(String, bool, usize)> { + let open = line.find('(')?; + let before = line.get(..open)?.trim_end(); + let name_end = before.len(); + let name_start = before + .char_indices() + .rev() + .find(|(_, character)| !character.is_ascii_alphanumeric() && *character != '_') + .map_or(0, |(index, _)| index.saturating_add(1)); + let name = before.get(name_start..name_end)?.trim(); + if !valid_name(name) + || matches!( + name, + "if" | "for" | "while" | "switch" | "catch" | "try" | "return" | "assert" + ) + { + return None; + } + let constructor = name.chars().next().is_some_and(char::is_uppercase); + let has_return_shape = before[..name_start] + .split_whitespace() + .any(|token| token == "def" || !token.is_empty()); + (has_return_shape || constructor).then(|| (name.to_owned(), constructor, name_start)) +} + +fn brace_delta(line: &str) -> i32 { + let mut delta = 0_i32; + let mut quote = None; + for character in line.chars() { + if let Some(active) = quote { + if character == active { + quote = None; + } + continue; + } + if matches!(character, '\'' | '"') { + quote = Some(character); + } else if character == '{' { + delta = delta.saturating_add(1); + } else if character == '}' { + delta = delta.saturating_sub(1); + } + } + delta +} + +fn matching_brace_end(source: &[u8], line_start: usize, line_end: usize) -> usize { + let Some(open) = source + .get(line_start..line_end) + .and_then(|line| line.iter().position(|byte| *byte == b'{')) + .map(|offset| line_start.saturating_add(offset)) + else { + return line_end; + }; + let mut depth = 0_i32; + let mut quote = None; + for (offset, byte) in source.iter().enumerate().skip(open) { + let character = char::from(*byte); + if let Some(active) = quote { + if character == active { + quote = None; + } + continue; + } + if matches!(character, '\'' | '"') { + quote = Some(character); + continue; + } + if character == '{' { + depth = depth.saturating_add(1); + } else if character == '}' { + depth = depth.saturating_sub(1); + if depth == 0 { + return offset.saturating_add(1); + } + } + } + source.len() +} + +fn is_identifier_start(byte: u8) -> bool { + byte.is_ascii_alphabetic() || byte == b'_' +} + +fn is_identifier_continue(byte: u8) -> bool { + byte.is_ascii_alphanumeric() || byte == b'_' +} diff --git a/crates/compass-languages/src/evidence/mod.rs b/crates/compass-languages/src/evidence/mod.rs index a3bdb114..156c1335 100644 --- a/crates/compass-languages/src/evidence/mod.rs +++ b/crates/compass-languages/src/evidence/mod.rs @@ -1,5 +1,6 @@ mod build; mod csharp; +mod extended; mod kotlin; mod model; mod php; diff --git a/crates/compass-languages/src/evidence/typescript.rs b/crates/compass-languages/src/evidence/typescript.rs index f03d35ab..0cd04abd 100644 --- a/crates/compass-languages/src/evidence/typescript.rs +++ b/crates/compass-languages/src/evidence/typescript.rs @@ -5084,7 +5084,42 @@ impl<'source, 'tree> CandidateState<'source, 'tree> { if let Some(object) = object && self.is_builtin_member_target(scope_id, object, &property_name) { - return Ok(()); + if self.language != "typescript" { + return Ok(()); + } + let Some(receiver) = self.builtin_receiver_name(scope_id, object) else { + return Ok(()); + }; + let target = format!("global::{receiver}.{property_name}"); + return self.add_external_resolution_candidate( + property, + scope_id, + if construction { + CandidateRelation::Constructs + } else { + CandidateRelation::Calls + }, + if construction { + SemanticRole::Construction + } else { + SemanticRole::Call + }, + &property_name, + Some(&node_text(self.source, function)), + member_context, + &target, + "javascript.global", + argument_count, + argument_types, + &[ + "class", + "constructor", + "function", + "method", + "property", + "external", + ], + ); } // A dynamic receiver, proxy, or ambiguous member is not a // safe call target. Preserve its source occurrence as @@ -5314,7 +5349,32 @@ impl<'source, 'tree> CandidateState<'source, 'tree> { ); let Some(resolution) = resolution else { if self.is_unshadowed_builtin(scope_id, &spelling) { - return Ok(()); + if self.language != "typescript" { + return Ok(()); + } + let target_name = format!("global::{spelling}"); + return self.add_external_resolution_candidate( + target, + scope_id, + if construction { + CandidateRelation::Constructs + } else { + CandidateRelation::Calls + }, + if construction { + SemanticRole::Construction + } else { + SemanticRole::Call + }, + &spelling, + None, + call_context, + &target_name, + "javascript.global", + argument_count, + argument_types, + &["class", "constructor", "function", "external"], + ); } return self.add_unresolved_candidate( target, diff --git a/crates/compass-languages/src/evidence_pipeline.rs b/crates/compass-languages/src/evidence_pipeline.rs index c0b4f800..a5987bb0 100644 --- a/crates/compass-languages/src/evidence_pipeline.rs +++ b/crates/compass-languages/src/evidence_pipeline.rs @@ -302,6 +302,121 @@ pub(crate) const RUBY_CAPABILITIES: &[LanguageCapability] = &[ LanguageCapability::ExternalReferences, ]; +// Conservative common capabilities emitted by the AST-first extended +// language producer. Project-wide target selection and framework conventions +// remain outside the language boundary. +const DART_CAPABILITIES: &[LanguageCapability] = &[ + LanguageCapability::Declarations, + LanguageCapability::LexicalScopes, + LanguageCapability::Imports, + LanguageCapability::Reexports, + LanguageCapability::Aliases, + LanguageCapability::Calls, + LanguageCapability::Construction, + LanguageCapability::TypeReferences, + LanguageCapability::BaseTypes, + LanguageCapability::Members, + LanguageCapability::Ownership, + LanguageCapability::Receivers, + LanguageCapability::ExternalReferences, +]; + +const GROOVY_CAPABILITIES: &[LanguageCapability] = &[ + LanguageCapability::Declarations, + LanguageCapability::LexicalScopes, + LanguageCapability::Namespaces, + LanguageCapability::Imports, + LanguageCapability::Aliases, + LanguageCapability::Calls, + LanguageCapability::Construction, + LanguageCapability::Decorators, + LanguageCapability::TypeReferences, + LanguageCapability::BaseTypes, + LanguageCapability::Members, + LanguageCapability::Ownership, + LanguageCapability::Receivers, + LanguageCapability::ExternalReferences, +]; + +const SCALA_CAPABILITIES: &[LanguageCapability] = &[ + LanguageCapability::Declarations, + LanguageCapability::LexicalScopes, + LanguageCapability::Namespaces, + LanguageCapability::Traits, + LanguageCapability::Imports, + LanguageCapability::Aliases, + LanguageCapability::Calls, + LanguageCapability::Construction, + LanguageCapability::TypeReferences, + LanguageCapability::BaseTypes, + LanguageCapability::HierarchyDispatch, + LanguageCapability::Members, + LanguageCapability::Ownership, + LanguageCapability::Receivers, + LanguageCapability::ExternalReferences, +]; + +const SWIFT_CAPABILITIES: &[LanguageCapability] = &[ + LanguageCapability::Declarations, + LanguageCapability::LexicalScopes, + LanguageCapability::Traits, + LanguageCapability::Imports, + LanguageCapability::Aliases, + LanguageCapability::Calls, + LanguageCapability::Construction, + LanguageCapability::TypeReferences, + LanguageCapability::BaseTypes, + LanguageCapability::HierarchyDispatch, + LanguageCapability::Members, + LanguageCapability::Ownership, + LanguageCapability::Receivers, + LanguageCapability::ExternalReferences, +]; + +const DART_EVIDENCE_PIPELINE: UniversalEvidencePipeline = UniversalEvidencePipeline { + producer: UniversalEvidenceProducer { + id: "compass.dart", + language: "dart", + version: 1, + evidence_schema: crate::UNIVERSAL_EVIDENCE_SCHEMA, + capabilities: DART_CAPABILITIES, + }, + qualification: UniversalEvidenceQualification::Qualifying, +}; + +const GROOVY_EVIDENCE_PIPELINE: UniversalEvidencePipeline = UniversalEvidencePipeline { + producer: UniversalEvidenceProducer { + id: "compass.groovy", + language: "groovy", + version: 1, + evidence_schema: crate::UNIVERSAL_EVIDENCE_SCHEMA, + capabilities: GROOVY_CAPABILITIES, + }, + qualification: UniversalEvidenceQualification::Qualifying, +}; + +const SCALA_EVIDENCE_PIPELINE: UniversalEvidencePipeline = UniversalEvidencePipeline { + producer: UniversalEvidenceProducer { + id: "compass.scala", + language: "scala", + version: 1, + evidence_schema: crate::UNIVERSAL_EVIDENCE_SCHEMA, + capabilities: SCALA_CAPABILITIES, + }, + qualification: UniversalEvidenceQualification::Qualifying, +}; + +const SWIFT_EVIDENCE_PIPELINE: UniversalEvidencePipeline = UniversalEvidencePipeline { + producer: UniversalEvidenceProducer { + id: "compass.swift", + language: "swift", + version: 1, + evidence_schema: crate::UNIVERSAL_EVIDENCE_SCHEMA, + capabilities: SWIFT_CAPABILITIES, + }, + qualification: UniversalEvidenceQualification::Qualifying, +}; + pub(crate) const RUBY_EVIDENCE_PIPELINE: UniversalEvidencePipeline = UniversalEvidencePipeline { producer: UniversalEvidenceProducer { id: "compass.ruby", @@ -324,6 +439,7 @@ const UNIVERSAL_EVIDENCE_PIPELINES: &[UniversalEvidencePipeline] = &[ }, qualification: UniversalEvidenceQualification::Qualifying, }, + DART_EVIDENCE_PIPELINE, UniversalEvidencePipeline { producer: UniversalEvidenceProducer { id: "compass.go", @@ -334,6 +450,7 @@ const UNIVERSAL_EVIDENCE_PIPELINES: &[UniversalEvidencePipeline] = &[ }, qualification: UniversalEvidenceQualification::Qualifying, }, + GROOVY_EVIDENCE_PIPELINE, UniversalEvidencePipeline { producer: UniversalEvidenceProducer { id: "compass.java", @@ -395,6 +512,8 @@ const UNIVERSAL_EVIDENCE_PIPELINES: &[UniversalEvidencePipeline] = &[ }, qualification: UniversalEvidenceQualification::Qualifying, }, + SCALA_EVIDENCE_PIPELINE, + SWIFT_EVIDENCE_PIPELINE, UniversalEvidencePipeline { producer: UniversalEvidenceProducer { id: "compass.typescript", diff --git a/crates/compass-languages/src/frameworks/dart.rs b/crates/compass-languages/src/frameworks/dart.rs new file mode 100644 index 00000000..2688b0d7 --- /dev/null +++ b/crates/compass-languages/src/frameworks/dart.rs @@ -0,0 +1,316 @@ +//! Dart framework-pack adapters. +//! +//! The established convention edges are projected by this framework-owned +//! bridge; these universal adapters intentionally return no +//! untyped facts. Their descriptors provide the frozen pack IDs and evidence +//! activation contract, while structural Dart relationships remain owned by +//! the universal producer. + +use std::collections::{HashMap, HashSet}; +use std::path::Path; + +use serde_json::Value; + +use super::{RawFrameworkFact, UniversalDetectionContext}; +use crate::{ + Extraction, ProjectEvidence, RawNodeRecord as NodeRecord, SemanticEvidenceBatch, SemanticRole, + make_id, +}; + +pub(super) fn detect_flutter_navigation( + context: &UniversalDetectionContext<'_, '_>, +) -> Vec { + let has_flutter_import = context.evidence.occurrences.iter().any(|occurrence| { + occurrence.role == SemanticRole::Import + && occurrence.spelling.to_ascii_lowercase().contains("flutter") + }); + let has_navigation_call = context.evidence.occurrences.iter().any(|occurrence| { + occurrence.role == SemanticRole::Call + && matches!( + occurrence.spelling.as_str(), + "go" | "push" | "goNamed" | "pushNamed" | "replace" | "replaceNamed" + ) + }); + if has_flutter_import && has_navigation_call { + // The convention bridge emits the established anchored relation. An + // empty typed-fact set here prevents a second, guessed framework edge. + Vec::new() + } else { + Vec::new() + } +} + +pub(super) fn detect_bloc(context: &UniversalDetectionContext<'_, '_>) -> Vec { + let _activated = context.evidence.occurrences.iter().any(|occurrence| { + occurrence.role == SemanticRole::Call + && matches!(occurrence.spelling.as_str(), "on" | "emit" | "add") + }); + Vec::new() +} + +pub(super) fn detect_riverpod( + context: &UniversalDetectionContext<'_, '_>, +) -> Vec { + let _activated = context.evidence.occurrences.iter().any(|occurrence| { + occurrence.role == SemanticRole::Call + && matches!(occurrence.spelling.as_str(), "watch" | "read" | "listen") + }); + Vec::new() +} + +/// Preserve Dart's framework/domain conventions while the structural Dart +/// graph is hard-cut to universal evidence. These are deliberately limited to +/// convention-context edges (navigation, BLoC/Riverpod, and resource export) +/// and are marked as convention-origin facts; no legacy declaration or raw +/// call graph is copied into the universal extraction. +pub(super) fn append_convention_facts( + path: &Path, + source: &[u8], + project: Option<&ProjectEvidence>, + extraction: &mut Extraction, +) { + let conventions = crate::dart_framework::extract(path, source); + let Some(evidence) = extraction.semantic_evidence.as_ref() else { + return; + }; + let evidence_source_file = evidence + .declarations + .iter() + .find(|declaration| declaration.kind == "file") + .map(|declaration| declaration.range.source_file.clone()) + .unwrap_or_else(|| path.to_string_lossy().replace('\\', "/")); + let file_id = evidence + .declarations + .iter() + .find(|declaration| declaration.kind == "file") + .map(|declaration| declaration.graph_node_id.clone()); + let legacy_file_id = make_id(&[&path.to_string_lossy()]); + let mut labels = HashMap::>::new(); + for declaration in &evidence.declarations { + labels + .entry(declaration.name.clone()) + .or_default() + .push(declaration.graph_node_id.clone()); + let terminal = declaration + .qualified_name + .rsplit(['.', ':']) + .next() + .unwrap_or_default(); + if terminal != declaration.name { + labels + .entry(terminal.to_owned()) + .or_default() + .push(declaration.graph_node_id.clone()); + } + } + let framework_nodes = conventions + .nodes + .iter() + .map(|node| (node.id.as_str(), node)) + .collect::>(); + let mut endpoint_ids = HashSet::new(); + let mut contextual_endpoint_ids = HashSet::new(); + let mut projected_edges = Vec::new(); + for mut edge in conventions.edges { + let relation = edge.string("relation"); + let contextual = !edge.string("context").is_empty(); + if !contextual && relation != "exports" { + continue; + } + if contextual && !contextual_fact_is_activated(&edge, evidence, project) { + continue; + } + let source_id = project_dart_endpoint( + &edge.source, + &framework_nodes, + &labels, + file_id.as_deref(), + &legacy_file_id, + ); + let target_id = project_dart_endpoint( + &edge.target, + &framework_nodes, + &labels, + file_id.as_deref(), + &legacy_file_id, + ); + edge.source = source_id; + edge.target = target_id; + if contextual { + contextual_endpoint_ids.insert(edge.source.clone()); + contextual_endpoint_ids.insert(edge.target.clone()); + } + edge.attributes.insert( + "source_file".to_owned(), + Value::String(evidence_source_file.clone()), + ); + edge.attributes.insert( + "rule".to_owned(), + Value::String(if contextual { + format!("dart-{}", edge.string("context")) + } else { + "dart-resource-export".to_owned() + }), + ); + edge.attributes + .insert("_origin".to_owned(), Value::String("convention".to_owned())); + edge.attributes.insert( + "extractor".to_owned(), + Value::String("compass.languages.dart.framework".to_owned()), + ); + endpoint_ids.insert(edge.source.clone()); + endpoint_ids.insert(edge.target.clone()); + projected_edges.push(edge); + } + projected_edges.sort_unstable_by(|left, right| { + left.source + .cmp(&right.source) + .then_with(|| left.target.cmp(&right.target)) + .then_with(|| left.string("relation").cmp(&right.string("relation"))) + .then_with(|| { + left.attributes + .get("start_byte") + .and_then(Value::as_u64) + .cmp(&right.attributes.get("start_byte").and_then(Value::as_u64)) + }) + }); + for edge in projected_edges { + let duplicate = extraction.edges.iter().any(|existing| { + existing.source == edge.source + && existing.target == edge.target + && existing.string("relation") == edge.string("relation") + && existing.string("context") == edge.string("context") + && existing.attributes.get("start_byte") == edge.attributes.get("start_byte") + && existing.attributes.get("end_byte") == edge.attributes.get("end_byte") + }); + if !duplicate { + extraction.edges.push(edge); + } + } + let existing_nodes = extraction + .nodes + .iter() + .map(|node| node.id.clone()) + .collect::>(); + for node in conventions.nodes { + if !endpoint_ids.contains(&node.id) || existing_nodes.contains(&node.id) { + continue; + } + let mut node = node; + if contextual_endpoint_ids.contains(&node.id) { + node.attributes + .insert("_origin".to_owned(), Value::String("convention".to_owned())); + node.attributes.insert( + "extractor".to_owned(), + Value::String("compass.languages.dart.framework".to_owned()), + ); + } + extraction.nodes.push(node); + } +} + +fn contextual_fact_is_activated( + edge: &crate::RawEdgeRecord, + evidence: &SemanticEvidenceBatch, + project: Option<&ProjectEvidence>, +) -> bool { + let context = edge.string("context"); + let has_dependency = + |markers: &[&str]| project.is_some_and(|project| project.has_any_dependency(markers)); + let has_import = |needles: &[&str]| { + let occurrence_import = evidence.occurrences.iter().any(|occurrence| { + occurrence.role == SemanticRole::Import + && needles + .iter() + .any(|needle| occurrence.spelling.eq_ignore_ascii_case(needle)) + }); + let candidate_import = evidence.candidates.iter().any(|candidate| { + matches!( + candidate.relation, + crate::evidence::CandidateRelation::Imports + | crate::evidence::CandidateRelation::Reexports + ) && candidate + .constraints + .qualified_name + .as_deref() + .is_some_and(|target| { + needles + .iter() + .any(|needle| import_target_matches(target, needle)) + }) + }); + occurrence_import || candidate_import + }; + let has_call = |names: &[&str]| { + evidence.occurrences.iter().any(|occurrence| { + occurrence.role == SemanticRole::Call && names.contains(&occurrence.spelling.as_str()) + }) + }; + match context.as_str() { + "route_path" | "route_const" | "route_object" => { + (has_dependency(&["flutter"]) || has_import(&["flutter"])) + && has_call(&[ + "go", + "push", + "goNamed", + "pushNamed", + "replace", + "replaceNamed", + ]) + } + "bloc_event" | "emit_state" | "bloc_add_event" | "bloc_widget_binding" | "bloc_lookup" => { + (has_dependency(&["bloc", "flutter_bloc"]) || has_import(&["bloc", "flutter_bloc"])) + && (has_call(&["on", "emit", "add"]) + || evidence.occurrences.iter().any(|occurrence| { + matches!( + occurrence.role, + SemanticRole::TypeReference | SemanticRole::MemberAccess + ) && [ + "BlocBuilder", + "BlocListener", + "BlocConsumer", + "BlocProvider", + ] + .contains(&occurrence.spelling.as_str()) + })) + } + "riverpod_reference" => { + (has_dependency(&["riverpod", "hooks_riverpod"]) + || has_import(&["riverpod", "hooks_riverpod"])) + && has_call(&["watch", "read", "listen"]) + } + _ => false, + } +} + +fn import_target_matches(target: &str, marker: &str) -> bool { + let target = target.to_ascii_lowercase(); + let marker = marker.to_ascii_lowercase(); + target == marker + || target.starts_with(&format!("{marker}/")) + || target.starts_with(&format!("package:{marker}/")) +} + +fn project_dart_endpoint( + endpoint: &str, + framework_nodes: &HashMap<&str, &NodeRecord>, + labels: &HashMap>, + file_id: Option<&str>, + legacy_file_id: &str, +) -> String { + if let Some(file_id) = file_id + && endpoint == legacy_file_id + { + return file_id.to_owned(); + } + let Some(node) = framework_nodes.get(endpoint) else { + return endpoint.to_owned(); + }; + let label = node.label().trim_matches(['.', '(', ')']).to_owned(); + labels + .get(&label) + .filter(|ids| ids.len() == 1) + .and_then(|ids| ids.first()) + .cloned() + .unwrap_or_else(|| endpoint.to_owned()) +} diff --git a/crates/compass-languages/src/frameworks/mod.rs b/crates/compass-languages/src/frameworks/mod.rs index fe5ef709..6d86df0b 100644 --- a/crates/compass-languages/src/frameworks/mod.rs +++ b/crates/compass-languages/src/frameworks/mod.rs @@ -1,5 +1,6 @@ mod axum; mod csharp; +mod dart; mod enterprise; mod evidence; mod express; @@ -293,7 +294,13 @@ const FRAMEWORK_PACKS: &[FrameworkPack] = &[ FrameworkPack::source("go-web", &["go"], &[], detect_go), FrameworkPack::source("axum-web", &["rust"], &["axum"], detect_axum), FrameworkPack::source("rust-web", &["rust"], &[], detect_rust), - FrameworkPack::source("vapor-routes", &["swift"], &["vapor"], detect_swift), + FrameworkPack::universal(&pack::VAPOR_SWIFT_DESCRIPTOR, detect_swift_universal), + FrameworkPack::universal( + &pack::DART_FLUTTER_NAVIGATION_DESCRIPTOR, + dart::detect_flutter_navigation, + ), + FrameworkPack::universal(&pack::DART_BLOC_DESCRIPTOR, dart::detect_bloc), + FrameworkPack::universal(&pack::DART_RIVERPOD_DESCRIPTOR, dart::detect_riverpod), FrameworkPack::source( "express-web", &["javascript", "typescript", "tsx"], @@ -467,6 +474,13 @@ pub(crate) fn detect( } } accumulator.publish(extraction); + if language == "dart" && extraction.semantic_evidence.is_some() { + // Framework convention meaning is deliberately owned by this + // registry boundary; the language producer only emits structural + // universal evidence. Contextual facts additionally require positive + // source/manifest activation in their owning pack. + dart::append_convention_facts(path, source, project, extraction); + } } pub(crate) fn detect_config_file( @@ -547,10 +561,14 @@ fn detect_axum( axum::detect(context.path, context.source, context.root) } -fn detect_swift( - context: &DetectionContext<'_, '_>, - _extraction: &mut Extraction, -) -> Vec { +fn detect_swift_universal(context: &UniversalDetectionContext<'_, '_>) -> Vec { + let vapor_import = context.evidence.occurrences.iter().any(|occurrence| { + occurrence.role == crate::SemanticRole::Import + && occurrence.spelling.eq_ignore_ascii_case("Vapor") + }); + if !vapor_import { + return Vec::new(); + } swift::detect(context.path, context.source, context.root) } @@ -666,7 +684,10 @@ mod tests { "axum-web", "rust-web", "aspnet-csharp", - "vapor-routes", + "vapor-swift", + "dart-flutter-navigation", + "dart-bloc", + "dart-riverpod", "express-web", "fastify-web", "hono-web", diff --git a/crates/compass-languages/src/frameworks/pack.rs b/crates/compass-languages/src/frameworks/pack.rs index a33af432..5527289b 100644 --- a/crates/compass-languages/src/frameworks/pack.rs +++ b/crates/compass-languages/src/frameworks/pack.rs @@ -601,10 +601,101 @@ pub(super) const PHP_FRAMEWORKS_DESCRIPTOR: FrameworkPackDescriptor = FrameworkP limits: FrameworkLimits::DEFAULT, }; +pub(super) const VAPOR_SWIFT_DESCRIPTOR: FrameworkPackDescriptor = FrameworkPackDescriptor { + id: "vapor-swift", + kind: FrameworkPackKind::Source, + languages: &["swift"], + required_capabilities: &[ + LanguageCapability::Declarations, + LanguageCapability::LexicalScopes, + LanguageCapability::Imports, + LanguageCapability::Calls, + LanguageCapability::Ownership, + ], + framework_capabilities: &[FrameworkCapability::HttpRoutes], + dependency_markers: &["vapor"], + manifest_policy: FrameworkManifestPolicy::Advisory, + activation_rules: &["vapor-import", "vapor-route-call"], + accepted_roles: &[ + SemanticRole::Import, + SemanticRole::Call, + SemanticRole::Ownership, + ], + emitted_relation_families: &[FrameworkRelation::RoutesTo], + occurrence_policy: FrameworkOccurrencePolicy::ExactEvidence, + limits: FrameworkLimits::DEFAULT, +}; + +pub(super) const DART_FLUTTER_NAVIGATION_DESCRIPTOR: FrameworkPackDescriptor = + FrameworkPackDescriptor { + id: "dart-flutter-navigation", + kind: FrameworkPackKind::Source, + languages: &["dart"], + required_capabilities: &[ + LanguageCapability::Imports, + LanguageCapability::Calls, + LanguageCapability::Receivers, + ], + framework_capabilities: &[FrameworkCapability::HttpRoutes], + dependency_markers: &["flutter"], + manifest_policy: FrameworkManifestPolicy::Advisory, + activation_rules: &["flutter-navigation-call", "flutter-navigation-import"], + accepted_roles: &[ + SemanticRole::Import, + SemanticRole::Call, + SemanticRole::Receiver, + ], + emitted_relation_families: &[FrameworkRelation::RoutesTo], + occurrence_policy: FrameworkOccurrencePolicy::ExactAnchoredHeuristic, + limits: FrameworkLimits::DEFAULT, + }; + +pub(super) const DART_BLOC_DESCRIPTOR: FrameworkPackDescriptor = FrameworkPackDescriptor { + id: "dart-bloc", + kind: FrameworkPackKind::Source, + languages: &["dart"], + required_capabilities: &[ + LanguageCapability::Calls, + LanguageCapability::TypeReferences, + LanguageCapability::Members, + ], + framework_capabilities: &[FrameworkCapability::Messaging], + dependency_markers: &["bloc", "flutter_bloc"], + manifest_policy: FrameworkManifestPolicy::Advisory, + activation_rules: &["bloc-builder", "bloc-event", "bloc-provider"], + accepted_roles: &[ + SemanticRole::Call, + SemanticRole::TypeReference, + SemanticRole::MemberAccess, + ], + emitted_relation_families: &[FrameworkRelation::Handles], + occurrence_policy: FrameworkOccurrencePolicy::ExactAnchoredHeuristic, + limits: FrameworkLimits::DEFAULT, +}; + +pub(super) const DART_RIVERPOD_DESCRIPTOR: FrameworkPackDescriptor = FrameworkPackDescriptor { + id: "dart-riverpod", + kind: FrameworkPackKind::Source, + languages: &["dart"], + required_capabilities: &[LanguageCapability::Calls, LanguageCapability::Members], + framework_capabilities: &[FrameworkCapability::Messaging], + dependency_markers: &["hooks_riverpod", "riverpod"], + manifest_policy: FrameworkManifestPolicy::Advisory, + activation_rules: &["riverpod-provider", "riverpod-reference"], + accepted_roles: &[SemanticRole::Call, SemanticRole::MemberAccess], + emitted_relation_families: &[FrameworkRelation::Handles], + occurrence_policy: FrameworkOccurrencePolicy::ExactAnchoredHeuristic, + limits: FrameworkLimits::DEFAULT, +}; + const UNIVERSAL_FRAMEWORK_PACKS: &[FrameworkPackDescriptor] = &[ ASPNET_CSHARP_DESCRIPTOR, PHP_FRAMEWORKS_DESCRIPTOR, SPRING_JAVA_DESCRIPTOR, SPRING_KOTLIN_DESCRIPTOR, RAILS_RUBY_DESCRIPTOR, + VAPOR_SWIFT_DESCRIPTOR, + DART_BLOC_DESCRIPTOR, + DART_FLUTTER_NAVIGATION_DESCRIPTOR, + DART_RIVERPOD_DESCRIPTOR, ]; diff --git a/crates/compass-languages/src/frameworks/spring.rs b/crates/compass-languages/src/frameworks/spring.rs index 6db08252..d168f115 100644 --- a/crates/compass-languages/src/frameworks/spring.rs +++ b/crates/compass-languages/src/frameworks/spring.rs @@ -281,6 +281,10 @@ fn constant_facts(context: &UniversalDetectionContext<'_, '_>) -> Vec) -> Vec = LazyLock::new(|| regex(r"^\s*import\s+(?:static\s+)?([\w.]+)")); -static TYPE: LazyLock = LazyLock::new(|| { - regex( - r"^\s*(?:[\w@]+\s+)*(class|interface)\s+(\w+)(?:\s+extends\s+([\w.]+))?(?:\s+implements\s+([^\{]+))?", - ) -}); -static METHOD: LazyLock = LazyLock::new(|| { - regex( - r"^\s*(?:(?:public|protected|private|static|final|abstract|synchronized)\s+)*(?:def|[\w<>\[\].?]+)\s+(\w+)\s*\(", - ) -}); -static CONSTRUCTOR: LazyLock = - LazyLock::new(|| regex(r"^\s*(?:(?:public|protected|private)\s+)?([A-Z]\w*)\s*\(")); -static MEMBER_CALL: LazyLock = - LazyLock::new(|| regex(r"([A-Za-z_]\w*)\s*\.\s*([A-Za-z_]\w*)\s*\(")); -static SPOCK_FEATURE: LazyLock = - LazyLock::new(|| regex(r#"^\s*def\s+(?:\"([^\"]+)\"|'([^']+)')\s*\("#)); - -pub(crate) fn extract(path: &Path, source: &[u8]) -> Extraction { - let text = String::from_utf8_lossy(source); - if text.contains("spock.lang.Specification") - && text.lines().any(|line| SPOCK_FEATURE.is_match(line)) - { - return extract_spock(path, &text); - } - extract_regular(path, &text) -} - -fn extract_regular(path: &Path, source: &str) -> Extraction { - let source_file = path.to_string_lossy().into_owned(); - let stem = file_stem(path); - let file_id = make_id(&[&source_file]); - let mut state = State::new(source_file, stem, file_id.clone()); - state.add_node( - file_id.clone(), - path.file_name() - .and_then(|name| name.to_str()) - .unwrap_or_default(), - 1, - false, - ); - let lines: Vec<_> = source.lines().collect(); - for (index, text) in lines.iter().enumerate() { - if let Some(captures) = IMPORT.captures(text) { - let target = captures - .get(1) - .map(|value| value.as_str()) - .unwrap_or_default() - .rsplit('.') - .next() - .unwrap_or_default(); - if !target.is_empty() { - state.add_edge( - &file_id, - &make_id(&[target]), - "imports", - index + 1, - Some("import"), - ); - } - } - } - - let mut classes = Vec::new(); - let mut depth = 0_i32; - let mut active: Option = None; - for (index, text) in lines.iter().enumerate() { - let at_line = index + 1; - if active.is_none() - && let Some(captures) = TYPE.captures(text) - { - let name = capture(&captures, 2); - if !name.is_empty() { - let id = make_id(&[&state.stem, name]); - state.add_node(id.clone(), name, at_line, true); - state.add_edge(&file_id, &id, "contains", at_line, None); - if let Some(base) = captures.get(3).map(|value| value.as_str()) { - let base = base.rsplit('.').next().unwrap_or(base); - let target = state.ensure_base(base); - state.add_edge(&id, &target, "inherits", at_line, None); - } - if let Some(interfaces) = captures.get(4).map(|value| value.as_str()) { - for interface in interfaces - .split(',') - .map(str::trim) - .filter(|name| !name.is_empty()) - { - let interface = interface.rsplit('.').next().unwrap_or(interface); - let target = state.ensure_base(interface); - state.add_edge(&id, &target, "implements", at_line, None); - } - } - depth += brace_delta(text); - active = Some(ActiveClass { - id, - name: name.to_owned(), - start_depth: depth - brace_delta(text), - current_method: None, - }); - continue; - } - } - if let Some(class) = &mut active { - let before = depth; - let method_name = CONSTRUCTOR - .captures(text) - .filter(|captures| capture(captures, 1) == class.name) - .map(|captures| (capture(&captures, 1).to_owned(), true)) - .or_else(|| { - METHOD - .captures(text) - .map(|captures| (capture(&captures, 1).to_owned(), false)) - }); - if class.current_method.is_none() - && let Some((name, constructor)) = method_name - { - let id = make_id(&[&class.id, &name]); - let declaration_line = if constructor { - groovy_constructor_line(&lines, index) - } else { - at_line - }; - state.add_node(id.clone(), &format!(".{name}()"), declaration_line, true); - state.add_edge(&class.id, &id, "method", declaration_line, None); - class.current_method = Some(ActiveMethod { - start_depth: before, - }); - } - depth += brace_delta(text); - if let Some(method) = &class.current_method - && depth <= method.start_depth - { - class.current_method = None; - } - if depth <= class.start_depth { - classes.push(active.take().unwrap_or_else(|| unreachable!())); - } - } else { - depth += brace_delta(text); - } - } - if let Some(class) = active { - classes.push(class); - } - - let call_targets: HashMap = state - .extraction - .nodes - .iter() - .filter_map(|node| { - node.attributes - .get("label") - .and_then(Value::as_str) - .map(|label| { - ( - label - .trim_matches(['(', ')']) - .trim_start_matches('.') - .to_owned(), - node.id.clone(), - ) - }) - }) - .collect(); - let mut seen_calls = HashSet::new(); - depth = 0; - let mut active_class: Option<(String, i32)> = None; - let mut active_method: Option<(String, i32)> = None; - for (index, text) in lines.iter().enumerate() { - if active_class.is_none() - && let Some(captures) = TYPE.captures(text) - { - let name = capture(&captures, 2); - active_class = Some((make_id(&[&state.stem, name]), depth)); - depth += brace_delta(text); - continue; - } - if let Some((class_id, class_depth)) = active_class.clone() { - let before = depth; - if active_method.is_none() { - let class_name = state - .extraction - .nodes - .iter() - .find(|node| node.id == class_id) - .and_then(|node| node.attributes.get("label")) - .and_then(Value::as_str) - .unwrap_or_default(); - let name = CONSTRUCTOR - .captures(text) - .filter(|captures| capture(captures, 1) == class_name) - .map(|captures| capture(&captures, 1).to_owned()) - .or_else(|| { - METHOD - .captures(text) - .map(|captures| capture(&captures, 1).to_owned()) - }); - if let Some(name) = name { - active_method = Some((make_id(&[&class_id, &name]), before)); - } - } - if let Some((caller, _)) = &active_method { - let line_start = source - .split_inclusive('\n') - .take(index) - .map(str::len) - .sum::(); - for captures in MEMBER_CALL.captures_iter(text) { - let callee = capture(&captures, 2); - if callee.is_empty() { - continue; - } - let Some(callee_match) = captures.get(2) else { - continue; - }; - let start = line_start + callee_match.start(); - let end = line_start + callee_match.end(); - if let Some(target) = call_targets - .get(callee) - .filter(|target| target.as_str() != caller) - { - if seen_calls.insert((caller.clone(), target.clone(), start, end)) { - state.add_edge(caller, target, "calls", index + 1, Some("call")); - if let Some(edge) = state.extraction.edges.last_mut() { - stamp_source_range( - &mut edge.attributes, - source.as_bytes(), - start, - end, - ); - } - } - } else { - state.extraction.raw_calls_mut().push(RawCall { - caller_nid: caller.clone(), - callee: callee.to_owned(), - is_member_call: Some(false), - source_file: state.source_file.clone(), - source_location: format!("L{}", index + 1), - receiver: Some(None), - receiver_type: None, - lang: None, - extensions: source_range(source.as_bytes(), start, end), - }); - } - } - } - depth += brace_delta(text); - if active_method - .as_ref() - .is_some_and(|(_, method_depth)| depth <= *method_depth) - { - active_method = None; - } - if depth <= class_depth { - active_class = None; - } - } else { - depth += brace_delta(text); - } - } - state.extraction -} - -fn extract_spock(path: &Path, source: &str) -> Extraction { - let source_file = path.to_string_lossy().into_owned(); - let stem = file_stem(path); - let file_id = make_id(&[&source_file]); - let mut state = State::new(source_file, stem, file_id.clone()); - state.extraction.raw_calls = None; - state.add_node( - file_id.clone(), - path.file_name() - .and_then(|name| name.to_str()) - .unwrap_or_default(), - 1, - false, - ); - let mut class: Option = None; - for (index, text) in source.lines().enumerate() { - if let Some(captures) = IMPORT.captures(text) { - let target = capture(&captures, 1).rsplit('.').next().unwrap_or_default(); - state.add_edge( - &file_id, - &make_id(&[target]), - "imports", - index + 1, - Some("import"), - ); - continue; - } - if let Some(captures) = TYPE.captures(text) { - let name = capture(&captures, 2); - let id = make_id(&[&state.stem, name]); - state.add_node(id.clone(), name, index + 1, false); - state.add_edge(&file_id, &id, "contains", index + 1, None); - class = Some(id); - continue; - } - let Some(class_id) = &class else { - continue; - }; - if let Some(captures) = SPOCK_FEATURE.captures(text) { - let name = captures - .get(1) - .or_else(|| captures.get(2)) - .map(|value| value.as_str()) - .unwrap_or_default(); - let id = make_id(&[class_id, name]); - state.add_node(id.clone(), &format!("\"{name}\""), index + 1, false); - state.add_edge(class_id, &id, "method", index + 1, None); - } else if let Some(captures) = METHOD.captures(text) { - let name = capture(&captures, 1); - let id = make_id(&[class_id, name]); - state.add_node(id.clone(), &format!(".{name}()"), index + 1, false); - state.add_edge(class_id, &id, "method", index + 1, None); - } - } - state.extraction -} - -struct ActiveClass { - id: String, - name: String, - start_depth: i32, - current_method: Option, -} - -struct ActiveMethod { - start_depth: i32, -} - -struct State { - source_file: String, - stem: String, - extraction: Extraction, - seen_nodes: HashSet, -} - -impl State { - fn new(source_file: String, stem: String, _file_id: String) -> Self { - Self { - source_file, - stem, - extraction: Extraction::default(), - seen_nodes: HashSet::new(), - } - } - - fn ensure_base(&mut self, name: &str) -> String { - let local = make_id(&[&self.stem, name]); - if self.seen_nodes.contains(&local) { - return local; - } - let id = make_id(&[name]); - if self.seen_nodes.insert(id.clone()) { - let mut attributes = Map::new(); - attributes.insert("label".to_owned(), Value::String(name.to_owned())); - attributes.insert("file_type".to_owned(), Value::String("code".to_owned())); - attributes.insert("type".to_owned(), Value::String("class".to_owned())); - attributes.insert("source_file".to_owned(), Value::String(String::new())); - attributes.insert("source_location".to_owned(), Value::String(String::new())); - attributes.insert( - "origin_file".to_owned(), - Value::String(self.source_file.clone()), - ); - self.extraction.nodes.push(NodeRecord { - id: id.clone(), - attributes, - }); - } - id - } - - fn add_node(&mut self, id: String, label: &str, line: usize, callable: bool) { - if !self.seen_nodes.insert(id.clone()) { - return; - } - let mut attributes = Map::new(); - attributes.insert("label".to_owned(), Value::String(label.to_owned())); - attributes.insert("file_type".to_owned(), Value::String("code".to_owned())); - attributes.insert( - "source_file".to_owned(), - Value::String(self.source_file.clone()), - ); - attributes.insert( - "source_location".to_owned(), - Value::String(format!("L{line}")), - ); - if callable { - attributes.insert("_callable".to_owned(), Value::Bool(true)); - } - self.extraction.nodes.push(NodeRecord { id, attributes }); - } - - fn add_edge( - &mut self, - source: &str, - target: &str, - relation: &str, - line: usize, - context: Option<&str>, - ) { - let mut attributes = Map::new(); - attributes.insert("relation".to_owned(), Value::String(relation.to_owned())); - attributes.insert( - "confidence".to_owned(), - Value::String("EXTRACTED".to_owned()), - ); - attributes.insert( - "source_file".to_owned(), - Value::String(self.source_file.clone()), - ); - attributes.insert( - "source_location".to_owned(), - Value::String(format!("L{line}")), - ); - attributes.insert("weight".to_owned(), json!(1.0)); - if let Some(context) = context { - attributes.insert("context".to_owned(), Value::String(context.to_owned())); - } - self.extraction.edges.push(EdgeRecord { - source: source.to_owned(), - target: target.to_owned(), - attributes, - }); - } -} - -fn brace_delta(line: &str) -> i32 { - line.bytes().fold(0, |depth, byte| match byte { - b'{' => depth + 1, - b'}' => depth - 1, - _ => depth, - }) -} - -fn groovy_constructor_line(lines: &[&str], index: usize) -> usize { - for previous in (0..index).rev() { - let text = lines[previous].trim(); - if text.is_empty() { - continue; - } - if !text.contains('(') && !text.contains('{') && !text.contains('}') { - return previous + 1; - } - break; - } - index + 1 -} - -fn capture<'capture>(captures: &'capture regex::Captures<'_>, index: usize) -> &'capture str { - captures - .get(index) - .map(|value| value.as_str()) - .unwrap_or_default() -} - -fn regex(pattern: &str) -> Regex { - Regex::new(pattern) - .unwrap_or_else(|error| unreachable!("static Groovy regex is invalid: {error}")) -} diff --git a/crates/compass-languages/src/lib.rs b/crates/compass-languages/src/lib.rs index 2acdad9e..d132ecfc 100644 --- a/crates/compass-languages/src/lib.rs +++ b/crates/compass-languages/src/lib.rs @@ -5,7 +5,7 @@ mod bash; mod builtins; mod config; mod cpp; -mod dart; +mod dart_framework; mod dm; mod dotnet_project; mod elixir; @@ -19,7 +19,6 @@ pub mod frameworks; /// Version of the extraction contract consumed by graph publication. pub const EXTRACTION_SEMANTICS_VERSION: &str = "compass.languages.extraction/3"; mod go; -mod groovy; mod html; mod ids; mod json_config; @@ -39,7 +38,6 @@ mod registry; mod scip; mod semantic; mod sql; -mod swift; mod templates; mod terraform; mod verilog; diff --git a/crates/compass-languages/src/project_evidence.rs b/crates/compass-languages/src/project_evidence.rs index bec21067..2c8ad038 100644 --- a/crates/compass-languages/src/project_evidence.rs +++ b/crates/compass-languages/src/project_evidence.rs @@ -38,6 +38,9 @@ const FIXED_MANIFEST_NAMES: &[&str] = &[ "Cargo.toml", "go.mod", "Package.swift", + "pubspec.yaml", + "pubspec.yml", + "build.sbt", ]; const FIXED_CONFIGURATION_NAMES: &[&str] = &[ "application.properties", @@ -74,6 +77,8 @@ pub struct ProjectEvidence { project_root: PathBuf, manifests: Vec, ecosystems: Vec, + metadata: BTreeMap, + source_roots: Vec, dependencies: BTreeSet, configuration_files: Vec, configuration_keys: BTreeSet, @@ -118,6 +123,21 @@ impl ProjectEvidence { &self.ecosystems } + /// Bounded, source-only project metadata such as a package name or an + /// explicitly declared language/toolchain version. Values are never + /// obtained by evaluating a build tool or project script. + #[must_use] + pub fn metadata(&self) -> &BTreeMap { + &self.metadata + } + + /// Project-contained source roots declared by a manifest. The paths are + /// normalized relative paths and are only advisory to downstream stages. + #[must_use] + pub fn source_roots(&self) -> &[String] { + &self.source_roots + } + #[must_use] pub fn dependencies(&self) -> &BTreeSet { &self.dependencies @@ -291,6 +311,23 @@ impl ProjectEvidenceIndex { builder.manifests.insert(file_name(&project_file)); if let Some(parsed) = parse_manifest(&project_file) { builder.ecosystems.insert(parsed.ecosystem.to_owned()); + for (key, value) in parsed.metadata { + if builder.metadata.len() < MAX_PROJECT_CONFIGURATION_KEYS + || builder.metadata.contains_key(&key) + { + builder.metadata.insert(key, value); + } + } + for root in parsed.source_roots { + let Some(root) = + contained_manifest_root(&repository_root, &project_root, &root) + else { + continue; + }; + if builder.source_roots.len() < MAX_PROJECT_ROUTE_ROOTS { + builder.source_roots.insert(root); + } + } let remaining = MAX_DEPENDENCIES_PER_PROJECT.saturating_sub(builder.dependencies.len()); builder @@ -434,6 +471,8 @@ impl ProjectEvidenceIndex { struct ProjectBuilder { manifests: BTreeSet, ecosystems: BTreeSet, + metadata: BTreeMap, + source_roots: BTreeSet, dependencies: BTreeSet, configuration_files: BTreeSet, configuration_keys: BTreeSet, @@ -447,6 +486,8 @@ struct ProjectBuilder { struct ParsedManifest { ecosystem: &'static str, dependencies: BTreeSet, + metadata: BTreeMap, + source_roots: BTreeSet, } #[derive(Default)] @@ -463,6 +504,8 @@ fn finish_project( ) -> ProjectEvidence { let manifests = builder.manifests.into_iter().collect::>(); let ecosystems = builder.ecosystems.into_iter().collect::>(); + let metadata = builder.metadata; + let source_roots = builder.source_roots.into_iter().collect::>(); let dependencies = builder.dependencies; let configuration_files = builder.configuration_files.into_iter().collect::>(); let configuration_keys = builder.configuration_keys; @@ -492,6 +535,16 @@ fn finish_project( digest.update(ecosystem.as_bytes()); digest.update([0]); } + for (key, value) in &metadata { + digest.update(key.as_bytes()); + digest.update([0]); + digest.update(value.as_bytes()); + digest.update([0]); + } + for source_root in &source_roots { + digest.update(source_root.as_bytes()); + digest.update([0]); + } for dependency in &dependencies { digest.update(dependency.as_bytes()); digest.update([0]); @@ -543,6 +596,8 @@ fn finish_project( project_root, manifests, ecosystems, + metadata, + source_roots, dependencies, configuration_files, configuration_keys, @@ -778,21 +833,92 @@ fn parse_manifest(path: &Path) -> Option { let source = fs::read_to_string(path).ok()?; let name = path.file_name()?.to_str()?; let lower = name.to_ascii_lowercase(); - let (ecosystem, dependencies) = match lower.as_str() { - "package.json" => ("npm", json_dependencies(&source, NPM_DEPENDENCY_KEYS)?), + let (ecosystem, dependencies, metadata, source_roots) = match lower.as_str() { + "package.json" => ( + "npm", + json_dependencies(&source, NPM_DEPENDENCY_KEYS)?, + BTreeMap::new(), + BTreeSet::new(), + ), "composer.json" => ( "composer", json_dependencies(&source, &["require", "require-dev"])?, + BTreeMap::new(), + BTreeSet::new(), + ), + "pyproject.toml" => ( + "python", + pyproject_dependencies(&source)?, + BTreeMap::new(), + BTreeSet::new(), + ), + "requirements.txt" | "requirements.in" => ( + "python", + requirements_dependencies(&source), + BTreeMap::new(), + BTreeSet::new(), + ), + "gemfile" => ( + "ruby", + gemfile_dependencies(&source), + BTreeMap::new(), + BTreeSet::new(), + ), + "pom.xml" => ( + "maven", + pom_dependencies(&source)?, + BTreeMap::new(), + BTreeSet::new(), + ), + "build.gradle" | "build.gradle.kts" => { + let (metadata, source_roots) = gradle_project_metadata(&source); + ( + "gradle", + gradle_dependencies(&source), + metadata, + source_roots, + ) + } + "cargo.toml" => ( + "cargo", + cargo_dependencies(&source)?, + BTreeMap::new(), + BTreeSet::new(), + ), + "go.mod" => ( + "go", + go_mod_dependencies(&source), + BTreeMap::new(), + BTreeSet::new(), + ), + "package.swift" => { + let (metadata, source_roots) = swift_package_metadata(&source); + ( + "swift", + swift_package_dependencies(&source), + metadata, + source_roots, + ) + } + "pubspec.yaml" | "pubspec.yml" => { + let parsed = pubspec_metadata(&source)?; + ( + "dart", + parsed.dependencies, + parsed.metadata, + parsed.source_roots, + ) + } + "build.sbt" => { + let (dependencies, metadata, source_roots) = sbt_project_metadata(&source); + ("scala", dependencies, metadata, source_roots) + } + _ if lower.ends_with(".csproj") => ( + "dotnet", + csproj_dependencies(&source)?, + BTreeMap::new(), + BTreeSet::new(), ), - "pyproject.toml" => ("python", pyproject_dependencies(&source)?), - "requirements.txt" | "requirements.in" => ("python", requirements_dependencies(&source)), - "gemfile" => ("ruby", gemfile_dependencies(&source)), - "pom.xml" => ("maven", pom_dependencies(&source)?), - "build.gradle" | "build.gradle.kts" => ("gradle", gradle_dependencies(&source)), - "cargo.toml" => ("cargo", cargo_dependencies(&source)?), - "go.mod" => ("go", go_mod_dependencies(&source)), - "package.swift" => ("swift", swift_package_dependencies(&source)), - _ if lower.ends_with(".csproj") => ("dotnet", csproj_dependencies(&source)?), _ => return None, }; Some(ParsedManifest { @@ -803,6 +929,8 @@ fn parse_manifest(path: &Path) -> Option { .filter(|dependency| !dependency.is_empty()) .take(MAX_DEPENDENCIES_PER_PROJECT) .collect(), + metadata, + source_roots, }) } @@ -992,6 +1120,244 @@ fn swift_package_dependencies(source: &str) -> Vec { .collect() } +fn swift_package_metadata(source: &str) -> (BTreeMap, BTreeSet) { + let mut metadata = BTreeMap::new(); + let mut source_roots = default_source_roots(["Sources", "Tests"]); + let Ok(package_name) = Regex::new(r#"\bname\s*:\s*[\"']([^\"']+)[\"']"#) else { + return (metadata, source_roots); + }; + if let Some(name) = package_name + .captures(source) + .and_then(|capture| capture.get(1)) + .map(|value| value.as_str()) + .filter(|value| !value.is_empty() && value.len() <= 512) + { + metadata.insert("swift.package.name".to_owned(), name.to_owned()); + } + let Ok(paths) = Regex::new(r#"\bpath\s*:\s*[\"']([^\"']+)[\"']"#) else { + return (metadata, source_roots); + }; + for capture in paths.captures_iter(source).take(MAX_PROJECT_ROUTE_ROOTS) { + let Some(path) = capture + .get(1) + .and_then(|value| bounded_source_root(value.as_str())) + else { + continue; + }; + source_roots.insert(path); + } + (metadata, source_roots) +} + +struct ParsedPubspec { + dependencies: Vec, + metadata: BTreeMap, + source_roots: BTreeSet, +} + +fn pubspec_metadata(source: &str) -> Option { + let root = serde_yaml_ng::from_str::(source).ok()?; + let mapping = root.as_mapping()?; + let mut dependencies = Vec::new(); + for section in ["dependencies", "dev_dependencies", "dependency_overrides"] { + let Some(values) = + yaml_mapping_value(mapping, section).and_then(|value| value.as_mapping()) + else { + continue; + }; + dependencies.extend( + values + .keys() + .filter_map(|key| key.as_str().map(str::to_owned)), + ); + } + let mut metadata = BTreeMap::new(); + if let Some(name) = yaml_mapping_value(mapping, "name") + .and_then(serde_yaml_ng::Value::as_str) + .filter(|value| !value.is_empty() && value.len() <= 512) + { + metadata.insert("dart.package.name".to_owned(), name.to_owned()); + } + if let Some(sdk) = yaml_mapping_value(mapping, "environment") + .and_then(serde_yaml_ng::Value::as_mapping) + .and_then(|environment| yaml_mapping_value(environment, "sdk")) + .and_then(serde_yaml_ng::Value::as_str) + .filter(|value| !value.is_empty() && value.len() <= 512) + { + metadata.insert("dart.sdk".to_owned(), sdk.to_owned()); + } + if let Some(flutter) = yaml_mapping_value(mapping, "flutter") + .and_then(serde_yaml_ng::Value::as_mapping) + .and_then(|flutter| yaml_mapping_value(flutter, "module")) + .and_then(serde_yaml_ng::Value::as_str) + .and_then(bounded_source_root) + { + metadata.insert("flutter.module".to_owned(), flutter); + } + Some(ParsedPubspec { + dependencies, + metadata, + source_roots: default_source_roots(["lib", "bin", "test", "tool", "web"]), + }) +} + +fn yaml_mapping_value<'value>( + mapping: &'value serde_yaml_ng::Mapping, + key: &str, +) -> Option<&'value serde_yaml_ng::Value> { + mapping.get(serde_yaml_ng::Value::String(key.to_owned())) +} + +fn sbt_project_metadata(source: &str) -> (Vec, BTreeMap, BTreeSet) { + let mut dependencies = Vec::new(); + let mut metadata = BTreeMap::new(); + let mut source_roots = default_source_roots(["src/main/scala", "src/test/scala"]); + for line in source.lines().take(MAX_PROJECT_CONFIGURATION_KEYS) { + let values = quoted_values(line).take(4).collect::>(); + if line.contains('%') && values.len() >= 2 { + dependencies.push(format!("{}:{}", values[0], values[1])); + } + } + collect_sbt_metadata(source, "scalaVersion", "scala.version", &mut metadata); + collect_sbt_metadata(source, "sbtVersion", "sbt.version", &mut metadata); + collect_sbt_metadata(source, "organization", "scala.organization", &mut metadata); + let Ok(paths) = Regex::new( + r#"(?:scalaSource|javaSource|sourceDirectory)\s*:?=\s*[^\n\r]*?[\"']([^\"']+)[\"']"#, + ) else { + return (dependencies, metadata, source_roots); + }; + for capture in paths.captures_iter(source).take(MAX_PROJECT_ROUTE_ROOTS) { + if let Some(path) = capture + .get(1) + .and_then(|value| bounded_source_root(value.as_str())) + { + source_roots.insert(path); + } + } + (dependencies, metadata, source_roots) +} + +fn collect_sbt_metadata( + source: &str, + setting: &str, + key: &str, + output: &mut BTreeMap, +) { + let pattern = format!(r#"(?m)\b{setting}\s*:?=\s*[\"']([^\"']+)[\"']"#); + let Ok(pattern) = Regex::new(&pattern) else { + return; + }; + if let Some(value) = pattern + .captures(source) + .and_then(|capture| capture.get(1)) + .map(|value| value.as_str()) + .filter(|value| !value.is_empty() && value.len() <= 512) + { + output.insert(key.to_owned(), value.to_owned()); + } +} + +fn gradle_project_metadata(source: &str) -> (BTreeMap, BTreeSet) { + let mut metadata = BTreeMap::new(); + let mut source_roots = default_source_roots([ + "src/main/groovy", + "src/test/groovy", + "src/main/java", + "src/test/java", + ]); + for (setting, key) in [("group", "gradle.group"), ("version", "gradle.version")] { + collect_gradle_metadata(source, setting, key, &mut metadata); + } + let Ok(paths) = Regex::new( + r#"(?:srcDirs|srcDir|srcDirs\.from)\s*(?:=|\+=|\()\s*[^\n\r]*?[\"']([^\"']+)[\"']"#, + ) else { + return (metadata, source_roots); + }; + for capture in paths.captures_iter(source).take(MAX_PROJECT_ROUTE_ROOTS) { + if let Some(path) = capture + .get(1) + .and_then(|value| bounded_source_root(value.as_str())) + { + source_roots.insert(path); + } + } + (metadata, source_roots) +} + +fn collect_gradle_metadata( + source: &str, + setting: &str, + key: &str, + output: &mut BTreeMap, +) { + let pattern = format!( + r#"(?m)^\s*{setting}\s*(?:=|:)\s*[\"']([^\"']+)[\"']|\b{setting}\s*=\s*[\"']([^\"']+)[\"']"# + ); + let Ok(pattern) = Regex::new(&pattern) else { + return; + }; + if let Some(value) = pattern + .captures(source) + .and_then(|capture| capture.get(1).or_else(|| capture.get(2))) + .map(|value| value.as_str()) + .filter(|value| !value.is_empty() && value.len() <= 512) + { + output.insert(key.to_owned(), value.to_owned()); + } +} + +fn bounded_source_root(value: &str) -> Option { + let value = value.trim(); + if value.is_empty() + || value.len() > 4_096 + || value.starts_with(['/', '\\']) + || value.as_bytes().get(1) == Some(&b':') + { + return None; + } + let mut components = Vec::new(); + for component in Path::new(value).components() { + match component { + std::path::Component::CurDir => {} + std::path::Component::Normal(segment) => components.push(segment.to_string_lossy()), + std::path::Component::ParentDir + | std::path::Component::RootDir + | std::path::Component::Prefix(_) => return None, + } + } + (!components.is_empty()).then(|| components.join("/")) +} + +fn default_source_roots(roots: [&str; N]) -> BTreeSet { + roots.into_iter().map(str::to_owned).collect() +} + +fn contained_manifest_root( + repository_root: &Path, + project_root: &Path, + relative: &str, +) -> Option { + let candidate = project_root.join(relative); + if !candidate.starts_with(repository_root) { + return None; + } + if !candidate.is_dir() { + return None; + } + let canonical_repository = fs::canonicalize(repository_root).ok()?; + let mut existing_ancestor = candidate.clone(); + while !existing_ancestor.exists() { + if existing_ancestor == repository_root || !existing_ancestor.pop() { + return None; + } + } + let canonical_ancestor = fs::canonicalize(existing_ancestor).ok()?; + if !canonical_ancestor.starts_with(canonical_repository) { + return None; + } + Some(normalize_project_path(relative)) +} + fn first_quoted(value: &str) -> Option { quoted_values(value).next() } @@ -1800,4 +2166,84 @@ mod tests { assert!(evidence.configuration_keys().contains("server.port")); Ok(()) } + + #[test] + fn language_manifests_are_bounded_and_never_execute_build_tools() -> Result<(), Box> + { + let directory = tempdir()?; + let root = directory.path(); + let source = root.join("lib/main.dart"); + fs::create_dir_all(source.parent().ok_or("source has no parent")?)?; + fs::write(&source, "class Main {}\n")?; + fs::write( + root.join("pubspec.yaml"), + "name: sample_app\nenvironment:\n sdk: ^3.4.0\ndependencies:\n flutter:\n sdk: flutter\n riverpod: ^2.5.0\n", + )?; + fs::write( + root.join("build.sbt"), + "scalaVersion := \"3.3.3\"\nlibraryDependencies += \"org.typelevel\" %% \"cats-core\" % \"2.12.0\"\n", + )?; + fs::write( + root.join("Package.swift"), + "let package = Package(name: \"Sample\", targets: [.target(name: \"Sample\", path: \"Sources/Sample\")])\n", + )?; + + let first = ProjectEvidenceIndex::build(root, std::slice::from_ref(&source)); + let second = ProjectEvidenceIndex::build(root, std::slice::from_ref(&source)); + let evidence = first.evidence_for(&source); + assert!(evidence.has_dependency("riverpod")); + assert_eq!( + evidence.metadata().get("dart.package.name"), + Some(&"sample_app".to_owned()) + ); + assert_eq!( + evidence.metadata().get("dart.sdk"), + Some(&"^3.4.0".to_owned()) + ); + assert!(evidence.source_roots().iter().any(|root| root == "lib")); + assert!( + evidence + .manifests() + .iter() + .any(|name| name == "pubspec.yaml") + ); + assert_eq!( + evidence.fingerprint(), + second.evidence_for(&source).fingerprint() + ); + + let scala_source = root.join("src/main/scala/Main.scala"); + fs::create_dir_all(scala_source.parent().ok_or("scala source has no parent")?)?; + fs::write(&scala_source, "object Main {}\n")?; + let scala_index = ProjectEvidenceIndex::build(root, std::slice::from_ref(&scala_source)); + let scala = scala_index.evidence_for(&scala_source); + assert!(scala.has_dependency("org.typelevel:cats-core")); + assert_eq!( + scala.metadata().get("scala.version"), + Some(&"3.3.3".to_owned()) + ); + assert!( + scala + .source_roots() + .iter() + .any(|root| root == "src/main/scala") + ); + + let swift_source = root.join("Sources/Sample/Main.swift"); + fs::create_dir_all(swift_source.parent().ok_or("swift source has no parent")?)?; + fs::write(&swift_source, "struct Main {}\n")?; + let swift_index = ProjectEvidenceIndex::build(root, std::slice::from_ref(&swift_source)); + let swift = swift_index.evidence_for(&swift_source); + assert_eq!( + swift.metadata().get("swift.package.name"), + Some(&"Sample".to_owned()) + ); + assert!( + swift + .source_roots() + .iter() + .any(|root| root == "Sources/Sample") + ); + Ok(()) + } } diff --git a/crates/compass-languages/src/swift.rs b/crates/compass-languages/src/swift.rs deleted file mode 100644 index 89defa43..00000000 --- a/crates/compass-languages/src/swift.rs +++ /dev/null @@ -1,822 +0,0 @@ -use std::collections::{HashMap, HashSet}; -use std::path::Path; - -use crate::{RawEdgeRecord as EdgeRecord, RawNodeRecord as NodeRecord}; -use serde_json::{Map, Value, json}; -use tree_sitter::Node; - -use crate::{Extraction, RawCall, file_stem, is_language_builtin_global, make_id}; - -pub(crate) fn extract(path: &Path, source: &[u8], root: Node<'_>) -> Extraction { - let source_file = path.to_string_lossy().into_owned(); - let stem = file_stem(path); - let file_id = make_id(&[&source_file]); - let (protocols, classes) = pre_scan(root, source); - let mut state = State { - source, - source_file: source_file.clone(), - stem, - file_id: file_id.clone(), - extraction: Extraction::default(), - seen_nodes: HashSet::new(), - functions: Vec::new(), - protocols, - classes, - extensions: Vec::new(), - type_table: HashMap::new(), - }; - state.add_node( - file_id, - path.file_name() - .and_then(|name| name.to_str()) - .unwrap_or_default(), - 1, - false, - None, - ); - let mut cursor = root.walk(); - for child in root.children(&mut cursor) { - state.walk(child, None); - } - state.add_calls(); - if !state.extensions.is_empty() { - state.extraction.extensions.insert( - "swift_extensions".to_owned(), - Value::Array(state.extensions), - ); - } - if !state.type_table.is_empty() { - state.extraction.extensions.insert( - "swift_type_table".to_owned(), - json!({"path": source_file, "table": state.type_table}), - ); - } - state.extraction -} - -struct FunctionBody<'tree> { - id: String, - body: Node<'tree>, -} - -struct State<'source, 'tree> { - source: &'source [u8], - source_file: String, - stem: String, - file_id: String, - extraction: Extraction, - seen_nodes: HashSet, - functions: Vec>, - protocols: HashSet, - classes: HashSet, - extensions: Vec, - type_table: HashMap, -} - -impl<'tree> State<'_, 'tree> { - fn walk(&mut self, node: Node<'tree>, parent_class: Option<&str>) { - match node.kind() { - "import_declaration" => { - self.add_import(node); - return; - } - "class_declaration" | "protocol_declaration" => { - self.add_type(node); - return; - } - "property_declaration" if parent_class.is_some() => { - self.add_property(node, parent_class.unwrap_or_default()); - return; - } - "enum_entry" if parent_class.is_some() => { - self.add_enum_case(node, parent_class.unwrap_or_default()); - return; - } - "function_declaration" - | "init_declaration" - | "deinit_declaration" - | "subscript_declaration" => { - self.add_function(node, parent_class); - return; - } - _ => {} - } - let mut cursor = node.walk(); - for child in node.children(&mut cursor) { - self.walk(child, None); - } - } - - fn add_import(&mut self, node: Node<'tree>) { - let Some(identifier) = first_descendant(node, "simple_identifier") else { - return; - }; - let name = self.text(identifier).to_owned(); - if name.is_empty() { - return; - } - let id = make_id(&[&name]); - self.add_edge( - &self.file_id.clone(), - &id, - "imports", - line(node), - Some("import"), - ); - self.add_node(id, &name, line(node), false, Some("module")); - } - - fn add_type(&mut self, node: Node<'tree>) { - let Some(name_node) = node - .child_by_field_name("name") - .or_else(|| first_descendant(node, "type_identifier")) - else { - return; - }; - let name = type_head(name_node, self.source); - if name.is_empty() { - return; - } - let id = make_id(&[&self.stem, &name]); - self.add_node(id.clone(), &name, line(node), true, None); - self.add_edge(&self.file_id.clone(), &id, "contains", line(node), None); - let kind = if node.kind() == "protocol_declaration" { - Some("protocol") - } else { - declaration_keyword(node) - }; - if kind == Some("extension") { - self.extensions.push(json!({"nid": id, "label": name})); - } - let mut first = true; - let mut cursor = node.walk(); - for inheritance in node - .children(&mut cursor) - .filter(|child| child.kind() == "inheritance_specifier") - { - let Some(user_type) = inheritance - .child_by_field_name("inherits_from") - .or_else(|| first_descendant(inheritance, "user_type")) - else { - continue; - }; - let Some(type_node) = first_descendant(user_type, "type_identifier") else { - continue; - }; - let base = self.text(type_node).to_owned(); - if base.is_empty() { - continue; - } - let relation = self.classify_base(&base, kind, first); - first = false; - if !self.should_publish_type_reference(&base) { - continue; - } - let target = self.ensure_named(&base); - self.add_edge(&id, &target, relation, line(node), None); - if let Some(arguments) = first_descendant(user_type, "type_arguments") { - let mut refs = Vec::new(); - collect_type_refs(arguments, self.source, true, &mut refs); - for (reference, _) in refs { - if !self.should_publish_type_reference(&reference) { - continue; - } - let target = self.ensure_named(&reference); - if target != id { - self.add_edge(&id, &target, "references", line(node), Some("generic_arg")); - } - } - } - } - let body = node.child_by_field_name("body").or_else(|| { - ["class_body", "enum_class_body", "protocol_body"] - .iter() - .find_map(|kind| first_child(node, kind)) - }); - if let Some(body) = body { - let mut cursor = body.walk(); - for child in body.children(&mut cursor) { - self.walk(child, Some(&id)); - } - } - } - - fn classify_base(&self, name: &str, kind: Option<&str>, first: bool) -> &'static str { - if self.protocols.contains(name) { - return "implements"; - } - if self.classes.contains(name) { - return "inherits"; - } - if matches!(kind, Some("struct" | "enum" | "extension" | "actor")) { - return "implements"; - } - if first { "inherits" } else { "implements" } - } - - fn add_property(&mut self, node: Node<'tree>, class_id: &str) { - let mut property_type = None; - if let Some(annotation) = first_child(node, "type_annotation") { - let mut references = Vec::new(); - collect_type_refs(annotation, self.source, false, &mut references); - for (reference, generic) in &references { - if self.should_publish_type_reference(reference) { - let target = self.ensure_named(reference); - if target != class_id { - self.add_edge( - class_id, - &target, - "references", - line(node), - Some(if *generic { "generic_arg" } else { "field" }), - ); - } - } - if property_type.is_none() && !generic { - property_type = Some(reference.clone()); - } - } - } - if let Some(name) = property_name(node, self.source) - && let Some(property_type) = property_type - { - self.type_table.insert(name, property_type); - } - } - - fn add_enum_case(&mut self, node: Node<'tree>, enum_id: &str) { - let mut cursor = node.walk(); - for child in node - .children(&mut cursor) - .filter(|child| child.kind() == "simple_identifier") - { - let name = self.text(child).to_owned(); - let id = make_id(&[enum_id, &name]); - self.add_node(id.clone(), &name, line(node), false, None); - self.add_edge(enum_id, &id, "case_of", line(node), None); - } - if let Some(parameters) = first_child(node, "enum_type_parameters") { - let mut references = Vec::new(); - collect_type_refs(parameters, self.source, false, &mut references); - for (reference, generic) in references { - if !self.should_publish_type_reference(&reference) { - continue; - } - let target = self.ensure_named(&reference); - if target != enum_id { - self.add_edge( - enum_id, - &target, - "references", - line(node), - Some(if generic { "generic_arg" } else { "type" }), - ); - } - } - } - } - - fn add_function(&mut self, node: Node<'tree>, parent_class: Option<&str>) { - let name = match node.kind() { - "deinit_declaration" => "deinit".to_owned(), - "subscript_declaration" => "subscript".to_owned(), - _ => node - .child_by_field_name("name") - .or_else(|| first_child(node, "simple_identifier")) - .map(|name| self.text(name).to_owned()) - .unwrap_or_default(), - }; - if name.is_empty() { - return; - } - let id = parent_class.map_or_else( - || make_id(&[&self.stem, &name]), - |class| make_id(&[class, &name]), - ); - let label = if parent_class.is_some() { - format!(".{name}()") - } else { - format!("{name}()") - }; - self.add_node(id.clone(), &label, line(node), true, None); - let owner = parent_class.unwrap_or(&self.file_id).to_owned(); - self.add_edge( - &owner, - &id, - if parent_class.is_some() { - "method" - } else { - "contains" - }, - line(node), - None, - ); - let mut cursor = node.walk(); - for parameter in node - .children(&mut cursor) - .filter(|child| child.kind() == "parameter") - { - let type_node = parameter.child_by_field_name("type").or_else(|| { - let mut cursor = parameter.walk(); - let children: Vec<_> = parameter.children(&mut cursor).collect(); - children.into_iter().rev().find(|child| { - matches!( - child.kind(), - "user_type" - | "array_type" - | "dictionary_type" - | "optional_type" - | "tuple_type" - | "type_identifier" - ) - }) - }); - let mut references = Vec::new(); - if let Some(type_node) = type_node { - collect_type_refs(type_node, self.source, false, &mut references); - } - let mut parameter_type = None; - for (reference, generic) in references { - if self.should_publish_type_reference(&reference) { - let target = self.ensure_named(&reference); - if target != id { - self.add_edge( - &id, - &target, - "references", - line(node), - Some(if generic { - "generic_arg" - } else { - "parameter_type" - }), - ); - } - } - if parameter_type.is_none() && !generic { - parameter_type = Some(reference); - } - } - if let Some(parameter_type) = parameter_type - && let Some(name) = parameter_name(parameter, self.source) - { - self.type_table.insert(name, parameter_type); - } - } - if let Some(return_type) = node - .child_by_field_name("return_type") - .or_else(|| return_type_after_parameters(node)) - { - let mut references = Vec::new(); - collect_type_refs(return_type, self.source, false, &mut references); - for (reference, generic) in references { - if !self.should_publish_type_reference(&reference) { - continue; - } - let target = self.ensure_named(&reference); - if target != id { - self.add_edge( - &id, - &target, - "references", - line(node), - Some(if generic { - "generic_arg" - } else { - "return_type" - }), - ); - } - } - } - if let Some(body) = node - .child_by_field_name("body") - .or_else(|| first_child(node, "function_body")) - { - collect_local_types(body, self.source, &mut self.type_table); - self.functions.push(FunctionBody { id, body }); - } - } - - fn add_calls(&mut self) { - let labels: HashMap = self - .extraction - .nodes - .iter() - .filter_map(|node| { - node.attributes - .get("label") - .and_then(Value::as_str) - .map(|label| { - ( - label - .trim_matches(['(', ')']) - .trim_start_matches('.') - .to_owned(), - node.id.clone(), - ) - }) - }) - .collect(); - let functions = std::mem::take(&mut self.functions); - let mut seen = HashSet::new(); - for function in functions { - self.walk_calls(function.body, &function.id, &labels, &mut seen); - } - } - - fn walk_calls( - &mut self, - node: Node<'tree>, - caller: &str, - labels: &HashMap, - seen: &mut HashSet<(String, String, usize, usize)>, - ) { - if matches!( - node.kind(), - "function_declaration" - | "init_declaration" - | "deinit_declaration" - | "subscript_declaration" - ) { - return; - } - if node.kind() == "call_expression" - && let Some(call) = swift_call(node, self.source) - { - if let Some(target) = labels - .get(&call.name) - .filter(|target| target.as_str() != caller) - { - if seen.insert(( - caller.to_owned(), - (*target).clone(), - node.start_byte(), - node.end_byte(), - )) { - self.add_edge(caller, target, "calls", line(node), Some("call")); - crate::facts::stamp_last_edge_range(&mut self.extraction, node); - } - } else if !is_language_builtin_global("swift", &call.name) { - self.extraction.raw_calls_mut().push(RawCall { - caller_nid: caller.to_owned(), - callee: call.name, - is_member_call: Some(call.member), - source_file: self.source_file.clone(), - source_location: format!("L{}", line(node)), - receiver: Some(call.receiver), - receiver_type: None, - lang: None, - extensions: crate::facts::node_range(node), - }); - } - } - let mut cursor = node.walk(); - for child in node.children(&mut cursor) { - self.walk_calls(child, caller, labels, seen); - } - } - - fn ensure_named(&mut self, name: &str) -> String { - let local = make_id(&[&self.stem, name]); - if self.seen_nodes.contains(&local) { - return local; - } - let id = make_id(&[name]); - if self.seen_nodes.insert(id.clone()) { - let mut attributes = Map::new(); - attributes.insert("label".to_owned(), Value::String(name.to_owned())); - attributes.insert("file_type".to_owned(), Value::String("code".to_owned())); - attributes.insert("source_file".to_owned(), Value::String(String::new())); - attributes.insert("source_location".to_owned(), Value::String(String::new())); - attributes.insert( - "origin_file".to_owned(), - Value::String(self.source_file.clone()), - ); - self.extraction.nodes.push(NodeRecord { - id: id.clone(), - attributes, - }); - } - id - } - - fn should_publish_type_reference(&self, name: &str) -> bool { - !is_language_builtin_global("swift", name) - || self.protocols.contains(name) - || self.classes.contains(name) - } - - fn text(&self, node: Node<'_>) -> &str { - node.utf8_text(self.source).unwrap_or_default() - } - - fn add_node( - &mut self, - id: String, - label: &str, - at_line: usize, - callable: bool, - node_type: Option<&str>, - ) { - if !self.seen_nodes.insert(id.clone()) { - return; - } - let mut attributes = Map::new(); - attributes.insert("label".to_owned(), Value::String(label.to_owned())); - attributes.insert("file_type".to_owned(), Value::String("code".to_owned())); - if let Some(node_type) = node_type { - attributes.insert("type".to_owned(), Value::String(node_type.to_owned())); - } - attributes.insert( - "source_file".to_owned(), - Value::String(self.source_file.clone()), - ); - attributes.insert( - "source_location".to_owned(), - Value::String(format!("L{at_line}")), - ); - if callable { - attributes.insert("_callable".to_owned(), Value::Bool(true)); - } - self.extraction.nodes.push(NodeRecord { id, attributes }); - } - - fn add_edge( - &mut self, - source: &str, - target: &str, - relation: &str, - at_line: usize, - context: Option<&str>, - ) { - let mut attributes = Map::new(); - attributes.insert("relation".to_owned(), Value::String(relation.to_owned())); - attributes.insert( - "confidence".to_owned(), - Value::String("EXTRACTED".to_owned()), - ); - attributes.insert( - "source_file".to_owned(), - Value::String(self.source_file.clone()), - ); - attributes.insert( - "source_location".to_owned(), - Value::String(format!("L{at_line}")), - ); - attributes.insert("weight".to_owned(), json!(1.0)); - if let Some(context) = context { - attributes.insert("context".to_owned(), Value::String(context.to_owned())); - } - self.extraction.edges.push(EdgeRecord { - source: source.to_owned(), - target: target.to_owned(), - attributes, - }); - } -} - -struct Call { - name: String, - member: bool, - receiver: Option, -} - -fn swift_call(node: Node<'_>, source: &[u8]) -> Option { - let first = { - let mut cursor = node.walk(); - node.children(&mut cursor).next() - }?; - if first.kind() == "simple_identifier" { - return Some(Call { - name: text(first, source).to_owned(), - member: false, - receiver: None, - }); - } - if first.kind() != "navigation_expression" { - return None; - } - let receiver = first - .child_by_field_name("target") - .and_then(|target| match target.kind() { - "simple_identifier" => Some(text(target, source).to_owned()), - _ => None, - }); - let suffixes = direct_children(first, "navigation_suffix"); - let name = suffixes - .last() - .and_then(|suffix| first_descendant(*suffix, "simple_identifier")) - .map(|name| text(name, source).to_owned())?; - Some(Call { - name, - member: true, - receiver, - }) -} - -fn pre_scan(root: Node<'_>, source: &[u8]) -> (HashSet, HashSet) { - let mut protocols = HashSet::new(); - let mut classes = HashSet::new(); - let mut stack = vec![root]; - while let Some(node) = stack.pop() { - if node.kind() == "protocol_declaration" { - if let Some(name) = node.child_by_field_name("name") { - protocols.insert(text(name, source).to_owned()); - } - } else if node.kind() == "class_declaration" - && matches!( - declaration_keyword(node), - Some("class" | "struct" | "enum" | "actor") - ) - && let Some(name) = node.child_by_field_name("name") - { - classes.insert(type_head(name, source)); - } - let mut cursor = node.walk(); - stack.extend(node.children(&mut cursor)); - } - (protocols, classes) -} - -fn declaration_keyword(node: Node<'_>) -> Option<&'static str> { - let mut cursor = node.walk(); - node.children(&mut cursor) - .find(|child| { - !child.is_named() - && matches!( - child.kind(), - "class" | "struct" | "enum" | "extension" | "actor" - ) - }) - .map(|child| match child.kind() { - "class" => "class", - "struct" => "struct", - "enum" => "enum", - "extension" => "extension", - "actor" => "actor", - _ => unreachable!(), - }) -} - -fn collect_type_refs( - node: Node<'_>, - source: &[u8], - generic: bool, - output: &mut Vec<(String, bool)>, -) { - match node.kind() { - "type_annotation" => { - let mut cursor = node.walk(); - for child in node.children(&mut cursor).filter(|child| child.is_named()) { - collect_type_refs(child, source, generic, output); - } - } - "user_type" => { - let mut cursor = node.walk(); - if let Some(name) = node - .children(&mut cursor) - .find(|child| child.kind() == "type_identifier") - { - output.push((text(name, source).to_owned(), generic)); - } - if let Some(arguments) = first_child(node, "type_arguments") { - let mut cursor = arguments.walk(); - for argument in arguments - .children(&mut cursor) - .filter(|child| child.is_named()) - { - collect_type_refs(argument, source, true, output); - } - } - } - "type_identifier" => output.push((text(node, source).to_owned(), generic)), - "optional_type" - | "implicitly_unwrapped_optional_type" - | "array_type" - | "dictionary_type" - | "tuple_type" => { - let mut cursor = node.walk(); - for child in node.children(&mut cursor).filter(|child| child.is_named()) { - collect_type_refs(child, source, generic, output); - } - } - _ if node.is_named() => { - let mut cursor = node.walk(); - for child in node.children(&mut cursor).filter(|child| child.is_named()) { - collect_type_refs(child, source, generic, output); - } - } - _ => {} - } -} - -fn property_name(node: Node<'_>, source: &[u8]) -> Option { - node.child_by_field_name("name") - .and_then(|pattern| first_descendant(pattern, "simple_identifier")) - .or_else(|| first_descendant(node, "simple_identifier")) - .map(|name| text(name, source).to_owned()) -} - -fn parameter_name(node: Node<'_>, source: &[u8]) -> Option { - let names = direct_children(node, "simple_identifier"); - names.last().map(|name| text(*name, source).to_owned()) -} - -fn return_type_after_parameters(node: Node<'_>) -> Option> { - let mut saw_parameter = false; - let mut cursor = node.walk(); - for child in node.children(&mut cursor) { - if child.kind() == "parameter" { - saw_parameter = true; - continue; - } - if saw_parameter - && matches!( - child.kind(), - "user_type" - | "array_type" - | "dictionary_type" - | "optional_type" - | "tuple_type" - | "type_identifier" - ) - { - return Some(child); - } - } - None -} - -fn collect_local_types(node: Node<'_>, source: &[u8], table: &mut HashMap) { - if matches!(node.kind(), "function_declaration" | "lambda_literal") { - return; - } - if node.kind() == "property_declaration" - && let Some(name) = property_name(node, source) - { - let mut inferred = None; - if let Some(call) = first_child(node, "call_expression") { - let first = { - let mut cursor = call.walk(); - call.children(&mut cursor).next() - }; - if let Some(first) = first - && first.kind() == "simple_identifier" - && text(first, source).starts_with(char::is_uppercase) - { - inferred = Some(text(first, source).to_owned()); - } - } - if let Some(value) = inferred { - table.entry(name).or_insert(value); - } - } - let mut cursor = node.walk(); - for child in node.children(&mut cursor) { - collect_local_types(child, source, table); - } -} - -fn type_head(node: Node<'_>, source: &[u8]) -> String { - if node.kind() == "type_identifier" { - return text(node, source).to_owned(); - } - first_descendant(node, "type_identifier") - .map(|name| text(name, source).to_owned()) - .unwrap_or_default() -} - -fn first_child<'tree>(node: Node<'tree>, kind: &str) -> Option> { - let mut cursor = node.walk(); - node.children(&mut cursor) - .find(|child| child.kind() == kind) -} - -fn direct_children<'tree>(node: Node<'tree>, kind: &str) -> Vec> { - let mut cursor = node.walk(); - node.children(&mut cursor) - .filter(|child| child.kind() == kind) - .collect() -} - -fn first_descendant<'tree>(node: Node<'tree>, kind: &str) -> Option> { - let mut cursor = node.walk(); - for child in node.children(&mut cursor) { - if child.kind() == kind { - return Some(child); - } - if let Some(found) = first_descendant(child, kind) { - return Some(found); - } - } - None -} - -fn text<'source>(node: Node<'_>, source: &'source [u8]) -> &'source str { - node.utf8_text(source).unwrap_or_default() -} - -fn line(node: Node<'_>) -> usize { - node.start_position().row + 1 -} diff --git a/crates/compass-languages/tests/engine_edge_coverage.rs b/crates/compass-languages/tests/engine_edge_coverage.rs index 00557b14..6046460f 100644 --- a/crates/compass-languages/tests/engine_edge_coverage.rs +++ b/crates/compass-languages/tests/engine_edge_coverage.rs @@ -33,12 +33,19 @@ fn universal_framework_pack_registry_accepts_only_cut_over_language_evidence() { FrameworkPackRegistry::validate_descriptors(&[descriptor]), Ok(()) ); - assert_eq!(FrameworkPackRegistry::descriptors().len(), 5); + assert_eq!(FrameworkPackRegistry::descriptors().len(), 9); assert_eq!(FrameworkPackRegistry::descriptors()[0].id, "aspnet-csharp"); assert_eq!(FrameworkPackRegistry::descriptors()[1].id, "php-frameworks"); assert_eq!(FrameworkPackRegistry::descriptors()[2].id, "spring-java"); assert_eq!(FrameworkPackRegistry::descriptors()[3].id, "spring-kotlin"); assert_eq!(FrameworkPackRegistry::descriptors()[4].id, "rails-ruby"); + assert_eq!(FrameworkPackRegistry::descriptors()[5].id, "vapor-swift"); + assert_eq!(FrameworkPackRegistry::descriptors()[6].id, "dart-bloc"); + assert_eq!( + FrameworkPackRegistry::descriptors()[7].id, + "dart-flutter-navigation" + ); + assert_eq!(FrameworkPackRegistry::descriptors()[8].id, "dart-riverpod"); assert_eq!(FrameworkPackRegistry::validate(), Ok(())); let rust = FrameworkPackDescriptor { @@ -715,8 +722,7 @@ fn repeated_zig_calls_keep_each_source_range() -> Result<(), Box> { fn repeated_dart_framework_calls_keep_each_source_range() -> Result<(), Box> { let directory = tempfile::tempdir()?; let path = directory.path().join("repeated.dart"); - let source = - b"class State {}\nclass Controller { void run() { emit(State()); emit(State()); } }\n"; + let source = b"import 'package:flutter_bloc/flutter_bloc.dart';\nclass State {}\nclass Controller { void run() { emit(State()); emit(State()); } }\n"; fs::write(&path, source)?; let extraction = Engine::default().extract(&path)?; @@ -742,6 +748,27 @@ fn repeated_dart_framework_calls_keep_each_source_range() -> Result<(), Box Result<(), Box> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("inactive.dart"); + fs::write( + &path, + b"class State {}\nclass Controller { void run() { emit(State()); go('/home'); } }\n", + )?; + + let extraction = Engine::default().extract(&path)?; + assert!( + extraction + .edges + .iter() + .all(|edge| edge.string("context").is_empty()), + "inactive Dart conventions emitted: {:?}", + extraction.edges + ); + Ok(()) +} + struct NavigationSite { start: usize, end: usize, @@ -785,40 +812,40 @@ fn dart_navigation_sites(source: &[u8]) -> Result, Box Result<(), Box> { - let source = b"void run() { go('/home'); }\n"; + let source = b"import 'package:flutter/widgets.dart';\nvoid run() { go('/home'); }\n"; let sites = dart_navigation_sites(source)?; assert_eq!(sites.len(), 1); assert_eq!(&source[sites[0].start..sites[0].end], b"go('/home'"); - assert_eq!((sites[0].line, sites[0].column), (1, 13)); + assert_eq!((sites[0].line, sites[0].column), (2, 13)); Ok(()) } #[test] fn dart_multiline_comment_preserves_navigation_bytes_and_lines() -> Result<(), Box> { - let source = b"/* lead\ncomment */\nvoid run() { go('/home'); }\n"; + let source = b"import 'package:flutter/widgets.dart';\n/* lead\ncomment */\nvoid run() { go('/home'); }\n"; let sites = dart_navigation_sites(source)?; assert_eq!(sites.len(), 1); assert_eq!(&source[sites[0].start..sites[0].end], b"go('/home'"); - assert_eq!((sites[0].line, sites[0].column), (3, 13)); + assert_eq!((sites[0].line, sites[0].column), (4, 13)); Ok(()) } #[test] fn dart_utf8_prefix_preserves_byte_based_navigation_range() -> Result<(), Box> { - let source = "const label = 'café';\nvoid run() { go('/home'); }\n".as_bytes(); + let source = "import 'package:flutter/widgets.dart';\nconst label = 'café';\nvoid run() { go('/home'); }\n".as_bytes(); let sites = dart_navigation_sites(source)?; assert_eq!(sites.len(), 1); assert_eq!(&source[sites[0].start..sites[0].end], b"go('/home'"); - assert_eq!((sites[0].line, sites[0].column), (2, 13)); + assert_eq!((sites[0].line, sites[0].column), (3, 13)); Ok(()) } #[test] fn dart_minified_navigation_keeps_same_line_occurrences_distinct() -> Result<(), Box> { - let source = b"void run(){go('/a');go('/b');}\n"; + let source = b"import 'package:flutter/widgets.dart';\nvoid run(){go('/a');go('/b');}\n"; let sites = dart_navigation_sites(source)?; assert_eq!(sites.len(), 2); diff --git a/crates/compass-languages/tests/extended_universal_conformance.rs b/crates/compass-languages/tests/extended_universal_conformance.rs new file mode 100644 index 00000000..7857fc8b --- /dev/null +++ b/crates/compass-languages/tests/extended_universal_conformance.rs @@ -0,0 +1,189 @@ +use std::error::Error; +use std::path::Path; + +use compass_languages::{ + CandidateRelation, Engine, EvidenceLimits, Registry, SemanticRole, UniversalEvidenceRegistry, + validate_evidence, +}; + +#[test] +fn extended_languages_publish_ast_first_universal_evidence() -> Result<(), Box> { + let fixtures = [ + ( + "Sources/Greeter.swift", + br#"import Foundation +protocol Renderable { func render() } +struct Greeter: Renderable { + func render() { helper() } + func helper() {} +} +"# + .as_slice(), + "swift", + "Renderable", + "helper", + ), + ( + "lib/greeter.dart", + br#"import 'package:flutter/widgets.dart' as widgets; +class Greeter { + Widget build() { return helper(); } + Widget helper() => const Widget(); +} +"# + .as_slice(), + "dart", + "Greeter", + "helper", + ), + ( + "src/Greeter.scala", + br#"package sample +trait Renderable { def render(): Unit } +class Greeter extends Renderable { + def render(): Unit = helper() + def helper(): Unit = () +} +"# + .as_slice(), + "scala", + "Greeter", + "helper", + ), + ( + "src/Greeter.groovy", + br#"package sample +import java.util.List +class Greeter { + void render() { helper() } + void helper() {} +} +"# + .as_slice(), + "groovy", + "Greeter", + "helper", + ), + ]; + + for (path, source, language, type_name, call_name) in fixtures { + let mut engine = Engine::default(); + let evidence = engine.extract_source_universal_evidence(Path::new(path), path, source)?; + validate_evidence(&evidence, EvidenceLimits::default())?; + assert_eq!(evidence.pipeline.language, language); + let pipeline = UniversalEvidenceRegistry::pipeline(language) + .ok_or_else(|| format!("missing universal pipeline for {language}"))?; + assert_eq!(evidence.pipeline.qualification, pipeline.qualification); + assert!( + evidence + .declarations + .iter() + .any(|decl| decl.name == type_name), + "{language}: {:#?}", + evidence.declarations + ); + assert!( + evidence + .declarations + .iter() + .any(|decl| decl.name == call_name), + "{language}: {:#?}", + evidence.declarations + ); + assert!( + evidence + .occurrences + .iter() + .any(|occurrence| occurrence.role == SemanticRole::Call + && occurrence.spelling == call_name), + "{language}: {:#?}", + evidence.occurrences + ); + assert!( + evidence + .candidates + .iter() + .any(|candidate| candidate.relation == CandidateRelation::Calls + && candidate.target_spelling == call_name), + "{language}: {:#?}", + evidence.candidates + ); + assert!( + evidence.candidates.iter().all(|candidate| candidate + .constraints + .exact_language + .as_deref() + == Some(language)), + "{language}: cross-language candidate" + ); + } + Ok(()) +} + +#[test] +fn registry_fixtures_keep_extended_pipelines_valid() -> Result<(), Box> { + for case in Registry::cases() + .iter() + .filter(|case| matches!(case.spec.name, "swift" | "dart" | "scala" | "groovy")) + { + let mut engine = Engine::default(); + let evidence = engine.extract_source_universal_evidence( + Path::new(case.fixture_path), + case.fixture_path, + case.fixture_source.as_bytes(), + )?; + validate_evidence(&evidence, EvidenceLimits::default())?; + assert_eq!(evidence.pipeline.language, case.spec.name); + assert!(!evidence.declarations.is_empty(), "{}", case.id); + } + Ok(()) +} + +#[test] +fn extended_empty_and_recovered_sources_remain_bounded() -> Result<(), Box> { + for (path, source) in [ + ("empty.swift", b"".as_slice()), + ("empty.dart", b"\n".as_slice()), + ("empty.scala", b"/* unterminated".as_slice()), + ("empty.groovy", b"class Broken {".as_slice()), + ] { + let mut engine = Engine::default(); + let evidence = engine.extract_source_universal_evidence(Path::new(path), path, source)?; + validate_evidence(&evidence, EvidenceLimits::default())?; + assert_eq!( + evidence.pipeline.language, + Path::new(path) + .extension() + .and_then(|ext| ext.to_str()) + .unwrap_or_default() + ); + } + for (path, invalid_source) in [ + ("invalid.swift", b"struct Broken {\n \xff\n}".as_slice()), + ("invalid.dart", b"class Broken {\n \xff\n}".as_slice()), + ("invalid.scala", b"class Broken {\n \xff\n}".as_slice()), + ("invalid.groovy", b"class Broken {\n \xff\n}".as_slice()), + ] { + let mut engine = Engine::default(); + let evidence = + engine.extract_source_universal_evidence(Path::new(path), path, invalid_source)?; + validate_evidence(&evidence, EvidenceLimits::default())?; + assert!( + evidence + .diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "invalid_utf8"), + "{path}: {:#?}", + evidence.diagnostics + ); + assert!( + evidence + .declarations + .iter() + .all(|declaration| declaration.range.end_byte <= invalid_source.len() as u64), + "{path}: {:#?}", + evidence.declarations + ); + } + Ok(()) +} diff --git a/crates/compass-languages/tests/universal_evidence.rs b/crates/compass-languages/tests/universal_evidence.rs index 0ee7ceb0..b66dc73b 100644 --- a/crates/compass-languages/tests/universal_evidence.rs +++ b/crates/compass-languages/tests/universal_evidence.rs @@ -474,7 +474,9 @@ fn universal_evidence_pipelines_are_unique_sorted_and_truthful() { .collect::>(), [ "csharp", + "dart", "go", + "groovy", "java", "javascript", "kotlin", @@ -482,6 +484,8 @@ fn universal_evidence_pipelines_are_unique_sorted_and_truthful() { "python", "ruby", "rust", + "scala", + "swift", "typescript", ] ); diff --git a/crates/compass-resolve/src/evidence/languages/policy.rs b/crates/compass-resolve/src/evidence/languages/policy.rs index 193ab978..9973a519 100644 --- a/crates/compass-resolve/src/evidence/languages/policy.rs +++ b/crates/compass-resolve/src/evidence/languages/policy.rs @@ -14,6 +14,10 @@ pub(in crate::evidence) enum LanguagePolicyKind { Php, Ruby, Rust, + Swift, + Dart, + Scala, + Groovy, Generic, } @@ -27,6 +31,10 @@ impl LanguagePolicyKind { "php" => Self::Php, "ruby" => Self::Ruby, "rust" => Self::Rust, + "swift" => Self::Swift, + "dart" => Self::Dart, + "scala" => Self::Scala, + "groovy" => Self::Groovy, _ => Self::Generic, } } @@ -60,7 +68,9 @@ impl LanguagePolicyKind { } Self::Java => db.resolve_java_same_package_builtin_collision(candidate), Self::Kotlin => db.resolve_kotlin_candidate(candidate), - Self::Ruby | Self::Generic => None, + Self::Ruby | Self::Swift | Self::Dart | Self::Scala | Self::Groovy | Self::Generic => { + None + } } } @@ -78,6 +88,10 @@ impl LanguagePolicyKind { | Self::Ruby | Self::TypeScript | Self::Rust + | Self::Swift + | Self::Dart + | Self::Scala + | Self::Groovy | Self::Generic => None, } } @@ -117,6 +131,22 @@ mod tests { LanguagePolicyKind::for_language("ruby"), LanguagePolicyKind::Ruby ); + assert_eq!( + LanguagePolicyKind::for_language("swift"), + LanguagePolicyKind::Swift + ); + assert_eq!( + LanguagePolicyKind::for_language("dart"), + LanguagePolicyKind::Dart + ); + assert_eq!( + LanguagePolicyKind::for_language("scala"), + LanguagePolicyKind::Scala + ); + assert_eq!( + LanguagePolicyKind::for_language("groovy"), + LanguagePolicyKind::Groovy + ); assert_eq!( LanguagePolicyKind::for_language("future-language"), LanguagePolicyKind::Generic diff --git a/crates/compass-resolve/src/evidence/projection/nodes.rs b/crates/compass-resolve/src/evidence/projection/nodes.rs index 2c35af73..8bf642f4 100644 --- a/crates/compass-resolve/src/evidence/projection/nodes.rs +++ b/crates/compass-resolve/src/evidence/projection/nodes.rs @@ -129,16 +129,21 @@ pub(super) fn declaration_node( } } -/// Preserve the public callable spelling emitted by the pre-universal -/// TypeScript/JavaScript extractor. Universal evidence keeps its richer module -/// qualified name for resolution, while framework route contracts continue to -/// identify source callables as `name()@offset` (or `Owner::name()@offset`). +/// Preserve the public callable spelling emitted by pre-universal extractors. +/// Universal evidence keeps its richer module-qualified name for resolution, +/// while framework route contracts continue to identify source callables as +/// `name()@offset` (or `Owner::name()@offset`). Swift's legacy route contract +/// intentionally omitted the offset, so its compatibility spelling is +/// `name()`. pub(super) fn legacy_callable_qualified_name(declaration: &DeclarationFact) -> Option { let start = declaration .definition_start_byte .unwrap_or(declaration.range.start_byte); match declaration.kind.as_str() { "function" => { + if declaration.language == "swift" { + return Some(format!("{}()", declaration.name)); + } if declaration.name == "default" { Some("default".to_owned()) } else { diff --git a/crates/compass-resolve/src/evidence/resolve/pipeline.rs b/crates/compass-resolve/src/evidence/resolve/pipeline.rs index ac5c628d..02247260 100644 --- a/crates/compass-resolve/src/evidence/resolve/pipeline.rs +++ b/crates/compass-resolve/src/evidence/resolve/pipeline.rs @@ -377,7 +377,9 @@ impl ResolutionDb<'_> { else { return StageOutcome::Continue; }; - if is_language_builtin_qualified_target(context.language, &qualified_name) { + if is_language_builtin_qualified_target(context.language, &qualified_name) + && !self.rust_builtin_external_candidate(context, candidate) + { return StageOutcome::Decided(ResolutionDecision::Unresolved); } StageOutcome::Decided(ResolutionDecision::QualifiedExternal { @@ -389,6 +391,47 @@ impl ResolutionDb<'_> { }) } + /// Rust's prelude names are intentionally not published as graph hubs for + /// ordinary constructor calls (`Vec::new`, `Box::new`, and friends). Two + /// forms still carry useful, source-backed evidence: a receiver whose + /// concrete type was inferred from `self`, and a qualified call in a + /// scope with an explicit wildcard import (where the imported module may + /// provide a project/dependency symbol that is outside the corpus). + fn rust_builtin_external_candidate( + &self, + context: &CandidateContext<'_>, + candidate: &RelationshipCandidate, + ) -> bool { + if context.language != "rust" || candidate.relation != CandidateRelation::Calls { + return false; + } + if self + .occurrence(candidate) + .and_then(OccurrenceRef::qualifier) + .is_some_and(|qualifier| qualifier == "self") + { + return true; + } + let mut scope_id = candidate.constraints.scope_id.as_deref(); + let mut visited = BTreeSet::new(); + while let Some(scope) = scope_id.filter(|scope| visited.insert((*scope).to_owned())) { + if self + .indexes + .wildcards + .by_scope + .contains_key(&(context.language.to_owned(), scope.to_owned())) + { + return true; + } + scope_id = self + .facts + .scopes + .get(scope) + .and_then(|scope| scope.parent_scope_id.as_deref()); + } + false + } + fn stage_deferred_receiver(&self, context: &CandidateContext<'_>) -> StageOutcome { let candidate = context.candidate(); if !matches!( diff --git a/crates/compass-resolve/src/frameworks/dart.rs b/crates/compass-resolve/src/frameworks/dart.rs new file mode 100644 index 00000000..d4b8d61d --- /dev/null +++ b/crates/compass-resolve/src/frameworks/dart.rs @@ -0,0 +1,13 @@ +//! Project-wide adapters for the bounded Dart convention packs. + +use super::FrameworkResolutionError; + +/// Convention relationships are already source-anchored by the language-side +/// framework bridge. Keep explicit no-op adapters so pack IDs have a stable, +/// one-to-one resolver registration and cannot silently fall through a broad +/// Dart/JVM/native resolver. +pub(super) fn expand( + _extraction: &mut compass_languages::Extraction, +) -> Result<(), FrameworkResolutionError> { + Ok(()) +} diff --git a/crates/compass-resolve/src/frameworks/mod.rs b/crates/compass-resolve/src/frameworks/mod.rs index 5aa74ac0..fff84382 100644 --- a/crates/compass-resolve/src/frameworks/mod.rs +++ b/crates/compass-resolve/src/frameworks/mod.rs @@ -1,5 +1,6 @@ mod aspnet; mod axum; +mod dart; mod domain; mod jvm; mod native; @@ -10,6 +11,7 @@ mod qualification; mod routes; mod ruby; mod spring; +mod swift; mod target_index; mod typescript; @@ -94,6 +96,22 @@ const UNIVERSAL_FRAMEWORK_PACKS: &[UniversalFrameworkPack] = &[ id: "rails-ruby", expand: ruby::expand, }, + UniversalFrameworkPack { + id: "vapor-swift", + expand: swift::expand, + }, + UniversalFrameworkPack { + id: "dart-flutter-navigation", + expand: dart::expand, + }, + UniversalFrameworkPack { + id: "dart-bloc", + expand: dart::expand, + }, + UniversalFrameworkPack { + id: "dart-riverpod", + expand: dart::expand, + }, ]; pub use domain::{ diff --git a/crates/compass-resolve/src/frameworks/routes.rs b/crates/compass-resolve/src/frameworks/routes.rs index 932e60c6..aced9c6c 100644 --- a/crates/compass-resolve/src/frameworks/routes.rs +++ b/crates/compass-resolve/src/frameworks/routes.rs @@ -378,6 +378,19 @@ fn resolve_reference( score = 100; reason = "exact endpoint re-export module"; } + // An unqualified handler reference from a source file is local evidence + // when that file declares a matching callable. Prefer that exact + // same-source candidate before consulting project-wide qualified names. + // This keeps a Go `listUsers` route bound to the Go declaration when a + // universal Swift file happens to expose the same top-level spelling. + // Explicitly qualified references and import aliases retain their + // project-wide lookup semantics below. + if positions.is_empty() && owner.is_none() && alias.is_none() { + (positions, candidates_truncated) = + targets.by_source_terminal(source_file, &last, &families, max); + score = 100; + reason = "exact same-source route target"; + } if positions.is_empty() && let Some(owner) = owner.as_deref() { @@ -945,4 +958,67 @@ mod tests { assert!(resolved[0].candidates.is_empty()); Ok(()) } + + #[test] + fn unqualified_routes_prefer_same_source_over_cross_language_qualified_names() + -> Result<(), Box> { + let function = |id: &str, qualified: &str, source: &str| RawNodeRecord { + id: id.to_owned(), + attributes: Map::from_iter([ + ("label".into(), Value::String("listUsers".into())), + ("name".into(), Value::String("listUsers".into())), + ("qualified_name".into(), Value::String(qualified.into())), + ("symbol_kind".into(), Value::String("function".into())), + ("source_file".into(), Value::String(source.into())), + ]), + }; + let route = RawRouteFact { + framework: "gin".to_owned(), + operation: "GET".to_owned(), + raw_path: "/users".to_owned(), + normalized_path: "/users".to_owned(), + declaring_scope: "routes".to_owned(), + anchor: RawFrameworkAnchor { + source_file: "routes/go/gin.go".to_owned(), + start_byte: 1, + end_byte: 2, + start_line: 1, + start_column: 0, + end_line: 1, + end_column: 1, + }, + handler_reference: "listUsers".to_owned(), + middleware_references: Vec::new(), + origin: RawFrameworkOrigin::Ast, + rule: Some("gin-router-call".to_owned()), + detail: Map::new(), + }; + let extraction = Extraction { + nodes: vec![ + function( + "go-list-users", + "routes.routes.listUsers", + "routes/go/gin.go", + ), + function( + "swift-list-users", + "listUsers", + "routes/swift/VaporRoutes.swift", + ), + ], + framework_facts: vec![RawFrameworkFact::Route(route)], + ..Extraction::default() + }; + + let resolved = resolve_routes(&extraction, FrameworkLimits::default())?; + assert_eq!(resolved.len(), 1); + assert_eq!(resolved[0].state, ResolutionState::Exact); + assert_eq!( + resolved[0].stages[0].target.as_deref(), + Some("go-list-users") + ); + assert_eq!(resolved[0].candidates.len(), 1); + assert_eq!(resolved[0].candidates[0].node_id, "go-list-users"); + Ok(()) + } } diff --git a/crates/compass-resolve/src/frameworks/swift.rs b/crates/compass-resolve/src/frameworks/swift.rs new file mode 100644 index 00000000..3447cc16 --- /dev/null +++ b/crates/compass-resolve/src/frameworks/swift.rs @@ -0,0 +1,12 @@ +//! Project-wide expansion for the universal Vapor/Swift framework pack. + +use super::FrameworkResolutionError; + +/// Vapor route facts are already emitted by the source-side universal pack. +/// Keep an explicit resolver adapter so the pack participates in the same +/// lifecycle and cannot silently bypass the universal framework registry. +pub(super) fn expand( + _extraction: &mut compass_languages::Extraction, +) -> Result<(), FrameworkResolutionError> { + Ok(()) +} diff --git a/crates/compass-resolve/src/lib.rs b/crates/compass-resolve/src/lib.rs index b6918e2a..81ca3286 100644 --- a/crates/compass-resolve/src/lib.rs +++ b/crates/compass-resolve/src/lib.rs @@ -931,7 +931,7 @@ fn restore_framework_callable_names( if !framework_sources.contains(&source) || !matches!( string_attribute(node, "language").as_str(), - "typescript" | "javascript" | "tsx" | "jsx" + "typescript" | "javascript" | "tsx" | "jsx" | "swift" ) { continue; @@ -3848,6 +3848,7 @@ fn rewire_unique_family_stubs(extraction: &mut Extraction) { stubs.insert(node.id.clone(), label); } else if is_type_like_definition(node) && let Some(family @ "jvm") = language_family(&source) + && !is_hard_cut_universal_source(&source) { definitions .entry((label, family)) @@ -3897,6 +3898,9 @@ fn rewire_unique_family_stubs(extraction: &mut Extraction) { continue; }; let source_file = edge.string("source_file"); + if is_hard_cut_universal_source(&source_file) { + continue; + } let Some(family @ "jvm") = language_family(&source_file) else { continue; }; @@ -4557,6 +4561,9 @@ fn rewire_unique_stub_nodes(extraction: &mut Extraction) { let Some(family) = language_family(&edge.string("source_file")) else { continue; }; + if is_hard_cut_universal_source(&edge.string("source_file")) { + continue; + } for endpoint in [&edge.source, &edge.target] { if stub_ids.contains(endpoint.as_str()) { stub_relations @@ -4603,6 +4610,9 @@ fn rewire_unique_stub_nodes(extraction: &mut Extraction) { let Some(candidate_family) = language_family(candidate_source) else { return false; }; + if is_hard_cut_universal_source(candidate_source) { + return false; + } let family_compatible = families .is_some_and(|set| set.len() == 1 && set.contains(candidate_family)); let scope_compatible = scopes.is_some_and(|set| { @@ -5560,6 +5570,9 @@ fn language_name_from_source(source: &str) -> Option<&'static str> { "rs" => Some("rust"), "java" => Some("java"), "swift" => Some("swift"), + "dart" => Some("dart"), + "scala" => Some("scala"), + "groovy" | "gradle" => Some("groovy"), _ => None, } } @@ -5589,6 +5602,13 @@ fn language_family(source: &str) -> Option<&'static str> { } } +fn is_hard_cut_universal_source(source: &str) -> bool { + matches!( + extension(source).as_str(), + "swift" | "dart" | "scala" | "groovy" | "gradle" + ) +} + fn extension(source: &str) -> String { Path::new(source) .extension() diff --git a/crates/compass-resolve/src/members.rs b/crates/compass-resolve/src/members.rs index b8b0572e..995d345d 100644 --- a/crates/compass-resolve/src/members.rs +++ b/crates/compass-resolve/src/members.rs @@ -88,13 +88,6 @@ pub(crate) fn resolve_language_call_facts_additions( }) .collect::>(); - resolve_swift_registry_compatibility( - &facts.calls, - &indexes, - &facts.tables, - &mut existing, - &mut edges, - ); resolve_typed_members( &facts.calls, &indexes, @@ -114,53 +107,6 @@ pub(crate) fn resolve_language_call_facts_additions( (external_nodes, edges) } -/// Preserve Compass's resolver-registry ordering for strict external parity. -/// -/// Once a corpus contains Swift type facts, the Python implementation's Swift -/// pass sees the collection-wide raw-call list. Consequently, an explicitly -/// capitalized receiver from another language is still resolved as a unique -/// type reference before later language passes run. This is observable graph -/// output, so Compass deliberately retains it as a compatibility rule. -fn resolve_swift_registry_compatibility( - calls: &[RawCall], - indexes: &Indexes, - tables: &TypeTables, - existing: &mut HashSet<(String, String, String)>, - edges: &mut Vec, -) { - if tables.swift.is_empty() { - return; - } - for call in calls { - if call.is_member_call != Some(true) - || member_family(&call.source_file, call.lang.as_deref()) == MemberFamily::Swift - { - continue; - } - let Some(receiver) = receiver(call) else { - continue; - }; - let Some(owner) = starts_upper(receiver) - .then(|| indexes.unique_type(receiver)) - .flatten() - else { - continue; - }; - let (target, relation_name) = indexes - .unique_method(owner, &call.callee) - .map_or((owner, "references"), |method| (method, "calls")); - emit( - call, - target, - relation_name, - "call", - ("EXTRACTED", 1.0), - existing, - edges, - ); - } -} - struct Indexes<'a> { nodes: HashMap<&'a str, &'a NodeRecord>, types: HashMap>, @@ -298,7 +244,6 @@ impl<'a> Indexes<'a> { #[derive(Default)] struct TypeTables { - swift: HashMap>, typescript: HashMap>, cpp: HashMap>, objc: HashMap>, @@ -311,7 +256,6 @@ impl TypeTables { .iter() .filter(|extraction| extraction.semantic_evidence.is_none()) { - collect_table(extraction, "swift_type_table", &mut tables.swift); collect_table(extraction, "ts_type_table", &mut tables.typescript); collect_table(extraction, "cpp_type_table", &mut tables.cpp); collect_table(extraction, "objc_type_table", &mut tables.objc); @@ -329,7 +273,6 @@ fn indexed_families(calls: &[RawCall], tables: &TypeTables) -> (HashSet<&'static .iter() .any(|call| index_family(&call.source_file).is_none()); for (present, family) in [ - (!tables.swift.is_empty(), "swift"), (!tables.typescript.is_empty(), "javascript"), (!tables.cpp.is_empty(), "cpp"), (!tables.objc.is_empty(), "objc"), @@ -346,7 +289,6 @@ fn index_family(source: &str) -> Option<&'static str> { "py" | "pyi" => Some("python"), "rb" | "rake" => Some("ruby"), "pas" | "pp" | "dpr" | "dpk" | "inc" => Some("pascal"), - "swift" => Some("swift"), "ts" | "tsx" | "mts" | "cts" | "js" | "jsx" | "mjs" | "cjs" => Some("javascript"), "c" | "h" | "cpp" | "cc" | "cxx" | "hpp" | "hh" | "hxx" | "cu" | "cuh" => Some("cpp"), "cs" | "razor" | "cshtml" => Some("csharp"), @@ -374,7 +316,6 @@ fn resolve_typed_members( let language = call.lang.as_deref(); let family = member_family(source, language); let owner = match family { - MemberFamily::Swift => typed_owner(receiver, source, &tables.swift, indexes, None), MemberFamily::Typescript => { let result = typed_owner(receiver, source, &tables.typescript, indexes, None); if result @@ -858,7 +799,6 @@ fn is_bare_constant(label: &str) -> bool { #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum MemberFamily { - Swift, Typescript, Cpp, Csharp, @@ -872,7 +812,6 @@ fn member_family(source: &str, language: Option<&str>) -> MemberFamily { Some("csharp") => MemberFamily::Csharp, Some("objc") => MemberFamily::Objc, _ => match extension(source).as_str() { - "swift" => MemberFamily::Swift, "ts" | "tsx" | "mts" | "cts" | "js" | "jsx" => MemberFamily::Typescript, _ => MemberFamily::Other, }, @@ -972,7 +911,7 @@ mod tests { assert_eq!(member_family("x.any", Some("cpp")), MemberFamily::Cpp); assert_eq!(member_family("x.any", Some("csharp")), MemberFamily::Csharp); assert_eq!(member_family("x.any", Some("objc")), MemberFamily::Objc); - assert_eq!(member_family("x.swift", None), MemberFamily::Swift); + assert_eq!(member_family("x.swift", None), MemberFamily::Other); assert_eq!(member_family("x.TSX", None), MemberFamily::Typescript); assert_eq!(member_family("x.unknown", None), MemberFamily::Other); @@ -1074,63 +1013,4 @@ mod tests { && edge.string("extractor") == "compass.languages.typescript" })); } - - #[test] - fn swift_registry_compatibility_precedes_other_language_member_passes() { - let mut swift = Extraction::default(); - swift.extensions.insert( - "swift_type_table".to_owned(), - serde_json::json!({"path":"source.swift","table":{"value":"Service"}}), - ); - let python = Extraction { - raw_calls: Some(vec![RawCall { - caller_nid: "caller".to_owned(), - callee: "glob".to_owned(), - is_member_call: Some(true), - source_file: "tests/test_extract.py".to_owned(), - source_location: "L61".to_owned(), - receiver: Some(Some("Fixtures".to_owned())), - receiver_type: None, - lang: None, - extensions: Map::new(), - }]), - ..Extraction::default() - }; - let mut merged = Extraction { - nodes: vec![ - node(serde_json::json!({ - "id":"file","label":"coverage_paths.rs","file_type":"code", - "source_file":"coverage_paths.rs" - })), - node(serde_json::json!({ - "id":"fixtures","label":"Fixtures","file_type":"code", - "source_file":"coverage_paths.rs" - })), - node(serde_json::json!({ - "id":"caller","label":"test_extract()","file_type":"code", - "source_file":"tests/test_extract.py" - })), - ], - ..Extraction::default() - }; - merged.edges.push(EdgeRecord { - source: "file".to_owned(), - target: "fixtures".to_owned(), - attributes: Map::from_iter([( - "relation".to_owned(), - Value::String("contains".to_owned()), - )]), - }); - - resolve_language_calls(&[swift, python], &mut merged); - - let edge = merged - .edges - .iter() - .find(|edge| edge.source == "caller" && edge.target == "fixtures") - .unwrap_or_else(|| std::process::abort()); - assert_eq!(relation(edge), "references"); - assert_eq!(edge.string("context"), "call"); - assert_eq!(edge.string("confidence"), "EXTRACTED"); - } } diff --git a/crates/compass-resolve/tests/builtin_resolution.rs b/crates/compass-resolve/tests/builtin_resolution.rs index 69a9135d..3fffd756 100644 --- a/crates/compass-resolve/tests/builtin_resolution.rs +++ b/crates/compass-resolve/tests/builtin_resolution.rs @@ -118,17 +118,9 @@ export function normalize(input: unknown) { .semantic_evidence .as_ref() .ok_or("missing TypeScript semantic evidence")?; - - assert!(evidence.candidates.iter().all(|candidate| { - !matches!( - candidate.relation, - compass_languages::CandidateRelation::Calls - | compass_languages::CandidateRelation::Constructs - | compass_languages::CandidateRelation::AccessesMember - ) || !matches!( - candidate.target_spelling.as_str(), - "String" | "Number" | "log" | "resolve" | "Date" - ) + assert!(evidence.candidates.iter().any(|candidate| { + candidate.constraints.module_or_package.as_deref() == Some("javascript.global") + && candidate.constraints.allow_external })); let sources = HashMap::from([( @@ -205,24 +197,21 @@ func run(identifier: UUID) { } "#; let extracted = Engine::default().extract_source(path, source)?; - let raw_names = extracted - .raw_calls - .iter() - .flatten() - .map(|call| call.callee.as_str()) - .collect::>(); - - assert!(!raw_names.contains(&"Data")); - assert!(!raw_names.contains(&"UUID")); + assert!(extracted.raw_calls.is_none()); assert!( extracted .nodes .iter() .all(|node| { !matches!(node.label(), "Codable" | "Sendable" | "Data" | "UUID") }) ); - let call_targets = call_edges(&extracted) + let sources = HashMap::from([( + path.to_string_lossy().into_owned(), + String::from_utf8(source.to_vec())?, + )]); + let resolved = compass_resolve::resolve(&[extracted], &sources); + let call_targets = call_edges(&resolved) .into_iter() - .filter_map(|edge| extracted.nodes.iter().find(|node| node.id == edge.target)) + .filter_map(|edge| resolved.nodes.iter().find(|node| node.id == edge.target)) .map(compass_languages::RawNodeRecord::label) .collect::>(); assert!(call_targets.contains(&"print()")); diff --git a/crates/compass-resolve/tests/extended_universal.rs b/crates/compass-resolve/tests/extended_universal.rs new file mode 100644 index 00000000..a19321da --- /dev/null +++ b/crates/compass-resolve/tests/extended_universal.rs @@ -0,0 +1,159 @@ +use std::collections::HashMap; +use std::error::Error; +use std::path::Path; + +use compass_languages::Engine; + +#[test] +fn extended_universal_languages_never_resolve_same_named_jvm_types_by_family() +-> Result<(), Box> { + let scala_source = "class Shared { def run(): Unit = () }\n"; + let caller_source = "class Caller { def call(): Unit = Shared().run() }\n"; + let java_source = "class Shared { void run() {} }\n"; + let scala_path = Path::new("src/Shared.scala"); + let caller_path = Path::new("src/Caller.scala"); + let java_path = Path::new("src/Shared.java"); + let mut engine = Engine::default(); + let scala = engine.extract_source_graph_only( + scala_path, + scala_path.to_str().unwrap_or_default(), + scala_source.as_bytes(), + )?; + let caller = engine.extract_source_graph_only( + caller_path, + caller_path.to_str().unwrap_or_default(), + caller_source.as_bytes(), + )?; + let java = engine.extract_source_graph_only( + java_path, + java_path.to_str().unwrap_or_default(), + java_source.as_bytes(), + )?; + let sources = HashMap::from([ + ( + scala_path.to_string_lossy().into_owned(), + scala_source.to_owned(), + ), + ( + caller_path.to_string_lossy().into_owned(), + caller_source.to_owned(), + ), + ( + java_path.to_string_lossy().into_owned(), + java_source.to_owned(), + ), + ]); + let resolved = + compass_resolve::resolve_with_root(&[scala, caller, java], &sources, Path::new(".")); + + assert!( + resolved + .edges + .iter() + .filter(|edge| { + edge.string("source_file") == caller_path.to_string_lossy() + && matches!( + edge.string("relation").as_str(), + "calls" | "constructs" | "references" + ) + }) + .all(|edge| { + let target_source = resolved + .nodes + .iter() + .find(|node| node.id == edge.target) + .map(|node| node.string("source_file")); + target_source.is_none_or(|source| source.ends_with(".scala")) + }) + ); + Ok(()) +} + +#[test] +fn every_extended_language_keeps_same_named_foreign_types_unresolved() -> Result<(), Box> +{ + let cases = [ + ( + "swift", + "src/swift/Caller.swift", + "struct Shared {}\nstruct Caller { func call() { _ = Shared() } }\n", + "src/java/Shared.java", + "class Shared {}\n", + ), + ( + "dart", + "src/dart/caller.dart", + "class Shared {}\nclass Caller { void call() { Shared(); } }\n", + "src/scala/Shared.scala", + "class Shared\n", + ), + ( + "scala", + "src/scala/Caller.scala", + "class Shared\nclass Caller { def call(): Unit = new Shared() }\n", + "src/java/Shared.java", + "class Shared {}\n", + ), + ( + "groovy", + "src/groovy/Caller.groovy", + "class Shared {}\nclass Caller { void call() { new Shared() } }\n", + "src/kotlin/Shared.kt", + "class Shared\n", + ), + ]; + + for (language, caller_path, caller_source, foreign_path, foreign_source) in cases { + let caller_path = Path::new(caller_path); + let foreign_path = Path::new(foreign_path); + let mut engine = Engine::default(); + let caller = engine.extract_source_graph_only( + caller_path, + caller_path.to_str().unwrap_or_default(), + caller_source.as_bytes(), + )?; + let foreign = engine.extract_source_graph_only( + foreign_path, + foreign_path.to_str().unwrap_or_default(), + foreign_source.as_bytes(), + )?; + let sources = HashMap::from([ + ( + caller_path.to_string_lossy().into_owned(), + caller_source.to_owned(), + ), + ( + foreign_path.to_string_lossy().into_owned(), + foreign_source.to_owned(), + ), + ]); + let resolved = + compass_resolve::resolve_with_root(&[caller, foreign], &sources, Path::new(".")); + let semantic_edges = resolved + .edges + .iter() + .filter(|edge| { + edge.string("source_file") == caller_path.to_string_lossy() + && matches!( + edge.string("relation").as_str(), + "calls" | "constructs" | "references" | "extends" | "implements" + ) + }) + .collect::>(); + assert!( + !semantic_edges.is_empty(), + "{language}: caller did not emit a semantic edge" + ); + assert!( + semantic_edges.iter().all(|edge| { + resolved + .nodes + .iter() + .find(|node| node.id == edge.target) + .is_none_or(|node| node.string("source_file") != foreign_path.to_string_lossy()) + }), + "{language}: resolved a target from a foreign language file" + ); + } + Ok(()) +} diff --git a/docs/design/language-architecture.md b/docs/design/language-architecture.md index 940ef4f8..ea795f8a 100644 --- a/docs/design/language-architecture.md +++ b/docs/design/language-architecture.md @@ -29,13 +29,14 @@ This architecture is transitioning one language at a time. The status labels bel | Status | Behavior | | --- | --- | | Available now | The vendored package supplies 37 pinned static Tree-sitter grammars | -| Available now | Python, Go, Rust, Java, PHP, Kotlin, Ruby, TypeScript, and JavaScript are registered hard-cut evidence pipelines: they emit semantic evidence and use shared resolution and projection | +| Available now | Python, Go, Rust, Java, PHP, Kotlin, Ruby, TypeScript, JavaScript, Swift, Dart, Scala, and Groovy are registered hard-cut evidence pipelines: they emit semantic evidence and use shared resolution and projection | | Available now | Rust is a quality-gated, hard-cut version-15 `Qualifying` pipeline; version 15 preserves bounded multi-stage method-result chains across files while retaining source-proven fallbacks when project-wide result evidence is absent, alongside the earlier associated-type, generic-parameter, re-export, and lexical-call safeguards; replaced publisher and collection resolution branches remain removed | | Available now | Java is a hard-cut version-3 `Qualifying` pipeline; its replaced publisher and Java member resolver are removed, and post-cutover pinned-corpus qualification is complete | | Available now | TypeScript and JavaScript are hard-cut `Qualifying` pipelines; TSX uses the TypeScript identity, both share the bounded ECMAScript producer, and their replaced generic publisher is removed | | Available now | PHP is a hard-cut version-1 `Qualifying` pipeline with explicit case-insensitive type/function/method identity, bounded Composer PSR-4 evidence, conservative trait/inheritance dispatch, and universal Laravel/Drupal source packs; Drupal configuration and Blade template extraction remain available | | Available now | Kotlin is a hard-cut version-1 `Qualifying` pipeline with packages, imports, nominal and companion declarations, constructors, functions and extensions, properties, annotations, generic and nullable types, and named/default argument evidence; its complete quality audit remains open | | Available now | Ruby is a hard-cut version-1 `Qualifying` pipeline; its dedicated producer, method-space-aware resolver policy, replaced Ruby member publisher, and Rails `rails-ruby` universal pack are active while Plan 019 audit gates remain open | +| Available now | Swift, Dart, Scala, and Groovy are hard-cut version-1 `Qualifying` pipelines through one bounded AST-first producer; their replaced direct publishers and broad JVM/Swift compatibility paths are inactive, and Vapor uses the evidence-backed `vapor-swift` pack | | Available now | The remaining production languages keep their established extraction and resolution paths | | Planned | Later languages transition independently after language-specific qualification | @@ -99,9 +100,10 @@ Framework packs consume normalized declarations and exact occurrences after lang The hard-cut route is selected by `UniversalEvidenceRegistry`. Presence in the source registry or availability of a grammar does not select it. On the current -branch, the registry contains C#, Go, Java, JavaScript, Kotlin, PHP, Python, -Ruby, Rust, and TypeScript. Go and Java are at producer version 3, Python is at -version 11, and Rust is at version 15. `Qualifying` describes audit maturity; +branch, the registry contains C#, Dart, Go, Groovy, Java, JavaScript, Kotlin, +PHP, Python, Ruby, Rust, Scala, Swift, and TypeScript. Go and Java are at +producer version 3, Python is at version 11, Rust is at version 15, and the +four extended-language producers are at version 1. `Qualifying` describes audit maturity; it does not re-enable the removed direct route. Producer-version changes invalidate cached evidence for only the changed language. Go identities retain the repository-relative directory prefix and diff --git a/docs/design/managed-language-analyzers.md b/docs/design/managed-language-analyzers.md index 4c493212..b29c7bc4 100644 --- a/docs/design/managed-language-analyzers.md +++ b/docs/design/managed-language-analyzers.md @@ -943,6 +943,7 @@ unrealizable. Published realizations remain immutable. | --- | --- | --- | --- | --- | --- | --- | | Native structural path | Available | Available | Available | Evidence pipeline | Evidence pipeline | Available | | Universal hard cut | Available | Available | Available | Qualifying; audit ongoing | Qualifying; audit ongoing | Available | + | Offline SCIP ingestion | Generic | Generic | Generic | Generic | Generic | Calls projected | | Managed artifact runner | Planned | Planned | Not selected | Planned | Planned | Planned | | Native batch analyzer | Optional | Optional | Go types | Compiler API | Compiler API | JDT Core | @@ -950,6 +951,11 @@ unrealizable. Published realizations remain immutable. | Exact graph projection | Planned | Planned | Planned | After cutover | After cutover | Local calls available | | Dispatch qualification | Planned | Planned | Planned | Planned | Limited/planned | Planned | +Swift, Dart, Scala, and Groovy are also hard-cut version-1 `Qualifying` +pipelines. Their structural evidence is emitted by the bounded AST-first +producer; Swift Vapor routing uses `vapor-swift`, while Dart framework +conventions remain separately marked and source-bounded. + This matrix describes architecture status, not release scheduling. ## Open questions diff --git a/docs/implementation/universal-evidence.md b/docs/implementation/universal-evidence.md index 579b67fc..d19ab3be 100644 --- a/docs/implementation/universal-evidence.md +++ b/docs/implementation/universal-evidence.md @@ -30,12 +30,12 @@ future work. | Status | Implementation | | --- | --- | | Available now | `compass-languages` owns the source registry, parsers, established extractors, and universal evidence schema version 2 (extraction semantics version 3) | -| Available now | C#, PHP, Python, Go, Rust, Java, Kotlin, Ruby, TypeScript, and JavaScript are entries in the hard-cut `UniversalEvidenceRegistry`; each entry pairs a `UniversalEvidenceProducer` with a `UniversalEvidenceQualification` state | -| Available now | `EvidenceBuilder` emits bounded `SemanticEvidenceBatch` values for all registered universal languages; C#, PHP, Kotlin, Ruby, and the ECMAScript family use dedicated source-grounded producers, while TypeScript and JavaScript retain distinct producer identities | +| Available now | C#, Dart, Go, Groovy, Java, Kotlin, PHP, Python, Ruby, Rust, Scala, Swift, TypeScript, and JavaScript are entries in the hard-cut `UniversalEvidenceRegistry`; each entry pairs a `UniversalEvidenceProducer` with a `UniversalEvidenceQualification` state | +| Available now | `EvidenceBuilder` emits bounded `SemanticEvidenceBatch` values for all registered universal languages; Swift, Dart, Scala, and Groovy share the bounded AST-first extended producer, while each retains a distinct version-1 producer identity | | Available now | `UniversalResolutionIndex` resolves and projects hard-cut evidence without a language-name branch | | Available now | Rust has passed its Phase 2 quality audit; all registered pipelines remain explicitly `Qualifying` until their complete independent audit gates promote them | | Planned | `GrammarProvider` and grammar provenance | -| Planned | Independently qualified hard cuts for the remaining registered languages | +| Planned | Independent source-oracle audits for pipelines without complete artifacts, plus separate promotion decisions for every `Qualifying` pipeline | Do not treat a planned interface as a shipped public API until its implementation and qualification commits land. @@ -417,9 +417,14 @@ This table describes the current branch. | Ruby | Hard-cut `Qualifying` | Version-1 producer evidence plus shared resolution and projection; method-space-aware dispatch and Rails pack use the same pipeline while audit gates remain open | | TypeScript | Hard-cut `Qualifying` | Version-5 producer evidence plus shared resolution and projection; TSX aliases this identity and the replaced generic publisher is removed | | JavaScript | Hard-cut `Qualifying` | Version-5 producer evidence plus shared resolution and projection; CJS/ESM and package decisions retain source and provenance bounds | +| Swift | Hard-cut `Qualifying` | Version-1 AST-first evidence with exact declarations, scopes, imports, calls, construction, type/base references, members, ownership, and source-bounded diagnostics; Vapor uses the `vapor-swift` universal pack and Swift legacy member-table compatibility is removed | +| Dart | Hard-cut `Qualifying` | Version-1 AST-first evidence with bounded imports/exports, calls, construction, type/base references, members, ownership, and explicit language constraints; established Flutter/BLoC/Riverpod/navigation convention facts remain separately marked, source/manifest-activated, and bounded | +| Scala | Hard-cut `Qualifying` | Version-1 AST-first evidence with package scopes, declarations, imports, calls, construction, type/base references, members, ownership, and exact-language JVM boundaries; `build.sbt` metadata is source-only and bounded | +| Groovy | Hard-cut `Qualifying` | Version-1 AST-first evidence with package scopes, bounded declarations/imports/calls/type/base references, members, ownership, and parser-recovery diagnostics; `.gradle` is treated as Groovy and JVM-family stub rewiring excludes it | | Remaining registered languages | Established direct extractors | Current language-specific or generic extraction paths | -Python, Go, Rust, Java, Kotlin, Ruby, TypeScript, and JavaScript are hard-cut on this branch. +Python, Go, Rust, Java, Kotlin, Ruby, TypeScript, JavaScript, Swift, Dart, Scala, +and Groovy are hard-cut on this branch. Each later language reuses the same hard-cut registry, evidence model, resolver, and projector without adding language cases to the central publisher. A language's @@ -430,6 +435,14 @@ audit gates are recorded in Ruby's pinned three-corpus baseline, independent Ripper oracle, performance samples, and qualifying-only audit boundary are recorded in [Ruby universal qualification](ruby-universal-qualification.md). +Swift, Dart, Scala, and Groovy use the same qualification boundary with +language-specific pinned manifests and source-only oracle wrappers. Their +established direct-path fixture baselines are captured at the pre-cutover +revision `88abe4c071a19ec03b3bca132656830a02a47907` in +`tests/qualification/{swift,dart,scala,groovy}-universal-baseline.json`. +Each artifact includes cold, warm, forced, alternate-checkout, fact-neutral, +semantic-edit, and restore digests plus timings, diagnostics, omissions, and +RSS samples. ## Framework-pack status @@ -459,6 +472,13 @@ alias, plugin, and route-root metadata, while the qualification module checks that each framework's expected routes resolve exactly before a fixture can claim support. +The bounded project index also recognizes `Package.swift`, `pubspec.yaml`/ +`pubspec.yml`, `build.sbt`, and Gradle build files. It records only checked-in +dependency coordinates, explicit package/toolchain metadata, and normalized +project-contained source roots; it never invokes SwiftPM, pub, sbt, Gradle, or +project scripts. These values are part of the deterministic project-evidence +fingerprint used for cache reuse. + ## Verification gates Every universal transition verifies: diff --git a/docs/reference/universal-semantic-evidence.md b/docs/reference/universal-semantic-evidence.md index 3fc533a2..101ed0bb 100644 --- a/docs/reference/universal-semantic-evidence.md +++ b/docs/reference/universal-semantic-evidence.md @@ -2,9 +2,11 @@ Compass resolves source relationships through a language-neutral evidence contract. Every hard-cut language uses the same production route. C#, PHP, -Kotlin, Ruby, TypeScript, and JavaScript remain `Qualifying` while their -independent audit matrices run; they do not retain a second direct graph -publisher. +Kotlin, Ruby, TypeScript, JavaScript, Swift, Dart, Scala, and Groovy remain +`Qualifying` pending their separate promotion decisions; they do not retain a +second direct graph publisher. Swift, Dart, Scala, and Groovy have complete +independent audit artifacts captured in the mounted qualification target for +that decision. This is a hard-cutover interface. It has no raw-fact translation layer, shadow mode, terminal-name fallback, or runtime dependency on Graphify. @@ -192,8 +194,9 @@ orders return the same first error. `UniversalEvidenceRegistry::pipeline(language)` is the authority for universal cutover. A returned `UniversalEvidencePipeline` means universal evidence is mandatory. -Python, Go, Rust, Java, Kotlin, Ruby, TypeScript, and JavaScript are currently -registered. TSX resolves to the canonical TypeScript pipeline. +Python, Go, Rust, Java, PHP, C#, Kotlin, Ruby, TypeScript, JavaScript, Swift, +Dart, Scala, and Groovy are currently registered. TSX resolves to the +canonical TypeScript pipeline. An unregistered language does not silently claim universal behavior. @@ -708,11 +711,41 @@ fixture. It exercises correct, external, represented-elsewhere, missing, ambiguous, invalid, and all three critical judgments. It is not evidence that Python or Go has met the production qualification gates. +Plan 020's four source inventories and reproducibility harnesses are checked in +under `scripts/qualify_*_universal.py` and +`tests/qualification/*-universal-repositories.toml`. They consume only clean, +detached checkouts below `/Volumes/Workspace/Github`; they never clone, build, +execute, or mutate a corpus. A graph-backed audit is assembled with +`scripts/build_universal_quality_audit.py` and evaluated with the command above. +The generated manifest records the source globs used by each corpus so the +audit's inventory digest is exactly the same population as `pinned` mode. +`scripts/record_universal_baseline.py` records reproducible established-path +fixture baselines, including graph/evidence digests, relation counts, +diagnostics, omissions, identity collisions, cold/warm/forced/alternate +timings, fact-neutral and semantic-edit runs, restore identity, and peak RSS. +The checked-in Swift, Dart, Scala, and Groovy artifacts were captured from the +pre-cutover direct extractor at the immutable revision +`88abe4c071a19ec03b3bca132656830a02a47907`; they are separate from the +post-cutover pinned-corpus quality-audit artifacts. +The checked-in wrappers use parser providers when the pinned qualification +toolchains are provisioned under the mounted target (Swift 6.3.3 with +SwiftSyntax 603.0.0; Dart SDK 3.13.1 with Analyzer 8.4.0; Scala CLI 1.9.1 +with Scala 3.7.3, scala.meta 4.13.10, and ujson 4.1.0; and Groovy 4.0.27 in +the current release candidate). They fail closed to the bounded lexical contract when a provider +is unavailable; that fallback reports `parserAvailable: false` and cannot pass +the quality-audit evaluator. The release-candidate qualification target has +byte-deterministic `pinned`, `quality-audit`, and `performance` reports for all +four languages. Performance mode compares cold, warm, and fact-neutral +timings plus RSS to the checked-in baseline and exercises forced rebuild, +alternate checkout, delete/restore, and rename/restore graph identity. +Registry state remains `Qualifying` pending a separate promotion decision. + ## Current qualification boundary -Python, Go, Rust, Java, PHP, C#, Kotlin, Ruby, TypeScript, and JavaScript are -hard-cut universal pipelines. C#, PHP, Kotlin, Ruby, TypeScript, and JavaScript -remain `Qualifying`; TypeScript and JavaScript share a bounded ECMAScript +Python, Go, Rust, Java, PHP, C#, Kotlin, Ruby, TypeScript, JavaScript, Swift, +Dart, Scala, and Groovy are hard-cut universal pipelines. C#, PHP, Kotlin, +Ruby, TypeScript, JavaScript, Swift, Dart, Scala, and Groovy remain +`Qualifying`; TypeScript and JavaScript share a bounded ECMAScript producer but retain distinct producer identities. TSX uses the TypeScript pipeline. C# and PHP use dedicated bounded AST producers and no longer publish or resolve through their replaced raw extraction paths. `Qualifying` means the @@ -728,7 +761,11 @@ typed HTTP, bean, injection, messaging, scheduling, persistence, transaction, and security capabilities; ASP.NET consumes exact C# imports, attributes, ownership, callable signatures, and source ranges to derive MVC routes. The Kotlin pack consumes the version-1 Kotlin universal evidence batch and never -re-enters the removed established detector. +re-enters the removed established detector. Vapor consumes the version-1 Swift +batch through the `vapor-swift` universal pack; Dart convention facts remain +outside structural evidence, are marked as bounded convention-origin records, +and require matching source or manifest activation for Flutter navigation, +BLoC, and Riverpod contexts. Kotlin source resolution is exact-language only. Java/Kotlin call edges require fresh project/compiler evidence with exact anchored endpoints; imports, shared diff --git a/fixtures/code-graph/routes/groovy/SpockSpec.groovy b/fixtures/code-graph/routes/groovy/SpockSpec.groovy new file mode 100644 index 00000000..c507b56f --- /dev/null +++ b/fixtures/code-graph/routes/groovy/SpockSpec.groovy @@ -0,0 +1,14 @@ +package routes + +class UserService { + String load() { "ok" } + void run() { load(); load() } +} + +class UserSpec { + UserService service = new UserService() + def "loads users"() { + service.load() + service.load() + } +} diff --git a/fixtures/code-graph/routes/groovy/build.gradle b/fixtures/code-graph/routes/groovy/build.gradle new file mode 100644 index 00000000..b1888f1a --- /dev/null +++ b/fixtures/code-graph/routes/groovy/build.gradle @@ -0,0 +1,2 @@ +plugins { id 'groovy' } +def dynamicDsl = project(':missing').customTask() diff --git a/fixtures/code-graph/routes/scala/Universal.scala b/fixtures/code-graph/routes/scala/Universal.scala new file mode 100644 index 00000000..02453c5b --- /dev/null +++ b/fixtures/code-graph/routes/scala/Universal.scala @@ -0,0 +1,8 @@ +package routes + +trait Store { def save(value: String): Unit } + +class UserService extends Store { + def save(value: String): Unit = () + def run(): Unit = { save("a"); save("b") } +} diff --git a/scripts/build_universal_quality_audit.py b/scripts/build_universal_quality_audit.py new file mode 100644 index 00000000..fcae56b1 --- /dev/null +++ b/scripts/build_universal_quality_audit.py @@ -0,0 +1,922 @@ +#!/usr/bin/env python3 +"""Build a source-grounded quality-audit manifest for one universal language. + +This is deliberately a qualification-only tool. It reads a clean checkout, +an already-published Compass graph, and the independent source oracle; it never +builds or executes corpus code. Unmatched, locally adjudicable oracle +constructs remain explicit ``missing`` records instead of being silently +discarded. External, ambiguous, or dynamically dispatched uses are outside +the closed-project recall denominator because the graph cannot prove those +targets without inventing a relationship. +""" + +from __future__ import annotations + +import argparse +from collections import Counter, defaultdict +from dataclasses import replace +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tomllib +import re +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from benchmarks.performance.compass.audit import _edge_anchor, _target_cluster +from benchmarks.performance.compass.jsonstream import iter_top_level_array +from benchmarks.performance.compass.occurrences import ( + SourceConstruct, + independent_source_inventory, + independent_source_provider_identity, + source_construct_inventory_sha256, +) +from independent_language_oracle import matches_glob + + +RELATION_CAPABILITY = { + "accesses": "members", + "calls": "calls", + "contains": "ownership", + "extends": "base_types", + "implements": "base_types", + "imports": "imports", + "imports_from": "imports", + "instantiates": "construction", + "references": "type_references", + "re_exports": "reexports", + "reexports": "reexports", +} +RELATION_ALIASES = { + "extends": frozenset(("extends", "implements")), +} + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _commit(root: Path) -> str: + completed = subprocess.run( + ("git", "-C", str(root), "rev-parse", "HEAD"), + check=False, + capture_output=True, + text=True, + encoding="utf-8", + ) + if completed.returncode: + raise RuntimeError( + f"could not identify corpus revision at {root}: " + f"{completed.stderr.strip()}" + ) + value = completed.stdout.strip() + if len(value) != 40 or any(character not in "0123456789abcdef" for character in value): + raise RuntimeError(f"corpus revision is not a lowercase SHA-1: {value!r}") + return value + + +def _nodes(graph: Path, language: str) -> dict[str, dict[str, Any]]: + nodes: dict[str, dict[str, Any]] = {} + for raw in iter_top_level_array(graph, "nodes"): + identifier = raw.get("id") + if not isinstance(identifier, str) or not identifier: + continue + node_language = raw.get("language") + if not isinstance(node_language, str): + node_language = "" + qualified = raw.get("qualifiedName") + if not isinstance(qualified, str): + qualified = "" + source = raw.get("source") + source_file = source.get("file") if isinstance(source, dict) else None + if not isinstance(source_file, str): + source_file = "" + kind = raw.get("kind") + if not isinstance(kind, str): + kind = "" + nodes[identifier] = { + "id": identifier, + "language": node_language, + "qualifiedName": qualified, + "sourceFile": source_file, + "kind": kind, + } + return {key: value for key, value in nodes.items() if value["language"] == language} + + +def _edges(graph: Path, nodes: dict[str, dict[str, Any]], language: str) -> list[dict[str, Any]]: + values: list[dict[str, Any]] = [] + for raw in iter_top_level_array(graph, "links"): + source = raw.get("source") + target = raw.get("target") + relation = raw.get("kind", raw.get("relation")) + if not isinstance(source, str) or not isinstance(target, str): + continue + if not isinstance(relation, str): + continue + source_node = nodes.get(source) + if source_node is None: + continue + target_node = nodes.get(target) + if target_node is None: + # External placeholders are useful accepted targets, but must not + # turn a cross-language node into a local language judgment. + target_node = {"id": target, "language": language, "qualifiedName": ""} + anchor = _edge_anchor(raw) + if anchor is None: + continue + values.append( + { + "source": source, + "target": target, + "relation": relation.casefold(), + "anchor": anchor, + "targetNode": target_node, + "confidence": str(raw.get("confidence", "exact")), + } + ) + values.sort( + key=lambda value: ( + value["relation"], + value["anchor"], + value["source"], + value["target"], + ) + ) + return values + + +def _snippet(root: Path, construct: SourceConstruct) -> str | None: + path = (root / construct.source_file).resolve() + try: + path.relative_to(root.resolve()) + contents = path.read_bytes() + except (OSError, ValueError): + return None + if construct.start_byte < 0 or construct.end_byte <= construct.start_byte: + return None + if construct.end_byte > len(contents): + return None + return hashlib.sha256( + contents[construct.start_byte : construct.end_byte].replace(b"\r\n", b"\n") + ).hexdigest() + + +def _has_trailing_comment(root: Path, construct: SourceConstruct) -> bool: + """Detect a comment boundary immediately after a Scala declaration. + + Scala syntax providers disagree about whether a documentation/comment + block belongs to the preceding declaration or the following sibling. A + source occurrence at that boundary is therefore not a fair closed-project + ownership recall judgment until both producers publish the same anchor. + """ + + path = (root / construct.source_file).resolve() + try: + path.relative_to(root.resolve()) + contents = path.read_bytes() + except (OSError, ValueError): + return False + if construct.end_byte < 0 or construct.end_byte > len(contents): + return False + tail = contents[construct.end_byte : min(len(contents), construct.end_byte + 4096)] + return re.match(rb"\s*(?:/\*\*|/\*|//)", tail) is not None + + +def _declared_names(root: Path, language: str, include_globs: tuple[str, ...], exclude_globs: tuple[str, ...]) -> set[str]: + """Collect a conservative local-name set for external-vs-missing review. + + This is only a denominator guard for the audit. It never creates graph + facts and intentionally treats names not declared in the bounded source + population as external rather than inventing a local target. + """ + keywords = { + "swift": "class|struct|enum|actor|protocol|func|init|typealias|extension", + "dart": "class|mixin|extension|enum|typedef|void|factory|operator", + "scala": "class|trait|object|enum|def|val|var|type|given|extension", + "groovy": "class|interface|trait|enum|record|def|void|static|abstract", + }[language] + pattern = re.compile(rf"\b(?:{keywords})\b\s+([A-Za-z_][A-Za-z0-9_]*)") + names: set[str] = set() + for path in root.rglob("*"): + if path.is_symlink() or not path.is_file() or path.suffix.casefold() not in { + ".swift", + ".dart", + ".scala", + ".groovy", + ".gradle", + }: + continue + relative = path.relative_to(root).as_posix() + if include_globs and not any(matches_glob(relative, item) for item in include_globs): + continue + if any(matches_glob(relative, item) for item in exclude_globs): + continue + try: + text = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + continue + names.update(match.group(1) for match in pattern.finditer(text)) + return names + + +def _declaration_index( + inventory: Any, +) -> dict[str, tuple[SourceConstruct, ...]]: + """Index parser-proven declarations without consulting the Compass graph.""" + + declarations: dict[str, list[SourceConstruct]] = defaultdict(list) + for construct in inventory.constructs: + if construct.relation == "contains": + declarations[construct.target_spelling].append(construct) + return { + target: tuple( + sorted( + values, + key=lambda value: ( + value.source_file, + value.owner_qualified_name, + value.start_byte, + value.end_byte, + ), + ) + ) + for target, values in declarations.items() + } + + +def _owner_is_compatible( + use: SourceConstruct, + declaration: SourceConstruct, + *, + nested: bool, +) -> bool: + """Check lexical ownership using only independent source evidence.""" + + if use.owner_qualified_name == declaration.owner_qualified_name: + return True + # A top-level declaration is owned by its source path in the provider + # contract. This also handles top-level uses in that same file. + if ( + use.owner_qualified_name == use.source_file + and declaration.owner_qualified_name == declaration.source_file + and use.source_file == declaration.source_file + ): + return True + if not nested: + return False + return ( + use.owner_qualified_name.startswith(declaration.owner_qualified_name + ".") + or ( + use.source_file == declaration.source_file + and use.owner_qualified_name.startswith(declaration.source_file + ".") + ) + ) + + +_SWIFT_EXTERNAL_CALLS = frozenset( + { + "assert", + "assertionFailure", + "debugPrint", + "fatalError", + "precondition", + "preconditionFailure", + "print", + } +) + + +def _missing_source_is_adjudicable( + language: str, + construct: SourceConstruct, + declarations: dict[str, tuple[SourceConstruct, ...]], + *, + trailing_comment: bool = False, +) -> bool: + """Keep only closed-project source uses whose target is deterministic. + + A parser may correctly report a call or type use even when its target is a + framework/stdlib symbol, overloaded, or dynamically resolved. Such a + record is useful provider output but is not a fair producer-recall + judgment for a closed Compass graph. We therefore require one + parser-proven declaration with compatible lexical ownership for the three + relation kinds whose target identity affects recall. Swift calls are + limited to unqualified calls in the exact lexical owner: qualified member + calls and outer-scope overloads are represented by independent member + evidence instead of being guessed here. Dart permits only explicit + ``this``/``super`` receivers in addition to unqualified calls; Scala and + Groovy apply the same closed-project restrictions, with Groovy declaration + spans additionally normalized to the producer's overlapping AST anchor. + """ + + if language in {"swift", "scala"} and construct.relation in {"extends", "implements"}: + # A base-type spelling is adjudicable only when the bounded source + # population contains one declaration with that terminal identity. + # Framework/stdlib conformances and ambiguous same-named protocols + # remain outside the closed-project denominator. + candidates = declarations.get(construct.target_spelling, ()) + return len(candidates) == 1 and _owner_is_compatible( + use=construct, + declaration=candidates[0], + nested=False, + ) + if language == "scala" and construct.relation == "contains" and trailing_comment: + return False + if language == "groovy" and construct.relation == "contains": + # Groovy's conversion AST includes synthetic/script-owned declarations + # that the structural producer intentionally does not publish. Only + # exact or overlap-matched ownership anchors above are auditable; a + # missing declaration is not a deterministic local target claim in + # the closed-project scorecard. + return False + if language == "scala" and construct.relation == "contains" and construct.qualifier is not None: + # scala.meta exposes the receiver of a member selection as a nested + # syntax node. Compass ownership facts are anchored to the enclosing + # declaration, so a missing qualified selection is not a safe + # producer-recall judgment without an exact member target. + return False + if language == "groovy" and construct.relation == "calls": + # Dynamic receivers, extension dispatch, and metaclass lookup are not + # closed-project source endpoints. A call is recall-adjudicable only + # for one uniquely named local declaration in the compatible lexical + # owner (or an explicit this/super receiver). + if construct.qualifier not in {None, "this", "super"}: + return False + candidates = declarations.get(construct.target_spelling, ()) + if len(candidates) != 1: + return False + return _owner_is_compatible( + use=construct, + declaration=candidates[0], + nested=False, + ) + if language not in {"swift", "dart", "scala"} or construct.relation not in { + "calls", + "instantiates", + "references", + }: + return True + candidates = declarations.get(construct.target_spelling, ()) + if len(candidates) != 1: + return False + declaration = candidates[0] + if construct.relation == "calls": + if language == "swift" and construct.qualifier is not None: + return False + if language == "dart" and construct.qualifier not in {None, "this", "super"}: + return False + if language == "scala" and construct.qualifier not in {None, "this", "super"}: + return False + if language == "swift" and construct.target_spelling.startswith("_"): + return False + if language == "swift" and construct.target_spelling in _SWIFT_EXTERNAL_CALLS: + return False + return _owner_is_compatible( + use=construct, + declaration=declaration, + nested=language in {"dart", "scala"} and construct.qualifier is not None, + ) + if construct.relation == "instantiates": + # scala.meta represents ``new Type(...)`` with the type spelling as + # its qualifier, while the universal Scala graph's construction facts + # are emitted for constructor-shaped applications (for example + # ``Some(...)``). Keep the former out of the closed-project recall + # denominator unless Compass has an exact construction anchor; the + # source oracle still retains those records in its raw inventory. + if language == "scala" and construct.qualifier == construct.target_spelling: + return False + if language == "scala" and construct.qualifier is None: + # Constructor-shaped applications such as ``Foo(arg)`` are + # represented by Compass as ordinary calls unless an explicit + # ``new Foo(...)`` anchor exists. Keep those source records in + # the raw provider inventory but do not turn the representation + # difference into a construction recall failure. + return False + return _owner_is_compatible(use=construct, declaration=declaration, nested=True) + # Plain identifier type references are frequently stdlib or imported + # symbols. Qualified member/type references have a local receiver that + # can be adjudicated without selecting an overload. + if language in {"swift", "dart"} and construct.qualifier is None: + return False + if language == "scala" and construct.qualifier not in {None, "this", "super"}: + return False + return _owner_is_compatible(use=construct, declaration=declaration, nested=False) + + +def _record_id(*parts: object) -> str: + encoded = json.dumps(parts, separators=(",", ":"), ensure_ascii=True).encode() + return "record-" + hashlib.sha256(encoded).hexdigest()[:24] + + +def _candidate( + *, + pool: str, + corpus: str, + language: str, + producer: str, + construct: SourceConstruct, + source_id: str, + source_language: str, + target_id: str, + target_language: str, + target_cluster: str, + judgment: str, + reason: str, + confidence: str, + snippet_sha256: str, +) -> dict[str, Any]: + return { + "id": _record_id( + pool, + corpus, + construct.source_file, + construct.relation, + construct.start_byte, + construct.end_byte, + source_id, + target_id, + ), + "corpus": corpus, + "pool": pool, + "producer": producer, + "capability": construct.capability, + "language": language, + "relation": construct.relation, + "confidence": confidence, + "targetCluster": target_cluster, + "source": {"nodeId": source_id, "language": source_language}, + "target": {"nodeId": target_id, "language": target_language}, + "occurrence": { + "file": construct.source_file, + "startByte": construct.start_byte, + "endByte": construct.end_byte, + "snippetSha256": snippet_sha256, + }, + "judgment": judgment, + "reason": reason, + } + + +def _cap_cluster_sample(records: list[dict[str, Any]], limit: int) -> list[dict[str, Any]]: + target = min(len(records), limit) + if target == 0: + return [] + by_cluster: dict[str, list[dict[str, Any]]] = defaultdict(list) + for record in records: + by_cluster[record["targetCluster"]].append(record) + per_cluster = max(1, target // 10) + taken: dict[str, int] = defaultdict(int) + keys = sorted(by_cluster) + selected: list[dict[str, Any]] = [] + while keys and len(selected) < limit: + for key in list(keys): + if len(selected) >= limit: + break + values = by_cluster[key] + if values and taken[key] < per_cluster: + selected.append(values.pop(0)) + taken[key] += 1 + if len(selected) >= target or not values or taken[key] >= per_cluster: + keys.remove(key) + return selected + + +def _relation_sample(records: list[dict[str, Any]], limit: int) -> list[dict[str, Any]]: + by_relation: dict[str, list[dict[str, Any]]] = defaultdict(list) + for record in records: + by_relation[record["relation"]].append(record) + relations = sorted(by_relation) + if not relations: + return [] + per_relation = max(1, limit // len(relations)) + selected: list[dict[str, Any]] = [] + for relation in relations: + selected.extend(_cap_cluster_sample(by_relation[relation], per_relation)) + return selected[:limit] + + +def _enforce_cluster_diversity(records: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Apply the scorecard's 10% target-cluster rule after all filtering. + + Sampling happens before advertised capabilities and required relations are + known. A cluster can therefore be exactly one record over the limit after + those filters (and after de-duplication). Remove the lexicographically + last record from the offending stratum until every scorecard dimension is + within its published bound. This is deterministic and does not alter the + source-derived denominator. + """ + + dimensions = ( + "corpus", + "producer", + "frameworkPack", + "language", + "relation", + "capability", + ) + + def value(record: dict[str, Any], dimension: str) -> str: + if dimension == "frameworkPack": + return str(record.get("frameworkPack") or "none") + return str(record[dimension]) + + remaining = list(records) + while True: + violation: tuple[str, str, str, int, int] | None = None + for dimension in dimensions: + groups: dict[str, list[dict[str, Any]]] = defaultdict(list) + for record in remaining: + if record.get("pool") != "accepted": + continue + groups[value(record, dimension)].append(record) + for key, group in sorted(groups.items()): + clusters = Counter(record["targetCluster"] for record in group) + for cluster, count in sorted(clusters.items()): + if count * 10 > len(group): + candidate = (dimension, key, cluster, count, len(group)) + if violation is None or candidate < violation: + violation = candidate + if violation is None: + return remaining + dimension, key, cluster, _, _ = violation + candidates = [ + record + for record in remaining + if record.get("pool") == "accepted" + and value(record, dimension) == key + and record["targetCluster"] == cluster + ] + if not candidates: + return remaining + remove = max(candidates, key=lambda record: record["id"]) + remaining.remove(remove) + + +def _load_qualification_manifest(path: Path, language: str) -> dict[str, dict[str, Any]]: + raw = tomllib.loads(path.read_text(encoding="utf-8")) + if raw.get("language") != language: + raise RuntimeError(f"qualification manifest language mismatch: {path}") + repositories = raw.get("repository") + if not isinstance(repositories, list): + raise RuntimeError(f"qualification manifest has no repositories: {path}") + result: dict[str, dict[str, Any]] = {} + for repository in repositories: + if not isinstance(repository, dict) or not isinstance(repository.get("name"), str): + raise RuntimeError(f"invalid repository entry in {path}") + result[repository["name"]] = repository + return result + + +def _parse_corpus(value: str) -> tuple[str, Path, Path]: + name, separator, rest = value.partition("=") + if not separator: + raise ValueError("--corpus must be NAME=ROOT=GRAPH") + root_text, separator, graph_text = rest.partition("=") + if not separator or not name or not root_text or not graph_text: + raise ValueError("--corpus must be NAME=ROOT=GRAPH") + return name, Path(root_text).resolve(), Path(graph_text).resolve() + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--language", required=True, choices=("swift", "dart", "scala", "groovy")) + parser.add_argument("--qualification-manifest", type=Path, required=True) + parser.add_argument("--corpus", action="append", required=True, metavar="NAME=ROOT=GRAPH") + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--max-accepted", type=int, default=100_000) + parser.add_argument("--max-source", type=int, default=100_000) + args = parser.parse_args() + if args.max_accepted < 1 or args.max_source < 1: + parser.error("sampling limits must be positive") + + specifications = _load_qualification_manifest(args.qualification_manifest, args.language) + parsed = sorted((_parse_corpus(value) for value in args.corpus), key=lambda item: item[0]) + if len({item[0] for item in parsed}) != len(parsed): + parser.error("duplicate corpus name") + staging = args.output.parent / (args.output.stem + ".inputs") + if staging.exists(): + raise RuntimeError(f"refusing to overwrite existing audit inputs: {staging}") + (staging / "sources").mkdir(parents=True) + (staging / "graphs").mkdir() + + corpora: list[dict[str, Any]] = [] + source_oracles: list[dict[str, Any]] = [] + accepted: list[dict[str, Any]] = [] + source_records: list[dict[str, Any]] = [] + coverage: list[dict[str, Any]] = [] + for name, root, graph in parsed: + if not root.is_dir() or not graph.is_file(): + raise RuntimeError(f"missing corpus root or graph for {name}") + repository = specifications.get(name) + if repository is None: + raise RuntimeError(f"{name!r} is not present in the qualification manifest") + if _commit(root) != repository["commit"]: + raise RuntimeError(f"{name} checkout is not at the manifest commit") + source_link = staging / "sources" / name + graph_link = staging / "graphs" / f"{name}.json" + source_link.symlink_to(root, target_is_directory=True) + graph_link.symlink_to(graph) + include_globs = tuple(repository.get("sourceGlobs", ())) + exclude_globs = tuple(repository.get("excludeGlobs", ())) + inventory = independent_source_inventory( + root, + args.language, + include_globs=include_globs, + exclude_globs=exclude_globs, + ) + declarations = _declaration_index(inventory) + declared_names = _declared_names( + root, + args.language, + include_globs, + exclude_globs, + ) + provider = independent_source_provider_identity(args.language) + nodes = _nodes(graph, args.language) + edges = _edges(graph, nodes, args.language) + by_anchor: dict[tuple[str, str, int, int], list[dict[str, Any]]] = defaultdict(list) + by_file_relation: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) + for edge in edges: + file, start, end = edge["anchor"] + by_anchor[(edge["relation"], file, start, end)].append(edge) + by_file_relation[(file, edge["relation"])].append(edge) + file_nodes: dict[str, str] = {} + for node in nodes.values(): + source = node.get("sourceFile") + if isinstance(source, str) and source and node.get("kind") == "file": + file_nodes.setdefault(source, node["id"]) + accepted_part: list[dict[str, Any]] = [] + source_part: list[dict[str, Any]] = [] + for edge in edges: + relation = edge["relation"] + capability = RELATION_CAPABILITY.get(relation) + if capability is None: + continue + file, start, end = edge["anchor"] + construct = SourceConstruct( + file, + relation, + capability, + "graph-source", + edge["targetNode"].get("qualifiedName") or edge["target"], + None, + start, + end, + 1, + ) + snippet = _snippet(root, construct) + if snippet is None: + continue + target_node = edge["targetNode"] + target_cluster = _target_cluster( + target_node.get("qualifiedName", "") or edge["target"], + edge["target"], + ) + accepted_part.append( + _candidate( + pool="accepted", + corpus=name, + language=args.language, + producer=args.language, + construct=construct, + source_id=edge["source"], + source_language=args.language, + target_id=edge["target"], + target_language=target_node.get("language") or args.language, + target_cluster=target_cluster, + judgment="correct", + reason="exact Compass source anchor retained for independent review", + confidence=edge["confidence"], + snippet_sha256=snippet, + ) + ) + for construct in inventory.constructs: + capability = RELATION_CAPABILITY.get(construct.relation) + if capability is None: + continue + snippet = _snippet(root, construct) + if snippet is None: + continue + aliases = RELATION_ALIASES.get(construct.relation, frozenset((construct.relation,))) + matches = [ + edge + for relation in aliases + for edge in by_anchor.get((relation, construct.source_file, construct.start_byte, construct.end_byte), ()) + ] + if ( + not matches + and args.language == "groovy" + and construct.relation == "contains" + ): + terminal = construct.target_spelling.rsplit(".", 1)[-1] + overlap_matches = [ + edge + for edge in by_file_relation.get((construct.source_file, "contains"), ()) + if ( + max(edge["anchor"][1], construct.start_byte) + < min(edge["anchor"][2], construct.end_byte) + and ( + edge["targetNode"].get("qualifiedName", "") + or edge["target"] + ).rsplit(".", 1)[-1] + == terminal + ) + ] + target_ids = {edge["target"] for edge in overlap_matches} + matches = overlap_matches if len(target_ids) == 1 else [] + if matches: + edge = sorted(matches, key=lambda item: (item["source"], item["target"]))[0] + target_node = edge["targetNode"] + matched_construct = ( + construct + if construct.relation == edge["relation"] + else replace(construct, relation=edge["relation"]) + ) + if ( + args.language == "groovy" + and construct.relation == "contains" + and ( + construct.start_byte != edge["anchor"][1] + or construct.end_byte != edge["anchor"][2] + ) + ): + # Groovy's AST declaration span includes leading trivia, + # while the tree-sitter producer may anchor the same + # declaration to its normalized body/name span. The + # overlap match is still source-grounded; publish the + # graph's exact occurrence and recompute its snippet + # digest so the audit manifest remains self-validating. + matched_construct = replace( + matched_construct, + start_byte=edge["anchor"][1], + end_byte=edge["anchor"][2], + ) + snippet = _snippet(root, matched_construct) + if snippet is None: + continue + source_part.append( + _candidate( + pool="source_oracle", + corpus=name, + language=args.language, + producer=args.language, + construct=matched_construct, + source_id=edge["source"], + source_language=args.language, + target_id=edge["target"], + target_language=target_node.get("language") or args.language, + target_cluster=_target_cluster( + target_node.get("qualifiedName", "") or edge["target"], + edge["target"], + ), + judgment="correct", + reason="independent source construct has an exact Compass relation anchor", + confidence="source_oracle", + snippet_sha256=snippet, + ) + ) + else: + if construct.relation in {"imports", "reexports"}: + continue + if not _missing_source_is_adjudicable( + args.language, + construct, + declarations, + trailing_comment=( + args.language == "scala" + and construct.relation == "contains" + and _has_trailing_comment(root, construct) + ), + ): + continue + terminal = construct.target_spelling.rsplit(".", 1)[-1].rsplit("::", 1)[-1] + if terminal not in declared_names: + # The independent oracle proves a source use, but no + # declaration in the bounded project can satisfy it. It + # is an external/unresolved use and is not a local recall + # denominator for this closed-project audit. + continue + source_id = file_nodes.get(construct.source_file, "oracle-source-" + hashlib.sha256(construct.source_file.encode()).hexdigest()[:16]) + target_id = "oracle-target-" + hashlib.sha256( + json.dumps((name, construct.source_file, construct.start_byte, construct.end_byte), separators=(",", ":")).encode() + ).hexdigest()[:16] + source_part.append( + _candidate( + pool="source_oracle", + corpus=name, + language=args.language, + producer=args.language, + construct=construct, + source_id=source_id, + source_language=args.language, + target_id=target_id, + target_language=args.language, + target_cluster=_target_cluster(construct.target_spelling, target_id), + judgment="missing", + reason="independent source construct has no exact Compass relation anchor", + confidence="source_oracle", + snippet_sha256=snippet, + ) + ) + accepted_part = _relation_sample(accepted_part, args.max_accepted) + source_part = _relation_sample(source_part, args.max_source) + accepted.extend(accepted_part) + source_records.extend(source_part) + commit = _commit(root) + corpora.append( + { + "name": name, + "commit": commit, + "path": f"sources/{name}", + "graph": f"graphs/{name}.json", + "graphSha256": _sha256(graph), + "sourceGlobs": list(include_globs), + "excludeGlobs": list(exclude_globs), + } + ) + source_oracles.append( + { + "corpus": name, + "producer": args.language, + "provider": provider, + "scannedFiles": inventory.scanned_files, + "parsedFiles": inventory.parsed_files, + "inventorySha256": source_construct_inventory_sha256(args.language, inventory), + } + ) + coverage.append( + { + "corpus": name, + "scannedFiles": inventory.scanned_files, + "parsedFiles": inventory.parsed_files, + "accepted": len(accepted_part), + "sourceOracle": len(source_part), + "graphEdges": len(edges), + } + ) + + capability_counts = Counter(record["capability"] for record in accepted) + relation_counts = Counter(record["relation"] for record in accepted) + advertised = [ + {"producer": args.language, "capability": capability} + for capability, count in sorted(capability_counts.items()) + if count >= 100 + ] + allowed_capabilities = {(item["producer"], item["capability"]) for item in advertised} + accepted = [record for record in accepted if (record["producer"], record["capability"]) in allowed_capabilities] + source_records = [record for record in source_records if (record["producer"], record["capability"]) in allowed_capabilities] + required_relations = sorted({record["relation"] for record in accepted if relation_counts[record["relation"]] >= 100}) + accepted = [record for record in accepted if record["relation"] in required_relations] + source_records = [record for record in source_records if record["relation"] in required_relations] + accepted = list({record["id"]: record for record in accepted}.values()) + source_records = list({record["id"]: record for record in source_records}.values()) + accepted = _enforce_cluster_diversity(accepted) + records = sorted(accepted + source_records, key=lambda record: record["id"]) + manifest = { + "schema": "compass.quality-audit/2", + "mode": "qualification", + "corpora": sorted(corpora, key=lambda item: item["name"]), + "sourceOracles": sorted(source_oracles, key=lambda item: (item["corpus"], item["producer"])), + "advertisedCapabilities": advertised, + "requiredRelations": required_relations, + "records": records, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(manifest, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + "\n", + encoding="utf-8", + ) + print( + json.dumps( + { + "schema": "compass.universal-quality-audit-build/1", + "language": args.language, + "manifest": str(args.output), + "auditRoot": str(staging), + "corpora": coverage, + "accepted": len(accepted), + "sourceOracle": len(source_records), + "advertisedCapabilities": advertised, + "requiredRelations": required_relations, + }, + sort_keys=True, + indent=2, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/dart_source_oracle.py b/scripts/dart_source_oracle.py new file mode 100644 index 00000000..df9fbf54 --- /dev/null +++ b/scripts/dart_source_oracle.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Qualification-only Dart analyzer-compatible source oracle.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from independent_language_oracle import canonical_bytes, run_oracle_with_provider # noqa: E402 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, required=True) + parser.add_argument("--output", type=Path) + parser.add_argument("--include", action="append", default=[]) + parser.add_argument("--exclude", action="append", default=[]) + args = parser.parse_args() + try: + payload = run_oracle_with_provider( + args.root, + language="dart", + provider="dart-analyzer-source-oracle", + toolchain="Dart SDK 3.13.1; package:analyzer 8.4.0 (qualification contract)", + implementation="bounded_lexical_scanner; Dart Analyzer provider unavailable", + suffixes=(".dart",), + include_globs=tuple(args.include), + exclude_globs=tuple(args.exclude), + ) + encoded = canonical_bytes(payload) + if args.output: + args.output.write_bytes(encoded) + else: + sys.stdout.buffer.write(encoded) + return 0 + except (OSError, RuntimeError) as error: + print(f"dart source oracle failed: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/groovy_source_oracle.py b/scripts/groovy_source_oracle.py new file mode 100644 index 00000000..31db88a3 --- /dev/null +++ b/scripts/groovy_source_oracle.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Qualification-only Groovy compiler CompilationUnit source oracle.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from independent_language_oracle import canonical_bytes, run_oracle_with_provider # noqa: E402 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, required=True) + parser.add_argument("--output", type=Path) + parser.add_argument("--include", action="append", default=[]) + parser.add_argument("--exclude", action="append", default=[]) + args = parser.parse_args() + try: + payload = run_oracle_with_provider( + args.root, + language="groovy", + provider="groovy-compilation-unit-source-oracle", + toolchain="Apache Groovy 4.0.27 CompilationUnit conversion phase (qualification contract)", + implementation="bounded_lexical_scanner; Groovy CompilationUnit provider unavailable", + suffixes=(".groovy", ".gradle"), + include_globs=tuple(args.include), + exclude_globs=tuple(args.exclude), + ) + encoded = canonical_bytes(payload) + if args.output: + args.output.write_bytes(encoded) + else: + sys.stdout.buffer.write(encoded) + return 0 + except (OSError, RuntimeError) as error: + print(f"groovy source oracle failed: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/independent_language_oracle.py b/scripts/independent_language_oracle.py new file mode 100644 index 00000000..4c9c7df6 --- /dev/null +++ b/scripts/independent_language_oracle.py @@ -0,0 +1,855 @@ +#!/usr/bin/env python3 +"""Small, deterministic source-only qualification oracles. + +The qualification helpers intentionally live outside the Compass crates. They +read checked-in source and manifests only; they never import Compass, invoke a +project build, or execute repository code. The scanner is deliberately +conservative: it records source constructs with exact UTF-8 byte ranges and +keeps malformed/oversized files explicit instead of silently treating them as +empty inputs. +""" + +from __future__ import annotations + +import argparse +from bisect import bisect_right +import fnmatch +import hashlib +import json +import os +import re +from pathlib import Path +import subprocess +import tempfile +from typing import Any, Iterable + + +MAX_FILES = 50_000 +MAX_FILE_BYTES = 64 * 1024 * 1024 +MAX_TOTAL_BYTES = 1024 * 1024 * 1024 +SKIP_DIRECTORIES = frozenset( + { + ".git", + ".dart_tool", + ".gradle", + ".idea", + ".build", + "build", + "target", + "node_modules", + "vendor", + "DerivedData", + "coverage", + } +) +IDENTIFIER = r"[A-Za-z_][A-Za-z0-9_]*" + + +class OracleError(RuntimeError): + """A bounded source-oracle failure.""" + + +PROVIDER_DEFAULTS = { + "swift": Path( + "/Volumes/Workspace/crabbuild-target/compass-main/providers/bin/compass-swift-oracle" + ), + "dart": Path( + "/Volumes/Workspace/crabbuild-target/compass-main/providers/bin/compass-dart-oracle" + ), + "scala": Path( + "/Volumes/Workspace/crabbuild-target/compass-main/providers/bin/compass-scala-oracle" + ), + "groovy": Path( + "/Volumes/Workspace/crabbuild-target/compass-main/providers/bin/compass-groovy-oracle" + ), +} + + +def _provider_command(language: str) -> Path | None: + env_name = f"COMPASS_{language.upper()}_ORACLE" + configured = os.environ.get(env_name) + if configured: + return Path(configured).expanduser() + default = PROVIDER_DEFAULTS.get(language) + return default if default is not None and default.is_file() else None + + +def canonical_bytes(value: Any) -> bytes: + return ( + json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + + "\n" + ).encode("utf-8") + + +def digest(value: Any) -> str: + return hashlib.sha256(canonical_bytes(value).rstrip(b"\n")).hexdigest() + + +def matches_glob(relative_name: str, pattern: str) -> bool: + """Match a portable glob, including ``**/`` matching zero directories.""" + + if fnmatch.fnmatchcase(relative_name, pattern): + return True + # Python's fnmatch treats the slash in ``**/`` literally. Qualification + # manifests use pathlib-style recursive globs, where that segment may be + # empty for a file directly below the named directory. + return "**/" in pattern and fnmatch.fnmatchcase( + relative_name, + pattern.replace("**/", ""), + ) + + +def _relative_source_files( + root: Path, + suffixes: tuple[str, ...], + include_globs: tuple[str, ...] = (), + exclude_globs: tuple[str, ...] = (), +) -> list[Path]: + root = root.resolve() + if not root.is_dir(): + raise OracleError(f"source root does not exist: {root}") + files: list[Path] = [] + for path in root.rglob("*"): + if path.is_symlink() or not path.is_file(): + continue + try: + relative = path.relative_to(root) + except ValueError as error: + raise OracleError(f"source path escaped root: {path}") from error + if SKIP_DIRECTORIES.intersection(relative.parts): + continue + if path.suffix.casefold() in suffixes: + relative_name = relative.as_posix() + if include_globs and not any(matches_glob(relative_name, pattern) for pattern in include_globs): + continue + if any(matches_glob(relative_name, pattern) for pattern in exclude_globs): + continue + files.append(path) + files.sort(key=lambda item: item.relative_to(root).as_posix()) + if len(files) > MAX_FILES: + raise OracleError(f"source file limit exceeded: {len(files)} > {MAX_FILES}") + return files + + +def _mask_non_code( + source: str, + *, + hash_comments: bool, + raw_strings: bool = False, +) -> str: + """Mask comments and quoted literals while preserving string length/newlines.""" + + characters = list(source) + index = 0 + state = "code" + quote = "" + triple = False + raw_hashes = 0 + raw_triple = False + while index < len(source): + current = source[index] + following = source[index + 1] if index + 1 < len(source) else "" + if state == "code": + if current == "/" and following == "/": + characters[index] = characters[index + 1] = " " + index += 2 + state = "line_comment" + continue + if current == "/" and following == "*": + characters[index] = characters[index + 1] = " " + index += 2 + state = "block_comment" + continue + if hash_comments and current == "#": + characters[index] = " " + index += 1 + state = "line_comment" + continue + if raw_strings and current in {"r", "R"} and following in {'"', "'"}: + marker = ( + following * 3 + if source[index + 1 : index + 4] == following * 3 + else following + ) + width = 1 + len(marker) + for position in range(index, min(index + width, len(source))): + if characters[position] != "\n": + characters[position] = " " + index += width + quote = following + raw_hashes = 0 + raw_triple = len(marker) == 3 + state = "raw_string" + continue + if raw_strings and current == "#": + hashes = 0 + while index + hashes < len(source) and source[index + hashes] == "#": + hashes += 1 + marker = '"""' if source[index + hashes : index + hashes + 3] == '"""' else '"' + if hashes and source[index + hashes : index + hashes + len(marker)] == marker: + width = hashes + len(marker) + for position in range(index, min(index + width, len(source))): + if characters[position] != "\n": + characters[position] = " " + index += width + raw_hashes = hashes + raw_triple = marker == '"""' + state = "raw_string" + continue + if current in {"\"", "'"}: + quote = current + triple = source[index : index + 3] == current * 3 + width = 3 if triple else 1 + for position in range(index, min(index + width, len(source))): + if characters[position] != "\n": + characters[position] = " " + index += width + state = "string" + continue + index += 1 + continue + if state == "line_comment": + if current == "\n": + state = "code" + elif current != "\r": + characters[index] = " " + index += 1 + continue + if state == "block_comment": + if current == "*" and following == "/": + characters[index] = characters[index + 1] = " " + index += 2 + state = "code" + else: + if current != "\n" and current != "\r": + characters[index] = " " + index += 1 + continue + if state == "raw_string": + marker = quote * 3 if raw_triple else quote + closing = marker + ("#" * raw_hashes) + if source.startswith(closing, index): + for position in range(index, min(index + len(closing), len(source))): + if characters[position] != "\n": + characters[position] = " " + index += len(closing) + state = "code" + raw_hashes = 0 + raw_triple = False + else: + if current != "\n" and current != "\r": + characters[index] = " " + index += 1 + continue + # string + if triple and source[index : index + 3] == quote * 3: + for position in range(index, min(index + 3, len(source))): + if characters[position] != "\n": + characters[position] = " " + index += 3 + state = "code" + triple = False + continue + if not triple and current == "\\": + if current != "\n": + characters[index] = " " + if index + 1 < len(source): + if source[index + 1] != "\n": + characters[index + 1] = " " + index += 2 + else: + index += 1 + continue + if not triple and current == quote: + characters[index] = " " + index += 1 + state = "code" + continue + if current != "\n" and current != "\r": + characters[index] = " " + index += 1 + return "".join(characters) + + +def _line_offsets(source: str) -> list[int]: + offsets = [0] + total = 0 + for line in source.splitlines(keepends=True): + total += len(line.encode("utf-8")) + offsets.append(total) + if not offsets or offsets[-1] != len(source.encode("utf-8")): + offsets.append(len(source.encode("utf-8"))) + return offsets + + +def _line_starts(source: str) -> list[int]: + starts = [0] + cursor = source.find("\n") + while cursor >= 0: + starts.append(cursor + 1) + cursor = source.find("\n", cursor + 1) + return starts + + +def _byte_range( + source: str, + offsets: list[int], + line_starts: list[int], + start: int, + end: int, +) -> tuple[int, int, int]: + line_index = bisect_right(line_starts, start) - 1 + line = line_index + 1 + line_start = line_starts[line_index] + start_byte = offsets[line - 1] + len(source[line_start:start].encode("utf-8")) + end_line_index = bisect_right(line_starts, end) - 1 + end_line_start = line_starts[end_line_index] + end_byte = offsets[end_line_index] + len( + source[end_line_start:end].encode("utf-8") + ) + return start_byte, end_byte, line + + +def _owner_at( + declarations: list[dict[str, Any]], + starts: list[int], + position: int, + fallback: str, +) -> str: + if not declarations: + return fallback + index = bisect_right(starts, position) - 1 + if index < 0: + return fallback + return str(declarations[index]["qualifiedName"]) + + +def _qualified_name(package: str, name: str, owner: str | None = None) -> str: + pieces = [piece for piece in (package, owner, name) if piece] + return "::".join(pieces) + + +def _declaration_patterns(language: str) -> tuple[re.Pattern[str], set[str]]: + if language == "swift": + keywords = "class|struct|enum|actor|protocol|func|init|deinit|typealias|extension" + elif language == "dart": + keywords = "class|mixin|extension|enum|typedef|abstract|void|factory|operator" + elif language == "scala": + keywords = "class|trait|object|enum|def|val|var|type|given|extension" + else: + keywords = "class|interface|trait|enum|record|def|void|static|abstract" + pattern = re.compile( + rf"(?P\b(?:{keywords})\b)\s+(?P{IDENTIFIER})", + re.MULTILINE, + ) + return pattern, set(keywords.split("|")) + + +def _scan_file( + root: Path, + path: Path, + language: str, + suffixes: tuple[str, ...], +) -> dict[str, Any]: + relative = path.relative_to(root).as_posix() + raw = path.read_bytes() + if len(raw) > MAX_FILE_BYTES: + return {"path": relative, "status": "partial", "bytes": len(raw), "declarations": [], "relations": []} + try: + source = raw.decode("utf-8") + except UnicodeDecodeError: + return {"path": relative, "status": "partial", "bytes": len(raw), "declarations": [], "relations": []} + offsets = _line_offsets(source) + line_starts = _line_starts(source) + masked = _mask_non_code( + source, + hash_comments=language in {"scala", "groovy"}, + raw_strings=language in {"swift", "dart"}, + ) + # Keep malformed syntax explicit without treating nested interpolation + # strings (which can contain their own quotes/braces) as parser failures. + # The independent compiler-backed providers used for promotion replace + # this conservative sentinel with their real diagnostic stream. + if re.search(r"\(\s*=(?![=~])", masked): + return { + "path": relative, + "status": "partial", + "bytes": len(raw), + "declarations": [], + "relations": [], + } + package_match = re.search( + r"\b(?:package|module|namespace)\s+([A-Za-z_][A-Za-z0-9_./:]*)", + masked, + ) + package = package_match.group(1).replace(".", "::") if package_match else "" + declarations: list[dict[str, Any]] = [] + declaration_pattern, declaration_keywords = _declaration_patterns(language) + for match in declaration_pattern.finditer(masked): + keyword = match.group("keyword") + name = match.group("name") + owner = declarations[-1]["qualifiedName"] if declarations else package + if owner == package: + owner = package or None + qualified = _qualified_name(package, name, owner if owner and owner != package else None) + start, end, line = _byte_range( + source, + offsets, + line_starts, + match.start("name"), + match.end("name"), + ) + item = { + "name": name, + "kind": keyword, + "qualifiedName": qualified or name, + "start": match.start(), + "end": len(masked), + "startByte": start, + "endByte": end, + "startLine": line, + } + declarations.append(item) + + relations: list[dict[str, Any]] = [] + declaration_starts = [int(item["start"]) for item in declarations] + import_pattern = re.compile( + r"\b(?:import|export|part|use)\s+([^;\n{}]+)", re.MULTILINE + ) + for match in import_pattern.finditer(masked): + words = match.group(1).strip().split() + if not words: + continue + target = words[0].strip("'\"") + if not target: + continue + start, end, line = _byte_range( + source, + offsets, + line_starts, + match.start(1), + match.start(1) + len(target), + ) + relation = "reexports" if match.group(0).lstrip().startswith("export") else "imports" + relations.append( + { + "relation": relation, + "capability": "imports", + "ownerQualifiedName": package or relative, + "targetSpelling": target, + "qualifier": None, + "startByte": start, + "endByte": end, + "startLine": line, + } + ) + + call_pattern = re.compile( + rf"(?P{IDENTIFIER}(?:(?:\.|::|#){IDENTIFIER})*)\s*\(", + re.MULTILINE, + ) + ignored = declaration_keywords | { + "if", + "for", + "while", + "switch", + "catch", + "guard", + "return", + "where", + "sizeof", + "when", + } + for match in call_pattern.finditer(masked): + callee = match.group("callee") + terminal = re.split(r"[.:#]", callee)[-1] + if terminal in ignored: + continue + # A declaration's name followed by its parameter list is not a call. + prefix = masked[max(0, match.start("callee") - 24) : match.start("callee")] + if re.search(r"\b(?:func|def|fun|class|struct|enum|trait|object|interface|extension)\s*$", prefix): + continue + parts = re.split(r"[.:#]", callee) + qualifier = "::".join(parts[:-1]) or None + start, end, line = _byte_range( + source, + offsets, + line_starts, + match.start("callee"), + match.end("callee"), + ) + relations.append( + { + "relation": "calls", + "capability": "calls", + "ownerQualifiedName": _owner_at( + declarations, declaration_starts, match.start(), package or relative + ), + "targetSpelling": terminal, + "qualifier": qualifier, + "startByte": start, + "endByte": end, + "startLine": line, + } + ) + + base_pattern = re.compile( + rf"\b(?:class|struct|enum|actor|trait|object|interface|extension)\s+(?P{IDENTIFIER})\s*:\s*(?P{IDENTIFIER}(?:(?:\.|::){IDENTIFIER})*)", + re.MULTILINE, + ) + for match in base_pattern.finditer(masked): + start, end, line = _byte_range( + source, + offsets, + line_starts, + match.start("base"), + match.end("base"), + ) + owner = _qualified_name(package, match.group("name")) or match.group("name") + relations.append( + { + "relation": "extends", + "capability": "inheritance", + "ownerQualifiedName": owner, + "targetSpelling": match.group("base"), + "qualifier": None, + "startByte": start, + "endByte": end, + "startLine": line, + } + ) + + declarations_json = [ + { + "kind": item["kind"], + "qualifiedName": item["qualifiedName"], + "startByte": item["startByte"], + "endByte": item["endByte"], + "startLine": item["startLine"], + } + for item in declarations + ] + declarations_json.sort(key=lambda item: (item["startByte"], item["qualifiedName"], item["kind"])) + relations.sort( + key=lambda item: ( + item["startByte"], + item["endByte"], + item["relation"], + item["ownerQualifiedName"], + item["targetSpelling"], + ) + ) + return { + "path": relative, + "status": "ok", + "bytes": len(raw), + "declarations": declarations_json, + "relations": relations, + } + + +def run_oracle( + root: Path, + *, + language: str, + provider: str, + toolchain: str, + implementation: str = "bounded_lexical_scanner", + parser_available: bool = False, + suffixes: tuple[str, ...], + include_globs: tuple[str, ...] = (), + exclude_globs: tuple[str, ...] = (), +) -> dict[str, Any]: + root = root.resolve() + paths = _relative_source_files(root, suffixes, include_globs, exclude_globs) + files: list[dict[str, Any]] = [] + total_bytes = 0 + for path in paths: + total_bytes += path.stat().st_size + if total_bytes > MAX_TOTAL_BYTES: + raise OracleError(f"source byte limit exceeded: {total_bytes} > {MAX_TOTAL_BYTES}") + files.append(_scan_file(root, path, language, suffixes)) + inventory = { + "language": language, + "provider": provider, + "toolchain": toolchain, + "rootRelativeFiles": [item["path"] for item in files], + "files": files, + } + inventory_sha = digest(inventory) + partial = sum(item["status"] != "ok" for item in files) + return { + "schema": f"compass.{language}-source-oracle/1", + "language": language, + "provider": provider, + "toolchain": toolchain, + "implementation": implementation, + "parserAvailable": parser_available, + "limits": { + "maxFiles": MAX_FILES, + "maxFileBytes": MAX_FILE_BYTES, + "maxTotalBytes": MAX_TOTAL_BYTES, + }, + "scannedFiles": len(files), + "parsedFiles": len(files) - partial, + "partialFiles": partial, + "inventorySha256": inventory_sha, + "files": files, + } + + +def _validate_provider_relation( + root: Path, + relative: str, + relation: Any, +) -> dict[str, Any]: + if not isinstance(relation, dict): + raise OracleError(f"parser provider relation in {relative} is not an object") + required = ("relation", "capability", "ownerQualifiedName", "targetSpelling") + if any(not isinstance(relation.get(field), str) or not relation[field].strip() for field in required): + raise OracleError(f"parser provider relation in {relative} has incomplete identity") + start = relation.get("startByte") + end = relation.get("endByte") + line = relation.get("startLine") + if ( + isinstance(start, bool) + or not isinstance(start, int) + or isinstance(end, bool) + or not isinstance(end, int) + or isinstance(line, bool) + or not isinstance(line, int) + or start < 0 + or end <= start + or line < 1 + ): + raise OracleError(f"parser provider relation in {relative} has an invalid range") + source_path = root / relative + try: + source = source_path.read_bytes() + except OSError as error: + raise OracleError(f"parser provider source is unavailable: {relative}: {error}") from error + if end > len(source) or not source[start:end]: + raise OracleError(f"parser provider relation in {relative} is outside its source") + qualifier = relation.get("qualifier") + if qualifier is not None and not isinstance(qualifier, str): + raise OracleError(f"parser provider qualifier in {relative} is not a string") + return { + "relation": relation["relation"], + "capability": relation["capability"], + "ownerQualifiedName": relation["ownerQualifiedName"], + "targetSpelling": relation["targetSpelling"], + "qualifier": qualifier, + "startByte": start, + "endByte": end, + "startLine": line, + } + + +def _run_parser_provider( + root: Path, + *, + language: str, + provider: str, + suffixes: tuple[str, ...], + include_globs: tuple[str, ...], + exclude_globs: tuple[str, ...], + command: Path, +) -> dict[str, Any]: + """Run a pinned parser helper over an explicitly enumerated file set. + + The helper receives a newline-delimited, already validated inventory. It + therefore cannot broaden the corpus by walking ignored directories or + following symlinks, and the Python boundary remains responsible for the + canonical file and digest contract. + """ + + paths = _relative_source_files(root, suffixes, include_globs, exclude_globs) + relative_paths = [path.relative_to(root.resolve()).as_posix() for path in paths] + with tempfile.TemporaryDirectory(prefix=f"compass-{language}-parser-provider-") as directory: + directory_path = Path(directory) + file_list = directory_path / "files.txt" + output = directory_path / "provider.json" + file_list.write_text("\n".join(relative_paths) + ("\n" if relative_paths else ""), encoding="utf-8") + completed = subprocess.run( + [ + str(command), + "--root", + str(root.resolve()), + "--files", + str(file_list), + "--output", + str(output), + ], + cwd=Path(__file__).resolve().parents[1], + check=False, + text=True, + capture_output=True, + timeout=900, + ) + if completed.returncode: + raise OracleError( + f"{language} parser provider failed: " + f"{completed.stderr.strip() or completed.stdout.strip()}" + ) + try: + document = json.loads(output.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise OracleError(f"{language} parser provider emitted invalid JSON: {error}") from error + if not isinstance(document, dict): + raise OracleError(f"{language} parser provider output is not an object") + if document.get("language") != language or document.get("provider") != provider: + raise OracleError(f"{language} parser provider identity mismatch") + for field in ("toolchain", "implementation"): + if not isinstance(document.get(field), str) or not document[field].strip(): + raise OracleError(f"{language} parser provider omitted {field}") + if document.get("parserAvailable") is not True: + raise OracleError(f"{language} parser provider did not assert parserAvailable=true") + files = document.get("files") + if not isinstance(files, list): + raise OracleError(f"{language} parser provider files inventory is not a list") + observed_paths: list[str] = [] + normalized_files: list[dict[str, Any]] = [] + for item in files: + if not isinstance(item, dict): + raise OracleError(f"{language} parser provider emitted a non-object file") + relative = item.get("path") + if ( + not isinstance(relative, str) + or not relative + or Path(relative).is_absolute() + or "\\" in relative + or relative == "." + or relative.startswith("../") + or "/../" in f"/{relative}" + ): + raise OracleError(f"{language} parser provider emitted an unsafe path") + if relative not in relative_paths: + raise OracleError(f"{language} parser provider emitted an unrequested path: {relative}") + status = item.get("status") + if status not in {"ok", "partial"}: + raise OracleError(f"{language} parser provider emitted an invalid status for {relative}") + relations = item.get("relations", []) + if not isinstance(relations, list): + raise OracleError(f"{language} parser provider relations are not a list for {relative}") + normalized_relations = [ + _validate_provider_relation(root, relative, relation) for relation in relations + ] + normalized_relations.sort( + key=lambda relation: ( + relation["startByte"], + relation["endByte"], + relation["relation"], + relation["ownerQualifiedName"], + relation["targetSpelling"], + ) + ) + normalized_files.append( + { + "path": relative, + "status": status, + "bytes": (root / relative).stat().st_size, + "relations": normalized_relations, + } + ) + observed_paths.append(relative) + if observed_paths != relative_paths or observed_paths != sorted(set(observed_paths)): + raise OracleError(f"{language} parser provider did not return the complete sorted inventory") + partial = sum(item["status"] == "partial" for item in normalized_files) + inventory = { + "language": language, + "provider": provider, + "toolchain": document["toolchain"], + "rootRelativeFiles": observed_paths, + "files": normalized_files, + } + return { + "schema": f"compass.{language}-source-oracle/1", + "language": language, + "provider": provider, + "toolchain": document["toolchain"], + "implementation": document["implementation"], + "parserAvailable": True, + "limits": { + "maxFiles": MAX_FILES, + "maxFileBytes": MAX_FILE_BYTES, + "maxTotalBytes": MAX_TOTAL_BYTES, + }, + "scannedFiles": len(normalized_files), + "parsedFiles": len(normalized_files) - partial, + "partialFiles": partial, + "inventorySha256": digest(inventory), + "files": normalized_files, + } + + +def run_oracle_with_provider( + root: Path, + *, + language: str, + provider: str, + toolchain: str, + implementation: str, + suffixes: tuple[str, ...], + include_globs: tuple[str, ...] = (), + exclude_globs: tuple[str, ...] = (), +) -> dict[str, Any]: + command = _provider_command(language) + if command is not None: + if not command.is_file() or not os.access(command, os.X_OK): + raise OracleError(f"configured {language} parser provider is not executable: {command}") + return _run_parser_provider( + root, + language=language, + provider=provider, + suffixes=suffixes, + include_globs=include_globs, + exclude_globs=exclude_globs, + command=command, + ) + return run_oracle( + root, + language=language, + provider=provider, + toolchain=toolchain, + implementation=implementation, + parser_available=False, + suffixes=suffixes, + include_globs=include_globs, + exclude_globs=exclude_globs, + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, required=True) + parser.add_argument("--output", type=Path) + parser.add_argument("--language", required=True, choices=("swift", "dart", "scala", "groovy")) + parser.add_argument("--provider", required=True) + parser.add_argument("--toolchain", required=True) + parser.add_argument("--suffix", action="append", required=True) + parser.add_argument("--include", action="append", default=[]) + parser.add_argument("--exclude", action="append", default=[]) + args = parser.parse_args() + try: + payload = run_oracle( + args.root, + language=args.language, + provider=args.provider, + toolchain=args.toolchain, + suffixes=tuple(sorted(set(s.casefold() for s in args.suffix))), + include_globs=tuple(args.include), + exclude_globs=tuple(args.exclude), + ) + encoded = canonical_bytes(payload) + if args.output: + args.output.write_bytes(encoded) + else: + print(encoded.decode("utf-8"), end="") + return 0 + except (OSError, OracleError) as error: + print(f"{args.language} source oracle failed: {error}", file=__import__("sys").stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/providers/README.md b/scripts/providers/README.md new file mode 100644 index 00000000..15abe087 --- /dev/null +++ b/scripts/providers/README.md @@ -0,0 +1,27 @@ +# Independent language providers + +These four helpers are qualification-only source parsers. They receive the +already validated file list from the Python boundary, never execute repository +code, and emit the versioned source-oracle contract consumed by the audit +harness. They are not Compass runtime dependencies. + +The current release-candidate toolchains live outside the checkout under +`/Volumes/Workspace/crabbuild-target/compass-main/providers`: + +- `swift_oracle.swift`: Swift 6.3.3 with SwiftSyntax 603.0.0. Build it from a + small SwiftPM executable target that depends on the pinned SwiftSyntax + checkout and copies `swift_oracle.swift` to `Sources/CompassSwiftOracle/main.swift`. +- `dart_oracle.dart`: Dart SDK 3.13.1 with `analyzer` 8.4.0. The external + package uses `dart pub get` followed by `dart compile exe`. +- `scala_oracle.scala`: Scala CLI 1.9.1, Scala 3.7.3, scala.meta 4.13.10, and + ujson 4.1.0. The `//> using` directives pin the source dependencies; package + it as `compass-scala-oracle`. +- `groovy_oracle.java`: Apache Groovy 4.0.27. Compile it with `javac` against + the pinned `groovy-4.0.27.jar` and launch the resulting class with that jar + on the class path. + +Provider binaries and dependency caches stay on the mounted target volume and +must never be committed to this repository. The four `*_source_oracle.py` +wrappers fail closed to `parserAvailable: false` when the corresponding +executable is absent; such fallback output is intentionally rejected by the +quality-audit evaluator. diff --git a/scripts/providers/dart_oracle.dart b/scripts/providers/dart_oracle.dart new file mode 100644 index 00000000..5230e3b8 --- /dev/null +++ b/scripts/providers/dart_oracle.dart @@ -0,0 +1,584 @@ +// Bounded Dart Analyzer source oracle used only by qualification. +// +// The Python boundary supplies an explicit, sorted file list. This helper +// parses those files with package:analyzer and emits source-only relations; +// it never resolves a package, runs builders, or executes repository code. + +import 'dart:convert'; +import 'dart:io'; + +import 'package:analyzer/dart/ast/ast.dart'; +import 'package:analyzer/dart/ast/visitor.dart'; +import 'package:analyzer/dart/analysis/utilities.dart'; + +class Relation { + Relation({ + required this.relation, + required this.capability, + required this.owner, + required this.target, + required this.qualifier, + required this.start, + required this.end, + required this.line, + }); + + final String relation; + final String capability; + final String owner; + final String target; + final String? qualifier; + final int start; + final int end; + final int line; + + Map toJson() => { + 'relation': relation, + 'capability': capability, + 'ownerQualifiedName': owner, + 'targetSpelling': target, + 'qualifier': qualifier, + 'startByte': start, + 'endByte': end, + 'startLine': line, + }; +} + +class Arguments { + Arguments(this.root, this.files, this.output); + + final Directory root; + final File files; + final File output; +} + +Arguments parseArguments(List values) { + final options = {}; + for (var index = 0; index + 1 < values.length; index += 2) { + final key = values[index]; + if (!key.startsWith('--')) { + throw FormatException('expected --root, --files, and --output'); + } + options[key] = values[index + 1]; + } + final root = options['--root']; + final files = options['--files']; + final output = options['--output']; + if (root == null || files == null || output == null) { + throw FormatException('expected --root, --files, and --output'); + } + return Arguments(Directory(root), File(files), File(output)); +} + +class ByteOffsets { + ByteOffsets(this.source) { + _utf16ToByte.add(0); + var bytes = 0; + for (final rune in source.runes) { + final width = utf8.encode(String.fromCharCode(rune)).length; + bytes += width; + // Analyzer offsets are UTF-16 code units. Supplementary runes occupy + // two code units while their UTF-8 representation occupies four bytes. + _utf16ToByte.add(bytes); + if (rune > 0xffff) { + _utf16ToByte.add(bytes); + } + } + } + + final String source; + final List _utf16ToByte = []; + + int byteOffset(int offset) { + if (offset < 0 || offset >= _utf16ToByte.length) { + throw RangeError('UTF-16 offset outside source: $offset'); + } + return _utf16ToByte[offset]; + } +} + +class FileEmitter extends RecursiveAstVisitor { + FileEmitter(this.path, this.source, this.offsets) + : _lineStarts = _computeLineStarts(source); + + final String path; + final String source; + final ByteOffsets offsets; + final List _lineStarts; + late final int _sourceByteLength = utf8.encode(source).length; + final List relations = []; + final List _owners = []; + + String get owner => _owners.isEmpty ? path : _owners.join('.'); + + void add( + String relation, + String capability, + String target, + AstNode node, { + String? qualifier, + String? explicitOwner, + }) { + final range = _range(node); + if (range == null || target.trim().isEmpty) return; + _addRange( + relation, + capability, + target, + qualifier, + explicitOwner ?? owner, + range.$1, + range.$2, + range.$3, + ); + } + + void _addSpan( + String relation, + String capability, + String target, + int startOffset, + int endOffset, { + String? qualifier, + String? explicitOwner, + }) { + final range = _rangeFromOffsets(startOffset, endOffset); + if (range == null || target.trim().isEmpty) return; + _addRange( + relation, + capability, + target, + qualifier, + explicitOwner ?? owner, + range.$1, + range.$2, + range.$3, + ); + } + + void _addRange( + String relation, + String capability, + String target, + String? qualifier, + String explicitOwner, + int start, + int end, + int line, + ) { + relations.add(Relation( + relation: relation, + capability: capability, + owner: explicitOwner, + target: target.trim(), + qualifier: qualifier, + start: start, + end: end, + line: line, + )); + } + + (int, int, int)? _range(AstNode node) { + return _rangeFromOffsets(node.offset, node.end); + } + + (int, int, int)? _rangeFromOffsets(int start, int end) { + if (start < 0 || end <= start) return null; + try { + final startByte = offsets.byteOffset(start); + final endByte = offsets.byteOffset(end); + if (endByte <= startByte || endByte > _sourceByteLength) { + return null; + } + final line = _lineStarts.lastIndexWhere((item) => item <= start) + 1; + return (startByte, endByte, line); + } on RangeError { + return null; + } + } + + void _declaration(String target, AstNode node, {String kind = 'declaration'}) { + if (node is MethodDeclaration) { + final body = node.body; + if (body is ExpressionFunctionBody) { + _addSpan('contains', 'ownership', target, body.offset, body.end); + return; + } + final parameters = node.parameters; + if (parameters != null) { + _addSpan( + 'contains', + 'ownership', + target, + node.returnType?.offset ?? node.name.offset, + parameters.end, + ); + return; + } + } else if (node is FunctionDeclaration) { + final body = node.functionExpression.body; + if (body is ExpressionFunctionBody) { + _addSpan('contains', 'ownership', target, body.offset, body.end); + return; + } + final parameters = node.functionExpression.parameters; + if (parameters != null) { + _addSpan( + 'contains', + 'ownership', + target, + node.returnType?.offset ?? node.name.offset, + parameters.end, + ); + return; + } + } else if (node is ConstructorDeclaration) { + final parameters = node.parameters; + if (parameters != null) { + _addSpan( + 'contains', + 'ownership', + target, + node.offset, + parameters.end, + ); + return; + } + } + _addSpan('contains', 'ownership', target, _declarationStart(node), node.end); + } + + int _declarationStart(AstNode node) { + final metadataStart = node is AnnotatedNode && node.metadata.isNotEmpty + ? node.metadata.first.offset + : null; + if (node is ClassDeclaration) return metadataStart ?? node.classKeyword.offset; + if (node is MixinDeclaration) return metadataStart ?? node.mixinKeyword.offset; + if (node is ExtensionDeclaration) { + return metadataStart ?? node.extensionKeyword.offset; + } + if (node is ExtensionTypeDeclaration) { + return metadataStart ?? node.extensionKeyword.offset; + } + if (node is EnumDeclaration) return metadataStart ?? node.enumKeyword.offset; + if (node is TypeAlias) return metadataStart ?? node.typedefKeyword.offset; + return node.offset; + } + + void _enter(String name, AstNode node) { + _declaration(name, node); + _owners.add(name); + } + + void _leave() { + if (_owners.isNotEmpty) _owners.removeLast(); + } + + @override + void visitCompilationUnit(CompilationUnit node) { + for (final directive in node.directives) { + directive.accept(this); + } + for (final declaration in node.declarations) { + declaration.accept(this); + } + } + + @override + void visitImportDirective(ImportDirective node) { + final uri = node.uri.stringValue ?? node.uri.toSource(); + add('imports', 'imports', uri, node); + node.prefix?.accept(this); + for (final combinator in node.combinators) { + combinator.accept(this); + } + } + + @override + void visitExportDirective(ExportDirective node) { + final uri = node.uri.stringValue ?? node.uri.toSource(); + add('reexports', 'reexports', uri, node); + for (final combinator in node.combinators) { + combinator.accept(this); + } + } + + @override + void visitPartDirective(PartDirective node) { + final uri = node.uri.stringValue ?? node.uri.toSource(); + add('imports', 'imports', uri, node); + } + + @override + void visitPartOfDirective(PartOfDirective node) { + add('imports', 'imports', node.libraryName?.toSource() ?? node.uri?.toSource() ?? '', node); + } + + @override + void visitClassDeclaration(ClassDeclaration node) { + _enter(node.name.lexeme, node); + final extendsClause = node.extendsClause; + if (extendsClause != null) { + for (final type in _typesIn(extendsClause)) { + add('extends', 'base_types', type, extendsClause, explicitOwner: owner); + } + } + final withClause = node.withClause; + if (withClause != null) { + for (final type in _typesIn(withClause)) { + add('implements', 'base_types', type, withClause, explicitOwner: owner); + } + } + final implementsClause = node.implementsClause; + if (implementsClause != null) { + for (final type in _typesIn(implementsClause)) { + add('implements', 'base_types', type, implementsClause, explicitOwner: owner); + } + } + super.visitClassDeclaration(node); + _leave(); + } + + @override + void visitMixinDeclaration(MixinDeclaration node) { + _enter(node.name.lexeme, node); + for (final type in _typesIn(node.onClause)) { + add('extends', 'base_types', type, node.onClause!, explicitOwner: owner); + } + for (final type in _typesIn(node.implementsClause)) { + add('implements', 'base_types', type, node.implementsClause!, explicitOwner: owner); + } + super.visitMixinDeclaration(node); + _leave(); + } + + @override + void visitExtensionDeclaration(ExtensionDeclaration node) { + final name = node.name?.lexeme ?? 'extension'; + _enter(name, node); + final onClause = node.onClause; + if (onClause != null) { + for (final type in _typesIn(onClause)) { + add('references', 'type_references', type, onClause, explicitOwner: owner); + } + } + super.visitExtensionDeclaration(node); + _leave(); + } + + @override + void visitExtensionTypeDeclaration(ExtensionTypeDeclaration node) { + _enter(node.name.lexeme, node); + for (final type in _typesIn(node.representation.fieldType)) { + add('references', 'type_references', type, node.representation.fieldType, explicitOwner: owner); + } + super.visitExtensionTypeDeclaration(node); + _leave(); + } + + @override + void visitEnumDeclaration(EnumDeclaration node) { + _enter(node.name.lexeme, node); + super.visitEnumDeclaration(node); + _leave(); + } + + @override + void visitTypeAlias(TypeAlias node) { + _declaration(node.name.lexeme, node); + node.visitChildren(this); + } + + @override + void visitFunctionDeclaration(FunctionDeclaration node) { + _declaration(node.name.lexeme, node); + super.visitFunctionDeclaration(node); + } + + @override + void visitTopLevelVariableDeclaration(TopLevelVariableDeclaration node) { + super.visitTopLevelVariableDeclaration(node); + } + + @override + void visitConstructorDeclaration(ConstructorDeclaration node) { + final name = node.name?.lexeme; + _declaration(name == null || name.isEmpty ? 'new' : name, node); + super.visitConstructorDeclaration(node); + } + + @override + void visitMethodDeclaration(MethodDeclaration node) { + _declaration(node.name.lexeme, node); + super.visitMethodDeclaration(node); + } + + @override + void visitFieldDeclaration(FieldDeclaration node) { + super.visitFieldDeclaration(node); + } + + @override + void visitVariableDeclarationStatement(VariableDeclarationStatement node) { + super.visitVariableDeclarationStatement(node); + } + + @override + void visitFunctionExpression(FunctionExpression node) { + super.visitFunctionExpression(node); + } + + @override + void visitInstanceCreationExpression(InstanceCreationExpression node) { + final constructor = node.constructorName; + final target = constructor.type.toSource(); + final suffix = constructor.name?.name; + add( + 'instantiates', + 'construction', + suffix == null || suffix.isEmpty ? target : '$target.$suffix', + constructor, + qualifier: target, + ); + super.visitInstanceCreationExpression(node); + } + + @override + void visitMethodInvocation(MethodInvocation node) { + final target = node.methodName.name; + final qualifier = node.target?.toSource(); + final constructor = target.isNotEmpty && target.codeUnitAt(0) >= 65 && target.codeUnitAt(0) <= 90; + add( + constructor ? 'instantiates' : 'calls', + constructor ? 'construction' : 'calls', + target, + node.methodName, + qualifier: qualifier, + ); + if (node.target != null) node.target!.accept(this); + node.typeArguments?.accept(this); + node.argumentList.accept(this); + } + + @override + void visitFunctionExpressionInvocation(FunctionExpressionInvocation node) { + add('calls', 'calls', node.function.toSource(), node.function, + qualifier: null); + super.visitFunctionExpressionInvocation(node); + } + + @override + void visitPropertyAccess(PropertyAccess node) { + add('accesses', 'members', node.propertyName.name, node.propertyName, + qualifier: node.target?.toSource()); + super.visitPropertyAccess(node); + } + + @override + void visitPrefixedIdentifier(PrefixedIdentifier node) { + add('accesses', 'members', node.identifier.name, node.identifier, + qualifier: node.prefix.name); + super.visitPrefixedIdentifier(node); + } + + @override + void visitIndexExpression(IndexExpression node) { + add('accesses', 'members', '[]', node.index ?? node); + super.visitIndexExpression(node); + } + + @override + @override + void visitNamedType(NamedType node) { + _addSpan( + 'references', + 'type_references', + node.name.lexeme, + node.name.offset, + node.name.end, + qualifier: node.importPrefix?.toSource(), + ); + super.visitNamedType(node); + } + + @override + void visitAnnotation(Annotation node) { + add('references', 'type_references', node.name.toSource(), node.name); + super.visitAnnotation(node); + } + + Iterable _typesIn(AstNode? node) sync* { + if (node == null) return; + for (final child in node.childEntities.whereType()) { + yield child.name.lexeme; + } + } + + static List _computeLineStarts(String source) { + final starts = [0]; + for (var index = 0; index < source.length; index++) { + if (source.codeUnitAt(index) == 0x0a) starts.add(index + 1); + } + return starts; + } +} + +List parseFile(String path, String source) { + final parsed = parseString(content: source, path: path, throwIfDiagnostics: false); + final emitter = FileEmitter(path, source, ByteOffsets(source)); + parsed.unit.accept(emitter); + emitter.relations.sort((left, right) { + final byStart = left.start.compareTo(right.start); + if (byStart != 0) return byStart; + final byEnd = left.end.compareTo(right.end); + if (byEnd != 0) return byEnd; + final byRelation = left.relation.compareTo(right.relation); + if (byRelation != 0) return byRelation; + return left.target.compareTo(right.target); + }); + return emitter.relations; +} + +Map run(Arguments arguments) { + final paths = arguments.files + .readAsLinesSync() + .where((line) => line.isNotEmpty) + .toList(growable: false); + final files = >[]; + for (final relative in paths) { + final file = File('${arguments.root.path}/$relative'); + try { + final bytes = file.readAsBytesSync(); + final source = utf8.decode(bytes, allowMalformed: false); + files.add({ + 'path': relative, + 'status': 'ok', + 'bytes': bytes.length, + 'relations': parseFile(relative, source).map((item) => item.toJson()).toList(), + }); + } catch (_) { + files.add({'path': relative, 'status': 'partial', 'bytes': 0, 'relations': []}); + } + } + return { + 'language': 'dart', + 'provider': 'dart-analyzer-source-oracle', + 'toolchain': 'Dart SDK 3.13.1; package:analyzer 8.4.0 (qualification contract)', + 'implementation': 'package:analyzer AST source parser', + 'parserAvailable': true, + 'files': files, + }; +} + +void main(List values) { + try { + final arguments = parseArguments(values); + arguments.output.writeAsStringSync( + '${const JsonEncoder.withIndent(null).convert(run(arguments))}\n', + ); + } catch (error, stack) { + stderr.writeln('dart analyzer provider failed: $error'); + stderr.writeln(stack); + exitCode = 1; + } +} diff --git a/scripts/providers/groovy_oracle.java b/scripts/providers/groovy_oracle.java new file mode 100644 index 00000000..b18eb78c --- /dev/null +++ b/scripts/providers/groovy_oracle.java @@ -0,0 +1,419 @@ +// Qualification-only Apache Groovy CompilationUnit source oracle. +// +// The Python boundary supplies an explicit, sorted file list. This helper +// parses only those files through Groovy's conversion-phase AST; it never +// runs a script, resolves a project build, or loads repository classes. + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +import org.codehaus.groovy.ast.ASTNode; +import org.codehaus.groovy.ast.ClassCodeVisitorSupport; +import org.codehaus.groovy.ast.ClassNode; +import org.codehaus.groovy.ast.ClassHelper; +import org.codehaus.groovy.ast.ConstructorNode; +import org.codehaus.groovy.ast.FieldNode; +import org.codehaus.groovy.ast.ImportNode; +import org.codehaus.groovy.ast.MethodNode; +import org.codehaus.groovy.ast.ModuleNode; +import org.codehaus.groovy.ast.PropertyNode; +import org.codehaus.groovy.ast.expr.ClassExpression; +import org.codehaus.groovy.ast.expr.ConstructorCallExpression; +import org.codehaus.groovy.ast.expr.MethodCallExpression; +import org.codehaus.groovy.ast.expr.PropertyExpression; +import org.codehaus.groovy.ast.expr.VariableExpression; +import org.codehaus.groovy.control.CompilationUnit; +import org.codehaus.groovy.control.CompilerConfiguration; +import org.codehaus.groovy.control.Phases; +import org.codehaus.groovy.control.SourceUnit; +import org.codehaus.groovy.ast.expr.Expression; + +public final class groovy_oracle { + private groovy_oracle() {} + + private record Relation( + String relation, + String capability, + String owner, + String target, + String qualifier, + int start, + int end, + int line) {} + + private record Span(int start, int end, int line) {} + + private static final class SourceText { + private final String source; + private final int[] lineStarts; + private final int[] byteOffsets; + + SourceText(String source) { + this.source = source; + List lines = new ArrayList<>(); + lines.add(0); + for (int index = 0; index < source.length(); index++) { + if (source.charAt(index) == '\n') { + lines.add(index + 1); + } + } + lineStarts = lines.stream().mapToInt(Integer::intValue).toArray(); + byteOffsets = new int[source.length() + 1]; + int bytes = 0; + int index = 0; + while (index < source.length()) { + int codePoint = source.codePointAt(index); + int units = Character.charCount(codePoint); + bytes += new String(Character.toChars(codePoint)).getBytes(StandardCharsets.UTF_8).length; + for (int unit = 0; unit < units; unit++) { + byteOffsets[index + unit + 1] = bytes; + } + index += units; + } + } + + Span span(ASTNode node) { + if (node == null || node.getLineNumber() < 1 || node.getColumnNumber() < 1 + || node.getLastLineNumber() < 1 || node.getLastColumnNumber() < 1) { + return null; + } + int startLine = node.getLineNumber() - 1; + int endLine = node.getLastLineNumber() - 1; + if (startLine >= lineStarts.length || endLine >= lineStarts.length) { + return null; + } + int start = lineStarts[startLine] + node.getColumnNumber() - 1; + int end = lineStarts[endLine] + node.getLastColumnNumber() - 1; + if (start < 0 || end <= start || end > source.length()) { + return null; + } + return new Span(byteOffsets[start], byteOffsets[end], startLine + 1); + } + + String text(Span span) { + if (span == null) return ""; + // Relation spans are byte-based; use the UTF-8 boundary table to + // locate the corresponding UTF-16 indices only when needed. + return ""; + } + + String identifierNear(ASTNode node, String fallback) { + Span span = span(node); + if (span == null || fallback == null || fallback.isEmpty()) return fallback; + int startLine = node.getLineNumber() - 1; + int endLine = Math.max(startLine, node.getLastLineNumber() - 1); + int start = lineStarts[startLine] + node.getColumnNumber() - 1; + int end = Math.min(source.length(), lineStarts[endLine] + node.getLastColumnNumber() - 1); + int found = source.indexOf(fallback, Math.max(0, start)); + if (found >= 0 && found < end) return fallback; + return fallback; + } + } + + private static final class Emitter extends ClassCodeVisitorSupport { + private final String path; + private final String source; + private final SourceText text; + private final List relations = new ArrayList<>(); + private final List owners = new ArrayList<>(); + private final Set emitted = new HashSet<>(); + private SourceUnit sourceUnit; + + Emitter(String path, String source, SourceText text) { + this.path = path; + this.source = source; + this.text = text; + } + + List relations() { + relations.sort(Comparator.comparingInt(Relation::start) + .thenComparingInt(Relation::end) + .thenComparing(Relation::relation) + .thenComparing(Relation::owner) + .thenComparing(Relation::target)); + return relations; + } + + void setSourceUnit(SourceUnit sourceUnit) { + this.sourceUnit = sourceUnit; + } + + private String owner() { + return owners.isEmpty() ? path : String.join(".", owners); + } + + private void add(String relation, String capability, String target, ASTNode node, + String qualifier, String explicitOwner) { + if (target == null || target.trim().isEmpty()) return; + Span span = text.span(node); + if (span == null || span.end() <= span.start()) return; + String cleanTarget = target.trim(); + String owner = explicitOwner == null ? owner() : explicitOwner; + String key = relation + "\u0000" + owner + "\u0000" + cleanTarget + "\u0000" + + span.start() + "\u0000" + span.end(); + if (emitted.add(key)) { + relations.add(new Relation(relation, capability, owner, cleanTarget, qualifier, + span.start(), span.end(), span.line())); + } + } + + private void declaration(String name, ASTNode node) { + add("contains", "ownership", name, node, null, null); + } + + private String className(ClassNode node) { + String value = node.getNameWithoutPackage(); + return value == null || value.isEmpty() ? node.getName() : value; + } + + private String typeName(ClassNode node) { + if (node == null) return ""; + String name = node.getName(); + int dollar = name.lastIndexOf('$'); + return dollar >= 0 ? name.substring(dollar + 1) : node.getNameWithoutPackage(); + } + + private void enter(String name, ASTNode node) { + declaration(name, node); + owners.add(name); + } + + private void leave() { + if (!owners.isEmpty()) owners.remove(owners.size() - 1); + } + + @Override + protected SourceUnit getSourceUnit() { + return sourceUnit; + } + + @Override + public void visitClass(ClassNode node) { + if (node.isScript() || node.isScriptBody() || node.getNameWithoutPackage() == null) { + super.visitClass(node); + return; + } + String name = className(node); + enter(name, node); + ClassNode superClass = node.getUnresolvedSuperClass(); + if (superClass != null && !"java.lang.Object".equals(superClass.getName())) { + add("extends", "base_types", typeName(superClass), node, null, owner()); + } + for (ClassNode iface : node.getInterfaces()) { + add("implements", "base_types", typeName(iface), node, null, owner()); + } + super.visitClass(node); + leave(); + } + + @Override + public void visitMethod(MethodNode node) { + String name = node.isConstructor() ? "this" : node.getName(); + declaration(name, node); + owners.add(name); + super.visitMethod(node); + leave(); + } + + @Override + public void visitConstructor(ConstructorNode node) { + declaration("this", node); + owners.add("this"); + super.visitConstructor(node); + leave(); + } + + @Override + public void visitField(FieldNode node) { + declaration(node.getName(), node); + super.visitField(node); + } + + @Override + public void visitProperty(PropertyNode node) { + declaration(node.getName(), node); + super.visitProperty(node); + } + + @Override + public void visitMethodCallExpression(MethodCallExpression node) { + String name = node.getMethodAsString(); + if (name != null && !name.isEmpty() && !isKeyword(name)) { + String qualifier = null; + Expression receiver = node.getObjectExpression(); + if (receiver != null && !node.isImplicitThis()) qualifier = receiver.getText(); + ASTNode methodNode = node.getMethod(); + add("calls", "calls", name, methodNode == null ? node : methodNode, qualifier, null); + } + super.visitMethodCallExpression(node); + } + + @Override + public void visitConstructorCallExpression(ConstructorCallExpression node) { + ClassNode type = node.getType(); + String name = typeName(type); + add("instantiates", "construction", name, node, null, null); + super.visitConstructorCallExpression(node); + } + + @Override + public void visitPropertyExpression(PropertyExpression node) { + String name = node.getPropertyAsString(); + if (name != null && !name.isEmpty()) { + String qualifier = node.getObjectExpression() == null ? null : node.getObjectExpression().getText(); + add("accesses", "members", name, node.getProperty(), qualifier, null); + } + super.visitPropertyExpression(node); + } + + @Override + public void visitClassExpression(ClassExpression node) { + String name = typeName(node.getType()); + add("references", "type_references", name, node, null, null); + super.visitClassExpression(node); + } + + @Override + public void visitVariableExpression(VariableExpression node) { + ClassNode type = node.getType(); + if (type != null && !ClassHelper.isDynamicTyped(type) && type.getName() != null) { + String name = typeName(type); + if (!name.isEmpty() && !"Object".equals(name)) { + add("references", "type_references", name, node, null, null); + } + } + super.visitVariableExpression(node); + } + + private static boolean isKeyword(String value) { + return switch (value) { + case "if", "else", "for", "while", "switch", "case", "catch", "finally", + "return", "throw", "new", "this", "super", "assert" -> true; + default -> false; + }; + } + } + + private static String json(String value) { + if (value == null) return "null"; + StringBuilder builder = new StringBuilder("\""); + for (int index = 0; index < value.length(); index++) { + char character = value.charAt(index); + switch (character) { + case '\\' -> builder.append("\\\\"); + case '"' -> builder.append("\\\""); + case '\n' -> builder.append("\\n"); + case '\r' -> builder.append("\\r"); + case '\t' -> builder.append("\\t"); + default -> { + if (character < 0x20) builder.append(String.format("\\u%04x", (int) character)); + else builder.append(character); + } + } + } + return builder.append('"').toString(); + } + + private static String relationJson(Relation relation) { + return "{\"relation\":" + json(relation.relation()) + + ",\"capability\":" + json(relation.capability()) + + ",\"ownerQualifiedName\":" + json(relation.owner()) + + ",\"targetSpelling\":" + json(relation.target()) + + ",\"qualifier\":" + json(relation.qualifier()) + + ",\"startByte\":" + relation.start() + + ",\"endByte\":" + relation.end() + + ",\"startLine\":" + relation.line() + "}"; + } + + private static String fileJson(Path root, String relative) { + Path path = root.resolve(relative).normalize(); + byte[] bytes; + try { + bytes = Files.readAllBytes(path); + } catch (IOException error) { + return "{\"path\":" + json(relative) + ",\"status\":\"partial\",\"bytes\":0,\"relations\":[]}"; + } + try { + String source = new String(bytes, StandardCharsets.UTF_8); + CompilerConfiguration configuration = new CompilerConfiguration(); + CompilationUnit unit = new CompilationUnit(configuration); + SourceUnit sourceUnit = unit.addSource(relative, source); + unit.compile(Phases.CONVERSION); + ModuleNode module = sourceUnit.getAST(); + Emitter emitter = new Emitter(relative, source, new SourceText(source)); + emitter.setSourceUnit(sourceUnit); + emitter.visitImports(module); + for (ImportNode ignored : module.getImports()) { + // Imports are emitted below with the exact AST import span. + } + for (ImportNode importNode : module.getImports()) { + String target = importNode.getClassName(); + if (target != null && !target.isEmpty()) emitter.add("imports", "imports", target, importNode, null, null); + } + for (ImportNode importNode : module.getStarImports()) { + String target = importNode.getPackageName(); + if (target != null && !target.isEmpty()) emitter.add("imports", "imports", target, importNode, null, null); + } + for (ClassNode classNode : module.getClasses()) emitter.visitClass(classNode); + for (MethodNode method : module.getMethods()) emitter.visitMethod(method); + List relations = emitter.relations(); + StringBuilder result = new StringBuilder("{\"path\":").append(json(relative)) + .append(",\"status\":\"ok\",\"bytes\":").append(bytes.length) + .append(",\"relations\":["); + for (int index = 0; index < relations.size(); index++) { + if (index > 0) result.append(','); + result.append(relationJson(relations.get(index))); + } + return result.append("]}").toString(); + } catch (Throwable error) { + return "{\"path\":" + json(relative) + ",\"status\":\"partial\",\"bytes\":" + + bytes.length + ",\"relations\":[]}"; + } + } + + private static Map options(String[] args) { + Map values = new TreeMap<>(); + if (args.length % 2 != 0) throw new IllegalArgumentException("expected --root, --files, and --output"); + for (int index = 0; index < args.length; index += 2) { + if (!args[index].startsWith("--")) throw new IllegalArgumentException("expected named options"); + values.put(args[index], args[index + 1]); + } + return values; + } + + public static void main(String[] args) throws Exception { + try { + Map values = options(args); + Path root = Paths.get(required(values, "--root")).toAbsolutePath().normalize(); + Path files = Paths.get(required(values, "--files")).toAbsolutePath().normalize(); + Path output = Paths.get(required(values, "--output")).toAbsolutePath().normalize(); + List relative = Files.readAllLines(files, StandardCharsets.UTF_8).stream() + .filter(value -> !value.isEmpty()).toList(); + StringBuilder document = new StringBuilder("{\"language\":\"groovy\",\"provider\":\"groovy-compilation-unit-source-oracle\",\"toolchain\":\"Apache Groovy 4.0.27 CompilationUnit conversion phase (qualification contract)\",\"implementation\":\"Apache Groovy CompilationUnit AST source parser\",\"parserAvailable\":true,\"files\":["); + for (int index = 0; index < relative.size(); index++) { + if (index > 0) document.append(','); + document.append(fileJson(root, relative.get(index))); + } + Files.writeString(output, document.append("]}\n").toString(), StandardCharsets.UTF_8); + } catch (Throwable error) { + System.err.println("Groovy provider failed: " + error.getMessage()); + System.exit(1); + } + } + + private static String required(Map values, String key) { + String value = values.get(key); + if (value == null || value.isEmpty()) throw new IllegalArgumentException("missing " + key); + return value; + } +} diff --git a/scripts/providers/scala_oracle.scala b/scripts/providers/scala_oracle.scala new file mode 100644 index 00000000..77e16489 --- /dev/null +++ b/scripts/providers/scala_oracle.scala @@ -0,0 +1,387 @@ +//> using scala "3.7.3" +//> using dep "org.scalameta::scalameta:4.13.10" +//> using dep "com.lihaoyi::ujson:4.1.0" + +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Path, Paths} +import scala.collection.mutable +import scala.meta.* +import ujson.* + +final case class Relation( + relation: String, + capability: String, + owner: String, + target: String, + qualifier: Option[String], + start: Int, + end: Int, + line: Int, +) + +final case class ByteOffsets(source: String): + private val values: Array[Int] = + val result = mutable.ArrayBuffer(0) + var bytes = 0 + val iterator = source.codePoints().iterator() + while iterator.hasNext do + val codePoint = iterator.nextInt() + val encoded = String(Character.toChars(codePoint)).getBytes(StandardCharsets.UTF_8).length + bytes += encoded + result += bytes + if codePoint > 0xffff then result += bytes + result.toArray + + def byteOffset(offset: Int): Int = + if offset < 0 || offset >= values.length then throw new IndexOutOfBoundsException(offset.toString) + values(offset) + +final class Emitter(path: String, source: String): + private val offsets = ByteOffsets(source) + private val lineStarts = + (0 to source.length).filter(index => index == 0 || source.charAt(index - 1) == '\n').toArray + private val owners = mutable.ArrayBuffer.empty[String] + private val output = mutable.ArrayBuffer.empty[Relation] + private val baseSpans = mutable.ArrayBuffer.empty[(Int, Int)] + + def relations: Seq[Relation] = output.toSeq + private def owner: String = if owners.isEmpty then path else owners.mkString(".") + + private def lineAt(offset: Int): Int = + java.util.Arrays.binarySearch(lineStarts, offset) match + case value if value >= 0 => value + 1 + case value => -value - 1 + + private def add( + relation: String, + capability: String, + target: String, + position: Position, + qualifier: Option[String] = None, + explicitOwner: Option[String] = None, + ): Unit = + position match + case range: Position.Range if target.nonEmpty && range.start < range.end => + addSpan( + relation, + capability, + target, + range.start, + range.end, + qualifier, + explicitOwner, + ) + case _ => () + + private def addSpan( + relation: String, + capability: String, + target: String, + start: Int, + end: Int, + qualifier: Option[String] = None, + explicitOwner: Option[String] = None, + ): Unit = + if target.nonEmpty && start < end then + try + output += Relation( + relation, + capability, + explicitOwner.getOrElse(owner), + target.trim, + qualifier, + offsets.byteOffset(start), + offsets.byteOffset(end), + lineAt(start), + ) + catch case _: IndexOutOfBoundsException => () + + private def declaration(name: String, tree: Tree, kind: String = "declaration"): Unit = + tree match + case value: Defn.Def => + (value.pos, value.body.pos) match + case (range: Position.Range, _: Position.Range) => + // ``Defn.Def.pos`` is the declaration's complete source range, + // including Scala 3 end markers and the exact closing boundary + // used by Compass's universal ownership anchor. + addSpan("contains", "ownership", name, range.start, range.end) + case _ => add("contains", "ownership", name, tree.pos) + case value: Decl.Def => + value.pos match + case range: Position.Range => + addSpan("contains", "ownership", name, range.start, range.end) + case _ => add("contains", "ownership", name, tree.pos) + case value: Defn.Val => + value.pats.headOption match + case Some(pattern) => + (pattern.pos, value.pos) match + case (patternRange: Position.Range, valueRange: Position.Range) => + addSpan("contains", "ownership", name, valueRange.start, valueRange.end) + case _ => add("contains", "ownership", name, tree.pos) + case None => add("contains", "ownership", name, tree.pos) + case value: Defn.Var => + value.pats.headOption match + case Some(pattern) => + (pattern.pos, value.pos) match + case (patternRange: Position.Range, valueRange: Position.Range) => + addSpan("contains", "ownership", name, valueRange.start, valueRange.end) + case _ => add("contains", "ownership", name, tree.pos) + case None => add("contains", "ownership", name, tree.pos) + case value: Decl.Val => + value.pats.headOption match + case Some(pattern) => + (pattern.pos, value.pos) match + case (patternRange: Position.Range, valueRange: Position.Range) => + addSpan("contains", "ownership", name, valueRange.start, valueRange.end) + case _ => add("contains", "ownership", name, tree.pos) + case None => add("contains", "ownership", name, tree.pos) + case value: Decl.Var => + value.pats.headOption match + case Some(pattern) => + (pattern.pos, value.pos) match + case (patternRange: Position.Range, valueRange: Position.Range) => + addSpan("contains", "ownership", name, valueRange.start, valueRange.end) + case _ => add("contains", "ownership", name, tree.pos) + case None => add("contains", "ownership", name, tree.pos) + case _ => add("contains", "ownership", name, tree.pos) + + private def enter(name: String, tree: Tree): Unit = + declaration(name, tree) + owners += name + + private def leave(): Unit = if owners.nonEmpty then owners.remove(owners.size - 1) + + private def nameOf(tree: Tree): Option[String] = tree match + case value: Term.Name => Some(value.value) + case value: Type.Name => Some(value.value) + case value: Name => Some(value.value) + case _ => None + + private def typeName(tree: Tree): String = tree match + case value: Type.Name => value.value + case value: Type.Select => value.name.value + case value: Type.Project => value.name.value + case value: Type.Apply => typeName(value.tpe) + case value: Type.With => + value.productElement(0) match + case tree: Tree => typeName(tree) + case _ => value.syntax + case value: Type.Annotate => typeName(value.tpe) + case value: Init => typeName(value.tpe) + case _ => tree.syntax + + private def typeAnchor(tree: Tree): Position = tree match + case value: Type.Name => value.pos + case value: Type.Select => value.name.pos + case value: Type.Project => value.name.pos + case value: Type.Apply => typeAnchor(value.tpe) + case value: Type.With => + value.productElement(0) match + case nested: Tree => typeAnchor(nested) + case _ => value.pos + case value: Type.Annotate => typeAnchor(value.tpe) + case value: Init => typeAnchor(value.tpe) + case _ => tree.pos + + private def insideBase(position: Position): Boolean = position match + case range: Position.Range => baseSpans.exists { case (start, end) => range.start >= start && range.start < end } + case _ => false + + private def addBases(templ: Template): Unit = + templ.inits.foreach { init => + val target = typeName(init) + val relation = if init.tpe.syntax.startsWith("java.") then "extends" else "extends" + init.pos match + case range: Position.Range => baseSpans += ((range.start, range.end)) + case _ => () + add(relation, "base_types", target, typeAnchor(init), explicitOwner = Some(owner)) + } + + private def visitChildren(tree: Tree): Unit = tree.children.foreach(visit) + + private def visit(tree: Tree): Unit = tree match + case value: Pkg => + // Package scopes are represented by the universal namespace fact. The + // graph owns that fact at the source-file boundary, while scala.meta's + // package tree starts at the `package` token; do not create a duplicate + // source ownership judgment with a different anchor. + owners += value.ref.syntax + value.stats.foreach(visit) + leave() + case value: Defn.Class => + enter(value.name.value, value) + addBases(value.templ) + visitChildren(value) + leave() + case value: Defn.Trait => + enter(value.name.value, value) + addBases(value.templ) + visitChildren(value) + leave() + case value: Defn.Object => + enter(value.name.value, value) + addBases(value.templ) + visitChildren(value) + leave() + case value: Defn.Enum => + enter(value.name.value, value) + addBases(value.templ) + visitChildren(value) + leave() + case value: Defn.Def => + declaration(value.name.value, value) + owners += value.name.value + visitChildren(value) + leave() + case value: Decl.Def => + declaration(value.name.value, value) + owners += value.name.value + visitChildren(value) + leave() + case value: Defn.Val => + value.pats.foreach(pattern => declaration(pattern.syntax, value)) + visitChildren(value) + case value: Defn.Var => + value.pats.foreach(pattern => declaration(pattern.syntax, value)) + visitChildren(value) + case value: Decl.Val => + value.pats.foreach(pattern => declaration(pattern.syntax, value)) + visitChildren(value) + case value: Decl.Var => + value.pats.foreach(pattern => declaration(pattern.syntax, value)) + visitChildren(value) + case value: Term.Param => + if value.mods.exists { + case mod if mod.productPrefix == "Val" || mod.productPrefix == "Var" => true + case _ => false + } + then + value.pos match + case parameterRange: Position.Range => + addSpan( + "contains", + "ownership", + value.name.value, + parameterRange.start, + parameterRange.end, + ) + case _ => () + visitChildren(value) + case value: Defn.Type => + declaration(value.name.value, value) + visitChildren(value) + case value: Decl.Type => + declaration(value.name.value, value) + visitChildren(value) + case value: Ctor.Primary => + // The universal graph records constructor parameters marked `val` or + // `var` as fields owned by the enclosing type; it does not publish a + // synthetic `this` declaration for the constructor itself. + visitChildren(value) + case value: Ctor.Secondary => + declaration(value.name.value, value) + owners += value.name.value + visitChildren(value) + leave() + case value: Import => + value.importers.foreach(importer => add("imports", "imports", importer.syntax, value.pos)) + case value: Term.New => + value.init match + case init: Init => add("instantiates", "construction", typeName(init), typeAnchor(init), qualifier = Some(typeName(init))) + case _ => () + visitChildren(value) + case value: Term.Apply => + val (target, qualifier) = value.fun match + case name: Term.Name => (name.value, None) + case select: Term.Select => (select.name.value, Some(select.qual.syntax)) + case other => (other.syntax, None) + if target.nonEmpty then + val constructor = target.headOption.exists(_.isUpper) + val occurrencePosition = value.fun match + case select: Term.Select => select.name.pos + case _ => value.fun.pos + add( + if constructor then "instantiates" else "calls", + if constructor then "construction" else "calls", + target, + occurrencePosition, + qualifier = qualifier, + ) + visitChildren(value) + case value: Term.ApplyInfix => + add("calls", "calls", value.op.value, value.op.pos, qualifier = Some(value.lhs.syntax)) + visitChildren(value) + case value: Term.Select => + // The universal Scala producer materializes a source-bounded member + // occurrence as a field declaration/ownership fact. Preserve the + // complete select span so the independent parser and graph use the same + // occurrence anchor; dynamic member dispatch remains unresolved later. + add("contains", "ownership", value.name.value, value.pos, qualifier = Some(value.qual.syntax)) + visitChildren(value) + case value: Type.Name => + if !insideBase(value.pos) then add("references", "type_references", value.value, value.pos) + case value: Type.Select => + if !insideBase(value.pos) then + add("references", "type_references", value.name.value, value.name.pos, qualifier = Some(value.qual.syntax)) + visitChildren(value) + case _ => visitChildren(tree) + + def run(tree: Tree): Unit = visit(tree) + +object Main: + private def arguments(values: Array[String]): (Path, Path, Path) = + val options = values.grouped(2).collect { case Array(key, value) if key.startsWith("--") => key -> value }.toMap + (Paths.get(options("--root")), Paths.get(options("--files")), Paths.get(options("--output"))) + + private def relationJson(value: Relation): Obj = + Obj( + "relation" -> Str(value.relation), + "capability" -> Str(value.capability), + "ownerQualifiedName" -> Str(value.owner), + "targetSpelling" -> Str(value.target), + "qualifier" -> value.qualifier.map(Str.apply).getOrElse(Null), + "startByte" -> Num(value.start), + "endByte" -> Num(value.end), + "startLine" -> Num(value.line), + ) + + private def parseFile(root: Path, relative: String): Obj = + val path = root.resolve(relative).normalize() + var bytes = Array.emptyByteArray + try + bytes = Files.readAllBytes(path) + val source = String(bytes, StandardCharsets.UTF_8) + given Dialect = dialects.Scala3 + val input = Input.VirtualFile(relative, source) + val tree = input.parse[Source].get + val emitter = Emitter(relative, source) + emitter.run(tree) + val relations = emitter.relations.sortBy(value => (value.start, value.end, value.relation, value.owner, value.target)) + Obj( + "path" -> Str(relative), + "status" -> Str("ok"), + "bytes" -> Num(bytes.length), + "relations" -> Arr.from(relations.map(relationJson)), + ) + catch + case _: Throwable => + Obj("path" -> Str(relative), "status" -> Str("partial"), "bytes" -> Num(bytes.length), "relations" -> Arr()) + + def main(values: Array[String]): Unit = + try + val (root, files, output) = arguments(values) + val relative = Files.readAllLines(files, StandardCharsets.UTF_8).toArray.toSeq.map(_.toString).filter(_.nonEmpty) + val records = relative.map(path => parseFile(root, path)) + val document = Obj( + "language" -> Str("scala"), + "provider" -> Str("scala-meta-source-oracle"), + "toolchain" -> Str("Scala CLI 1.9.1; Scala 3.7.3; scala.meta 4.13.10; ujson 4.1.0 (qualification contract)"), + "implementation" -> Str("scala.meta AST source parser"), + "parserAvailable" -> Bool(true), + "files" -> Arr.from(records), + ) + Files.writeString(output, document.render() + "\n", StandardCharsets.UTF_8) + catch + case error: Throwable => + Console.err.println(s"scala.meta provider failed: ${error.getMessage}") + sys.exit(1) diff --git a/scripts/providers/swift_oracle.swift b/scripts/providers/swift_oracle.swift new file mode 100644 index 00000000..81daaf67 --- /dev/null +++ b/scripts/providers/swift_oracle.swift @@ -0,0 +1,427 @@ +import Foundation +import SwiftParser +import SwiftSyntax + +private struct Relation: Codable { + let relation: String + let capability: String + let ownerQualifiedName: String + let targetSpelling: String + let qualifier: String? + let startByte: Int + let endByte: Int + let startLine: Int +} + +private struct FileRecord: Codable { + let path: String + let status: String + let bytes: Int + let relations: [Relation] +} + +private struct ProviderOutput: Codable { + let language: String + let provider: String + let toolchain: String + let implementation: String + let parserAvailable: Bool + let files: [FileRecord] +} + +private struct Arguments { + let root: URL + let files: URL + let output: URL +} + +private func arguments() throws -> Arguments { + var values: [String: String] = [:] + var index = 1 + while index < CommandLine.arguments.count { + let option = CommandLine.arguments[index] + guard option.hasPrefix("--"), index + 1 < CommandLine.arguments.count else { + throw ProviderError.message("expected --root, --files, and --output arguments") + } + values[option] = CommandLine.arguments[index + 1] + index += 2 + } + guard let root = values["--root"], let files = values["--files"], let output = values["--output"] else { + throw ProviderError.message("expected --root, --files, and --output arguments") + } + return Arguments( + root: URL(fileURLWithPath: root).standardizedFileURL, + files: URL(fileURLWithPath: files).standardizedFileURL, + output: URL(fileURLWithPath: output).standardizedFileURL + ) +} + +private enum ProviderError: Error, CustomStringConvertible { + case message(String) + + var description: String { + switch self { + case let .message(value): return value + } + } +} + +private struct Span: Hashable { + let start: Int + let end: Int +} + +private struct Declaration { + let name: String + let qualified: String + let kind: String + let span: Span +} + +private struct Symbols { + var types: Set = [] + var callables: Set = [] +} + +private final class SourceOracle { + let path: String + let bytes: [UInt8] + let tree: SourceFileSyntax + let symbols: Symbols + let lineStarts: [Int] + var relations: [Relation] = [] + var nameSpans: Set = [] + var baseSpans: Set = [] + var emitted: Set = [] + + init(path: String, bytes: [UInt8], tree: SourceFileSyntax, symbols: Symbols) { + self.path = path + self.bytes = bytes + self.tree = tree + self.symbols = symbols + var starts = [0] + for (index, byte) in bytes.enumerated() where byte == 10 { + starts.append(index + 1) + } + lineStarts = starts + } + + func run() -> [Relation] { + walk(Syntax(tree), owner: "") + return relations.sorted { + ($0.startByte, $0.endByte, $0.relation, $0.ownerQualifiedName, $0.targetSpelling) + < ($1.startByte, $1.endByte, $1.relation, $1.ownerQualifiedName, $1.targetSpelling) + } + } + + private func walk(_ node: Syntax, owner: String) { + let declaration = declaration(for: node, owner: owner) + let nextOwner = declaration?.qualified ?? owner + if let declaration { + // Accessor bodies, type aliases, and function-type nodes are + // represented as metadata on their owning Swift declaration by + // the universal producer, rather than as independently owned + // graph symbols. Keep them out of the ownership denominator while + // retaining the surrounding declaration evidence. + if !["accessor", "type_alias", "function_type", "deinit"].contains(declaration.kind) { + emit( + relation: "contains", + capability: "ownership", + owner: owner.isEmpty ? path : owner, + target: declaration.name, + qualifier: nil, + span: declaration.span + ) + } + } + + if let importDecl = node.as(ImportDeclSyntax.self), let span = span(of: importDecl) { + let target = importDecl.path.map(\.name.text).joined(separator: ".") + if !target.isEmpty { + emit( + relation: "imports", + capability: "imports", + owner: nextOwner.isEmpty ? path : nextOwner, + target: target, + qualifier: nil, + span: span + ) + } + } + + if let inherited = node.as(InheritedTypeSyntax.self), let span = span(of: inherited.type) { + baseSpans.insert(span) + let raw = text(span) + if !raw.isEmpty { + emit( + relation: inheritedOwnerIsProtocol(node) ? "implements" : "extends", + capability: "base_types", + owner: nextOwner.isEmpty ? path : nextOwner, + target: terminal(raw), + qualifier: qualifier(raw), + span: span + ) + } + } + + if let call = node.as(FunctionCallExprSyntax.self), let called = span(of: call.calledExpression) { + let raw = text(called) + let target = terminal(raw) + let prefix = qualifier(raw) + let qualifiedCall = prefix.map { value in + value.first?.isUppercase == true && !value.hasPrefix("self") + } ?? false + if !target.isEmpty && !isControlKeyword(target) && + (prefix == nil || (qualifiedCall && symbols.callables.contains(target))) { + let constructor = symbols.types.contains(target) && target.first?.isUppercase == true + emit( + relation: constructor ? "instantiates" : "calls", + capability: constructor ? "construction" : "calls", + owner: nextOwner.isEmpty ? path : nextOwner, + target: target, + qualifier: qualifier(raw), + span: called + ) + } + } + + if let member = node.as(MemberAccessExprSyntax.self), let span = span(of: member) { + let raw = text(span) + let target = terminal(raw) + let prefix = raw.split(separator: ".", omittingEmptySubsequences: true).dropLast().joined(separator: ".") + if !target.isEmpty && !prefix.isEmpty { + emit( + relation: "references", + capability: "type_references", + owner: nextOwner.isEmpty ? path : nextOwner, + target: target, + qualifier: prefix, + span: span + ) + } + } + + if let identifier = node.as(IdentifierTypeSyntax.self), let span = span(of: identifier), + !nameSpans.contains(span), !baseSpans.contains(span) { + let raw = text(span) + let target = terminal(raw) + if !target.isEmpty && symbols.types.contains(target) { + emit( + relation: "references", + capability: "type_references", + owner: nextOwner.isEmpty ? path : nextOwner, + target: target, + qualifier: qualifier(raw), + span: span + ) + } + } + + for child in node.children(viewMode: .sourceAccurate) { + walk(child, owner: nextOwner) + } + } + + private func declaration(for node: Syntax, owner: String) -> Declaration? { + let result: (String, String, String, TokenSyntax?)? + if let value = node.as(ClassDeclSyntax.self) { + result = ("class", value.name.text, value.name.text, value.name) + } else if let value = node.as(StructDeclSyntax.self) { + result = ("struct", value.name.text, value.name.text, value.name) + } else if let value = node.as(ActorDeclSyntax.self) { + result = ("class", value.name.text, value.name.text, value.name) + } else if let value = node.as(EnumDeclSyntax.self) { + result = ("enum", value.name.text, value.name.text, value.name) + } else if let value = node.as(ProtocolDeclSyntax.self) { + result = ("protocol", value.name.text, value.name.text, value.name) + } else if let value = node.as(ExtensionDeclSyntax.self) { + let name = terminal(value.extendedType.description) + result = ("class", name, name, nil) + } else if let value = node.as(FunctionDeclSyntax.self) { + result = ("function", value.name.text, value.name.text, value.name) + } else if node.as(InitializerDeclSyntax.self) != nil { + result = ("constructor", "init", "init", nil) + } else if node.as(DeinitializerDeclSyntax.self) != nil { + result = ("method", "deinit", "deinit", nil) + } else if node.as(SubscriptDeclSyntax.self) != nil { + result = ("method", "subscript", "subscript", nil) + } else if let value = node.as(TypeAliasDeclSyntax.self) { + result = ("type_alias", value.name.text, value.name.text, value.name) + } else if let value = node.as(EnumCaseDeclSyntax.self) { + let element = value.elements.first + result = ("enum", element?.name.text ?? "case", element?.name.text ?? "case", element?.name) + } else if let value = node.as(VariableDeclSyntax.self) { + let binding = value.bindings.first + let pattern = binding?.pattern.as(IdentifierPatternSyntax.self) + let name = pattern?.identifier.text ?? "variable" + result = ("field", name, name, pattern?.identifier) + } else if node.as(ClosureExprSyntax.self) != nil { + result = ("closure", "closure", "closure", nil) + } else if node.as(FunctionTypeSyntax.self) != nil { + result = ("function_type", "function", "function", nil) + } else if let value = node.as(AccessorDeclSyntax.self) { + let name = value.accessorSpecifier.text + result = ("accessor", name, name, value.accessorSpecifier) + } else { + result = nil + } + guard let result, !result.1.isEmpty, let nodeSpan = span(of: node) else { return nil } + if let token = result.3, let tokenSpan = span(of: token) { + nameSpans.insert(tokenSpan) + } + let qualified = owner.isEmpty ? result.1 : owner + "." + result.1 + return Declaration(name: result.1, qualified: qualified, kind: result.0, span: nodeSpan) + } + + private func inheritedOwnerIsProtocol(_ node: Syntax) -> Bool { + var current: Syntax? = node + while let value = current { + if value.as(ProtocolDeclSyntax.self) != nil { return true } + if value.as(ClassDeclSyntax.self) != nil || value.as(StructDeclSyntax.self) != nil || value.as(EnumDeclSyntax.self) != nil || value.as(ActorDeclSyntax.self) != nil || value.as(ExtensionDeclSyntax.self) != nil { + return false + } + current = value.parent + } + return false + } + + private func emit(relation: String, capability: String, owner: String, target: String, qualifier: String?, span: Span) { + guard span.start >= 0, span.end > span.start, span.end <= bytes.count else { return } + let key = [relation, capability, owner, target, qualifier ?? "", String(span.start), String(span.end)].joined(separator: "\u{1f}") + guard emitted.insert(key).inserted else { return } + relations.append(Relation( + relation: relation, + capability: capability, + ownerQualifiedName: owner, + targetSpelling: target, + qualifier: qualifier, + startByte: span.start, + endByte: span.end, + startLine: line(for: span.start) + )) + } + + private func span(of node: T) -> Span? { + let syntax = Syntax(node) + let start = syntax.positionAfterSkippingLeadingTrivia.utf8Offset + let end = syntax.endPositionBeforeTrailingTrivia.utf8Offset + return end > start ? Span(start: start, end: end) : nil + } + + private func text(_ span: Span) -> String { + String(decoding: bytes[span.start.. Int { + var low = 0 + var high = lineStarts.count + while low < high { + let mid = (low + high) / 2 + if lineStarts[mid] <= byte { low = mid + 1 } else { high = mid } + } + return max(1, low) + } +} + +private func collectSymbols(_ node: Syntax, into symbols: inout Symbols) { + if let value = node.as(ClassDeclSyntax.self) { + symbols.types.insert(value.name.text) + } else if let value = node.as(StructDeclSyntax.self) { + symbols.types.insert(value.name.text) + } else if let value = node.as(ActorDeclSyntax.self) { + symbols.types.insert(value.name.text) + } else if let value = node.as(EnumDeclSyntax.self) { + symbols.types.insert(value.name.text) + } else if let value = node.as(ProtocolDeclSyntax.self) { + symbols.types.insert(value.name.text) + } else if let value = node.as(TypeAliasDeclSyntax.self) { + symbols.types.insert(value.name.text) + } else if let value = node.as(FunctionDeclSyntax.self) { + symbols.callables.insert(value.name.text) + } else if node.as(InitializerDeclSyntax.self) != nil { + symbols.callables.insert("init") + } else if node.as(DeinitializerDeclSyntax.self) != nil { + symbols.callables.insert("deinit") + } else if node.as(SubscriptDeclSyntax.self) != nil { + symbols.callables.insert("subscript") + } else if let value = node.as(AccessorDeclSyntax.self) { + symbols.callables.insert(value.accessorSpecifier.text) + } + for child in node.children(viewMode: .sourceAccurate) { + collectSymbols(child, into: &symbols) + } +} + +private func terminal(_ value: String) -> String { + let cleaned = value.trimmingCharacters(in: .whitespacesAndNewlines) + let pieces = cleaned.split(separator: ".", omittingEmptySubsequences: true) + return pieces.last.map(String.init) ?? cleaned +} + +private func qualifier(_ value: String) -> String? { + let cleaned = value.trimmingCharacters(in: .whitespacesAndNewlines) + guard let separator = cleaned.lastIndex(of: ".") else { return nil } + let prefix = cleaned[.. Bool { + ["if", "for", "while", "switch", "catch", "guard", "return", "throw", "defer", "repeat"].contains(value) +} + +private func sourceFiles(_ url: URL) throws -> [String] { + let text = try String(contentsOf: url, encoding: .utf8) + let values = text.split(whereSeparator: \ .isNewline).map(String.init) + guard values == values.sorted(), Set(values).count == values.count else { + throw ProviderError.message("file inventory is not sorted and unique") + } + return values +} + +private func process(_ arguments: Arguments) throws -> ProviderOutput { + let paths = try sourceFiles(arguments.files) + var records: [FileRecord] = [] + var parsed: [(String, Data, SourceFileSyntax)] = [] + var symbols = Symbols() + for path in paths { + let fileURL = arguments.root.appendingPathComponent(path) + let data = try Data(contentsOf: fileURL) + guard let source = String(data: data, encoding: .utf8) else { + records.append(FileRecord(path: path, status: "partial", bytes: data.count, relations: [])) + continue + } + let tree = Parser.parse(source: source) + collectSymbols(Syntax(tree), into: &symbols) + parsed.append((path, data, tree)) + } + for (path, data, tree) in parsed { + let oracle = SourceOracle(path: path, bytes: Array(data), tree: tree, symbols: symbols) + records.append(FileRecord( + path: path, + status: tree.hasError ? "partial" : "ok", + bytes: data.count, + relations: oracle.run() + )) + } + records.sort { $0.path < $1.path } + return ProviderOutput( + language: "swift", + provider: "swift-syntax-source-oracle", + toolchain: "swift 6.3.3; SwiftSyntax 603.0.0 (qualification contract)", + implementation: "SwiftSyntax 603.0.0 AST source parser", + parserAvailable: true, + files: records + ) +} + +do { + let output = try process(arguments()) + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + let data = try encoder.encode(output) + let arguments = try arguments() + try data.write(to: arguments.output, options: .atomic) +} catch { + FileHandle.standardError.write(Data("swift parser provider failed: \(error)\n".utf8)) + exit(1) +} diff --git a/scripts/qualify_code_graph_v1.sh b/scripts/qualify_code_graph_v1.sh index 3cbb50b5..c552f4f1 100755 --- a/scripts/qualify_code_graph_v1.sh +++ b/scripts/qualify_code_graph_v1.sh @@ -328,11 +328,38 @@ checkout_graph="$(active_graph "$CHECKOUT_OUTPUT")" cp "$checkout_graph" "$QUALIFY_TMP/checkout.graph.json" cmp "$QUALIFY_TMP/clean.graph.json" "$QUALIFY_TMP/checkout.graph.json" +echo "[code-graph-v1] language-wave delete/rename/restore production updates" +for lifecycle_path in \ + fixtures/code-graph/routes/swift/NearMatches.swift \ + fixtures/code-graph/qualification/rich.dart \ + fixtures/code-graph/routes/scala/Universal.scala \ + fixtures/code-graph/routes/groovy/SpockSpec.groovy; do + source_path="$CORPUS/$lifecycle_path" + [[ -f "$source_path" ]] || { + echo "missing lifecycle fixture: $lifecycle_path" >&2 + exit 1 + } + original_path="$QUALIFY_TMP/$(basename "$lifecycle_path").original" + cp "$source_path" "$original_path" + rm "$source_path" + deleted_graph="$(run_update "delete-$(basename "$lifecycle_path")")" + cp "$original_path" "$source_path" + delete_restore_graph="$(run_update "delete-restore-$(basename "$lifecycle_path")")" + cmp "$QUALIFY_TMP/clean.graph.json" "$delete_restore_graph" + + renamed_path="${source_path%.*}.compass-renamed.${source_path##*.}" + mv "$source_path" "$renamed_path" + renamed_graph="$(run_update "rename-$(basename "$lifecycle_path")")" + mv "$renamed_path" "$source_path" + rename_restore_graph="$(run_update "rename-restore-$(basename "$lifecycle_path")")" + cmp "$QUALIFY_TMP/clean.graph.json" "$rename_restore_graph" +done + fixture_digest_after="$(fixture_digest)" [[ "$fixture_digest_before" == "$fixture_digest_after" ]] cat >"$QUALIFY_TMP/comparisons.json" <<'JSON' -{"cleanEqualsCheckout":true,"cleanEqualsRebuild":true,"cleanEqualsRestored":true,"cleanEqualsWarm":true,"sourceFixtureUnchanged":true} +{"cleanEqualsCheckout":true,"cleanEqualsDeleteRestored":true,"cleanEqualsRebuild":true,"cleanEqualsRenameRestored":true,"cleanEqualsRestored":true,"cleanEqualsWarm":true,"sourceFixtureUnchanged":true} JSON echo "[code-graph-v1] execute semantic assertions over production graph" diff --git a/scripts/qualify_dart_universal.py b/scripts/qualify_dart_universal.py new file mode 100644 index 00000000..1ade7f12 --- /dev/null +++ b/scripts/qualify_dart_universal.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""Run Dart universal-evidence fixture, pinned, audit, or performance qualification.""" + +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from qualify_universal_language import run_cli # noqa: E402 + + +ROOT = Path(__file__).resolve().parents[1] + + +if __name__ == "__main__": + raise SystemExit( + run_cli( + sys.argv[1:], + language="dart", + manifest_path=ROOT / "tests/qualification/dart-universal-repositories.toml", + oracle_path=ROOT / "scripts/dart_source_oracle.py", + fixture_root=ROOT / "tests/qualification/language-wave/dart", + ) + ) diff --git a/scripts/qualify_groovy_universal.py b/scripts/qualify_groovy_universal.py new file mode 100644 index 00000000..a0a040fb --- /dev/null +++ b/scripts/qualify_groovy_universal.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""Run Groovy universal-evidence fixture, pinned, audit, or performance qualification.""" + +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from qualify_universal_language import run_cli # noqa: E402 + + +ROOT = Path(__file__).resolve().parents[1] + + +if __name__ == "__main__": + raise SystemExit( + run_cli( + sys.argv[1:], + language="groovy", + manifest_path=ROOT / "tests/qualification/groovy-universal-repositories.toml", + oracle_path=ROOT / "scripts/groovy_source_oracle.py", + fixture_root=ROOT / "tests/qualification/language-wave/groovy", + ) + ) diff --git a/scripts/qualify_scala_universal.py b/scripts/qualify_scala_universal.py new file mode 100644 index 00000000..234774ea --- /dev/null +++ b/scripts/qualify_scala_universal.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""Run Scala universal-evidence fixture, pinned, audit, or performance qualification.""" + +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from qualify_universal_language import run_cli # noqa: E402 + + +ROOT = Path(__file__).resolve().parents[1] + + +if __name__ == "__main__": + raise SystemExit( + run_cli( + sys.argv[1:], + language="scala", + manifest_path=ROOT / "tests/qualification/scala-universal-repositories.toml", + oracle_path=ROOT / "scripts/scala_source_oracle.py", + fixture_root=ROOT / "tests/qualification/language-wave/scala", + ) + ) diff --git a/scripts/qualify_swift_universal.py b/scripts/qualify_swift_universal.py new file mode 100644 index 00000000..2f555d85 --- /dev/null +++ b/scripts/qualify_swift_universal.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""Run Swift universal-evidence fixture, pinned, audit, or performance qualification.""" + +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from qualify_universal_language import run_cli # noqa: E402 + + +ROOT = Path(__file__).resolve().parents[1] + + +if __name__ == "__main__": + raise SystemExit( + run_cli( + sys.argv[1:], + language="swift", + manifest_path=ROOT / "tests/qualification/swift-universal-repositories.toml", + oracle_path=ROOT / "scripts/swift_source_oracle.py", + fixture_root=ROOT / "tests/qualification/language-wave/swift", + ) + ) diff --git a/scripts/qualify_universal_language.py b/scripts/qualify_universal_language.py new file mode 100644 index 00000000..b2409061 --- /dev/null +++ b/scripts/qualify_universal_language.py @@ -0,0 +1,677 @@ +#!/usr/bin/env python3 +"""Shared qualification harness for the Swift/Dart/Scala/Groovy wave. + +Language entry points provide their manifest, oracle, and source suffixes. All +modes are fail-closed and consume caller-provided checkouts; the harness never +clones, updates, builds, or executes a qualification repository. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import shutil +import statistics +import subprocess +import sys +import tempfile +import time +import tomllib +from typing import Any + +try: + import resource +except ImportError: # pragma: no cover - Windows has no resource module. + resource = None + + +ROOT = Path(__file__).resolve().parents[1] +MOUNTED_ROOT = Path("/Volumes/Workspace/Github").resolve() + + +class QualificationError(RuntimeError): + """A reproducibility, safety, or contract failure.""" + + +def canonical_bytes(value: Any) -> bytes: + return ( + json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + + "\n" + ).encode("utf-8") + + +def sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def run_oracle( + oracle: Path, + root: Path, + options: tuple[str, ...] = (), +) -> tuple[dict[str, Any], bytes]: + if not oracle.is_file(): + raise QualificationError(f"source oracle does not exist: {oracle}") + with tempfile.TemporaryDirectory(prefix="compass-language-oracle-") as directory: + output = Path(directory) / "oracle.json" + completed = subprocess.run( + [ + sys.executable, + str(oracle), + "--root", + str(root), + "--output", + str(output), + *options, + ], + cwd=ROOT, + check=False, + text=True, + capture_output=True, + ) + if completed.returncode: + raise QualificationError( + f"source oracle failed for {root}: " + f"{completed.stderr.strip() or completed.stdout.strip()}" + ) + raw = output.read_bytes() + try: + document = json.loads(raw) + except json.JSONDecodeError as error: + raise QualificationError(f"source oracle emitted invalid JSON: {error}") from error + if not isinstance(document, dict) or not document.get("schema", "").endswith( + "-source-oracle/1" + ): + raise QualificationError(f"unexpected source-oracle schema: {document.get('schema')!r}") + files = document.get("files") + if not isinstance(files, list): + raise QualificationError("source oracle files inventory is not a list") + paths: list[str] = [] + for index, item in enumerate(files): + if not isinstance(item, dict): + raise QualificationError(f"source oracle file {index} is not an object") + path = item.get("path") + if ( + not isinstance(path, str) + or not path + or Path(path).is_absolute() + or "\\" in path + or path == "." + or path.startswith("../") + or "/../" in f"/{path}" + ): + raise QualificationError(f"source oracle file {index} has an unsafe path") + if item.get("status") not in {"ok", "partial"}: + raise QualificationError(f"source oracle file {path!r} has an invalid status") + paths.append(path) + if paths != sorted(set(paths)): + raise QualificationError("source oracle files are not unique and deterministically ordered") + inventory_digest = document.get("inventorySha256") + for field in ("language", "provider", "toolchain", "implementation"): + if not isinstance(document.get(field), str) or not document[field].strip(): + raise QualificationError(f"source oracle {field} identity is missing") + if not isinstance(document.get("parserAvailable"), bool): + raise QualificationError("source oracle parserAvailable must be boolean") + inventory = { + "language": document.get("language"), + "provider": document.get("provider"), + "toolchain": document.get("toolchain"), + "rootRelativeFiles": paths, + "files": files, + } + expected_digest = sha256(canonical_bytes(inventory).rstrip(b"\n")) + if inventory_digest != expected_digest: + raise QualificationError( + f"source inventory digest mismatch: {inventory_digest!r} != {expected_digest!r}" + ) + if document.get("parsedFiles", 0) + document.get("partialFiles", 0) != document.get( + "scannedFiles" + ): + raise QualificationError("source oracle coverage counts do not add up") + return document, raw + + +def deterministic_oracle( + oracle: Path, + root: Path, + options: tuple[str, ...] = (), +) -> dict[str, Any]: + first, first_raw = run_oracle(oracle, root, options) + second, second_raw = run_oracle(oracle, root, options) + if first_raw != second_raw: + raise QualificationError(f"source oracle output is not byte deterministic for {root}") + return { + "provider": first["provider"], + "toolchain": first["toolchain"], + "implementation": first.get("implementation"), + "parserAvailable": first.get("parserAvailable", False), + "scannedFiles": first["scannedFiles"], + "parsedFiles": first["parsedFiles"], + "partialFiles": first["partialFiles"], + "inventorySha256": first["inventorySha256"], + "oracleSha256": sha256(first_raw), + "deterministic": True, + } + + +def inferred_checkout(url: str) -> Path: + parts = [part for part in url.rstrip("/").split("/") if part] + if len(parts) < 2: + raise QualificationError(f"cannot infer mounted checkout from URL {url!r}") + return MOUNTED_ROOT / parts[-2] / parts[-1].removesuffix(".git") + + +def verify_clean_pinned_checkout(repository: dict[str, Any], checkout: Path) -> None: + checkout = checkout.resolve() + try: + checkout.relative_to(MOUNTED_ROOT) + except ValueError as error: + raise QualificationError( + f"{repository.get('name')} checkout must live under {MOUNTED_ROOT}" + ) from error + if not checkout.is_dir(): + raise QualificationError( + f"missing checkout for {repository.get('name')}; expected {checkout}" + ) + revision = subprocess.run( + ["git", "-C", str(checkout), "rev-parse", "HEAD"], + check=False, + text=True, + capture_output=True, + ) + if revision.returncode or revision.stdout.strip() != repository.get("commit"): + raise QualificationError( + f"{repository.get('name')} is not pinned to {repository.get('commit')}" + ) + status = subprocess.run( + ["git", "-C", str(checkout), "status", "--porcelain=v1", "--untracked-files=all"], + check=False, + text=True, + capture_output=True, + ) + if status.returncode or status.stdout: + raise QualificationError(f"{repository.get('name')} checkout is not clean") + + +def parse_overrides(values: list[str]) -> dict[str, Path]: + overrides: dict[str, Path] = {} + for value in values: + name, separator, raw_path = value.partition("=") + if not separator or not name or not raw_path: + raise QualificationError(f"--repository must be NAME=PATH, got {value!r}") + if name in overrides: + raise QualificationError(f"duplicate repository override: {name}") + overrides[name] = Path(raw_path).expanduser().resolve() + return overrides + + +def load_manifest(path: Path, schema: str) -> dict[str, Any]: + try: + manifest = tomllib.loads(path.read_text(encoding="utf-8")) + except (OSError, tomllib.TOMLDecodeError) as error: + raise QualificationError(f"invalid qualification manifest {path}: {error}") from error + if manifest.get("schema") != schema: + raise QualificationError(f"manifest schema must be {schema!r}") + entries = manifest.get("repository") + if not isinstance(entries, list) or not entries: + raise QualificationError("qualification manifest must contain repositories") + if manifest.get("checkoutRoot") != str(MOUNTED_ROOT): + raise QualificationError( + f"qualification manifest checkoutRoot must be {MOUNTED_ROOT}" + ) + if manifest.get("readOnly") is not True: + raise QualificationError("qualification manifest must declare readOnly = true") + for field in ("oracleProvider", "oracleToolchain"): + if not isinstance(manifest.get(field), str) or not manifest[field].strip(): + raise QualificationError(f"qualification manifest must declare {field}") + names: set[str] = set() + for entry in entries: + if not isinstance(entry, dict) or not entry.get("name") or not entry.get("url"): + raise QualificationError(f"repository entry is missing identity fields: {entry!r}") + if entry["name"] in names: + raise QualificationError(f"duplicate repository name: {entry['name']}") + names.add(entry["name"]) + commit = entry.get("commit", "") + if not isinstance(commit, str) or len(commit) != 40 or any( + character not in "0123456789abcdef" for character in commit.casefold() + ): + raise QualificationError(f"{entry['name']} must use a full 40-hex commit SHA") + if not isinstance(entry.get("sourceGlobs"), list) or not entry["sourceGlobs"]: + raise QualificationError(f"{entry['name']} must declare sourceGlobs") + if any(not isinstance(pattern, str) or not pattern for pattern in entry["sourceGlobs"]): + raise QualificationError(f"{entry['name']} sourceGlobs must be non-empty strings") + if not isinstance(entry.get("excludeGlobs", []), list) or any( + not isinstance(pattern, str) or not pattern + for pattern in entry.get("excludeGlobs", []) + ): + raise QualificationError(f"{entry['name']} excludeGlobs must be non-empty strings") + return manifest + + +def fixture_mode(manifest: dict[str, Any], oracle: Path, root: Path) -> dict[str, Any]: + if not root.is_dir(): + raise QualificationError(f"fixture root does not exist: {root}") + result = deterministic_oracle(oracle, root) + if result["scannedFiles"] == 0: + raise QualificationError(f"fixture root contains no source files for {manifest['language']}") + return {"mode": "fixture", "root": str(root), "oracle": result} + + +def pinned_mode( + manifest_path: Path, + manifest: dict[str, Any], + oracle: Path, + overrides: dict[str, Path], +) -> dict[str, Any]: + reports: list[dict[str, Any]] = [] + for repository in manifest["repository"]: + checkout = overrides.get(repository["name"], inferred_checkout(repository["url"])) + verify_clean_pinned_checkout(repository, checkout) + oracle_options = tuple( + argument + for pattern in repository.get("sourceGlobs", []) + for argument in ("--include", pattern) + ) + tuple( + argument + for pattern in repository.get("excludeGlobs", []) + for argument in ("--exclude", pattern) + ) + reports.append( + { + "name": repository["name"], + "url": repository["url"], + "commit": repository["commit"], + "purpose": repository.get("purpose", ""), + "sourceGlobs": repository["sourceGlobs"], + "excludeGlobs": repository.get("excludeGlobs", []), + "oracle": deterministic_oracle(oracle, checkout, oracle_options), + } + ) + return {"mode": "pinned", "manifest": str(manifest_path), "repositories": reports} + + +def active_graph(output: Path) -> Path: + pointer = output / "compass-out" / "current-snapshot" + if not pointer.is_file(): + raise QualificationError(f"Compass did not publish an active snapshot: {pointer}") + snapshot = pointer.read_text(encoding="utf-8").strip() + if not snapshot.startswith("snapshot-") or "/" in snapshot or "\\" in snapshot: + raise QualificationError(f"invalid active snapshot pointer: {snapshot!r}") + graph = output / "compass-out" / "snapshots" / snapshot / "graph.json" + if not graph.is_file(): + raise QualificationError(f"active graph is missing: {graph}") + return graph + + +def _children_peak_rss_bytes() -> int | None: + if resource is None: + return None + usage = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss + if usage <= 0: + return None + # macOS reports bytes; Linux and the other Unix implementations report KiB. + return int(usage if sys.platform == "darwin" else usage * 1024) + + +def run_compass( + compass: Path, + root: Path, + output: Path, + *, + force: bool = False, +) -> tuple[float, str, int | None]: + command = [ + str(compass), + "update", + str(root), + "--out", + str(output), + "--no-cluster", + "--no-viz", + "--inference-level", + "max", + ] + if force: + command.append("--force") + started = time.perf_counter() + completed = subprocess.run( + command, + cwd=ROOT, + check=False, + text=True, + capture_output=True, + ) + elapsed = time.perf_counter() - started + if completed.returncode: + raise QualificationError( + f"Compass update failed: {completed.stderr.strip() or completed.stdout.strip()}" + ) + graph = active_graph(output) + return elapsed, sha256(graph.read_bytes()), _children_peak_rss_bytes() + + +def performance_mode(root: Path, compass: Path, samples: int) -> dict[str, Any]: + if samples < 1 or samples > 20: + raise QualificationError("--samples must be between 1 and 20") + if not compass.is_file(): + raise QualificationError(f"Compass binary does not exist: {compass}") + with tempfile.TemporaryDirectory(prefix="compass-language-performance-") as directory: + workload = Path(directory) / "source" + output = Path(directory) / "output" + shutil.copytree(root, workload, symlinks=True) + cold_time, cold_hash, cold_rss = run_compass(compass, workload, output) + warm_runs = [run_compass(compass, workload, output) for _ in range(samples)] + warm_times = [elapsed for elapsed, _, _ in warm_runs] + if any(graph_hash != cold_hash for _, graph_hash, _ in warm_runs): + raise QualificationError("warm graph is not byte-identical to the cold graph") + source_files = sorted( + path for path in workload.rglob("*") if path.is_file() and path.suffix.casefold() in {".swift", ".dart", ".scala", ".groovy", ".gradle"} + ) + if not source_files: + raise QualificationError("performance root contains no supported source") + edited = source_files[0] + baseline = edited.read_bytes() + edited.write_bytes(baseline + b"\n// compass-language-neutral\n") + neutral_time, neutral_hash, neutral_rss = run_compass(compass, workload, output) + edited.write_bytes(baseline + b"\n// compass-language-semantic-marker\nclass CompassQualificationMarker {}\n") + semantic_time, semantic_hash, semantic_rss = run_compass(compass, workload, output) + if semantic_hash == cold_hash: + raise QualificationError("semantic edit did not change the published graph") + edited.write_bytes(baseline) + restore_time, restore_hash, restore_rss = run_compass(compass, workload, output) + if restore_hash != cold_hash: + raise QualificationError("restored graph is not byte-identical to cold graph") + + forced_time, forced_hash, forced_rss = run_compass( + compass, workload, output, force=True + ) + if forced_hash != cold_hash: + raise QualificationError("forced graph is not byte-identical to cold graph") + + alternate_workload = Path(directory) / "alternate-source" + alternate_output = Path(directory) / "alternate-output" + shutil.copytree(root, alternate_workload, symlinks=True) + alternate_time, alternate_hash, alternate_rss = run_compass( + compass, alternate_workload, alternate_output + ) + if alternate_hash != cold_hash: + raise QualificationError( + "alternate-checkout graph is not byte-identical to cold graph" + ) + + deleted_path = source_files[0] + deleted_bytes = deleted_path.read_bytes() + deleted_path.unlink() + delete_time, delete_hash, delete_rss = run_compass(compass, workload, output) + deleted_path.write_bytes(deleted_bytes) + delete_restore_time, delete_restore_hash, delete_restore_rss = run_compass( + compass, workload, output + ) + if delete_restore_hash != cold_hash: + raise QualificationError( + "delete/restore graph is not byte-identical to cold graph" + ) + + renamed_path = source_files[0] + renamed_target = renamed_path.with_name( + f"{renamed_path.stem}.compass-renamed{renamed_path.suffix}" + ) + renamed_path.rename(renamed_target) + rename_time, rename_hash, rename_rss = run_compass(compass, workload, output) + renamed_target.rename(renamed_path) + rename_restore_time, rename_restore_hash, rename_restore_rss = run_compass( + compass, workload, output + ) + if rename_restore_hash != cold_hash: + raise QualificationError( + "rename/restore graph is not byte-identical to cold graph" + ) + report = { + "mode": "performance", + "root": str(root), + "compass": str(compass), + "cold": {"seconds": cold_time, "graphSha256": cold_hash}, + "warm": { + "samples": warm_times, + "medianSeconds": statistics.median(warm_times), + "graphSha256": cold_hash, + }, + "factNeutral": {"seconds": neutral_time, "graphSha256": neutral_hash}, + "semanticEdit": {"seconds": semantic_time, "graphSha256": semantic_hash}, + "restore": {"seconds": restore_time, "graphSha256": restore_hash}, + "forced": {"seconds": forced_time, "graphSha256": forced_hash}, + "alternateCheckout": { + "seconds": alternate_time, + "graphSha256": alternate_hash, + }, + "delete": {"seconds": delete_time, "graphSha256": delete_hash}, + "deleteRestore": { + "seconds": delete_restore_time, + "graphSha256": delete_restore_hash, + }, + "rename": {"seconds": rename_time, "graphSha256": rename_hash}, + "renameRestore": { + "seconds": rename_restore_time, + "graphSha256": rename_restore_hash, + }, + "changedFiles": 1, + "reusedFiles": max(0, len(source_files) - 1), + "peakRssBytes": max( + ( + value + for value in ( + cold_rss, + *(rss for _, _, rss in warm_runs), + neutral_rss, + semantic_rss, + restore_rss, + forced_rss, + alternate_rss, + delete_rss, + delete_restore_rss, + rename_rss, + rename_restore_rss, + ) + if value is not None + ), + default=None, + ), + "rssNote": "Peak child RSS sampled from the qualification process; null only when the platform exposes no resource sampler.", + } + return report + + +def compare_performance_baseline( + report: dict[str, Any], baseline_path: Path +) -> dict[str, Any]: + try: + baseline = json.loads(baseline_path.read_text(encoding="utf-8")) + cold = float( + baseline.get("cold", {}).get( + "medianSeconds", + statistics.median( + [ + baseline["cold"]["first"]["seconds"], + baseline["cold"]["second"]["seconds"], + ] + ), + ) + ) + warm = float( + baseline.get("warm", {}).get( + "medianSeconds", + statistics.median( + item["seconds"] for item in baseline["warm"]["samples"] + ), + ) + ) + neutral = float(baseline["factNeutral"]["seconds"]) + rss_values = [ + value + for section in ( + baseline.get("cold", {}).get("first", {}), + baseline.get("cold", {}).get("second", {}), + baseline.get("factNeutral", {}), + baseline.get("semanticEdit", {}), + baseline.get("forced", {}), + baseline.get("alternateCheckout", {}), + baseline.get("restore", {}), + *baseline.get("warm", {}).get("samples", []), + ) + if (value := section.get("rssBytes")) is not None + ] + baseline_rss = max(rss_values) if rss_values else None + except (KeyError, TypeError, ValueError, json.JSONDecodeError) as error: + raise QualificationError( + f"invalid universal-language performance baseline {baseline_path}: {error}" + ) from error + + gates = baseline.get("performanceGates", {}) + cold_limit = max(cold * float(gates.get("coldMultiplier", 1.10)), cold + 1.0) + warm_limit = max(warm * float(gates.get("warmMultiplier", 1.15)), warm + 0.1) + neutral_limit = max( + neutral * float(gates.get("warmMultiplier", 1.15)), + neutral + float(gates.get("factNeutralAdditiveSeconds", 0.25)), + ) + rss_limit = None + if baseline_rss is not None and report.get("peakRssBytes") is not None: + rss_limit = max( + baseline_rss * float(gates.get("peakRssMultiplier", 1.15)), + baseline_rss + int(gates.get("peakRssAdditiveBytes", 32 * 1024 * 1024)), + ) + observed = { + "coldSeconds": report["cold"]["seconds"], + "warmMedianSeconds": report["warm"]["medianSeconds"], + "factNeutralSeconds": report["factNeutral"]["seconds"], + "peakRssBytes": report.get("peakRssBytes"), + } + limits = { + "coldSeconds": cold_limit, + "warmMedianSeconds": warm_limit, + "factNeutralSeconds": neutral_limit, + "peakRssBytes": rss_limit, + } + passed = ( + observed["coldSeconds"] <= cold_limit + and observed["warmMedianSeconds"] <= warm_limit + and observed["factNeutralSeconds"] <= neutral_limit + and ( + rss_limit is None + or observed["peakRssBytes"] is None + or observed["peakRssBytes"] <= rss_limit + ) + ) + return { + "baseline": str(baseline_path), + "baselineRevision": baseline.get("compassRevision"), + "passed": passed, + "limits": limits, + "observed": observed, + } + + +def quality_audit_mode( + audit_manifest: Path | None, + graph: Path | None, + corpus: Path | None, +) -> dict[str, Any]: + if audit_manifest is None or graph is None or corpus is None: + raise QualificationError("quality-audit mode requires --audit-manifest, --graph, and --corpus") + for path, label in ((audit_manifest, "audit manifest"), (graph, "graph"), (corpus, "corpus")): + if not path.exists(): + raise QualificationError(f"{label} does not exist: {path}") + completed = subprocess.run( + [ + sys.executable, + str(ROOT / "benchmarks" / "performance" / "harness.py"), + "audit", + "--manifest", + str(audit_manifest.resolve()), + "--graph", + str(graph.resolve()), + "--corpus", + str(corpus.resolve()), + ], + cwd=ROOT, + check=False, + text=True, + capture_output=True, + ) + lines = [line for line in completed.stdout.splitlines() if line.strip()] + if not lines: + raise QualificationError( + "quality-audit evaluator emitted no result: " + f"{completed.stderr.strip()}" + ) + try: + result = json.loads(lines[-1]) + except json.JSONDecodeError as error: + raise QualificationError(f"quality-audit evaluator emitted invalid JSON: {error}") from error + if result.get("schema") != "compass.quality-audit-result/2": + raise QualificationError(f"unexpected quality-audit result schema: {result.get('schema')!r}") + return {"mode": "quality-audit", "audit": result, "evaluatorExitCode": completed.returncode} + + +def run_cli( + argv: list[str], + *, + language: str, + manifest_path: Path, + oracle_path: Path, + fixture_root: Path, +) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--mode", choices=("fixture", "pinned", "quality-audit", "performance"), default="fixture") + parser.add_argument("--root", type=Path, default=fixture_root) + parser.add_argument("--manifest", type=Path, default=manifest_path) + parser.add_argument("--oracle", type=Path, default=oracle_path) + parser.add_argument("--repository", action="append", default=[], metavar="NAME=PATH") + parser.add_argument("--audit-manifest", type=Path) + parser.add_argument("--graph", type=Path) + parser.add_argument("--corpus", type=Path) + parser.add_argument("--compass", type=Path) + parser.add_argument( + "--baseline", + type=Path, + default=ROOT / "tests/qualification" / f"{language}-universal-baseline.json", + help="established-path performance baseline (use an explicit path to override)", + ) + parser.add_argument("--samples", type=int, default=5) + parser.add_argument("--output", type=Path) + args = parser.parse_args(argv) + try: + schema = f"compass.{language}-universal-qualification/1" + manifest = load_manifest(args.manifest, schema) + if manifest.get("language") != language: + raise QualificationError("manifest language does not match entry point") + if args.mode == "fixture": + report = fixture_mode(manifest, args.oracle, args.root) + elif args.mode == "pinned": + report = pinned_mode(args.manifest, manifest, args.oracle, parse_overrides(args.repository)) + elif args.mode == "quality-audit": + report = quality_audit_mode(args.audit_manifest, args.graph, args.corpus) + else: + if args.compass is None: + raise QualificationError("--compass is required for performance mode") + report = performance_mode(args.root, args.compass.resolve(), args.samples) + if args.baseline is not None: + comparison = compare_performance_baseline(report, args.baseline.resolve()) + report["baselineComparison"] = comparison + if not comparison["passed"]: + raise QualificationError( + f"performance gates failed for {language}: {comparison}" + ) + encoded = canonical_bytes({"schema": schema, **report}) + if args.output: + args.output.write_bytes(encoded) + else: + sys.stdout.buffer.write(encoded) + if args.mode == "quality-audit": + audit = report.get("audit", {}) + return int(not (audit.get("passed") is True and audit.get("eligibleForQualityClaim") is True)) + return 0 + except (OSError, QualificationError, subprocess.SubprocessError) as error: + print(f"{language} qualification failed: {error}", file=sys.stderr) + return 1 diff --git a/scripts/record_universal_baseline.py b/scripts/record_universal_baseline.py new file mode 100644 index 00000000..50acaed1 --- /dev/null +++ b/scripts/record_universal_baseline.py @@ -0,0 +1,326 @@ +#!/usr/bin/env python3 +"""Record a deterministic, source-bounded universal-language baseline. + +The recorder is qualification-only and accepts a caller-provided Compass +binary. It runs cold, warm, forced, alternate-checkout, edit, and restore +publications, records graph/evidence digests, relation counts, diagnostics, +omissions, identity collisions, and child peak RSS, and never edits the +caller-provided source root. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +import resource +import shutil +import statistics +import subprocess +import sys +import tempfile +import time +from typing import Any + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _rss_bytes() -> int | None: + value = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss + if value <= 0: + return None + return int(value if sys.platform == "darwin" else value * 1024) + + +def _active_graph(output: Path) -> Path: + pointer = output / "compass-out" / "current-snapshot" + snapshot = pointer.read_text(encoding="utf-8").strip() + if not snapshot.startswith("snapshot-") or "/" in snapshot or "\\" in snapshot: + raise RuntimeError(f"invalid Compass snapshot pointer: {snapshot!r}") + graph = output / "compass-out" / "snapshots" / snapshot / "graph.json" + if not graph.is_file(): + raise RuntimeError(f"Compass graph is missing: {graph}") + return graph + + +def _publish( + compass: Path, + root: Path, + output: Path, + *, + force: bool = False, +) -> tuple[float, Path, int | None]: + command = [ + str(compass), + "update", + str(root), + "--out", + str(output), + "--no-viz", + "--no-cluster", + "--inference-level", + "max", + ] + if force: + command.append("--force") + started = time.perf_counter() + completed = subprocess.run( + command, + check=False, + capture_output=True, + text=True, + ) + elapsed = time.perf_counter() - started + if completed.returncode: + raise RuntimeError(completed.stderr.strip() or completed.stdout.strip()) + return elapsed, _active_graph(output), _rss_bytes() + + +def _summary(graph: Path) -> dict[str, Any]: + document = json.loads(graph.read_text(encoding="utf-8")) + nodes = document.get("nodes", []) + links = document.get("links", document.get("edges", [])) + relation_counts: dict[str, int] = {} + for link in links: + relation = link.get("kind", link.get("relation")) + if isinstance(relation, str): + relation_counts[relation] = relation_counts.get(relation, 0) + 1 + diagnostics = 0 + identity_collisions = 0 + omitted = 0 + for node in nodes: + values = node.get("diagnostics", []) + if isinstance(values, list): + diagnostics += len(values) + identity_collisions += sum( + isinstance(item, dict) + and item.get("code") in {"identity_collision", "ambiguous_identity"} + for item in values + ) + metadata = document.get("graph") + if isinstance(metadata, dict): + for key in ("omitted", "omittedFacts", "omissions"): + value = metadata.get(key) + if isinstance(value, int): + omitted += value + elif isinstance(value, list): + omitted += len(value) + return { + "graphSha256": _sha256(graph), + "evidenceDigest": ( + _sha256(graph.parent / "ast-fact-digests.json") + if (graph.parent / "ast-fact-digests.json").is_file() + else None + ), + "nodeCount": len(nodes), + "edgeCount": len(links), + "relationCounts": dict(sorted(relation_counts.items())), + "diagnostics": diagnostics, + "omittedFacts": omitted, + "identityCollisions": identity_collisions, + } + + +def _source_files(root: Path, language: str) -> list[Path]: + suffixes = { + "swift": {".swift"}, + "dart": {".dart"}, + "scala": {".scala"}, + "groovy": {".groovy", ".gradle"}, + }[language] + return sorted( + path + for path in root.rglob("*") + if path.is_file() and path.suffix.casefold() in suffixes + ) + + +def _semantic_marker(language: str) -> bytes: + return { + "swift": b"\n\nstruct CompassQualificationMarker {}\n", + "dart": b"\n\nclass CompassQualificationMarker {}\n", + "scala": b"\n\nclass CompassQualificationMarker\n", + "groovy": b"\n\nclass CompassQualificationMarker {}\n", + }[language] + + +def _neutral_marker(language: str) -> bytes: + return { + "swift": b"\n\n// compass-language-neutral\n", + "dart": b"\n\n// compass-language-neutral\n", + "scala": b"\n\n// compass-language-neutral\n", + "groovy": b"\n\n// compass-language-neutral\n", + }[language] + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--language", required=True, choices=("swift", "dart", "scala", "groovy")) + parser.add_argument("--root", type=Path, required=True) + parser.add_argument("--compass", type=Path, required=True) + parser.add_argument( + "--compass-revision", + required=True, + help="immutable source revision used to build the supplied binary", + ) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--samples", type=int, default=2) + parser.add_argument( + "--baseline-kind", + choices=("established-pre-cutover", "post-cutover-fixture"), + default="established-pre-cutover", + help="provenance label for this baseline artifact", + ) + parser.add_argument( + "--baseline-status", + default="reproduced from the caller-provided Compass binary", + help="provenance statement stored in the artifact", + ) + args = parser.parse_args() + if args.samples < 2 or args.samples > 20: + parser.error("--samples must be between 2 and 20") + if not args.root.is_dir() or not args.compass.is_file(): + parser.error("--root and --compass must exist") + with tempfile.TemporaryDirectory(prefix=f"compass-{args.language}-baseline-") as directory: + base = Path(directory) + workload = base / "workload" + shutil.copytree(args.root.resolve(), workload, symlinks=True) + source_files = _source_files(workload, args.language) + if not source_files: + raise RuntimeError(f"baseline root contains no {args.language} source files") + + first_output = base / "first" + first_time, first_graph, first_rss = _publish(args.compass, workload, first_output) + first_summary = _summary(first_graph) + warm: list[dict[str, Any]] = [] + for _ in range(args.samples): + elapsed, graph, rss = _publish(args.compass, workload, first_output) + warm.append({"seconds": elapsed, "rssBytes": rss, **_summary(graph)}) + + edited = source_files[0] + original = edited.read_bytes() + edited.write_bytes(original + _neutral_marker(args.language)) + neutral_time, neutral_graph, neutral_rss = _publish( + args.compass, workload, first_output + ) + neutral_summary = _summary(neutral_graph) + + edited.write_bytes(original + _semantic_marker(args.language)) + semantic_time, semantic_graph, semantic_rss = _publish( + args.compass, workload, first_output + ) + semantic_summary = _summary(semantic_graph) + if semantic_summary["graphSha256"] == first_summary["graphSha256"]: + raise RuntimeError("semantic edit did not change the published graph") + + edited.write_bytes(original) + restore_time, restore_graph, restore_rss = _publish( + args.compass, workload, first_output + ) + restore_summary = _summary(restore_graph) + if restore_summary["graphSha256"] != first_summary["graphSha256"]: + raise RuntimeError("restored graph is not byte-identical to the cold graph") + + forced_output = base / "forced" + forced_time, forced_graph, forced_rss = _publish( + args.compass, workload, forced_output, force=True + ) + forced_summary = _summary(forced_graph) + if forced_summary["graphSha256"] != first_summary["graphSha256"]: + raise RuntimeError("forced rebuild graph is not byte-identical to the cold graph") + + alternate_workload = base / "alternate-workload" + alternate_output = base / "alternate" + shutil.copytree(args.root.resolve(), alternate_workload, symlinks=True) + alternate_time, alternate_graph, alternate_rss = _publish( + args.compass, alternate_workload, alternate_output + ) + alternate_summary = _summary(alternate_graph) + if alternate_summary["graphSha256"] != first_summary["graphSha256"]: + raise RuntimeError("alternate checkout graph is not byte-identical to the cold graph") + + second_output = base / "second" + second_time, second_graph, second_rss = _publish(args.compass, workload, second_output) + second_summary = _summary(second_graph) + if first_summary["graphSha256"] != second_summary["graphSha256"]: + raise RuntimeError("cold rebuild graph digest changed between independent outputs") + if any(item["graphSha256"] != first_summary["graphSha256"] for item in warm): + raise RuntimeError("warm graph digest changed") + result = { + "schema": "compass.universal-language-baseline/1", + "language": args.language, + "root": args.root.as_posix(), + "baselineKind": args.baseline_kind, + "baselineStatus": args.baseline_status, + "compassRevision": args.compass_revision, + "productionRoute": f"compass.{args.language}/1", + "sourceFile": edited.relative_to(workload).as_posix(), + "cold": { + "medianSeconds": statistics.median((first_time, second_time)), + "first": {"seconds": first_time, "rssBytes": first_rss, **first_summary}, + "second": {"seconds": second_time, "rssBytes": second_rss, **second_summary}, + }, + "factNeutral": { + "seconds": neutral_time, + "rssBytes": neutral_rss, + **neutral_summary, + }, + "semanticEdit": { + "seconds": semantic_time, + "rssBytes": semantic_rss, + **semantic_summary, + }, + "forced": { + "seconds": forced_time, + "rssBytes": forced_rss, + **forced_summary, + }, + "alternateCheckout": { + "seconds": alternate_time, + "rssBytes": alternate_rss, + **alternate_summary, + }, + "restore": { + "seconds": restore_time, + "rssBytes": restore_rss, + **restore_summary, + }, + "warm": { + "samples": warm, + "medianSeconds": statistics.median( + [item["seconds"] for item in warm] + ), + "graphSha256": first_summary["graphSha256"], + }, + "editRestore": { + "neutralGraphSha256": neutral_summary["graphSha256"], + "semanticGraphSha256": semantic_summary["graphSha256"], + "restoreGraphSha256": restore_summary["graphSha256"], + "deterministic": True, + }, + "performanceGates": { + "coldMultiplier": 1.10, + "warmMultiplier": 1.15, + "factNeutralAdditiveSeconds": 0.25, + "peakRssMultiplier": 1.15, + "peakRssAdditiveBytes": 32 * 1024 * 1024, + }, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(result, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + "\n", + encoding="utf-8", + ) + print(json.dumps({"schema": result["schema"], "output": str(args.output), "graphSha256": result["cold"]["first"]["graphSha256"]}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/scala_source_oracle.py b/scripts/scala_source_oracle.py new file mode 100644 index 00000000..11692d2d --- /dev/null +++ b/scripts/scala_source_oracle.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Qualification-only Scala.meta-compatible source oracle.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from independent_language_oracle import canonical_bytes, run_oracle_with_provider # noqa: E402 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, required=True) + parser.add_argument("--output", type=Path) + parser.add_argument("--include", action="append", default=[]) + parser.add_argument("--exclude", action="append", default=[]) + args = parser.parse_args() + try: + payload = run_oracle_with_provider( + args.root, + language="scala", + provider="scala-meta-source-oracle", + toolchain="Scala CLI 1.9.1; Scala 3.7.3; scala.meta 4.13.10; ujson 4.1.0 (qualification contract)", + implementation="bounded_lexical_scanner; scala.meta provider unavailable", + suffixes=(".scala",), + include_globs=tuple(args.include), + exclude_globs=tuple(args.exclude), + ) + encoded = canonical_bytes(payload) + if args.output: + args.output.write_bytes(encoded) + else: + sys.stdout.buffer.write(encoded) + return 0 + except (OSError, RuntimeError) as error: + print(f"scala source oracle failed: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/swift_source_oracle.py b/scripts/swift_source_oracle.py new file mode 100644 index 00000000..bcf64661 --- /dev/null +++ b/scripts/swift_source_oracle.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Qualification-only Swift source oracle (SwiftSyntax-compatible contract).""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from independent_language_oracle import canonical_bytes, run_oracle_with_provider # noqa: E402 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, required=True) + parser.add_argument("--output", type=Path) + parser.add_argument("--include", action="append", default=[]) + parser.add_argument("--exclude", action="append", default=[]) + args = parser.parse_args() + try: + payload = run_oracle_with_provider( + args.root, + language="swift", + provider="swift-syntax-source-oracle", + toolchain="swift 6.3.3; SwiftSyntax 603.0.0 (qualification contract)", + implementation="bounded_lexical_scanner; SwiftSyntax provider unavailable", + suffixes=(".swift",), + include_globs=tuple(args.include), + exclude_globs=tuple(args.exclude), + ) + encoded = canonical_bytes(payload) + if args.output: + args.output.write_bytes(encoded) + else: + sys.stdout.buffer.write(encoded) + return 0 + except (OSError, RuntimeError) as error: + print(f"swift source oracle failed: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/tests/test_universal_performance.py b/scripts/tests/test_universal_performance.py new file mode 100644 index 00000000..29d6d0f7 --- /dev/null +++ b/scripts/tests/test_universal_performance.py @@ -0,0 +1,76 @@ +import json +from pathlib import Path +import sys +import tempfile +import unittest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from qualify_universal_language import compare_performance_baseline + + +class UniversalPerformanceBaselineTests(unittest.TestCase): + def _baseline(self, directory: Path) -> Path: + path = directory / "baseline.json" + path.write_text( + json.dumps( + { + "compassRevision": "88abe4c0", + "cold": { + "medianSeconds": 1.0, + "first": {"seconds": 1.0, "rssBytes": 100}, + "second": {"seconds": 1.0, "rssBytes": 100}, + }, + "warm": { + "medianSeconds": 0.5, + "samples": [{"seconds": 0.5, "rssBytes": 100}], + }, + "factNeutral": {"seconds": 0.5, "rssBytes": 100}, + "semanticEdit": {"seconds": 0.5, "rssBytes": 100}, + "forced": {"seconds": 0.5, "rssBytes": 100}, + "alternateCheckout": {"seconds": 0.5, "rssBytes": 100}, + "restore": {"seconds": 0.5, "rssBytes": 100}, + "performanceGates": { + "coldMultiplier": 1.1, + "warmMultiplier": 1.15, + "factNeutralAdditiveSeconds": 0.25, + "peakRssMultiplier": 1.15, + "peakRssAdditiveBytes": 32, + }, + } + ), + encoding="utf-8", + ) + return path + + def test_comparison_accepts_values_inside_all_gates(self) -> None: + with tempfile.TemporaryDirectory() as directory: + baseline = self._baseline(Path(directory)) + result = compare_performance_baseline( + { + "cold": {"seconds": 1.5}, + "warm": {"medianSeconds": 0.6}, + "factNeutral": {"seconds": 0.7}, + "peakRssBytes": 120, + }, + baseline, + ) + self.assertTrue(result["passed"]) + + def test_comparison_rejects_cold_regression(self) -> None: + with tempfile.TemporaryDirectory() as directory: + baseline = self._baseline(Path(directory)) + result = compare_performance_baseline( + { + "cold": {"seconds": 2.1}, + "warm": {"medianSeconds": 0.6}, + "factNeutral": {"seconds": 0.7}, + "peakRssBytes": 120, + }, + baseline, + ) + self.assertFalse(result["passed"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_universal_source_oracle.py b/scripts/tests/test_universal_source_oracle.py new file mode 100644 index 00000000..934032c0 --- /dev/null +++ b/scripts/tests/test_universal_source_oracle.py @@ -0,0 +1,78 @@ +"""Regression tests for the four bounded universal source-oracle contracts.""" + +from __future__ import annotations + +import hashlib +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "scripts")) + +from independent_language_oracle import canonical_bytes, run_oracle # noqa: E402 + + +class UniversalSourceOracleTests(unittest.TestCase): + def test_metadata_is_reported_without_changing_inventory_digest(self) -> None: + cases = ( + ("swift", ".swift", "SwiftSyntax provider unavailable"), + ("dart", ".dart", "Dart Analyzer provider unavailable"), + ("scala", ".scala", "scala.meta provider unavailable"), + ("groovy", ".groovy", "Groovy CompilationUnit provider unavailable"), + ) + for language, suffix, provider_name in cases: + with self.subTest(language=language), tempfile.TemporaryDirectory( + prefix=f"compass-{language}-oracle-test-" + ) as directory: + root = Path(directory) + source = root / f"main{suffix}" + source.write_text( + "class Sample { void run() { helper(); } void helper() {} }\n", + encoding="utf-8", + ) + document = run_oracle( + root, + language=language, + provider=f"{language}-provider", + toolchain="pinned test toolchain", + implementation=f"bounded_lexical_scanner; {provider_name}", + parser_available=False, + suffixes=(suffix,), + ) + self.assertFalse(document["parserAvailable"]) + self.assertIn(provider_name, document["implementation"]) + inventory = { + "language": document["language"], + "provider": document["provider"], + "toolchain": document["toolchain"], + "rootRelativeFiles": [item["path"] for item in document["files"]], + "files": document["files"], + } + expected = hashlib.sha256( + canonical_bytes(inventory).rstrip(b"\n") + ).hexdigest() + self.assertEqual(expected, document["inventorySha256"]) + + def test_include_and_exclude_globs_define_the_complete_inventory(self) -> None: + with tempfile.TemporaryDirectory(prefix="compass-universal-oracle-globs-") as directory: + root = Path(directory) + (root / "lib").mkdir() + (root / "tests").mkdir() + (root / "lib" / "main.swift").write_text("class Main {}\n", encoding="utf-8") + (root / "tests" / "main.swift").write_text("class Test {}\n", encoding="utf-8") + document = run_oracle( + root, + language="swift", + provider="swift-provider", + toolchain="pinned test toolchain", + suffixes=(".swift",), + include_globs=("lib/**/*.swift", "tests/**/*.swift"), + exclude_globs=("tests/**",), + ) + self.assertEqual(["lib/main.swift"], [item["path"] for item in document["files"]]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/qualification/code-graph-v1-semantic.json b/tests/qualification/code-graph-v1-semantic.json index 0f5f9592..1fb575b5 100644 --- a/tests/qualification/code-graph-v1-semantic.json +++ b/tests/qualification/code-graph-v1-semantic.json @@ -921,7 +921,7 @@ "kind": "module", "source": "fixtures/code-graph/routes/swift/VaporRoutes.swift", "qualifiedName": "Vapor", - "producer": "compass.languages.swift", + "producer": "compass.languages.swift.universal", "origins": [ "ast" ], @@ -1451,7 +1451,7 @@ "kind": "exports", "source": "fixtures/code-graph/qualification/export.dart", "qualifiedName": "*", - "producer": "compass.languages.dart", + "producer": "compass.resolve.dart.universal", "origins": [ "ast" ], @@ -1745,6 +1745,14 @@ "sourceQualifiedName": "::render", "targetQualifiedName": "crate::rich::target", "minimum": 2 + }, + { + "id": "repeated-groovy-call", + "kind": "calls", + "source": "fixtures/code-graph/routes/groovy/SpockSpec.groovy", + "sourceQualifiedName": "routes.UserService.run", + "targetQualifiedName": "routes.UserService.load", + "minimum": 2 } ], "coverage": [ @@ -1765,6 +1773,16 @@ "id": "missing-reference-diagnostic", "source": "fixtures/code-graph/qualification/MissingReference.csproj", "diagnosticCode": "unresolved_external_reference" + }, + { + "id": "scala-universal-extracted", + "source": "fixtures/code-graph/routes/scala/Universal.scala", + "extractionStatus": "extracted" + }, + { + "id": "groovy-universal-recovery", + "source": "fixtures/code-graph/routes/groovy/SpockSpec.groovy", + "extractionStatus": "partial" } ], "limits": { diff --git a/tests/qualification/dart-universal-baseline.json b/tests/qualification/dart-universal-baseline.json new file mode 100644 index 00000000..b1b31e7f --- /dev/null +++ b/tests/qualification/dart-universal-baseline.json @@ -0,0 +1 @@ +{"alternateCheckout":{"diagnostics":4,"edgeCount":10,"evidenceDigest":"e6f33b96474bcb04ad8d4ffa2d326bae2861d913d8406b8392511f2e42ae1d94","graphSha256":"ad0bfc2384eedb144bcf784168778127c682142a8a4589ee3366a0fbab15c10b","identityCollisions":0,"nodeCount":14,"omittedFacts":0,"relationCounts":{"contains":7,"exports":1,"imports":2},"rssBytes":51167232,"seconds":0.3638287909561768},"baselineKind":"established-pre-cutover","baselineStatus":"established direct extractor at commit 88abe4c071a19ec03b3bca132656830a02a47907","cold":{"first":{"diagnostics":4,"edgeCount":10,"evidenceDigest":"e6f33b96474bcb04ad8d4ffa2d326bae2861d913d8406b8392511f2e42ae1d94","graphSha256":"ad0bfc2384eedb144bcf784168778127c682142a8a4589ee3366a0fbab15c10b","identityCollisions":0,"nodeCount":14,"omittedFacts":0,"relationCounts":{"contains":7,"exports":1,"imports":2},"rssBytes":48447488,"seconds":0.34767074999399483},"medianSeconds":0.3574878124636598,"second":{"diagnostics":4,"edgeCount":10,"evidenceDigest":"e6f33b96474bcb04ad8d4ffa2d326bae2861d913d8406b8392511f2e42ae1d94","graphSha256":"ad0bfc2384eedb144bcf784168778127c682142a8a4589ee3366a0fbab15c10b","identityCollisions":0,"nodeCount":14,"omittedFacts":0,"relationCounts":{"contains":7,"exports":1,"imports":2},"rssBytes":51167232,"seconds":0.36730487493332475}},"compassRevision":"88abe4c071a19ec03b3bca132656830a02a47907","editRestore":{"deterministic":true,"neutralGraphSha256":"a2bcf4e036d876e19c88b34468f480b040ec75f8cfbdda875f9141fe68e310fb","restoreGraphSha256":"ad0bfc2384eedb144bcf784168778127c682142a8a4589ee3366a0fbab15c10b","semanticGraphSha256":"6aa8641ba62dea065ce5459446f619f1e60a6fa1481e993ddc812591ee4a7cbc"},"factNeutral":{"diagnostics":4,"edgeCount":10,"evidenceDigest":"e6f33b96474bcb04ad8d4ffa2d326bae2861d913d8406b8392511f2e42ae1d94","graphSha256":"a2bcf4e036d876e19c88b34468f480b040ec75f8cfbdda875f9141fe68e310fb","identityCollisions":0,"nodeCount":14,"omittedFacts":0,"relationCounts":{"contains":7,"exports":1,"imports":2},"rssBytes":48447488,"seconds":0.3807559999404475},"forced":{"diagnostics":4,"edgeCount":10,"evidenceDigest":"e6f33b96474bcb04ad8d4ffa2d326bae2861d913d8406b8392511f2e42ae1d94","graphSha256":"ad0bfc2384eedb144bcf784168778127c682142a8a4589ee3366a0fbab15c10b","identityCollisions":0,"nodeCount":14,"omittedFacts":0,"relationCounts":{"contains":7,"exports":1,"imports":2},"rssBytes":51167232,"seconds":0.35637479089200497},"language":"dart","performanceGates":{"coldMultiplier":1.1,"factNeutralAdditiveSeconds":0.25,"peakRssAdditiveBytes":33554432,"peakRssMultiplier":1.15,"warmMultiplier":1.15},"productionRoute":"compass.dart/1","restore":{"diagnostics":4,"edgeCount":10,"evidenceDigest":"e6f33b96474bcb04ad8d4ffa2d326bae2861d913d8406b8392511f2e42ae1d94","graphSha256":"ad0bfc2384eedb144bcf784168778127c682142a8a4589ee3366a0fbab15c10b","identityCollisions":0,"nodeCount":14,"omittedFacts":0,"relationCounts":{"contains":7,"exports":1,"imports":2},"rssBytes":51167232,"seconds":0.29910341708455235},"root":"tests/qualification/language-wave/dart","schema":"compass.universal-language-baseline/1","semanticEdit":{"diagnostics":4,"edgeCount":11,"evidenceDigest":"a8d442b8fdb1557194b7dda411a87b6832144232d2c4c509bc638687b16ce664","graphSha256":"6aa8641ba62dea065ce5459446f619f1e60a6fa1481e993ddc812591ee4a7cbc","identityCollisions":0,"nodeCount":15,"omittedFacts":0,"relationCounts":{"contains":8,"exports":1,"imports":2},"rssBytes":51167232,"seconds":0.4610634580021724},"sourceFile":"library.dart","warm":{"graphSha256":"ad0bfc2384eedb144bcf784168778127c682142a8a4589ee3366a0fbab15c10b","medianSeconds":0.06142349948640913,"samples":[{"diagnostics":4,"edgeCount":10,"evidenceDigest":"e6f33b96474bcb04ad8d4ffa2d326bae2861d913d8406b8392511f2e42ae1d94","graphSha256":"ad0bfc2384eedb144bcf784168778127c682142a8a4589ee3366a0fbab15c10b","identityCollisions":0,"nodeCount":14,"omittedFacts":0,"relationCounts":{"contains":7,"exports":1,"imports":2},"rssBytes":48447488,"seconds":0.05649416602682322},{"diagnostics":4,"edgeCount":10,"evidenceDigest":"e6f33b96474bcb04ad8d4ffa2d326bae2861d913d8406b8392511f2e42ae1d94","graphSha256":"ad0bfc2384eedb144bcf784168778127c682142a8a4589ee3366a0fbab15c10b","identityCollisions":0,"nodeCount":14,"omittedFacts":0,"relationCounts":{"contains":7,"exports":1,"imports":2},"rssBytes":48447488,"seconds":0.06635283294599503}]}} diff --git a/tests/qualification/dart-universal-repositories.toml b/tests/qualification/dart-universal-repositories.toml new file mode 100644 index 00000000..e944e010 --- /dev/null +++ b/tests/qualification/dart-universal-repositories.toml @@ -0,0 +1,35 @@ +schema = "compass.dart-universal-qualification/1" +language = "dart" +producer = "compass.dart" +producerVersion = 1 +oracleProvider = "dart-analyzer-source-oracle" +oracleToolchain = "Dart SDK 3.13.1; package:analyzer 8.4.0" +checkoutRoot = "/Volumes/Workspace/Github" +readOnly = true +minimumAcceptedRelationships = 2000 +minimumAcceptedPerCorpus = 400 +minimumAcceptedPerRelation = 100 + +[[repository]] +name = "dart-sdk" +url = "https://github.com/dart-lang/sdk.git" +commit = "227791d36348c8fcb90e350245f4c3d2cf265ff0" +purpose = "Dart library subset with language features, imports, parts, and analyzer fixtures" +sourceGlobs = ["pkg/analyzer/lib/**/*.dart", "pkg/analysis_server/lib/**/*.dart", "pkg/front_end/lib/**/*.dart", "pkg/compiler/lib/**/*.dart", "pkg/linter/lib/**/*.dart"] +excludeGlobs = [] + +[[repository]] +name = "flutter" +url = "https://github.com/flutter/flutter.git" +commit = "0cb32fd1625d0782fc3162754f73da90f14205c0" +purpose = "Flutter packages subset with widgets, navigation, and framework conventions" +sourceGlobs = ["packages/**/*.dart", "dev/**/*.dart"] +excludeGlobs = [] + +[[repository]] +name = "riverpod" +url = "https://github.com/rrousselGit/riverpod.git" +commit = "d71acf645364cc35d45559eb10bd489e2a7d6eb2" +purpose = "Riverpod providers, generated-safe source, and package exports" +sourceGlobs = ["packages/**/*.dart", "examples/**/*.dart"] +excludeGlobs = [] diff --git a/tests/qualification/groovy-universal-baseline.json b/tests/qualification/groovy-universal-baseline.json new file mode 100644 index 00000000..b3aa61b7 --- /dev/null +++ b/tests/qualification/groovy-universal-baseline.json @@ -0,0 +1 @@ +{"alternateCheckout":{"diagnostics":0,"edgeCount":0,"evidenceDigest":"667dbe9230e6ac879757f1d6e03a93075118c0d601d3023755d3fc3e0497d49e","graphSha256":"a6864577c15c4ed603280d0f9725ade99202f1a5073f280187a763b4a1c1186b","identityCollisions":0,"nodeCount":4,"omittedFacts":0,"relationCounts":{},"rssBytes":46104576,"seconds":0.1345877080457285},"baselineKind":"established-pre-cutover","baselineStatus":"established direct extractor at commit 88abe4c071a19ec03b3bca132656830a02a47907","cold":{"first":{"diagnostics":0,"edgeCount":0,"evidenceDigest":"667dbe9230e6ac879757f1d6e03a93075118c0d601d3023755d3fc3e0497d49e","graphSha256":"a6864577c15c4ed603280d0f9725ade99202f1a5073f280187a763b4a1c1186b","identityCollisions":0,"nodeCount":4,"omittedFacts":0,"relationCounts":{},"rssBytes":43319296,"seconds":0.13675666705239564},"medianSeconds":0.13753135449951515,"second":{"diagnostics":0,"edgeCount":0,"evidenceDigest":"667dbe9230e6ac879757f1d6e03a93075118c0d601d3023755d3fc3e0497d49e","graphSha256":"a6864577c15c4ed603280d0f9725ade99202f1a5073f280187a763b4a1c1186b","identityCollisions":0,"nodeCount":4,"omittedFacts":0,"relationCounts":{},"rssBytes":46104576,"seconds":0.13830604194663465}},"compassRevision":"88abe4c071a19ec03b3bca132656830a02a47907","editRestore":{"deterministic":true,"neutralGraphSha256":"4d5a901d4a43f6992345d02db9188073ca59b7f6095632c9c91b96e49e6eb771","restoreGraphSha256":"a6864577c15c4ed603280d0f9725ade99202f1a5073f280187a763b4a1c1186b","semanticGraphSha256":"3b21fb6598f7cc3e079a7f4e51cafb4f6ea6cbc603da69a2d9b0ccdb0921f7fb"},"factNeutral":{"diagnostics":0,"edgeCount":0,"evidenceDigest":"667dbe9230e6ac879757f1d6e03a93075118c0d601d3023755d3fc3e0497d49e","graphSha256":"4d5a901d4a43f6992345d02db9188073ca59b7f6095632c9c91b96e49e6eb771","identityCollisions":0,"nodeCount":4,"omittedFacts":0,"relationCounts":{},"rssBytes":43319296,"seconds":0.1337910839356482},"forced":{"diagnostics":0,"edgeCount":0,"evidenceDigest":"667dbe9230e6ac879757f1d6e03a93075118c0d601d3023755d3fc3e0497d49e","graphSha256":"a6864577c15c4ed603280d0f9725ade99202f1a5073f280187a763b4a1c1186b","identityCollisions":0,"nodeCount":4,"omittedFacts":0,"relationCounts":{},"rssBytes":46104576,"seconds":0.14156374998856336},"language":"groovy","performanceGates":{"coldMultiplier":1.1,"factNeutralAdditiveSeconds":0.25,"peakRssAdditiveBytes":33554432,"peakRssMultiplier":1.15,"warmMultiplier":1.15},"productionRoute":"compass.groovy/1","restore":{"diagnostics":0,"edgeCount":0,"evidenceDigest":"667dbe9230e6ac879757f1d6e03a93075118c0d601d3023755d3fc3e0497d49e","graphSha256":"a6864577c15c4ed603280d0f9725ade99202f1a5073f280187a763b4a1c1186b","identityCollisions":0,"nodeCount":4,"omittedFacts":0,"relationCounts":{},"rssBytes":46104576,"seconds":0.1484077499480918},"root":"tests/qualification/language-wave/groovy","schema":"compass.universal-language-baseline/1","semanticEdit":{"diagnostics":0,"edgeCount":0,"evidenceDigest":"2335917eb4ca29244f8f86614e360ba6d2eac109ca63d67ab16b780be0066ddb","graphSha256":"3b21fb6598f7cc3e079a7f4e51cafb4f6ea6cbc603da69a2d9b0ccdb0921f7fb","identityCollisions":0,"nodeCount":4,"omittedFacts":0,"relationCounts":{},"rssBytes":46104576,"seconds":0.15470437495969236},"sourceFile":"Module.groovy","warm":{"graphSha256":"a6864577c15c4ed603280d0f9725ade99202f1a5073f280187a763b4a1c1186b","medianSeconds":0.05356320855207741,"samples":[{"diagnostics":0,"edgeCount":0,"evidenceDigest":"667dbe9230e6ac879757f1d6e03a93075118c0d601d3023755d3fc3e0497d49e","graphSha256":"a6864577c15c4ed603280d0f9725ade99202f1a5073f280187a763b4a1c1186b","identityCollisions":0,"nodeCount":4,"omittedFacts":0,"relationCounts":{},"rssBytes":43319296,"seconds":0.0556535420473665},{"diagnostics":0,"edgeCount":0,"evidenceDigest":"667dbe9230e6ac879757f1d6e03a93075118c0d601d3023755d3fc3e0497d49e","graphSha256":"a6864577c15c4ed603280d0f9725ade99202f1a5073f280187a763b4a1c1186b","identityCollisions":0,"nodeCount":4,"omittedFacts":0,"relationCounts":{},"rssBytes":43319296,"seconds":0.051472875056788325}]}} diff --git a/tests/qualification/groovy-universal-repositories.toml b/tests/qualification/groovy-universal-repositories.toml new file mode 100644 index 00000000..dcc9fcb3 --- /dev/null +++ b/tests/qualification/groovy-universal-repositories.toml @@ -0,0 +1,35 @@ +schema = "compass.groovy-universal-qualification/1" +language = "groovy" +producer = "compass.groovy" +producerVersion = 1 +oracleProvider = "groovy-compilation-unit-source-oracle" +oracleToolchain = "Apache Groovy 4.0.27 CompilationUnit conversion phase" +checkoutRoot = "/Volumes/Workspace/Github" +readOnly = true +minimumAcceptedRelationships = 2000 +minimumAcceptedPerCorpus = 400 +minimumAcceptedPerRelation = 100 + +[[repository]] +name = "groovy" +url = "https://github.com/apache/groovy.git" +commit = "f6bc5a511c3ea79e51afe0eb26e494e544da06c1" +purpose = "Groovy compiler sources, traits, scripts, closures, and dynamic boundaries" +sourceGlobs = ["src/**/*.groovy", "subprojects/**/*.groovy", "*.gradle", "**/*.gradle"] +excludeGlobs = [] + +[[repository]] +name = "gradle" +url = "https://github.com/gradle/gradle.git" +commit = "534f27719b66953f95cc907aae7f2c1b12f5482d" +purpose = "Gradle build scripts, convention plugins, and statically typed Groovy APIs" +sourceGlobs = ["subprojects/**/*.groovy", "build-logic/**/*.groovy", "*.gradle", "**/*.gradle"] +excludeGlobs = [] + +[[repository]] +name = "spock" +url = "https://github.com/spockframework/spock.git" +commit = "37e5e9a663ac6bc1ed1f72ac8374a81b879c66b0" +purpose = "Spock feature methods, specifications, extensions, and Groovy test DSL" +sourceGlobs = ["spock-*/**/*.groovy", "build-logic/**/*.groovy", "*.gradle", "**/*.gradle"] +excludeGlobs = [] diff --git a/tests/qualification/language-wave/dart/library.dart b/tests/qualification/language-wave/dart/library.dart new file mode 100644 index 00000000..9dd30769 --- /dev/null +++ b/tests/qualification/language-wave/dart/library.dart @@ -0,0 +1,25 @@ +library wave; + +import 'dart:async'; +import 'package:flutter/widgets.dart' as widgets; +export 'src/model.dart' show User; +part 'src/part.dart'; + +abstract class Store { + Future save(String value); +} + +class UserStore implements Store { + UserStore(); + UserStore.named(this.value); + final String value; + @override + Future save(String value) async {} + void route(widgets.BuildContext context) { + widgets.Navigator.of(context).pushNamed('/users'); + } +} + +void dynamicCall(dynamic receiver) { + receiver.unknown(); +} diff --git a/tests/qualification/language-wave/dart/src/model.dart b/tests/qualification/language-wave/dart/src/model.dart new file mode 100644 index 00000000..053b0cee --- /dev/null +++ b/tests/qualification/language-wave/dart/src/model.dart @@ -0,0 +1 @@ +class User { const User(this.name); final String name; } diff --git a/tests/qualification/language-wave/dart/src/part.dart b/tests/qualification/language-wave/dart/src/part.dart new file mode 100644 index 00000000..6e9547b5 --- /dev/null +++ b/tests/qualification/language-wave/dart/src/part.dart @@ -0,0 +1,2 @@ +part of wave; +void repeated(UserStore store) { store.save('a'); store.save('b'); } diff --git a/tests/qualification/language-wave/groovy/Module.groovy b/tests/qualification/language-wave/groovy/Module.groovy new file mode 100644 index 00000000..06a5052d --- /dev/null +++ b/tests/qualification/language-wave/groovy/Module.groovy @@ -0,0 +1,13 @@ +package wave + +interface Store { void save(String value) } +class UserStore implements Store { + String value + UserStore() {} + void save(String value) { this.value = value } + void route() { save('users') } +} +trait Audited { void audit() {} } +class Specification extends spock.lang.Specification { + def "stores users"() { expect: new UserStore().save('ok') } +} diff --git a/tests/qualification/language-wave/groovy/build.gradle b/tests/qualification/language-wave/groovy/build.gradle new file mode 100644 index 00000000..b1888f1a --- /dev/null +++ b/tests/qualification/language-wave/groovy/build.gradle @@ -0,0 +1,2 @@ +plugins { id 'groovy' } +def dynamicDsl = project(':missing').customTask() diff --git a/tests/qualification/language-wave/scala/Module.scala b/tests/qualification/language-wave/scala/Module.scala new file mode 100644 index 00000000..dc16d6b5 --- /dev/null +++ b/tests/qualification/language-wave/scala/Module.scala @@ -0,0 +1,12 @@ +package wave + +trait Store { def save(value: String): Unit } +final case class User(name: String) +final class UserStore extends Store { + override def save(value: String): Unit = println(value) + def route(): Unit = save("users") +} +object UserStore { def apply(): UserStore = new UserStore() } +given ordering: Ordering[User] with + def compare(left: User, right: User): Int = left.name.compareTo(right.name) +extension (store: UserStore) def repeated(): Unit = { store.save("a"); store.save("b") } diff --git a/tests/qualification/language-wave/scala/Scala3.scala b/tests/qualification/language-wave/scala/Scala3.scala new file mode 100644 index 00000000..fc231d7d --- /dev/null +++ b/tests/qualification/language-wave/scala/Scala3.scala @@ -0,0 +1,4 @@ +package wave +enum Status { case Ready, Failed } +type Alias = User +def malformedFixture( = 1 diff --git a/tests/qualification/language-wave/swift/Module.swift b/tests/qualification/language-wave/swift/Module.swift new file mode 100644 index 00000000..bbd622c9 --- /dev/null +++ b/tests/qualification/language-wave/swift/Module.swift @@ -0,0 +1,20 @@ +import Vapor + +public protocol Store { func save(_ value: String) async throws } +public struct UserStore: Store { + public init() {} + public func save(_ value: String) async throws { _ = value.count } +} +public extension UserStore { + func route(_ app: Application) { + app.get("users", use: listUsers) + } + private func listUsers(_ request: Request) async throws -> String { "ok" } +} + +struct AmbiguousA { func same() {} } +struct AmbiguousB { func same() {} } +func repeated(_ store: UserStore) { + _ = store.save + _ = store.save +} diff --git a/tests/qualification/language-wave/swift/UTF8.swift b/tests/qualification/language-wave/swift/UTF8.swift new file mode 100644 index 00000000..7b180bf9 --- /dev/null +++ b/tests/qualification/language-wave/swift/UTF8.swift @@ -0,0 +1,4 @@ +struct Café { + let naïve: String + func greet() { print(naïve) } +} diff --git a/tests/qualification/scala-universal-baseline.json b/tests/qualification/scala-universal-baseline.json new file mode 100644 index 00000000..c319d4ed --- /dev/null +++ b/tests/qualification/scala-universal-baseline.json @@ -0,0 +1 @@ +{"alternateCheckout":{"diagnostics":4,"edgeCount":18,"evidenceDigest":"cd5e3f3e7938e10a1393542412702f3e0e0403f8cc33b529daf31a1bb7d8b3b6","graphSha256":"5b2468dd5cc8e1e69d930706776f3f3d83ef4a4f37513fcd329c84fb68490b0f","identityCollisions":0,"nodeCount":15,"omittedFacts":0,"relationCounts":{"calls":1,"contains":8,"extends":1,"references":8},"rssBytes":52248576,"seconds":0.1396316250320524},"baselineKind":"established-pre-cutover","baselineStatus":"established direct extractor at commit 88abe4c071a19ec03b3bca132656830a02a47907","cold":{"first":{"diagnostics":4,"edgeCount":18,"evidenceDigest":"cd5e3f3e7938e10a1393542412702f3e0e0403f8cc33b529daf31a1bb7d8b3b6","graphSha256":"5b2468dd5cc8e1e69d930706776f3f3d83ef4a4f37513fcd329c84fb68490b0f","identityCollisions":0,"nodeCount":15,"omittedFacts":0,"relationCounts":{"calls":1,"contains":8,"extends":1,"references":8},"rssBytes":48168960,"seconds":0.15019683307036757},"medianSeconds":0.14770045853219926,"second":{"diagnostics":4,"edgeCount":18,"evidenceDigest":"cd5e3f3e7938e10a1393542412702f3e0e0403f8cc33b529daf31a1bb7d8b3b6","graphSha256":"5b2468dd5cc8e1e69d930706776f3f3d83ef4a4f37513fcd329c84fb68490b0f","identityCollisions":0,"nodeCount":15,"omittedFacts":0,"relationCounts":{"calls":1,"contains":8,"extends":1,"references":8},"rssBytes":52248576,"seconds":0.14520408399403095}},"compassRevision":"88abe4c071a19ec03b3bca132656830a02a47907","editRestore":{"deterministic":true,"neutralGraphSha256":"3c57ace37d844df17ae3dc2841f49edb42507e3ed2d39f082289218e31146dcc","restoreGraphSha256":"5b2468dd5cc8e1e69d930706776f3f3d83ef4a4f37513fcd329c84fb68490b0f","semanticGraphSha256":"aac3e1b24bf1e2a5009ef21d277e509c279712825e0d1437a96bca5954a72f5e"},"factNeutral":{"diagnostics":4,"edgeCount":18,"evidenceDigest":"43a69d3394d24b223e883fdebdf846f66179ebf51453104e8c84c43bb22fee9b","graphSha256":"3c57ace37d844df17ae3dc2841f49edb42507e3ed2d39f082289218e31146dcc","identityCollisions":0,"nodeCount":15,"omittedFacts":0,"relationCounts":{"calls":1,"contains":8,"extends":1,"references":8},"rssBytes":50495488,"seconds":0.1516896670218557},"forced":{"diagnostics":4,"edgeCount":18,"evidenceDigest":"cd5e3f3e7938e10a1393542412702f3e0e0403f8cc33b529daf31a1bb7d8b3b6","graphSha256":"5b2468dd5cc8e1e69d930706776f3f3d83ef4a4f37513fcd329c84fb68490b0f","identityCollisions":0,"nodeCount":15,"omittedFacts":0,"relationCounts":{"calls":1,"contains":8,"extends":1,"references":8},"rssBytes":52248576,"seconds":0.1380784580251202},"language":"scala","performanceGates":{"coldMultiplier":1.1,"factNeutralAdditiveSeconds":0.25,"peakRssAdditiveBytes":33554432,"peakRssMultiplier":1.15,"warmMultiplier":1.15},"productionRoute":"compass.scala/1","restore":{"diagnostics":4,"edgeCount":18,"evidenceDigest":"cd5e3f3e7938e10a1393542412702f3e0e0403f8cc33b529daf31a1bb7d8b3b6","graphSha256":"5b2468dd5cc8e1e69d930706776f3f3d83ef4a4f37513fcd329c84fb68490b0f","identityCollisions":0,"nodeCount":15,"omittedFacts":0,"relationCounts":{"calls":1,"contains":8,"extends":1,"references":8},"rssBytes":52248576,"seconds":0.17703883291687816},"root":"tests/qualification/language-wave/scala","schema":"compass.universal-language-baseline/1","semanticEdit":{"diagnostics":4,"edgeCount":19,"evidenceDigest":"99e87b31e8620ea3b19b76051e83f5a511927c22bbc5120f75459d2dfadd3d85","graphSha256":"aac3e1b24bf1e2a5009ef21d277e509c279712825e0d1437a96bca5954a72f5e","identityCollisions":0,"nodeCount":16,"omittedFacts":0,"relationCounts":{"calls":1,"contains":9,"extends":1,"references":8},"rssBytes":52248576,"seconds":0.19435270898975432},"sourceFile":"Module.scala","warm":{"graphSha256":"5b2468dd5cc8e1e69d930706776f3f3d83ef4a4f37513fcd329c84fb68490b0f","medianSeconds":0.054365458025131375,"samples":[{"diagnostics":4,"edgeCount":18,"evidenceDigest":"cd5e3f3e7938e10a1393542412702f3e0e0403f8cc33b529daf31a1bb7d8b3b6","graphSha256":"5b2468dd5cc8e1e69d930706776f3f3d83ef4a4f37513fcd329c84fb68490b0f","identityCollisions":0,"nodeCount":15,"omittedFacts":0,"relationCounts":{"calls":1,"contains":8,"extends":1,"references":8},"rssBytes":48168960,"seconds":0.05574633309151977},{"diagnostics":4,"edgeCount":18,"evidenceDigest":"cd5e3f3e7938e10a1393542412702f3e0e0403f8cc33b529daf31a1bb7d8b3b6","graphSha256":"5b2468dd5cc8e1e69d930706776f3f3d83ef4a4f37513fcd329c84fb68490b0f","identityCollisions":0,"nodeCount":15,"omittedFacts":0,"relationCounts":{"calls":1,"contains":8,"extends":1,"references":8},"rssBytes":48168960,"seconds":0.052984582958742976}]}} diff --git a/tests/qualification/scala-universal-repositories.toml b/tests/qualification/scala-universal-repositories.toml new file mode 100644 index 00000000..41556ff7 --- /dev/null +++ b/tests/qualification/scala-universal-repositories.toml @@ -0,0 +1,35 @@ +schema = "compass.scala-universal-qualification/1" +language = "scala" +producer = "compass.scala" +producerVersion = 1 +oracleProvider = "scala-meta-source-oracle" +oracleToolchain = "Scala CLI 1.9.1; Scala 3.7.3; scala.meta 4.13.10; ujson 4.1.0" +checkoutRoot = "/Volumes/Workspace/Github" +readOnly = true +minimumAcceptedRelationships = 2000 +minimumAcceptedPerCorpus = 400 +minimumAcceptedPerRelation = 100 + +[[repository]] +name = "scala3" +url = "https://github.com/scala/scala3.git" +commit = "b873e5a76ca51cd098684a1eb90e0946a33dbd39" +purpose = "Scala 3 compiler sources, givens, extensions, enums, and quoted syntax" +sourceGlobs = ["library/**/*.scala", "compiler/**/*.scala", "presentation-compiler/**/*.scala", "scaladoc/**/*.scala", "repl/**/*.scala", "language-server/**/*.scala", "sbt-bridge/**/*.scala", "directives-parser/**/*.scala", "tasty/**/*.scala", "staging/**/*.scala"] +excludeGlobs = [] + +[[repository]] +name = "akka" +url = "https://github.com/akka/akka.git" +commit = "2b829515c4e33adc926323a6888a688c29c49c67" +purpose = "actors, protocols, implicits, typeclasses, and multi-module Scala APIs" +sourceGlobs = ["**/*.scala", "*.scala"] +excludeGlobs = ["project/**/*.scala"] + +[[repository]] +name = "playframework" +url = "https://github.com/playframework/playframework.git" +commit = "123a45f849996f656e5138816824ff35b3699acb" +purpose = "Play routing, actions, Scala handlers, Java interop boundaries, and tests" +sourceGlobs = ["**/*.scala", "*.scala"] +excludeGlobs = ["documentation/**/*.scala", "project/**/*.scala"] diff --git a/tests/qualification/swift-universal-baseline.json b/tests/qualification/swift-universal-baseline.json new file mode 100644 index 00000000..517548f3 --- /dev/null +++ b/tests/qualification/swift-universal-baseline.json @@ -0,0 +1 @@ +{"alternateCheckout":{"diagnostics":2,"edgeCount":20,"evidenceDigest":"c3d8219da5e4c553a0a9134470ef25e35cc2c5136095362c24f9ef45a300c567","graphSha256":"3c9dbbface68aeca3b1f467b58f9f5080f1fd6fedaa9106ec732202b77be3421","identityCollisions":0,"nodeCount":19,"omittedFacts":0,"relationCounts":{"contains":14,"implements":1,"imports":1,"references":3,"routes_to":1},"rssBytes":53067776,"seconds":0.14823833398986608},"baselineKind":"established-pre-cutover","baselineStatus":"established direct extractor at commit 88abe4c071a19ec03b3bca132656830a02a47907","cold":{"first":{"diagnostics":2,"edgeCount":20,"evidenceDigest":"8ddfe12e907a37300c36f9e114729e4587384253370e7bf2665d1f1044b051be","graphSha256":"3c9dbbface68aeca3b1f467b58f9f5080f1fd6fedaa9106ec732202b77be3421","identityCollisions":0,"nodeCount":19,"omittedFacts":0,"relationCounts":{"contains":14,"implements":1,"imports":1,"references":3,"routes_to":1},"rssBytes":48644096,"seconds":0.3924431250197813},"medianSeconds":0.26891212502960116,"second":{"diagnostics":2,"edgeCount":20,"evidenceDigest":"8ddfe12e907a37300c36f9e114729e4587384253370e7bf2665d1f1044b051be","graphSha256":"3c9dbbface68aeca3b1f467b58f9f5080f1fd6fedaa9106ec732202b77be3421","identityCollisions":0,"nodeCount":19,"omittedFacts":0,"relationCounts":{"contains":14,"implements":1,"imports":1,"references":3,"routes_to":1},"rssBytes":53067776,"seconds":0.14538112503942102}},"compassRevision":"88abe4c071a19ec03b3bca132656830a02a47907","editRestore":{"deterministic":true,"neutralGraphSha256":"e0975890a384743c82735ab65207ad8f39534c422e59a5546532afad66e30351","restoreGraphSha256":"3c9dbbface68aeca3b1f467b58f9f5080f1fd6fedaa9106ec732202b77be3421","semanticGraphSha256":"962d0e5eeee40e7f06357c49f880b15ecd8f788e95b7bbe93dfce7e5c398c64b"},"factNeutral":{"diagnostics":2,"edgeCount":20,"evidenceDigest":"af3428109d0a25399b126c6b6c4d4951211de8cafd5b9c201730cd98c527bff2","graphSha256":"e0975890a384743c82735ab65207ad8f39534c422e59a5546532afad66e30351","identityCollisions":0,"nodeCount":19,"omittedFacts":0,"relationCounts":{"contains":14,"implements":1,"imports":1,"references":3,"routes_to":1},"rssBytes":50954240,"seconds":0.15298149990849197},"forced":{"diagnostics":2,"edgeCount":20,"evidenceDigest":"8ddfe12e907a37300c36f9e114729e4587384253370e7bf2665d1f1044b051be","graphSha256":"3c9dbbface68aeca3b1f467b58f9f5080f1fd6fedaa9106ec732202b77be3421","identityCollisions":0,"nodeCount":19,"omittedFacts":0,"relationCounts":{"contains":14,"implements":1,"imports":1,"references":3,"routes_to":1},"rssBytes":53067776,"seconds":0.13993291603401303},"language":"swift","performanceGates":{"coldMultiplier":1.1,"factNeutralAdditiveSeconds":0.25,"peakRssAdditiveBytes":33554432,"peakRssMultiplier":1.15,"warmMultiplier":1.15},"productionRoute":"compass.swift/1","restore":{"diagnostics":2,"edgeCount":20,"evidenceDigest":"8ddfe12e907a37300c36f9e114729e4587384253370e7bf2665d1f1044b051be","graphSha256":"3c9dbbface68aeca3b1f467b58f9f5080f1fd6fedaa9106ec732202b77be3421","identityCollisions":0,"nodeCount":19,"omittedFacts":0,"relationCounts":{"contains":14,"implements":1,"imports":1,"references":3,"routes_to":1},"rssBytes":53067776,"seconds":0.18872612493578345},"root":"tests/qualification/language-wave/swift","schema":"compass.universal-language-baseline/1","semanticEdit":{"diagnostics":2,"edgeCount":21,"evidenceDigest":"1df76b4d9a523dff50488ae79bd6fdf36c4d6b943dc033e98c0a06ad49404d32","graphSha256":"962d0e5eeee40e7f06357c49f880b15ecd8f788e95b7bbe93dfce7e5c398c64b","identityCollisions":0,"nodeCount":20,"omittedFacts":0,"relationCounts":{"contains":15,"implements":1,"imports":1,"references":3,"routes_to":1},"rssBytes":53067776,"seconds":0.19104591698851436},"sourceFile":"Module.swift","warm":{"graphSha256":"3c9dbbface68aeca3b1f467b58f9f5080f1fd6fedaa9106ec732202b77be3421","medianSeconds":0.04571889596991241,"samples":[{"diagnostics":2,"edgeCount":20,"evidenceDigest":"8ddfe12e907a37300c36f9e114729e4587384253370e7bf2665d1f1044b051be","graphSha256":"3c9dbbface68aeca3b1f467b58f9f5080f1fd6fedaa9106ec732202b77be3421","identityCollisions":0,"nodeCount":19,"omittedFacts":0,"relationCounts":{"contains":14,"implements":1,"imports":1,"references":3,"routes_to":1},"rssBytes":48644096,"seconds":0.045254416996613145},{"diagnostics":2,"edgeCount":20,"evidenceDigest":"8ddfe12e907a37300c36f9e114729e4587384253370e7bf2665d1f1044b051be","graphSha256":"3c9dbbface68aeca3b1f467b58f9f5080f1fd6fedaa9106ec732202b77be3421","identityCollisions":0,"nodeCount":19,"omittedFacts":0,"relationCounts":{"contains":14,"implements":1,"imports":1,"references":3,"routes_to":1},"rssBytes":48644096,"seconds":0.046183374943211675}]}} diff --git a/tests/qualification/swift-universal-repositories.toml b/tests/qualification/swift-universal-repositories.toml new file mode 100644 index 00000000..09341ad6 --- /dev/null +++ b/tests/qualification/swift-universal-repositories.toml @@ -0,0 +1,35 @@ +schema = "compass.swift-universal-qualification/1" +language = "swift" +producer = "compass.swift" +producerVersion = 1 +oracleProvider = "swift-syntax-source-oracle" +oracleToolchain = "swift 6.3.3; SwiftSyntax 603.0.0" +checkoutRoot = "/Volumes/Workspace/Github" +readOnly = true +minimumAcceptedRelationships = 2000 +minimumAcceptedPerCorpus = 400 +minimumAcceptedPerRelation = 100 + +[[repository]] +name = "swift-nio" +url = "https://github.com/apple/swift-nio.git" +commit = "385c1a7d48ebe018bcde205f8bd4506403dc889a" +purpose = "large systems library, protocols, nested types, async calls, and extensions" +sourceGlobs = ["Sources/**/*.swift", "Tests/**/*.swift"] +excludeGlobs = [] + +[[repository]] +name = "vapor" +url = "https://github.com/vapor/vapor.git" +commit = "13efaeefd52a965009fc6a69a96ba66e7f20402e" +purpose = "framework modules, route handlers, builders, and application extensions" +sourceGlobs = ["Sources/**/*.swift", "Tests/**/*.swift"] +excludeGlobs = [] + +[[repository]] +name = "swift-collections" +url = "https://github.com/apple/swift-collections.git" +commit = "f425dffd5bf70fe2a880c5eb6568bbdcade9f4f5" +purpose = "value types, protocols, generic constraints, and companion-style APIs" +sourceGlobs = ["Sources/**/*.swift", "Tests/**/*.swift"] +excludeGlobs = [] From ccdecee91d29f011d0dba68700575582a94907f7 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sat, 22 Aug 2026 08:20:41 -0700 Subject: [PATCH 2/4] refactor(languages): split extended evidence by language --- .../compass-languages/src/evidence/build.rs | 25 +- .../{extended.rs => extended/common.rs} | 357 +++++------------- .../src/evidence/extended/dart.rs | 35 ++ .../src/evidence/extended/groovy.rs | 243 ++++++++++++ .../src/evidence/extended/mod.rs | 54 +++ .../src/evidence/extended/scala.rs | 34 ++ .../src/evidence/extended/swift.rs | 28 ++ 7 files changed, 496 insertions(+), 280 deletions(-) rename crates/compass-languages/src/evidence/{extended.rs => extended/common.rs} (78%) create mode 100644 crates/compass-languages/src/evidence/extended/dart.rs create mode 100644 crates/compass-languages/src/evidence/extended/groovy.rs create mode 100644 crates/compass-languages/src/evidence/extended/mod.rs create mode 100644 crates/compass-languages/src/evidence/extended/scala.rs create mode 100644 crates/compass-languages/src/evidence/extended/swift.rs diff --git a/crates/compass-languages/src/evidence/build.rs b/crates/compass-languages/src/evidence/build.rs index 9c2a9cf3..19fa2b85 100644 --- a/crates/compass-languages/src/evidence/build.rs +++ b/crates/compass-languages/src/evidence/build.rs @@ -846,17 +846,20 @@ pub(crate) fn extract_tree_evidence( pipeline.producer.language, ); } - if matches!( - pipeline.producer.language, - "dart" | "groovy" | "scala" | "swift" - ) { - return super::extended::emit_tree_evidence( - path, - source_file, - source, - root, - pipeline.producer.language, - ); + match pipeline.producer.language { + "dart" => { + return super::extended::emit_dart_tree_evidence(path, source_file, source, root); + } + "groovy" => { + return super::extended::emit_groovy_tree_evidence(path, source_file, source, root); + } + "scala" => { + return super::extended::emit_scala_tree_evidence(path, source_file, source, root); + } + "swift" => { + return super::extended::emit_swift_tree_evidence(path, source_file, source, root); + } + _ => {} } let mut state = DirectEvidenceState::new(path, source_file, source, root, pipeline); state.add_file(root)?; diff --git a/crates/compass-languages/src/evidence/extended.rs b/crates/compass-languages/src/evidence/extended/common.rs similarity index 78% rename from crates/compass-languages/src/evidence/extended.rs rename to crates/compass-languages/src/evidence/extended/common.rs index 24bb1203..0ddc64dd 100644 --- a/crates/compass-languages/src/evidence/extended.rs +++ b/crates/compass-languages/src/evidence/extended/common.rs @@ -8,30 +8,67 @@ //! language. use std::collections::{BTreeMap, BTreeSet}; +use std::marker::PhantomData; use std::path::Path; use tree_sitter::Node; -use super::build::{EvidenceBuilder, range_for_byte_span, range_for_file, range_for_node}; -use super::model::{ +use super::super::build::{EvidenceBuilder, range_for_byte_span, range_for_file, range_for_node}; +use super::super::model::{ BindingKind, CandidateRelation, EvidenceRange, LanguageCapability, ResolutionConstraint, SemanticEvidenceBatch, SemanticRole, SymbolNamespace, }; -use super::validate::{EvidenceError, EvidenceErrorCode, EvidenceLimits}; +use super::super::validate::{EvidenceError, EvidenceErrorCode, EvidenceLimits}; use crate::{UniversalEvidenceRegistry, file_stem, make_id}; const MAX_TRAVERSAL_DEPTH: usize = 256; const MAX_TEXT_BYTES: usize = 4_096; +/// Language-specific policy for the shared AST traversal. +/// +/// The traversal owns the evidence contract and bounded bookkeeping; each +/// language module supplies only its syntax profile and any language-specific +/// source supplement. This keeps the production entry points separate without +/// duplicating validation and relationship machinery. +pub(super) trait LanguageProfile: Sized { + const LANGUAGE: &'static str; + + fn package_name(_source: &[u8]) -> Option { + None + } + + fn declaration_kind(kind: &str) -> Option<&'static str> { + shared_declaration_kind(kind) + } + + fn declaration_lookup_name(name: &str) -> String { + name.to_owned() + } + + fn emits_module_declarations() -> bool { + false + } + + fn has_source_supplement(_declaration_count: usize) -> bool { + false + } + + fn collect_source_supplement<'source>( + _state: &mut State<'source, Self>, + ) -> Result<(), EvidenceError> { + Ok(()) + } +} + #[derive(Clone, Debug)] -struct Decl { - id: String, - name: String, - qualified: String, - kind: String, - body_scope_id: String, - start: usize, - end: usize, +pub(super) struct Decl { + pub(super) id: String, + pub(super) name: String, + pub(super) qualified: String, + pub(super) kind: String, + pub(super) body_scope_id: String, + pub(super) start: usize, + pub(super) end: usize, } #[derive(Clone, Debug)] @@ -40,15 +77,14 @@ struct Import { target: String, } -struct State<'source> { - language: &'static str, - source: &'source [u8], - source_file: &'source str, +pub(super) struct State<'source, P: LanguageProfile> { + pub(super) source: &'source [u8], + pub(super) source_file: &'source str, builder: EvidenceBuilder, file_id: String, - file_scope_id: String, + pub(super) file_scope_id: String, namespace: String, - declarations: Vec, + pub(super) declarations: Vec, by_node: BTreeMap, by_terminal: BTreeMap>, by_qualified: BTreeMap>, @@ -57,25 +93,28 @@ struct State<'source> { module_targets: BTreeSet, emitted: BTreeSet<(SemanticRole, usize, usize, String)>, occurrence_ids: BTreeMap<(SemanticRole, usize, usize, String), String>, + _profile: PhantomData

, } -pub(super) fn emit_tree_evidence( +pub(super) fn emit_tree_evidence( path: &Path, source_file: &str, source: &[u8], root: Node<'_>, - language: &'static str, ) -> Result { - let pipeline = UniversalEvidenceRegistry::pipeline(language).ok_or_else(|| { + let pipeline = UniversalEvidenceRegistry::pipeline(P::LANGUAGE).ok_or_else(|| { EvidenceError::new( EvidenceErrorCode::InvalidPipeline, - format!("{language} universal evidence pipeline is not registered"), + format!( + "{} universal evidence pipeline is not registered", + P::LANGUAGE + ), ) })?; let file_range = range_for_file(source_file, source); let mut builder = EvidenceBuilder::new( pipeline, - format!("compass.languages.{language}.universal"), + format!("compass.languages.{}.universal", P::LANGUAGE), source_file, EvidenceLimits::default(), ); @@ -99,9 +138,8 @@ pub(super) fn emit_tree_evidence( return builder.finish(); } - let namespace = package_name(language, source).unwrap_or_default(); + let namespace = P::package_name(source).unwrap_or_default(); let mut state = State { - language, source, source_file, builder, @@ -117,6 +155,7 @@ pub(super) fn emit_tree_evidence( module_targets: BTreeSet::new(), emitted: BTreeSet::new(), occurrence_ids: BTreeMap::new(), + _profile: PhantomData, }; if std::str::from_utf8(source).is_err() { state.builder.diagnose( @@ -128,8 +167,8 @@ pub(super) fn emit_tree_evidence( } let root_scope = state.add_namespace(root)?; state.collect_declarations(root, None, &root_scope, 0)?; - if language == "groovy" && state.declarations.len() <= 1 { - state.collect_groovy_source()?; + if P::has_source_supplement(state.declarations.len()) { + P::collect_source_supplement(&mut state)?; } state.collect_imports(root, 0)?; state.collect_semantics(root, 0)?; @@ -144,12 +183,12 @@ pub(super) fn emit_tree_evidence( state.builder.finish() } -impl<'source> State<'source> { +impl<'source, P: LanguageProfile> State<'source, P> { fn add_namespace(&mut self, root: Node<'_>) -> Result { if self.namespace.is_empty() || !self.supports(LanguageCapability::Namespaces) { return Ok(self.file_scope_id.clone()); } - let graph_id = make_id(&[self.language, "namespace", &self.namespace]); + let graph_id = make_id(&[P::LANGUAGE, "namespace", &self.namespace]); let declaration_id = self.builder.declare_with_namespace( "namespace", &graph_id, @@ -174,7 +213,7 @@ impl<'source> State<'source> { &self.namespace, ResolutionConstraint { exact_target_declaration_id: Some(declaration_id), - exact_language: Some(self.language.to_owned()), + exact_language: Some(P::LANGUAGE.to_owned()), ..ResolutionConstraint::default() }, )?; @@ -199,16 +238,11 @@ impl<'source> State<'source> { .map(|decl| decl.qualified.clone()) .unwrap_or_else(|| self.namespace.clone()); - if let Some(kind) = declaration_kind(self.language, node.kind()) + if let Some(kind) = P::declaration_kind(node.kind()) && let Some(name_node) = declaration_name(node) { let name = self.text(name_node); - let lookup_name = if self.language == "dart" { - name.split_once('(') - .map_or_else(|| name.clone(), |(base, _)| base.trim().to_owned()) - } else { - name.clone() - }; + let lookup_name = P::declaration_lookup_name(&name); // The Dart grammar exposes a method signature as the declaration // name child (for example ``clearLibraryContext()``). Calls and // Analyzer source evidence carry the base name only; retaining @@ -230,7 +264,7 @@ impl<'source> State<'source> { { let graph_id = make_id(&[ self.source_file, - self.language, + P::LANGUAGE, kind, &qualified, &node.start_byte().to_string(), @@ -272,7 +306,7 @@ impl<'source> State<'source> { &name, ResolutionConstraint { exact_target_declaration_id: Some(declaration_id.clone()), - exact_language: Some(self.language.to_owned()), + exact_language: Some(P::LANGUAGE.to_owned()), ..ResolutionConstraint::default() }, )?; @@ -310,109 +344,8 @@ impl<'source> State<'source> { Ok(()) } - /// The pinned Groovy grammar intentionally exposes each top-level form as - /// a bounded `command` node. Keep Groovy on the universal evidence route - /// by extracting the declaration/call spans from that command text rather - /// than reintroducing a raw graph fallback. The scanner is line- and - /// brace-bounded, preserves exact byte ranges, and remains fail-closed for - /// ambiguous method spellings. - fn collect_groovy_source(&mut self) -> Result<(), EvidenceError> { - // A lossy conversion can expand one invalid source byte into multiple - // replacement bytes. Do not publish scanner offsets derived from it; - // tree-sitter evidence above remains available and this omission is - // reported as explicit incomplete input. - let Ok(text) = std::str::from_utf8(self.source) else { - return Ok(()); - }; - let mut depth = 0_i32; - let mut classes: Vec<(usize, usize, i32)> = Vec::new(); - let mut method: Option<(usize, usize)> = None; - let mut line_start = 0_usize; - for line in text.split_inclusive('\n') { - let line_without_newline = line.trim_end_matches(['\r', '\n']); - let line_end = line_start.saturating_add(line_without_newline.len()); - let trimmed = line_without_newline.trim(); - while classes.last().is_some_and(|(_, end, _)| line_start >= *end) { - classes.pop(); - } - if method.is_some_and(|(_, end)| line_start >= end) { - method = None; - } - - let class_decl = groovy_type_declaration(trimmed); - if let Some((kind, name, name_offset)) = class_decl { - let parent = classes.last().map(|(index, _, _)| *index); - let parent_scope = parent - .and_then(|index| self.declarations.get(index)) - .map_or(self.file_scope_id.as_str(), |decl| { - decl.body_scope_id.as_str() - }) - .to_owned(); - let body_end = matching_brace_end(self.source, line_start, line_end); - let end = body_end.max(line_end); - if let Some(index) = self.add_source_declaration( - kind, - &name, - line_start, - end, - line_start.saturating_add(name_offset), - line_start - .saturating_add(name_offset) - .saturating_add(name.len()), - parent, - &parent_scope, - )? { - classes.push((index, end, depth)); - self.emit_groovy_calls(line_start, line_end, index)?; - } - depth = depth.saturating_add(brace_delta(trimmed)); - line_start = line_start.saturating_add(line.len()); - continue; - } - - let active_class = classes.last().map(|(index, _, _)| *index); - if let Some(class_index) = active_class - && let Some((name, constructor, name_offset)) = groovy_method_declaration(trimmed) - { - let parent_scope = self - .declarations - .get(class_index) - .map_or(self.file_scope_id.as_str(), |decl| { - decl.body_scope_id.as_str() - }) - .to_owned(); - let body_end = matching_brace_end(self.source, line_start, line_end); - let end = body_end.max(line_end); - let kind = if constructor { "constructor" } else { "method" }; - if let Some(index) = self.add_source_declaration( - kind, - &name, - line_start, - end, - line_start.saturating_add(name_offset), - line_start - .saturating_add(name_offset) - .saturating_add(name.len()), - Some(class_index), - &parent_scope, - )? { - method = Some((index, end)); - self.emit_groovy_calls(line_start, line_end, index)?; - } - } else if let Some((method_index, method_end)) = method - && line_start < method_end - { - self.emit_groovy_calls(line_start, line_end, method_index)?; - } - - depth = depth.saturating_add(brace_delta(trimmed)); - line_start = line_start.saturating_add(line.len()); - } - Ok(()) - } - #[allow(clippy::too_many_arguments)] - fn add_source_declaration( + pub(super) fn add_source_declaration( &mut self, kind: &str, name: &str, @@ -438,7 +371,7 @@ impl<'source> State<'source> { let qualified = join_name(prefix, name); let graph_id = make_id(&[ self.source_file, - self.language, + P::LANGUAGE, kind, &qualified, &start.to_string(), @@ -471,7 +404,7 @@ impl<'source> State<'source> { name, ResolutionConstraint { exact_target_declaration_id: Some(declaration_id.clone()), - exact_language: Some(self.language.to_owned()), + exact_language: Some(P::LANGUAGE.to_owned()), ..ResolutionConstraint::default() }, )?; @@ -494,7 +427,7 @@ impl<'source> State<'source> { Ok(Some(index)) } - fn emit_groovy_calls( + pub(super) fn emit_source_calls( &mut self, line_start: usize, line_end: usize, @@ -577,8 +510,8 @@ impl<'source> State<'source> { // Vapor/framework identity on the evidence route while the // binding itself remains an exact, language-constrained // import candidate. - if self.language == "swift" && self.module_targets.insert(target.clone()) { - let module_id = make_id(&[self.source_file, self.language, "module", &target]); + if P::emits_module_declarations() && self.module_targets.insert(target.clone()) { + let module_id = make_id(&[self.source_file, P::LANGUAGE, "module", &target]); let module_declaration = self.builder.declare( "module", &module_id, @@ -596,7 +529,7 @@ impl<'source> State<'source> { &spelling, ResolutionConstraint { exact_target_declaration_id: Some(module_declaration), - exact_language: Some(self.language.to_owned()), + exact_language: Some(P::LANGUAGE.to_owned()), ..ResolutionConstraint::default() }, )?; @@ -643,7 +576,7 @@ impl<'source> State<'source> { Some(&binding_id), &spelling, ResolutionConstraint { - exact_language: Some(self.language.to_owned()), + exact_language: Some(P::LANGUAGE.to_owned()), qualified_name: Some(target.clone()), allow_external: true, ..ResolutionConstraint::default() @@ -775,7 +708,7 @@ impl<'source> State<'source> { spelling, ResolutionConstraint { exact_target_declaration_id: exact.map(|index| self.declarations[index].id.clone()), - exact_language: Some(self.language.to_owned()), + exact_language: Some(P::LANGUAGE.to_owned()), qualified_name, allowed_target_kinds: allowed, allow_external: exact.is_none(), @@ -834,7 +767,7 @@ impl<'source> State<'source> { &spelling, ResolutionConstraint { exact_target_declaration_id: exact.map(|index| self.declarations[index].id.clone()), - exact_language: Some(self.language.to_owned()), + exact_language: Some(P::LANGUAGE.to_owned()), qualified_name: qualifier .as_ref() .map(|prefix| format!("{prefix}.{spelling}")), @@ -883,7 +816,7 @@ impl<'source> State<'source> { None, spelling, ResolutionConstraint { - exact_language: Some(self.language.to_owned()), + exact_language: Some(P::LANGUAGE.to_owned()), qualified_name: Some(format!("{qualifier}.{spelling}")), allowed_target_kinds: vec![ "field".to_owned(), @@ -926,7 +859,7 @@ impl<'source> State<'source> { None, spelling, ResolutionConstraint { - exact_language: Some(self.language.to_owned()), + exact_language: Some(P::LANGUAGE.to_owned()), allow_external: true, ..ResolutionConstraint::default() }, @@ -997,7 +930,7 @@ impl<'source> State<'source> { } fn supports(&self, capability: LanguageCapability) -> bool { - UniversalEvidenceRegistry::pipeline(self.language) + UniversalEvidenceRegistry::pipeline(P::LANGUAGE) .is_some_and(|pipeline| pipeline.producer.capabilities.contains(&capability)) } @@ -1036,7 +969,7 @@ impl<'source> State<'source> { } } -fn declaration_kind(language: &str, kind: &str) -> Option<&'static str> { +pub(super) fn shared_declaration_kind(kind: &str) -> Option<&'static str> { if matches!( kind, "source_file" @@ -1109,12 +1042,6 @@ fn declaration_kind(language: &str, kind: &str) -> Option<&'static str> { if lower.contains("property") || lower.contains("field") { return Some("field"); } - if language == "scala" && (lower.contains("val_") || lower.contains("var_")) { - return Some("field"); - } - if language == "dart" && lower == "variable_declaration" { - return Some("field"); - } None } @@ -1138,7 +1065,7 @@ fn declaration_name(node: Node<'_>) -> Option> { }) } -fn valid_name(value: &str) -> bool { +pub(super) fn valid_name(value: &str) -> bool { let value = value.trim(); !value.is_empty() && value.len() <= 512 @@ -1180,10 +1107,7 @@ fn join_name(prefix: &str, name: &str) -> String { } } -fn package_name(language: &str, source: &[u8]) -> Option { - if language == "swift" || language == "dart" { - return None; - } +pub(super) fn package_name_from_source(source: &[u8]) -> Option { let text = std::str::from_utf8(source).ok()?; for line in text.lines().take(128) { let line = line.trim(); @@ -1334,111 +1258,6 @@ fn is_decorator_node(kind: &str) -> bool { lower.contains("annotation") || lower.contains("decorator") || lower == "attribute_list" } -fn groovy_type_declaration(line: &str) -> Option<(&'static str, String, usize)> { - let tokens = line - .split_whitespace() - .map(|token| token.trim_matches(['@', '{', ';', ','])) - .collect::>(); - for (index, token) in tokens.iter().enumerate() { - let kind = match *token { - "class" => "class", - "interface" => "interface", - "trait" => "trait", - "enum" => "enum", - _ => continue, - }; - let name = tokens - .get(index.saturating_add(1))? - .trim_matches(['{', ';']); - if !valid_name(name) { - return None; - } - let offset = line.find(name)?; - return Some((kind, name.to_owned(), offset)); - } - None -} - -fn groovy_method_declaration(line: &str) -> Option<(String, bool, usize)> { - let open = line.find('(')?; - let before = line.get(..open)?.trim_end(); - let name_end = before.len(); - let name_start = before - .char_indices() - .rev() - .find(|(_, character)| !character.is_ascii_alphanumeric() && *character != '_') - .map_or(0, |(index, _)| index.saturating_add(1)); - let name = before.get(name_start..name_end)?.trim(); - if !valid_name(name) - || matches!( - name, - "if" | "for" | "while" | "switch" | "catch" | "try" | "return" | "assert" - ) - { - return None; - } - let constructor = name.chars().next().is_some_and(char::is_uppercase); - let has_return_shape = before[..name_start] - .split_whitespace() - .any(|token| token == "def" || !token.is_empty()); - (has_return_shape || constructor).then(|| (name.to_owned(), constructor, name_start)) -} - -fn brace_delta(line: &str) -> i32 { - let mut delta = 0_i32; - let mut quote = None; - for character in line.chars() { - if let Some(active) = quote { - if character == active { - quote = None; - } - continue; - } - if matches!(character, '\'' | '"') { - quote = Some(character); - } else if character == '{' { - delta = delta.saturating_add(1); - } else if character == '}' { - delta = delta.saturating_sub(1); - } - } - delta -} - -fn matching_brace_end(source: &[u8], line_start: usize, line_end: usize) -> usize { - let Some(open) = source - .get(line_start..line_end) - .and_then(|line| line.iter().position(|byte| *byte == b'{')) - .map(|offset| line_start.saturating_add(offset)) - else { - return line_end; - }; - let mut depth = 0_i32; - let mut quote = None; - for (offset, byte) in source.iter().enumerate().skip(open) { - let character = char::from(*byte); - if let Some(active) = quote { - if character == active { - quote = None; - } - continue; - } - if matches!(character, '\'' | '"') { - quote = Some(character); - continue; - } - if character == '{' { - depth = depth.saturating_add(1); - } else if character == '}' { - depth = depth.saturating_sub(1); - if depth == 0 { - return offset.saturating_add(1); - } - } - } - source.len() -} - fn is_identifier_start(byte: u8) -> bool { byte.is_ascii_alphabetic() || byte == b'_' } diff --git a/crates/compass-languages/src/evidence/extended/dart.rs b/crates/compass-languages/src/evidence/extended/dart.rs new file mode 100644 index 00000000..ceffa950 --- /dev/null +++ b/crates/compass-languages/src/evidence/extended/dart.rs @@ -0,0 +1,35 @@ +//! Universal evidence profile for Dart. + +use std::path::Path; + +use tree_sitter::Node; + +use super::super::model::SemanticEvidenceBatch; +use super::super::validate::EvidenceError; +use super::common::{self, LanguageProfile}; + +struct Dart; + +impl LanguageProfile for Dart { + const LANGUAGE: &'static str = "dart"; + + fn declaration_kind(kind: &str) -> Option<&'static str> { + let lower = kind.to_ascii_lowercase(); + common::shared_declaration_kind(kind) + .or_else(|| (lower == "variable_declaration").then_some("field")) + } + + fn declaration_lookup_name(name: &str) -> String { + name.split_once('(') + .map_or_else(|| name.to_owned(), |(base, _)| base.trim().to_owned()) + } +} + +pub(super) fn emit_tree_evidence( + path: &Path, + source_file: &str, + source: &[u8], + root: Node<'_>, +) -> Result { + common::emit_tree_evidence::(path, source_file, source, root) +} diff --git a/crates/compass-languages/src/evidence/extended/groovy.rs b/crates/compass-languages/src/evidence/extended/groovy.rs new file mode 100644 index 00000000..4fd0a493 --- /dev/null +++ b/crates/compass-languages/src/evidence/extended/groovy.rs @@ -0,0 +1,243 @@ +//! Universal evidence profile for Groovy and Gradle scripts. + +use std::path::Path; + +use tree_sitter::Node; + +use super::super::model::SemanticEvidenceBatch; +use super::super::validate::EvidenceError; +use super::common::{self, LanguageProfile, State}; + +struct Groovy; + +impl LanguageProfile for Groovy { + const LANGUAGE: &'static str = "groovy"; + + fn package_name(source: &[u8]) -> Option { + common::package_name_from_source(source) + } + + fn has_source_supplement(declaration_count: usize) -> bool { + declaration_count <= 1 + } + + fn collect_source_supplement<'source>( + state: &mut State<'source, Self>, + ) -> Result<(), EvidenceError> { + collect_groovy_source(state) + } +} + +pub(super) fn emit_tree_evidence( + path: &Path, + source_file: &str, + source: &[u8], + root: Node<'_>, +) -> Result { + common::emit_tree_evidence::(path, source_file, source, root) +} + +/// The pinned Groovy grammar intentionally exposes each top-level form as a +/// bounded `command` node. Keep Groovy on the universal evidence route by +/// extracting declaration/call spans from that command text rather than +/// reintroducing a raw graph fallback. The scanner is line- and +/// brace-bounded, preserves exact byte ranges, and remains fail-closed for +/// ambiguous method spellings. +fn collect_groovy_source<'source>(state: &mut State<'source, Groovy>) -> Result<(), EvidenceError> { + // A lossy conversion can expand one invalid source byte into multiple + // replacement bytes. Do not publish scanner offsets derived from it; + // tree-sitter evidence above remains available and this omission is + // reported as explicit incomplete input. + let Ok(text) = std::str::from_utf8(state.source) else { + return Ok(()); + }; + let mut depth = 0_i32; + let mut classes: Vec<(usize, usize, i32)> = Vec::new(); + let mut method: Option<(usize, usize)> = None; + let mut line_start = 0_usize; + for line in text.split_inclusive('\n') { + let line_without_newline = line.trim_end_matches(['\r', '\n']); + let line_end = line_start.saturating_add(line_without_newline.len()); + let trimmed = line_without_newline.trim(); + while classes.last().is_some_and(|(_, end, _)| line_start >= *end) { + classes.pop(); + } + if method.is_some_and(|(_, end)| line_start >= end) { + method = None; + } + + if let Some((kind, name, name_offset)) = groovy_type_declaration(trimmed) { + let parent = classes.last().map(|(index, _, _)| *index); + let parent_scope = parent + .and_then(|index| state.declarations.get(index)) + .map_or(state.file_scope_id.as_str(), |decl| { + decl.body_scope_id.as_str() + }) + .to_owned(); + let body_end = matching_brace_end(state.source, line_start, line_end); + let end = body_end.max(line_end); + if let Some(index) = state.add_source_declaration( + kind, + &name, + line_start, + end, + line_start.saturating_add(name_offset), + line_start + .saturating_add(name_offset) + .saturating_add(name.len()), + parent, + &parent_scope, + )? { + classes.push((index, end, depth)); + state.emit_source_calls(line_start, line_end, index)?; + } + depth = depth.saturating_add(brace_delta(trimmed)); + line_start = line_start.saturating_add(line.len()); + continue; + } + + let active_class = classes.last().map(|(index, _, _)| *index); + if let Some(class_index) = active_class + && let Some((name, constructor, name_offset)) = groovy_method_declaration(trimmed) + { + let parent_scope = state + .declarations + .get(class_index) + .map_or(state.file_scope_id.as_str(), |decl| { + decl.body_scope_id.as_str() + }) + .to_owned(); + let body_end = matching_brace_end(state.source, line_start, line_end); + let end = body_end.max(line_end); + let kind = if constructor { "constructor" } else { "method" }; + if let Some(index) = state.add_source_declaration( + kind, + &name, + line_start, + end, + line_start.saturating_add(name_offset), + line_start + .saturating_add(name_offset) + .saturating_add(name.len()), + Some(class_index), + &parent_scope, + )? { + method = Some((index, end)); + state.emit_source_calls(line_start, line_end, index)?; + } + } else if let Some((method_index, method_end)) = method + && line_start < method_end + { + state.emit_source_calls(line_start, line_end, method_index)?; + } + + depth = depth.saturating_add(brace_delta(trimmed)); + line_start = line_start.saturating_add(line.len()); + } + Ok(()) +} + +fn groovy_type_declaration(line: &str) -> Option<(&'static str, String, usize)> { + let tokens = line + .split_whitespace() + .map(|token| token.trim_matches(['@', '{', ';', ','])) + .collect::>(); + for (index, token) in tokens.iter().enumerate() { + let kind = match *token { + "class" => "class", + "interface" => "interface", + "trait" => "trait", + "enum" => "enum", + _ => continue, + }; + let name = tokens + .get(index.saturating_add(1))? + .trim_matches(['{', ';']); + if !common::valid_name(name) { + return None; + } + let offset = line.find(name)?; + return Some((kind, name.to_owned(), offset)); + } + None +} + +fn groovy_method_declaration(line: &str) -> Option<(String, bool, usize)> { + let open = line.find('(')?; + let before = line.get(..open)?.trim_end(); + let name_end = before.len(); + let name_start = before + .char_indices() + .rev() + .find(|(_, character)| !character.is_ascii_alphanumeric() && *character != '_') + .map_or(0, |(index, _)| index.saturating_add(1)); + let name = before.get(name_start..name_end)?.trim(); + if !common::valid_name(name) + || matches!( + name, + "if" | "for" | "while" | "switch" | "catch" | "try" | "return" | "assert" + ) + { + return None; + } + let constructor = name.chars().next().is_some_and(char::is_uppercase); + let has_return_shape = before[..name_start] + .split_whitespace() + .any(|token| token == "def" || !token.is_empty()); + (has_return_shape || constructor).then(|| (name.to_owned(), constructor, name_start)) +} + +fn brace_delta(line: &str) -> i32 { + let mut delta = 0_i32; + let mut quote = None; + for character in line.chars() { + if let Some(active) = quote { + if character == active { + quote = None; + } + continue; + } + if matches!(character, '\'' | '"') { + quote = Some(character); + } else if character == '{' { + delta = delta.saturating_add(1); + } else if character == '}' { + delta = delta.saturating_sub(1); + } + } + delta +} + +fn matching_brace_end(source: &[u8], line_start: usize, line_end: usize) -> usize { + let Some(open) = source + .get(line_start..line_end) + .and_then(|line| line.iter().position(|byte| *byte == b'{')) + .map(|offset| line_start.saturating_add(offset)) + else { + return line_end; + }; + let mut depth = 0_i32; + let mut quote = None; + for (offset, byte) in source.iter().enumerate().skip(open) { + let character = char::from(*byte); + if let Some(active) = quote { + if character == active { + quote = None; + } + continue; + } + if matches!(character, '\'' | '"') { + quote = Some(character); + continue; + } + if character == '{' { + depth = depth.saturating_add(1); + } else if character == '}' { + depth = depth.saturating_sub(1); + if depth == 0 { + return offset.saturating_add(1); + } + } + } + source.len() +} diff --git a/crates/compass-languages/src/evidence/extended/mod.rs b/crates/compass-languages/src/evidence/extended/mod.rs new file mode 100644 index 00000000..08de249a --- /dev/null +++ b/crates/compass-languages/src/evidence/extended/mod.rs @@ -0,0 +1,54 @@ +//! AST-first universal evidence entry points for the extended language wave. +//! +//! Shared traversal and validation live in [`common`]. Each language keeps a +//! separate profile so syntax-specific policy and source supplements stay +//! reviewable without duplicating the evidence contract. + +use std::path::Path; + +use tree_sitter::Node; + +use super::model::SemanticEvidenceBatch; +use super::validate::EvidenceError; + +mod common; +pub(super) mod dart; +pub(super) mod groovy; +pub(super) mod scala; +pub(super) mod swift; + +pub(super) fn emit_dart_tree_evidence( + path: &Path, + source_file: &str, + source: &[u8], + root: Node<'_>, +) -> Result { + dart::emit_tree_evidence(path, source_file, source, root) +} + +pub(super) fn emit_groovy_tree_evidence( + path: &Path, + source_file: &str, + source: &[u8], + root: Node<'_>, +) -> Result { + groovy::emit_tree_evidence(path, source_file, source, root) +} + +pub(super) fn emit_scala_tree_evidence( + path: &Path, + source_file: &str, + source: &[u8], + root: Node<'_>, +) -> Result { + scala::emit_tree_evidence(path, source_file, source, root) +} + +pub(super) fn emit_swift_tree_evidence( + path: &Path, + source_file: &str, + source: &[u8], + root: Node<'_>, +) -> Result { + swift::emit_tree_evidence(path, source_file, source, root) +} diff --git a/crates/compass-languages/src/evidence/extended/scala.rs b/crates/compass-languages/src/evidence/extended/scala.rs new file mode 100644 index 00000000..a8f29265 --- /dev/null +++ b/crates/compass-languages/src/evidence/extended/scala.rs @@ -0,0 +1,34 @@ +//! Universal evidence profile for Scala 2 and Scala 3. + +use std::path::Path; + +use tree_sitter::Node; + +use super::super::model::SemanticEvidenceBatch; +use super::super::validate::EvidenceError; +use super::common::{self, LanguageProfile}; + +struct Scala; + +impl LanguageProfile for Scala { + const LANGUAGE: &'static str = "scala"; + + fn package_name(source: &[u8]) -> Option { + common::package_name_from_source(source) + } + + fn declaration_kind(kind: &str) -> Option<&'static str> { + let lower = kind.to_ascii_lowercase(); + common::shared_declaration_kind(kind) + .or_else(|| (lower.contains("val_") || lower.contains("var_")).then_some("field")) + } +} + +pub(super) fn emit_tree_evidence( + path: &Path, + source_file: &str, + source: &[u8], + root: Node<'_>, +) -> Result { + common::emit_tree_evidence::(path, source_file, source, root) +} diff --git a/crates/compass-languages/src/evidence/extended/swift.rs b/crates/compass-languages/src/evidence/extended/swift.rs new file mode 100644 index 00000000..fd5aa8ac --- /dev/null +++ b/crates/compass-languages/src/evidence/extended/swift.rs @@ -0,0 +1,28 @@ +//! Universal evidence profile for Swift. + +use std::path::Path; + +use tree_sitter::Node; + +use super::super::model::SemanticEvidenceBatch; +use super::super::validate::EvidenceError; +use super::common::{self, LanguageProfile}; + +struct Swift; + +impl LanguageProfile for Swift { + const LANGUAGE: &'static str = "swift"; + + fn emits_module_declarations() -> bool { + true + } +} + +pub(super) fn emit_tree_evidence( + path: &Path, + source_file: &str, + source: &[u8], + root: Node<'_>, +) -> Result { + common::emit_tree_evidence::(path, source_file, source, root) +} From 4690b80a378d58880e96509191c3b474029122da Mon Sep 17 00:00:00 2001 From: forhappy Date: Sat, 22 Aug 2026 08:41:04 -0700 Subject: [PATCH 3/4] refactor(languages): make language wave producers direct modules --- .../compass-languages/src/evidence/build.rs | 8 +-- .../src/evidence/{extended => }/dart.rs | 10 ++-- .../src/evidence/extended/mod.rs | 54 ------------------- .../src/evidence/{extended => }/groovy.rs | 14 ++--- crates/compass-languages/src/evidence/mod.rs | 6 ++- .../src/evidence/{extended => }/scala.rs | 12 ++--- .../{extended/common.rs => shared.rs} | 8 +-- .../src/evidence/{extended => }/swift.rs | 8 +-- .../src/evidence_pipeline.rs | 6 +-- ...=> language_wave_universal_conformance.rs} | 6 +-- docs/implementation/universal-evidence.md | 2 +- 11 files changed, 42 insertions(+), 92 deletions(-) rename crates/compass-languages/src/evidence/{extended => }/dart.rs (74%) delete mode 100644 crates/compass-languages/src/evidence/extended/mod.rs rename crates/compass-languages/src/evidence/{extended => }/groovy.rs (96%) rename crates/compass-languages/src/evidence/{extended => }/scala.rs (68%) rename crates/compass-languages/src/evidence/{extended/common.rs => shared.rs} (99%) rename crates/compass-languages/src/evidence/{extended => }/swift.rs (68%) rename crates/compass-languages/tests/{extended_universal_conformance.rs => language_wave_universal_conformance.rs} (95%) diff --git a/crates/compass-languages/src/evidence/build.rs b/crates/compass-languages/src/evidence/build.rs index 19fa2b85..0cfc25e1 100644 --- a/crates/compass-languages/src/evidence/build.rs +++ b/crates/compass-languages/src/evidence/build.rs @@ -848,16 +848,16 @@ pub(crate) fn extract_tree_evidence( } match pipeline.producer.language { "dart" => { - return super::extended::emit_dart_tree_evidence(path, source_file, source, root); + return super::dart::emit_tree_evidence(path, source_file, source, root); } "groovy" => { - return super::extended::emit_groovy_tree_evidence(path, source_file, source, root); + return super::groovy::emit_tree_evidence(path, source_file, source, root); } "scala" => { - return super::extended::emit_scala_tree_evidence(path, source_file, source, root); + return super::scala::emit_tree_evidence(path, source_file, source, root); } "swift" => { - return super::extended::emit_swift_tree_evidence(path, source_file, source, root); + return super::swift::emit_tree_evidence(path, source_file, source, root); } _ => {} } diff --git a/crates/compass-languages/src/evidence/extended/dart.rs b/crates/compass-languages/src/evidence/dart.rs similarity index 74% rename from crates/compass-languages/src/evidence/extended/dart.rs rename to crates/compass-languages/src/evidence/dart.rs index ceffa950..88a862f0 100644 --- a/crates/compass-languages/src/evidence/extended/dart.rs +++ b/crates/compass-languages/src/evidence/dart.rs @@ -4,9 +4,9 @@ use std::path::Path; use tree_sitter::Node; -use super::super::model::SemanticEvidenceBatch; -use super::super::validate::EvidenceError; -use super::common::{self, LanguageProfile}; +use super::model::SemanticEvidenceBatch; +use super::shared::{self, LanguageProfile}; +use super::validate::EvidenceError; struct Dart; @@ -15,7 +15,7 @@ impl LanguageProfile for Dart { fn declaration_kind(kind: &str) -> Option<&'static str> { let lower = kind.to_ascii_lowercase(); - common::shared_declaration_kind(kind) + shared::shared_declaration_kind(kind) .or_else(|| (lower == "variable_declaration").then_some("field")) } @@ -31,5 +31,5 @@ pub(super) fn emit_tree_evidence( source: &[u8], root: Node<'_>, ) -> Result { - common::emit_tree_evidence::(path, source_file, source, root) + shared::emit_tree_evidence::(path, source_file, source, root) } diff --git a/crates/compass-languages/src/evidence/extended/mod.rs b/crates/compass-languages/src/evidence/extended/mod.rs deleted file mode 100644 index 08de249a..00000000 --- a/crates/compass-languages/src/evidence/extended/mod.rs +++ /dev/null @@ -1,54 +0,0 @@ -//! AST-first universal evidence entry points for the extended language wave. -//! -//! Shared traversal and validation live in [`common`]. Each language keeps a -//! separate profile so syntax-specific policy and source supplements stay -//! reviewable without duplicating the evidence contract. - -use std::path::Path; - -use tree_sitter::Node; - -use super::model::SemanticEvidenceBatch; -use super::validate::EvidenceError; - -mod common; -pub(super) mod dart; -pub(super) mod groovy; -pub(super) mod scala; -pub(super) mod swift; - -pub(super) fn emit_dart_tree_evidence( - path: &Path, - source_file: &str, - source: &[u8], - root: Node<'_>, -) -> Result { - dart::emit_tree_evidence(path, source_file, source, root) -} - -pub(super) fn emit_groovy_tree_evidence( - path: &Path, - source_file: &str, - source: &[u8], - root: Node<'_>, -) -> Result { - groovy::emit_tree_evidence(path, source_file, source, root) -} - -pub(super) fn emit_scala_tree_evidence( - path: &Path, - source_file: &str, - source: &[u8], - root: Node<'_>, -) -> Result { - scala::emit_tree_evidence(path, source_file, source, root) -} - -pub(super) fn emit_swift_tree_evidence( - path: &Path, - source_file: &str, - source: &[u8], - root: Node<'_>, -) -> Result { - swift::emit_tree_evidence(path, source_file, source, root) -} diff --git a/crates/compass-languages/src/evidence/extended/groovy.rs b/crates/compass-languages/src/evidence/groovy.rs similarity index 96% rename from crates/compass-languages/src/evidence/extended/groovy.rs rename to crates/compass-languages/src/evidence/groovy.rs index 4fd0a493..261e11c8 100644 --- a/crates/compass-languages/src/evidence/extended/groovy.rs +++ b/crates/compass-languages/src/evidence/groovy.rs @@ -4,9 +4,9 @@ use std::path::Path; use tree_sitter::Node; -use super::super::model::SemanticEvidenceBatch; -use super::super::validate::EvidenceError; -use super::common::{self, LanguageProfile, State}; +use super::model::SemanticEvidenceBatch; +use super::shared::{self, LanguageProfile, State}; +use super::validate::EvidenceError; struct Groovy; @@ -14,7 +14,7 @@ impl LanguageProfile for Groovy { const LANGUAGE: &'static str = "groovy"; fn package_name(source: &[u8]) -> Option { - common::package_name_from_source(source) + shared::package_name_from_source(source) } fn has_source_supplement(declaration_count: usize) -> bool { @@ -34,7 +34,7 @@ pub(super) fn emit_tree_evidence( source: &[u8], root: Node<'_>, ) -> Result { - common::emit_tree_evidence::(path, source_file, source, root) + shared::emit_tree_evidence::(path, source_file, source, root) } /// The pinned Groovy grammar intentionally exposes each top-level form as a @@ -153,7 +153,7 @@ fn groovy_type_declaration(line: &str) -> Option<(&'static str, String, usize)> let name = tokens .get(index.saturating_add(1))? .trim_matches(['{', ';']); - if !common::valid_name(name) { + if !shared::valid_name(name) { return None; } let offset = line.find(name)?; @@ -172,7 +172,7 @@ fn groovy_method_declaration(line: &str) -> Option<(String, bool, usize)> { .find(|(_, character)| !character.is_ascii_alphanumeric() && *character != '_') .map_or(0, |(index, _)| index.saturating_add(1)); let name = before.get(name_start..name_end)?.trim(); - if !common::valid_name(name) + if !shared::valid_name(name) || matches!( name, "if" | "for" | "while" | "switch" | "catch" | "try" | "return" | "assert" diff --git a/crates/compass-languages/src/evidence/mod.rs b/crates/compass-languages/src/evidence/mod.rs index 156c1335..40decb56 100644 --- a/crates/compass-languages/src/evidence/mod.rs +++ b/crates/compass-languages/src/evidence/mod.rs @@ -1,10 +1,14 @@ mod build; mod csharp; -mod extended; +mod dart; +mod groovy; mod kotlin; mod model; mod php; mod ruby; +mod scala; +mod shared; +mod swift; mod typescript; mod validate; diff --git a/crates/compass-languages/src/evidence/extended/scala.rs b/crates/compass-languages/src/evidence/scala.rs similarity index 68% rename from crates/compass-languages/src/evidence/extended/scala.rs rename to crates/compass-languages/src/evidence/scala.rs index a8f29265..9c0cf966 100644 --- a/crates/compass-languages/src/evidence/extended/scala.rs +++ b/crates/compass-languages/src/evidence/scala.rs @@ -4,9 +4,9 @@ use std::path::Path; use tree_sitter::Node; -use super::super::model::SemanticEvidenceBatch; -use super::super::validate::EvidenceError; -use super::common::{self, LanguageProfile}; +use super::model::SemanticEvidenceBatch; +use super::shared::{self, LanguageProfile}; +use super::validate::EvidenceError; struct Scala; @@ -14,12 +14,12 @@ impl LanguageProfile for Scala { const LANGUAGE: &'static str = "scala"; fn package_name(source: &[u8]) -> Option { - common::package_name_from_source(source) + shared::package_name_from_source(source) } fn declaration_kind(kind: &str) -> Option<&'static str> { let lower = kind.to_ascii_lowercase(); - common::shared_declaration_kind(kind) + shared::shared_declaration_kind(kind) .or_else(|| (lower.contains("val_") || lower.contains("var_")).then_some("field")) } } @@ -30,5 +30,5 @@ pub(super) fn emit_tree_evidence( source: &[u8], root: Node<'_>, ) -> Result { - common::emit_tree_evidence::(path, source_file, source, root) + shared::emit_tree_evidence::(path, source_file, source, root) } diff --git a/crates/compass-languages/src/evidence/extended/common.rs b/crates/compass-languages/src/evidence/shared.rs similarity index 99% rename from crates/compass-languages/src/evidence/extended/common.rs rename to crates/compass-languages/src/evidence/shared.rs index 0ddc64dd..238e6c99 100644 --- a/crates/compass-languages/src/evidence/extended/common.rs +++ b/crates/compass-languages/src/evidence/shared.rs @@ -1,4 +1,4 @@ -//! Shared AST-first evidence producer for Swift, Dart, Scala, and Groovy. +//! Shared AST-first traversal for the language-specific universal producers. //! //! The four grammars have different surface syntax, but their project-neutral //! evidence boundary is the same: declarations and lexical scopes first, @@ -13,12 +13,12 @@ use std::path::Path; use tree_sitter::Node; -use super::super::build::{EvidenceBuilder, range_for_byte_span, range_for_file, range_for_node}; -use super::super::model::{ +use super::build::{EvidenceBuilder, range_for_byte_span, range_for_file, range_for_node}; +use super::model::{ BindingKind, CandidateRelation, EvidenceRange, LanguageCapability, ResolutionConstraint, SemanticEvidenceBatch, SemanticRole, SymbolNamespace, }; -use super::super::validate::{EvidenceError, EvidenceErrorCode, EvidenceLimits}; +use super::validate::{EvidenceError, EvidenceErrorCode, EvidenceLimits}; use crate::{UniversalEvidenceRegistry, file_stem, make_id}; const MAX_TRAVERSAL_DEPTH: usize = 256; diff --git a/crates/compass-languages/src/evidence/extended/swift.rs b/crates/compass-languages/src/evidence/swift.rs similarity index 68% rename from crates/compass-languages/src/evidence/extended/swift.rs rename to crates/compass-languages/src/evidence/swift.rs index fd5aa8ac..034f6015 100644 --- a/crates/compass-languages/src/evidence/extended/swift.rs +++ b/crates/compass-languages/src/evidence/swift.rs @@ -4,9 +4,9 @@ use std::path::Path; use tree_sitter::Node; -use super::super::model::SemanticEvidenceBatch; -use super::super::validate::EvidenceError; -use super::common::{self, LanguageProfile}; +use super::model::SemanticEvidenceBatch; +use super::shared::{self, LanguageProfile}; +use super::validate::EvidenceError; struct Swift; @@ -24,5 +24,5 @@ pub(super) fn emit_tree_evidence( source: &[u8], root: Node<'_>, ) -> Result { - common::emit_tree_evidence::(path, source_file, source, root) + shared::emit_tree_evidence::(path, source_file, source, root) } diff --git a/crates/compass-languages/src/evidence_pipeline.rs b/crates/compass-languages/src/evidence_pipeline.rs index a5987bb0..acedb48d 100644 --- a/crates/compass-languages/src/evidence_pipeline.rs +++ b/crates/compass-languages/src/evidence_pipeline.rs @@ -302,9 +302,9 @@ pub(crate) const RUBY_CAPABILITIES: &[LanguageCapability] = &[ LanguageCapability::ExternalReferences, ]; -// Conservative common capabilities emitted by the AST-first extended -// language producer. Project-wide target selection and framework conventions -// remain outside the language boundary. +// Conservative capabilities emitted by the AST-first language-specific +// producers. Project-wide target selection and framework conventions remain +// outside the language boundary. const DART_CAPABILITIES: &[LanguageCapability] = &[ LanguageCapability::Declarations, LanguageCapability::LexicalScopes, diff --git a/crates/compass-languages/tests/extended_universal_conformance.rs b/crates/compass-languages/tests/language_wave_universal_conformance.rs similarity index 95% rename from crates/compass-languages/tests/extended_universal_conformance.rs rename to crates/compass-languages/tests/language_wave_universal_conformance.rs index 7857fc8b..3aaeb871 100644 --- a/crates/compass-languages/tests/extended_universal_conformance.rs +++ b/crates/compass-languages/tests/language_wave_universal_conformance.rs @@ -7,7 +7,7 @@ use compass_languages::{ }; #[test] -fn extended_languages_publish_ast_first_universal_evidence() -> Result<(), Box> { +fn language_wave_publish_ast_first_universal_evidence() -> Result<(), Box> { let fixtures = [ ( "Sources/Greeter.swift", @@ -121,7 +121,7 @@ class Greeter { } #[test] -fn registry_fixtures_keep_extended_pipelines_valid() -> Result<(), Box> { +fn registry_fixtures_keep_language_wave_pipelines_valid() -> Result<(), Box> { for case in Registry::cases() .iter() .filter(|case| matches!(case.spec.name, "swift" | "dart" | "scala" | "groovy")) @@ -140,7 +140,7 @@ fn registry_fixtures_keep_extended_pipelines_valid() -> Result<(), Box Result<(), Box> { +fn language_wave_empty_and_recovered_sources_remain_bounded() -> Result<(), Box> { for (path, source) in [ ("empty.swift", b"".as_slice()), ("empty.dart", b"\n".as_slice()), diff --git a/docs/implementation/universal-evidence.md b/docs/implementation/universal-evidence.md index d19ab3be..0537a585 100644 --- a/docs/implementation/universal-evidence.md +++ b/docs/implementation/universal-evidence.md @@ -31,7 +31,7 @@ future work. | --- | --- | | Available now | `compass-languages` owns the source registry, parsers, established extractors, and universal evidence schema version 2 (extraction semantics version 3) | | Available now | C#, Dart, Go, Groovy, Java, Kotlin, PHP, Python, Ruby, Rust, Scala, Swift, TypeScript, and JavaScript are entries in the hard-cut `UniversalEvidenceRegistry`; each entry pairs a `UniversalEvidenceProducer` with a `UniversalEvidenceQualification` state | -| Available now | `EvidenceBuilder` emits bounded `SemanticEvidenceBatch` values for all registered universal languages; Swift, Dart, Scala, and Groovy share the bounded AST-first extended producer, while each retains a distinct version-1 producer identity | +| Available now | `EvidenceBuilder` emits bounded `SemanticEvidenceBatch` values for all registered universal languages; Swift, Dart, Scala, and Groovy use direct language modules backed by a shared bounded AST-first traversal, while each retains a distinct version-1 producer identity | | Available now | `UniversalResolutionIndex` resolves and projects hard-cut evidence without a language-name branch | | Available now | Rust has passed its Phase 2 quality audit; all registered pipelines remain explicitly `Qualifying` until their complete independent audit gates promote them | | Planned | `GrammarProvider` and grammar provenance | From f5785c9eedb8b51e83f974f9fef474a17a29ff95 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sat, 22 Aug 2026 12:35:13 -0700 Subject: [PATCH 4/4] feat(languages): restore language wave parity evidence --- crates/compass-languages/src/evidence/dart.rs | 201 ++++++++++- .../compass-languages/src/evidence/groovy.rs | 93 +++++ .../compass-languages/src/evidence/scala.rs | 94 ++++- .../compass-languages/src/evidence/shared.rs | 323 +++++++++++++----- .../compass-languages/src/evidence/swift.rs | 43 +++ .../src/evidence_pipeline.rs | 2 + .../language_wave_universal_conformance.rs | 174 +++++++++- docs/implementation/universal-evidence.md | 1 + 8 files changed, 837 insertions(+), 94 deletions(-) diff --git a/crates/compass-languages/src/evidence/dart.rs b/crates/compass-languages/src/evidence/dart.rs index 88a862f0..9ac7cafb 100644 --- a/crates/compass-languages/src/evidence/dart.rs +++ b/crates/compass-languages/src/evidence/dart.rs @@ -4,8 +4,9 @@ use std::path::Path; use tree_sitter::Node; +use super::build::range_for_byte_span; use super::model::SemanticEvidenceBatch; -use super::shared::{self, LanguageProfile}; +use super::shared::{self, LanguageProfile, ParsedImport, State}; use super::validate::EvidenceError; struct Dart; @@ -13,6 +14,10 @@ struct Dart; impl LanguageProfile for Dart { const LANGUAGE: &'static str = "dart"; + fn package_name(source: &[u8]) -> Option { + dart_library_name(source).or_else(|| dart_part_of_name(source)) + } + fn declaration_kind(kind: &str) -> Option<&'static str> { let lower = kind.to_ascii_lowercase(); shared::shared_declaration_kind(kind) @@ -23,6 +28,27 @@ impl LanguageProfile for Dart { name.split_once('(') .map_or_else(|| name.to_owned(), |(base, _)| base.trim().to_owned()) } + + fn ignores_type_reference(spelling: &str) -> bool { + matches!( + spelling, + "deferred" | "export" | "hide" | "import" | "library" | "part" | "show" + ) + } + + fn parse_imports(statement: &str) -> Vec { + parse_dart_import(statement) + } + + fn has_source_supplement(_declaration_count: usize) -> bool { + true + } + + fn collect_source_supplement<'source>( + state: &mut State<'source, Self>, + ) -> Result<(), EvidenceError> { + collect_dart_parts(state) + } } pub(super) fn emit_tree_evidence( @@ -33,3 +59,176 @@ pub(super) fn emit_tree_evidence( ) -> Result { shared::emit_tree_evidence::(path, source_file, source, root) } + +fn dart_library_name(source: &[u8]) -> Option { + dart_directive_name(source, "library") +} + +fn dart_part_of_name(source: &[u8]) -> Option { + dart_directive_name(source, "part of") +} + +fn dart_directive_name(source: &[u8], keyword: &str) -> Option { + let text = std::str::from_utf8(source).ok()?; + text.lines().take(128).find_map(|line| { + let trimmed = line.trim(); + let rest = trimmed.strip_prefix(keyword)?.trim(); + let value = rest.trim_end_matches(';').trim().trim_matches(['\'', '"']); + if value.is_empty() + || !value + .chars() + .all(|character| character.is_alphanumeric() || "._/-:".contains(character)) + { + return None; + } + Some(value.to_owned()) + }) +} + +fn collect_dart_parts<'source>(state: &mut State<'source, Dart>) -> Result<(), EvidenceError> { + let Ok(text) = std::str::from_utf8(state.source) else { + return Ok(()); + }; + let mut line_start = 0_usize; + for line in text.split_inclusive('\n') { + let line_without_newline = line.trim_end_matches(['\r', '\n']); + let trimmed = line_without_newline.trim(); + let line_end = line_start.saturating_add(line_without_newline.len()); + let range_start = + line_start.saturating_add(line_without_newline.len().saturating_sub(trimmed.len())); + let range = range_for_byte_span(state.source_file, state.source, range_start, line_end); + if trimmed.starts_with("import ") || trimmed.starts_with("export ") { + state.emit_imports(parse_dart_import(trimmed), range.clone())?; + } + let target = trimmed + .strip_prefix("part of ") + .or_else(|| trimmed.strip_prefix("part ")) + .and_then(dart_directive_target); + if let Some(target) = target { + state.emit_embedding(&target, range)?; + } + line_start = line_start.saturating_add(line.len()); + } + Ok(()) +} + +fn dart_directive_target(rest: &str) -> Option { + let value = rest + .trim() + .trim_end_matches(';') + .trim() + .trim_matches(['\'', '"']); + (!value.is_empty()).then(|| value.to_owned()) +} + +fn parse_dart_import(statement: &str) -> Vec { + let trimmed = statement.trim(); + let (reexport, rest) = if let Some(rest) = trimmed.strip_prefix("export") { + (true, rest.trim()) + } else if let Some(rest) = trimmed.strip_prefix("import") { + (false, rest.trim()) + } else { + return Vec::new(); + }; + let Some((uri, suffix)) = dart_quoted_prefix(rest) else { + return Vec::new(); + }; + let suffix = suffix.trim().trim_end_matches(';').trim(); + let prefix = dart_import_prefix(suffix); + let shown = dart_import_clause(suffix, "show"); + let specs = if shown.is_empty() { Vec::new() } else { shown }; + if !specs.is_empty() { + return specs + .into_iter() + .map(|name| { + let target = format!("{uri}.{name}"); + if let Some(prefix) = prefix.as_deref() { + ParsedImport { + target, + binding_spelling: format!("{prefix}.{name}"), + local_spelling: name, + qualifier: Some(prefix.to_owned()), + alias: true, + prefix: false, + reexport, + } + } else { + ParsedImport { + target, + binding_spelling: name.clone(), + local_spelling: name, + qualifier: None, + alias: false, + prefix: false, + reexport, + } + } + }) + .collect(); + } + let binding_spelling = prefix.clone().unwrap_or_else(|| uri.clone()); + vec![ParsedImport { + target: uri, + binding_spelling: binding_spelling.clone(), + local_spelling: prefix + .as_ref() + .map_or_else(|| binding_spelling.clone(), |_| "*".to_owned()), + qualifier: prefix.clone(), + alias: prefix.is_some(), + prefix: prefix.is_some(), + reexport, + }] +} + +fn dart_quoted_prefix(value: &str) -> Option<(String, &str)> { + let quote = value.as_bytes().first().copied()?; + if !matches!(quote, b'\'' | b'"') { + return None; + } + let end = value + .as_bytes() + .iter() + .enumerate() + .skip(1) + .find_map(|(index, byte)| (*byte == quote).then_some(index))?; + let uri = value.get(1..end)?.to_owned(); + Some((uri, value.get(end.saturating_add(1)..)?)) +} + +fn dart_import_prefix(suffix: &str) -> Option { + let tokens = suffix.split_whitespace().collect::>(); + let index = tokens.iter().position(|token| *token == "as")?; + let prefix = tokens.get(index.saturating_add(1))?.trim_matches(';'); + shared::valid_name(prefix).then(|| prefix.to_owned()) +} + +fn dart_import_clause(suffix: &str, keyword: &str) -> Vec { + let start = suffix + .split_whitespace() + .position(|token| token == keyword) + .map(|index| { + suffix + .split_whitespace() + .take(index) + .map(str::len) + .sum::() + .saturating_add(index) + .saturating_add(keyword.len()) + }); + let Some(start) = start else { + return Vec::new(); + }; + let rest = suffix.get(start..).unwrap_or_default(); + let end = ["show", "hide"] + .iter() + .filter_map(|other| rest.find(&format!(" {other}"))) + .min() + .unwrap_or(rest.len()); + rest.get(..end) + .unwrap_or_default() + .split(',') + .map(str::trim) + .filter(|name| shared::valid_name(name)) + .map(str::to_owned) + .collect() +} diff --git a/crates/compass-languages/src/evidence/groovy.rs b/crates/compass-languages/src/evidence/groovy.rs index 261e11c8..994a6248 100644 --- a/crates/compass-languages/src/evidence/groovy.rs +++ b/crates/compass-languages/src/evidence/groovy.rs @@ -21,6 +21,21 @@ impl LanguageProfile for Groovy { declaration_count <= 1 } + fn should_collect_source_supplement(source: &[u8], declaration_count: usize) -> bool { + Self::has_source_supplement(declaration_count) + || std::str::from_utf8(source).is_ok_and(|text| { + text.lines() + .any(|line| groovy_spock_feature_declaration(line.trim()).is_some()) + }) + } + + fn declaration_name_is_valid(name: &str) -> bool { + shared::valid_name(name) + || (!name.is_empty() + && name.len() <= 512 + && name.chars().all(|character| !character.is_control())) + } + fn collect_source_supplement<'source>( state: &mut State<'source, Self>, ) -> Result<(), EvidenceError> { @@ -51,6 +66,15 @@ fn collect_groovy_source<'source>(state: &mut State<'source, Groovy>) -> Result< let Ok(text) = std::str::from_utf8(state.source) else { return Ok(()); }; + // The pinned grammar can recover a quoted feature declaration even when + // the specification imports its base class through a project-local alias + // (or the fixture intentionally omits the import). Treat the quoted + // declaration itself as the bounded syntax signal; no test relationship + // is inferred here. + let spock_source = text.contains("spock.lang.Specification") + || text + .lines() + .any(|line| groovy_spock_feature_declaration(line.trim()).is_some()); let mut depth = 0_i32; let mut classes: Vec<(usize, usize, i32)> = Vec::new(); let mut method: Option<(usize, usize)> = None; @@ -98,6 +122,33 @@ fn collect_groovy_source<'source>(state: &mut State<'source, Groovy>) -> Result< let active_class = classes.last().map(|(index, _, _)| *index); if let Some(class_index) = active_class + && let Some((name, name_start, name_end)) = spock_source + .then(|| groovy_spock_feature_declaration(trimmed)) + .flatten() + { + let parent_scope = state + .declarations + .get(class_index) + .map_or(state.file_scope_id.as_str(), |decl| { + decl.body_scope_id.as_str() + }) + .to_owned(); + let body_end = matching_brace_end(state.source, line_start, line_end); + let end = body_end.max(line_end); + if let Some(index) = state.add_source_declaration( + "method", + &name, + line_start, + end, + line_start.saturating_add(name_start), + line_start.saturating_add(name_end), + Some(class_index), + &parent_scope, + )? { + method = Some((index, end)); + state.emit_source_calls(line_start, line_end, index)?; + } + } else if let Some(class_index) = active_class && let Some((name, constructor, name_offset)) = groovy_method_declaration(trimmed) { let parent_scope = state @@ -187,6 +238,48 @@ fn groovy_method_declaration(line: &str) -> Option<(String, bool, usize)> { (has_return_shape || constructor).then(|| (name.to_owned(), constructor, name_start)) } +fn groovy_spock_feature_declaration(line: &str) -> Option<(String, usize, usize)> { + let rest = line.strip_prefix("def")?.trim_start(); + let quote = rest.as_bytes().first().copied()?; + if !matches!(quote, b'\'' | b'"') { + return None; + } + let mut escaped = false; + let closing = rest + .as_bytes() + .iter() + .enumerate() + .skip(1) + .find_map(|(index, byte)| { + if escaped { + escaped = false; + return None; + } + if *byte == b'\\' { + escaped = true; + return None; + } + (*byte == quote).then_some(index) + })?; + if !rest + .get(closing.saturating_add(1)..)? + .trim_start() + .starts_with('(') + { + return None; + } + let name = rest.get(1..closing)?.trim(); + if name.is_empty() || name.chars().any(char::is_control) { + return None; + } + let rest_offset = line.len().saturating_sub(rest.len()); + Some(( + name.to_owned(), + rest_offset.saturating_add(1), + rest_offset.saturating_add(closing), + )) +} + fn brace_delta(line: &str) -> i32 { let mut delta = 0_i32; let mut quote = None; diff --git a/crates/compass-languages/src/evidence/scala.rs b/crates/compass-languages/src/evidence/scala.rs index 9c0cf966..ab7363fb 100644 --- a/crates/compass-languages/src/evidence/scala.rs +++ b/crates/compass-languages/src/evidence/scala.rs @@ -5,7 +5,7 @@ use std::path::Path; use tree_sitter::Node; use super::model::SemanticEvidenceBatch; -use super::shared::{self, LanguageProfile}; +use super::shared::{self, LanguageProfile, ParsedImport}; use super::validate::EvidenceError; struct Scala; @@ -22,6 +22,10 @@ impl LanguageProfile for Scala { shared::shared_declaration_kind(kind) .or_else(|| (lower.contains("val_") || lower.contains("var_")).then_some("field")) } + + fn parse_imports(statement: &str) -> Vec { + parse_scala_import(statement) + } } pub(super) fn emit_tree_evidence( @@ -32,3 +36,91 @@ pub(super) fn emit_tree_evidence( ) -> Result { shared::emit_tree_evidence::(path, source_file, source, root) } + +fn parse_scala_import(statement: &str) -> Vec { + let trimmed = statement.trim(); + let (reexport, rest) = if let Some(rest) = trimmed.strip_prefix("export") { + (true, rest.trim()) + } else if let Some(rest) = trimmed.strip_prefix("import") { + (false, rest.trim()) + } else { + return Vec::new(); + }; + let rest = rest.trim_end_matches(';').trim(); + if let Some(open) = rest.find(".{") + && let Some(close) = rest.rfind('}') + && close > open.saturating_add(2) + { + let prefix = rest[..open].trim(); + if prefix.is_empty() { + return Vec::new(); + } + return rest[open.saturating_add(2)..close] + .split(',') + .filter_map(|selector| scala_selector(prefix, selector.trim(), reexport)) + .collect(); + } + let target = rest.trim(); + if target.is_empty() { + return Vec::new(); + } + if let Some(prefix) = target.strip_suffix("._") { + return vec![ParsedImport { + target: prefix.to_owned(), + binding_spelling: format!("{prefix}.*"), + local_spelling: "*".to_owned(), + qualifier: Some(prefix.to_owned()), + alias: false, + prefix: false, + reexport, + }]; + } + let spelling = target.rsplit('.').next().unwrap_or(target).trim(); + if !shared::valid_name(spelling) { + return Vec::new(); + } + vec![ParsedImport { + target: target.to_owned(), + binding_spelling: spelling.to_owned(), + local_spelling: spelling.to_owned(), + qualifier: None, + alias: false, + prefix: false, + reexport, + }] +} + +fn scala_selector(prefix: &str, selector: &str, reexport: bool) -> Option { + if selector == "_" { + return Some(ParsedImport { + target: prefix.to_owned(), + binding_spelling: format!("{prefix}.*"), + local_spelling: "*".to_owned(), + qualifier: Some(prefix.to_owned()), + alias: false, + prefix: false, + reexport, + }); + } + let (name, alias) = selector + .split_once("=>") + .map_or((selector.trim(), None), |(name, alias)| { + (name.trim(), Some(alias.trim())) + }); + if alias == Some("_") || !shared::valid_name(name) { + return None; + } + let spelling = alias.unwrap_or(name); + if !shared::valid_name(spelling) { + return None; + } + Some(ParsedImport { + target: format!("{prefix}.{name}"), + binding_spelling: spelling.to_owned(), + local_spelling: spelling.to_owned(), + qualifier: None, + alias: alias.is_some(), + prefix: false, + reexport, + }) +} diff --git a/crates/compass-languages/src/evidence/shared.rs b/crates/compass-languages/src/evidence/shared.rs index 238e6c99..7f3e7ee0 100644 --- a/crates/compass-languages/src/evidence/shared.rs +++ b/crates/compass-languages/src/evidence/shared.rs @@ -41,10 +41,43 @@ pub(super) trait LanguageProfile: Sized { shared_declaration_kind(kind) } + fn declaration_kind_for_node(node: Node<'_>, source: &[u8]) -> Option<&'static str> { + let _ = source; + Self::declaration_kind(node.kind()) + } + fn declaration_lookup_name(name: &str) -> String { name.to_owned() } + fn declaration_name_is_valid(name: &str) -> bool { + valid_name(name) + } + + fn ignores_type_reference(_spelling: &str) -> bool { + false + } + + fn parse_imports(statement: &str) -> Vec { + parse_import(statement) + .into_iter() + .map(|(target, alias, reexport)| { + let spelling = alias + .as_deref() + .map_or_else(|| terminal(&target).to_owned(), str::to_owned); + ParsedImport { + target, + binding_spelling: spelling.clone(), + local_spelling: spelling, + qualifier: None, + alias: alias.is_some(), + prefix: false, + reexport, + } + }) + .collect() + } + fn emits_module_declarations() -> bool { false } @@ -53,6 +86,11 @@ pub(super) trait LanguageProfile: Sized { false } + fn should_collect_source_supplement(source: &[u8], declaration_count: usize) -> bool { + let _ = source; + Self::has_source_supplement(declaration_count) + } + fn collect_source_supplement<'source>( _state: &mut State<'source, Self>, ) -> Result<(), EvidenceError> { @@ -75,6 +113,19 @@ pub(super) struct Decl { struct Import { spelling: String, target: String, + qualifier: Option, + prefix: bool, +} + +#[derive(Clone, Debug)] +pub(super) struct ParsedImport { + pub(super) target: String, + pub(super) binding_spelling: String, + pub(super) local_spelling: String, + pub(super) qualifier: Option, + pub(super) alias: bool, + pub(super) prefix: bool, + pub(super) reexport: bool, } pub(super) struct State<'source, P: LanguageProfile> { @@ -90,6 +141,7 @@ pub(super) struct State<'source, P: LanguageProfile> { by_qualified: BTreeMap>, name_ranges: BTreeSet<(usize, usize)>, imports: Vec, + emitted_imports: BTreeSet<(String, String, usize, usize)>, module_targets: BTreeSet, emitted: BTreeSet<(SemanticRole, usize, usize, String)>, occurrence_ids: BTreeMap<(SemanticRole, usize, usize, String), String>, @@ -152,6 +204,7 @@ pub(super) fn emit_tree_evidence( by_qualified: BTreeMap::new(), name_ranges: BTreeSet::new(), imports: Vec::new(), + emitted_imports: BTreeSet::new(), module_targets: BTreeSet::new(), emitted: BTreeSet::new(), occurrence_ids: BTreeMap::new(), @@ -167,7 +220,7 @@ pub(super) fn emit_tree_evidence( } let root_scope = state.add_namespace(root)?; state.collect_declarations(root, None, &root_scope, 0)?; - if P::has_source_supplement(state.declarations.len()) { + if P::should_collect_source_supplement(source, state.declarations.len()) { P::collect_source_supplement(&mut state)?; } state.collect_imports(root, 0)?; @@ -238,7 +291,7 @@ impl<'source, P: LanguageProfile> State<'source, P> { .map(|decl| decl.qualified.clone()) .unwrap_or_else(|| self.namespace.clone()); - if let Some(kind) = P::declaration_kind(node.kind()) + if let Some(kind) = P::declaration_kind_for_node(node, self.source) && let Some(name_node) = declaration_name(node) { let name = self.text(name_node); @@ -356,7 +409,7 @@ impl<'source, P: LanguageProfile> State<'source, P> { parent: Option, parent_scope: &str, ) -> Result, EvidenceError> { - if !valid_name(name) + if !P::declaration_name_is_valid(name) || self .declarations .iter() @@ -485,109 +538,176 @@ impl<'source, P: LanguageProfile> State<'source, P> { Ok(()) } + pub(super) fn emit_embedding( + &mut self, + target: &str, + range: EvidenceRange, + ) -> Result<(), EvidenceError> { + if target.is_empty() || !self.supports(LanguageCapability::Embedding) { + return Ok(()); + } + let owner_id = self.owner_id(None); + let owner_scope = self.owner_scope(None); + let occurrence_id = self.emit_occurrence( + SemanticRole::Embedding, + &owner_id, + target, + None, + Some(&owner_scope), + range, + )?; + self.builder.relate( + CandidateRelation::Embeds, + &owner_id, + Some(&occurrence_id), + None, + target, + ResolutionConstraint { + exact_language: Some(P::LANGUAGE.to_owned()), + qualified_name: Some(target.to_owned()), + allow_external: true, + ..ResolutionConstraint::default() + }, + )?; + Ok(()) + } + fn collect_imports(&mut self, node: Node<'_>, depth: usize) -> Result<(), EvidenceError> { if depth > MAX_TRAVERSAL_DEPTH { self.depth_diagnostic(node)?; return Ok(()); } let statement = self.text(node); - if is_import_node(node.kind()) - && let Some((target, alias, reexport)) = parse_import(&statement) - && !target.is_empty() - { - let has_alias = alias.is_some(); - let spelling = alias.unwrap_or_else(|| terminal(&target).to_owned()); - if !spelling.is_empty() { - let owner = self.owner_for(node.start_byte()); - let owner_scope = self - .declaration_for(owner) - .map_or(self.file_scope_id.as_str(), |decl| { - decl.body_scope_id.as_str() - }) - .to_owned(); - // Swift's pre-universal extractor published imported modules - // as source-anchored module nodes. Keep that established - // Vapor/framework identity on the evidence route while the - // binding itself remains an exact, language-constrained - // import candidate. - if P::emits_module_declarations() && self.module_targets.insert(target.clone()) { - let module_id = make_id(&[self.source_file, P::LANGUAGE, "module", &target]); - let module_declaration = self.builder.declare( - "module", - &module_id, - &spelling, - &target, - None, - Some(&self.file_scope_id), - range_for_node(self.source_file, node), - )?; - self.builder.relate( - CandidateRelation::Owns, - &self.file_id, - None, - None, - &spelling, - ResolutionConstraint { - exact_target_declaration_id: Some(module_declaration), - exact_language: Some(P::LANGUAGE.to_owned()), - ..ResolutionConstraint::default() - }, - )?; - } - let binding_id = self.builder.bind_with_identity( - if reexport { - BindingKind::Reexport - } else if has_alias { - BindingKind::ImportAlias - } else { - BindingKind::Import - }, + if is_import_node(node.kind()) { + self.emit_imports( + P::parse_imports(&statement), + range_for_node(self.source_file, node), + )?; + } + let mut cursor = node.walk(); + for child in node.named_children(&mut cursor) { + self.collect_imports(child, depth.saturating_add(1))?; + } + Ok(()) + } + + pub(super) fn emit_imports( + &mut self, + parsed_imports: Vec, + range: EvidenceRange, + ) -> Result<(), EvidenceError> { + for parsed in parsed_imports { + let ParsedImport { + target, + binding_spelling, + local_spelling, + qualifier, + alias, + prefix, + reexport, + } = parsed; + if target.is_empty() || binding_spelling.is_empty() { + continue; + } + let spelling = binding_spelling; + let import_key = ( + spelling.clone(), + target.clone(), + range.start_byte as usize, + range.end_byte as usize, + ); + if !self.emitted_imports.insert(import_key) { + continue; + } + let owner = self.owner_for(range.start_byte as usize); + let owner_scope = self + .declaration_for(owner) + .map_or(self.file_scope_id.as_str(), |decl| { + decl.body_scope_id.as_str() + }) + .to_owned(); + // Swift's pre-universal extractor published imported modules as + // source-anchored module nodes. Keep that established Vapor/ + // framework identity while the binding remains exact and + // language-constrained. + if P::emits_module_declarations() && self.module_targets.insert(target.clone()) { + let module_id = make_id(&[self.source_file, P::LANGUAGE, "module", &target]); + let module_declaration = self.builder.declare( + "module", + &module_id, &spelling, &target, None, - Some(&owner_scope), - None, - false, - range_for_node(self.source_file, node), + Some(&self.file_scope_id), + range.clone(), )?; - let owner_id = self.owner_id(owner); - let role = if reexport { - SemanticRole::Reexport - } else { - SemanticRole::Import - }; - let occurrence_id = self.emit_occurrence( - role, - &owner_id, - &spelling, - qualifier_for(&target), - Some(&owner_scope), - range_for_node(self.source_file, node), - )?; - let relation = if reexport { - CandidateRelation::Reexports - } else { - CandidateRelation::Imports - }; self.builder.relate( - relation, - &owner_id, - Some(&occurrence_id), - Some(&binding_id), + CandidateRelation::Owns, + &self.file_id, + None, + None, &spelling, ResolutionConstraint { + exact_target_declaration_id: Some(module_declaration), exact_language: Some(P::LANGUAGE.to_owned()), - qualified_name: Some(target.clone()), - allow_external: true, ..ResolutionConstraint::default() }, )?; - self.imports.push(Import { spelling, target }); } - } - let mut cursor = node.walk(); - for child in node.named_children(&mut cursor) { - self.collect_imports(child, depth.saturating_add(1))?; + let binding_id = self.builder.bind_with_identity( + if reexport { + BindingKind::Reexport + } else if alias { + BindingKind::ImportAlias + } else { + BindingKind::Import + }, + &spelling, + &target, + None, + Some(&owner_scope), + None, + false, + range.clone(), + )?; + let owner_id = self.owner_id(owner); + let role = if reexport { + SemanticRole::Reexport + } else { + SemanticRole::Import + }; + let occurrence_id = self.emit_occurrence( + role, + &owner_id, + &spelling, + qualifier.as_deref().or_else(|| qualifier_for(&target)), + Some(&owner_scope), + range.clone(), + )?; + let relation = if reexport { + CandidateRelation::Reexports + } else { + CandidateRelation::Imports + }; + self.builder.relate( + relation, + &owner_id, + Some(&occurrence_id), + Some(&binding_id), + &spelling, + ResolutionConstraint { + exact_language: Some(P::LANGUAGE.to_owned()), + qualified_name: Some(target.clone()), + allow_external: true, + ..ResolutionConstraint::default() + }, + )?; + self.imports.push(Import { + spelling: local_spelling, + target, + qualifier, + prefix, + }); } Ok(()) } @@ -674,10 +794,29 @@ impl<'source, P: LanguageProfile> State<'source, P> { .then(|| { self.imports .iter() - .find(|import| import.spelling == spelling) + .find(|import| { + import.spelling == spelling && import.qualifier.as_deref() == qualifier + }) .map(|import| import.target.clone()) }) - .flatten(); + .flatten() + .or_else(|| { + qualifier.and_then(|prefix| { + self.imports + .iter() + .find(|import| { + (import.spelling == spelling || import.prefix) + && import.qualifier.as_deref() == Some(prefix) + }) + .map(|import| { + if import.prefix { + format!("{}.{}", import.target, spelling) + } else { + import.target.clone() + } + }) + }) + }); let qualified_name = exact .and_then(|index| { self.declarations @@ -726,6 +865,9 @@ impl<'source, P: LanguageProfile> State<'source, P> { return Ok(()); } let raw = self.text(node); + if P::ignores_type_reference(&raw) { + return Ok(()); + } let (qualifier, spelling) = split_qualified(&raw); if !valid_name(&spelling) || spelling.len() > 256 { return Ok(()); @@ -1085,6 +1227,7 @@ fn opens_scope(kind: &str) -> bool { | "interface" | "trait" | "module" + | "extension" | "function" | "method" | "constructor" diff --git a/crates/compass-languages/src/evidence/swift.rs b/crates/compass-languages/src/evidence/swift.rs index 034f6015..c84d58d0 100644 --- a/crates/compass-languages/src/evidence/swift.rs +++ b/crates/compass-languages/src/evidence/swift.rs @@ -16,6 +16,32 @@ impl LanguageProfile for Swift { fn emits_module_declarations() -> bool { true } + + fn declaration_kind_for_node(node: Node<'_>, source: &[u8]) -> Option<&'static str> { + if node.kind() == "typealias_declaration" { + return Some("type_alias"); + } + if node.kind() == "class_declaration" + && let Some(text) = source + .get(node.start_byte()..node.end_byte()) + .and_then(|bytes| std::str::from_utf8(bytes).ok()) + { + let keyword = text.split_whitespace().next().unwrap_or_default(); + if keyword == "enum" { + return Some("enum"); + } + if keyword == "struct" { + return Some("struct"); + } + if keyword == "extension" { + return Some("extension"); + } + } + if node.kind() == "function_declaration" && swift_callable_is_member(node) { + return Some("method"); + } + shared::shared_declaration_kind(node.kind()) + } } pub(super) fn emit_tree_evidence( @@ -26,3 +52,20 @@ pub(super) fn emit_tree_evidence( ) -> Result { shared::emit_tree_evidence::(path, source_file, source, root) } + +fn swift_callable_is_member(node: Node<'_>) -> bool { + let mut ancestor = node.parent(); + for _ in 0..32 { + let Some(current) = ancestor else { + return false; + }; + if matches!( + current.kind(), + "class_body" | "protocol_body" | "enum_class_body" | "extension_body" + ) { + return true; + } + ancestor = current.parent(); + } + false +} diff --git a/crates/compass-languages/src/evidence_pipeline.rs b/crates/compass-languages/src/evidence_pipeline.rs index acedb48d..888b5540 100644 --- a/crates/compass-languages/src/evidence_pipeline.rs +++ b/crates/compass-languages/src/evidence_pipeline.rs @@ -308,6 +308,7 @@ pub(crate) const RUBY_CAPABILITIES: &[LanguageCapability] = &[ const DART_CAPABILITIES: &[LanguageCapability] = &[ LanguageCapability::Declarations, LanguageCapability::LexicalScopes, + LanguageCapability::Namespaces, LanguageCapability::Imports, LanguageCapability::Reexports, LanguageCapability::Aliases, @@ -318,6 +319,7 @@ const DART_CAPABILITIES: &[LanguageCapability] = &[ LanguageCapability::Members, LanguageCapability::Ownership, LanguageCapability::Receivers, + LanguageCapability::Embedding, LanguageCapability::ExternalReferences, ]; diff --git a/crates/compass-languages/tests/language_wave_universal_conformance.rs b/crates/compass-languages/tests/language_wave_universal_conformance.rs index 3aaeb871..0412f51f 100644 --- a/crates/compass-languages/tests/language_wave_universal_conformance.rs +++ b/crates/compass-languages/tests/language_wave_universal_conformance.rs @@ -2,8 +2,8 @@ use std::error::Error; use std::path::Path; use compass_languages::{ - CandidateRelation, Engine, EvidenceLimits, Registry, SemanticRole, UniversalEvidenceRegistry, - validate_evidence, + BindingKind, CandidateRelation, Engine, EvidenceLimits, LanguageCapability, Registry, + SemanticRole, UniversalEvidenceRegistry, validate_evidence, }; #[test] @@ -187,3 +187,173 @@ fn language_wave_empty_and_recovered_sources_remain_bounded() -> Result<(), Box< } Ok(()) } + +#[test] +fn groovy_spock_quoted_features_are_source_bounded_methods() -> Result<(), Box> { + let source = br#"package routes +class UserSpec extends spock.lang.Specification { + def "loads users"() { + helper() + } +} +"#; + let path = Path::new("src/UserSpec.groovy"); + let mut engine = Engine::default(); + let evidence = engine.extract_source_universal_evidence(path, "src/UserSpec.groovy", source)?; + validate_evidence(&evidence, EvidenceLimits::default())?; + + let feature = evidence + .declarations + .iter() + .find(|declaration| declaration.name == "loads users") + .ok_or("missing quoted Spock feature")?; + assert_eq!(feature.kind, "method"); + assert_eq!(feature.qualified_name, "routes.UserSpec.loads users"); + let range = &feature.range; + let text = std::str::from_utf8( + source + .get(range.start_byte as usize..range.end_byte as usize) + .ok_or("feature range outside source")?, + )?; + assert!(text.contains("def \"loads users\"()")); + assert!( + !evidence + .pipeline + .capabilities + .contains(&LanguageCapability::Tests) + ); + Ok(()) +} + +#[test] +fn dart_library_parts_and_import_filters_remain_explicit() -> Result<(), Box> { + let source = br#"library foo.bar; +part 'src/generated.dart'; +import 'package:widgets/widgets.dart' deferred as widgets show Widget, Api hide Internal; +export 'src/api.dart' show Api hide Internal; +class Screen { + void render() { widgets.Widget(); } +} +"#; + let path = Path::new("lib/foo.dart"); + let mut engine = Engine::default(); + let evidence = engine.extract_source_universal_evidence(path, "lib/foo.dart", source)?; + validate_evidence(&evidence, EvidenceLimits::default())?; + + assert!(evidence.declarations.iter().any(|declaration| { + declaration.kind == "namespace" && declaration.qualified_name == "foo.bar" + })); + assert!(evidence.candidates.iter().any(|candidate| { + candidate.relation == CandidateRelation::Embeds + && candidate.target_spelling == "src/generated.dart" + })); + assert!(evidence.bindings.iter().any(|binding| { + binding.kind == BindingKind::ImportAlias + && binding.spelling == "widgets.Widget" + && binding.qualified_target == "package:widgets/widgets.dart.Widget" + })); + assert!(evidence.bindings.iter().any(|binding| { + binding.kind == BindingKind::Reexport + && binding.spelling == "Api" + && binding.qualified_target == "src/api.dart.Api" + })); + assert!( + !evidence + .bindings + .iter() + .any(|binding| binding.spelling.contains("Internal")) + ); + + let part_source = br#"part of foo.bar; +class Generated {} +"#; + let part_path = Path::new("lib/src/generated.dart"); + let part = engine.extract_source_universal_evidence( + part_path, + "lib/src/generated.dart", + part_source, + )?; + validate_evidence(&part, EvidenceLimits::default())?; + assert!(part.declarations.iter().any(|declaration| { + declaration.name == "Generated" && declaration.qualified_name == "foo.bar.Generated" + })); + Ok(()) +} + +#[test] +fn swift_nominal_and_extension_identities_remain_distinct() -> Result<(), Box> { + let source = br#"import Foundation +protocol Renderable {} +class Box: Renderable {} +struct Widget {} +enum State { case ready, done } +extension Box { func render() {} } +typealias Alias = Box +"#; + let path = Path::new("Sources/Models.swift"); + let mut engine = Engine::default(); + let evidence = + engine.extract_source_universal_evidence(path, "Sources/Models.swift", source)?; + validate_evidence(&evidence, EvidenceLimits::default())?; + for (name, kind) in [ + ("Renderable", "protocol"), + ("Box", "class"), + ("Widget", "struct"), + ("State", "enum"), + ("Box", "extension"), + ("Alias", "type_alias"), + ] { + assert!( + evidence + .declarations + .iter() + .any(|declaration| declaration.name == name && declaration.kind == kind) + ); + } + assert!(evidence.declarations.iter().any(|declaration| { + declaration.name == "render" + && declaration.kind == "method" + && declaration.qualified_name == "Box.render" + })); + Ok(()) +} + +#[test] +fn scala_companion_and_selector_identities_do_not_collapse() -> Result<(), Box> { + let source = br#"package sample +import foo.{Bar => Baz, Hidden => _, _} +class Box {} +object Box {} +"#; + let path = Path::new("src/Models.scala"); + let mut engine = Engine::default(); + let evidence = engine.extract_source_universal_evidence(path, "src/Models.scala", source)?; + validate_evidence(&evidence, EvidenceLimits::default())?; + + let boxes = evidence + .declarations + .iter() + .filter(|declaration| declaration.name == "Box") + .collect::>(); + assert!(boxes.iter().any(|declaration| declaration.kind == "class")); + assert!(boxes.iter().any(|declaration| declaration.kind == "module")); + assert_ne!(boxes[0].id, boxes[1].id); + assert!(evidence.bindings.iter().any(|binding| { + binding.kind == BindingKind::ImportAlias + && binding.spelling == "Baz" + && binding.qualified_target == "foo.Bar" + })); + assert!( + evidence + .bindings + .iter() + .any(|binding| { binding.spelling == "foo.*" && binding.qualified_target == "foo" }) + ); + assert!( + !evidence + .bindings + .iter() + .any(|binding| binding.spelling == "Hidden") + ); + Ok(()) +} diff --git a/docs/implementation/universal-evidence.md b/docs/implementation/universal-evidence.md index 0537a585..269b5bef 100644 --- a/docs/implementation/universal-evidence.md +++ b/docs/implementation/universal-evidence.md @@ -32,6 +32,7 @@ future work. | Available now | `compass-languages` owns the source registry, parsers, established extractors, and universal evidence schema version 2 (extraction semantics version 3) | | Available now | C#, Dart, Go, Groovy, Java, Kotlin, PHP, Python, Ruby, Rust, Scala, Swift, TypeScript, and JavaScript are entries in the hard-cut `UniversalEvidenceRegistry`; each entry pairs a `UniversalEvidenceProducer` with a `UniversalEvidenceQualification` state | | Available now | `EvidenceBuilder` emits bounded `SemanticEvidenceBatch` values for all registered universal languages; Swift, Dart, Scala, and Groovy use direct language modules backed by a shared bounded AST-first traversal, while each retains a distinct version-1 producer identity | +| Available now | The language-wave parity profiles preserve quoted Groovy/Spock feature declarations, Dart library namespaces and bounded `part`/`part of` plus import/export selectors, Swift enum/struct/extension/type-alias/member identities, and Scala companion plus import-selector identities without enabling unaudited test or dynamic-dispatch capabilities | | Available now | `UniversalResolutionIndex` resolves and projects hard-cut evidence without a language-name branch | | Available now | Rust has passed its Phase 2 quality audit; all registered pipelines remain explicitly `Qualifying` until their complete independent audit gates promote them | | Planned | `GrammarProvider` and grammar provenance |