diff --git a/docs/superpowers/EXPECTED_TEST_FAILURES.md b/docs/superpowers/EXPECTED_TEST_FAILURES.md new file mode 100644 index 00000000..2ddc9cc0 --- /dev/null +++ b/docs/superpowers/EXPECTED_TEST_FAILURES.md @@ -0,0 +1,22 @@ +# Expected test failures (fork: exaforce schema pruning) + +These upstream tests are kept at upstream parity on purpose and therefore +assert the *un-pruned* schema, which the exaforce runtime patch removes. They +are expected to FAIL. A failure here is only a problem if the failure is NOT an +assertion about a pruned key (e.g. an import/collection error). + +Captured from: +`uv run pytest tests/nodes/test_llm_analyzer_base.py tests/nodes/test_semantic_quality_policy.py -q` + +- tests/nodes/test_llm_analyzer_base.py::TestLLMAnalysisResult::test_to_finding +- tests/nodes/test_llm_analyzer_base.py::TestLLMAnalysisResult::test_model_dump +- tests/nodes/test_llm_analyzer_base.py::TestMetaAnalyzerResult::test_intent_validation +- tests/nodes/test_semantic_quality_policy.py::TestFixtureMaliciousSkill::test_malicious_skill_findings_preserve_metadata + +All four fail with an `AssertionError` (or `KeyError`) about a pruned key +(`explanation`, `intent`) being absent — not an import/collection error. +Confirmed bounded to these two files via `uv run pytest -q -rf`: + +``` +4 failed, 1261 passed, 13 skipped, 34 deselected, 6 xfailed +``` diff --git a/src/skillspector/__init__.py b/src/skillspector/__init__.py index 30ce9322..27d7c776 100644 --- a/src/skillspector/__init__.py +++ b/src/skillspector/__init__.py @@ -35,3 +35,7 @@ from skillspector.graph import create_graph, graph # noqa: E402 (after filter setup) __all__ = ["create_graph", "graph", "__version__"] + +# ExaForce fork: apply runtime schema/prompt patches (kept out of upstream files). +from skillspector import exaforce as _exaforce # noqa: E402 +_exaforce.apply_patches() diff --git a/src/skillspector/exaforce/__init__.py b/src/skillspector/exaforce/__init__.py new file mode 100644 index 00000000..15b7b27f --- /dev/null +++ b/src/skillspector/exaforce/__init__.py @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: Apache-2.0 +"""ExaForce fork-local runtime patches. + +Keeps fork behavior — pruning unused LLM structured-output keys and prompt text +to shrink requests and reduce LLM timeouts — out of upstream-tracked source +files. All mutations are guarded: an upstream rename/rewrite raises +``PatchDriftError`` at import time rather than silently going stale. +""" + +from __future__ import annotations + +from . import _prompt_patches, _schema_patches + +_PATCHED = False + + +def apply_patches() -> None: + """Idempotently apply all fork-local runtime patches.""" + global _PATCHED + if _PATCHED: + return + _schema_patches.apply() + _prompt_patches.apply() + _PATCHED = True diff --git a/src/skillspector/exaforce/_patchlib.py b/src/skillspector/exaforce/_patchlib.py new file mode 100644 index 00000000..1505c9f7 --- /dev/null +++ b/src/skillspector/exaforce/_patchlib.py @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Guarded monkeypatch primitives for the ExaForce fork. + +Each primitive asserts its upstream target still exists before mutating it, so +an upstream rename/rewrite fails loudly (``PatchDriftError``) at import time +instead of silently leaving the fork's runtime behavior stale. +""" + +from __future__ import annotations + +from types import ModuleType + +from pydantic import BaseModel + + +class PatchDriftError(RuntimeError): + """An upstream target a fork patch depends on has changed or is missing.""" + + +def remove_model_fields(model: type[BaseModel], field_names: list[str]) -> None: + """Delete ``field_names`` from ``model.model_fields``. + + The caller must run ``model.model_rebuild(force=True)`` afterward (and on any + container model that nests ``model``) for the change to reach the emitted + JSON schema. Raises ``PatchDriftError`` if a field is absent. + """ + for name in field_names: + if name not in model.model_fields: + raise PatchDriftError( + f"{model.__module__}.{model.__qualname__} has no field {name!r}; " + "upstream changed — update the exaforce patch." + ) + del model.model_fields[name] + + +def pop_field_validator(model: type[BaseModel], validator_name: str) -> None: + """Remove a ``field_validator`` decorator by its function name (no-op if absent).""" + model.__pydantic_decorators__.field_validators.pop(validator_name, None) + + +def replace_module_str(module: ModuleType, attr: str, old: str, new: str) -> None: + """Replace substring ``old`` with ``new`` in module-global ``attr``. + + Raises ``PatchDriftError`` if ``old`` is not present in the current value. + """ + current = getattr(module, attr) + if old not in current: + raise PatchDriftError( + f"{module.__name__}.{attr} does not contain the expected text; " + "upstream changed — update the exaforce patch." + ) + setattr(module, attr, current.replace(old, new)) diff --git a/src/skillspector/exaforce/_prompt_patches.py b/src/skillspector/exaforce/_prompt_patches.py new file mode 100644 index 00000000..77e68991 --- /dev/null +++ b/src/skillspector/exaforce/_prompt_patches.py @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Trim prompt text the pruned schema no longer needs (fork behavior).""" + +from __future__ import annotations + +import skillspector.nodes.analyzers.semantic_developer_intent as dev_intent +import skillspector.nodes.analyzers.semantic_quality_policy as quality +import skillspector.nodes.meta_analyzer as meta + +from ._patchlib import replace_module_str + +# Present verbatim in both semantic analyzers' ANALYZER_PROMPT (note the two +# spaces after "listed." and the mid-sentence newline). +_LINE_NUMBER_OLD = ( + "Use the rule IDs exactly as listed. Reference the L-prefixed line numbers\n" + "when reporting findings." +) +_LINE_NUMBER_NEW = "Use the rule IDs exactly as listed." + +# The meta-analyzer's "Your Task" list: drop the intent (2) and impact (3) items. +_META_TASK_OLD = ( + "1. Is this a true vulnerability or a false positive?\n" + "2. What is the likely intent (malicious, negligent, or benign)?\n" + "3. What is the potential impact if exploited?\n" + "4. Does the skill context make this more or less dangerous?" +) +_META_TASK_NEW = ( + "1. Is this a true vulnerability or a false positive?\n" + "2. Does the skill context make this more or less dangerous?" +) + + +def apply() -> None: + replace_module_str(dev_intent, "ANALYZER_PROMPT", _LINE_NUMBER_OLD, _LINE_NUMBER_NEW) + replace_module_str(quality, "ANALYZER_PROMPT", _LINE_NUMBER_OLD, _LINE_NUMBER_NEW) + replace_module_str(meta, "PER_FILE_ANALYSIS_PROMPT", _META_TASK_OLD, _META_TASK_NEW) diff --git a/src/skillspector/exaforce/_schema_patches.py b/src/skillspector/exaforce/_schema_patches.py new file mode 100644 index 00000000..5d8c6c45 --- /dev/null +++ b/src/skillspector/exaforce/_schema_patches.py @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Prune unused keys from the LLM structured-output schemas (fork behavior). + +Reproduces the schema pruning that would otherwise live in upstream +``llm_analyzer_base`` / ``meta_analyzer``, keeping those upstream files at +parity with NVIDIA/SkillSpector. +""" + +from __future__ import annotations + +import skillspector.llm_analyzer_base as llm_base +import skillspector.nodes.meta_analyzer as meta +from skillspector.models import Finding + +from ._patchlib import pop_field_validator, remove_model_fields + + +def _pruned_to_finding(self: "llm_base.LLMFinding", file: str) -> Finding: + """``LLMFinding.to_finding`` without the removed explanation/remediation.""" + return Finding( + rule_id=self.rule_id, + message=self.message, + severity=self.severity, + confidence=self.confidence, + file=file, + start_line=self.start_line, + end_line=self.end_line, + ) + + +def apply() -> None: + # LLMFinding: drop explanation + remediation, and stop forwarding them. + remove_model_fields(llm_base.LLMFinding, ["explanation", "remediation"]) + llm_base.LLMFinding.to_finding = _pruned_to_finding + llm_base.LLMFinding.model_rebuild(force=True) + llm_base.LLMAnalysisResult.model_rebuild(force=True) + + # MetaAnalyzerFinding: drop intent + impact. + remove_model_fields(meta.MetaAnalyzerFinding, ["intent", "impact"]) + meta.MetaAnalyzerFinding.model_rebuild(force=True) + + # MetaAnalyzerResult: drop overall_assessment field + its validator. + remove_model_fields(meta.MetaAnalyzerResult, ["overall_assessment"]) + pop_field_validator(meta.MetaAnalyzerResult, "_parse_stringified_assessment") + meta.MetaAnalyzerResult.model_rebuild(force=True) diff --git a/tests/exaforce/__init__.py b/tests/exaforce/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/exaforce/conftest.py b/tests/exaforce/conftest.py new file mode 100644 index 00000000..6664ab27 --- /dev/null +++ b/tests/exaforce/conftest.py @@ -0,0 +1,21 @@ +import subprocess +import sys +import textwrap + +import pytest + + +@pytest.fixture +def run_in_subprocess(): + """Run *code* in a fresh interpreter; return stdout, assert clean exit.""" + + def _run(code: str) -> str: + result = subprocess.run( + [sys.executable, "-c", textwrap.dedent(code)], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + return result.stdout + + return _run diff --git a/tests/exaforce/test_activation.py b/tests/exaforce/test_activation.py new file mode 100644 index 00000000..88c5e130 --- /dev/null +++ b/tests/exaforce/test_activation.py @@ -0,0 +1,19 @@ +def test_bare_import_activates_patches(run_in_subprocess): + # No explicit apply_patches() call — importing skillspector must patch. + out = run_in_subprocess( + """ + import skillspector + from skillspector.llm_analyzer_base import LLMFinding + from skillspector.nodes.meta_analyzer import ( + MetaAnalyzerFinding, + MetaAnalyzerResult, + ) + from skillspector.nodes import meta_analyzer as m + assert "explanation" not in LLMFinding.model_json_schema()["properties"] + assert "intent" not in MetaAnalyzerFinding.model_json_schema()["properties"] + assert "overall_assessment" not in MetaAnalyzerResult.model_json_schema()["properties"] + assert "What is the likely intent" not in m.PER_FILE_ANALYSIS_PROMPT + print("OK") + """ + ) + assert "OK" in out diff --git a/tests/exaforce/test_patches.py b/tests/exaforce/test_patches.py new file mode 100644 index 00000000..e089e84e --- /dev/null +++ b/tests/exaforce/test_patches.py @@ -0,0 +1,80 @@ +def test_apply_patches_prunes_model_schemas(run_in_subprocess): + out = run_in_subprocess( + """ + import skillspector + from skillspector.exaforce import apply_patches + apply_patches() + from skillspector.llm_analyzer_base import LLMFinding, LLMAnalysisResult + from skillspector.nodes.meta_analyzer import ( + MetaAnalyzerFinding, + MetaAnalyzerResult, + ) + lf = set(LLMFinding.model_json_schema()["properties"]) + assert "explanation" not in lf and "remediation" not in lf, lf + mf = set(MetaAnalyzerFinding.model_json_schema()["properties"]) + assert "intent" not in mf and "impact" not in mf, mf + mr = set(MetaAnalyzerResult.model_json_schema()["properties"]) + assert "overall_assessment" not in mr, mr + # Container schema (what the LLM actually receives) is pruned too: + nested = LLMAnalysisResult.model_json_schema()["$defs"]["LLMFinding"]["properties"] + assert "explanation" not in nested, nested + nested_meta = MetaAnalyzerResult.model_json_schema()["$defs"]["MetaAnalyzerFinding"]["properties"] + assert "intent" not in nested_meta and "impact" not in nested_meta, nested_meta + print("OK") + """ + ) + assert "OK" in out + + +def test_apply_patches_prunes_to_finding_and_dump(run_in_subprocess): + out = run_in_subprocess( + """ + import skillspector + from skillspector.exaforce import apply_patches + apply_patches() + from skillspector.llm_analyzer_base import LLMFinding + f = LLMFinding(rule_id="R", message="m", severity="LOW", start_line=3) + assert "explanation" not in f.model_dump() + assert "remediation" not in f.model_dump() + fin = f.to_finding("x.py") + assert fin.explanation is None + assert fin.remediation is None + assert fin.rule_id == "R" and fin.start_line == 3 + print("OK") + """ + ) + assert "OK" in out + + +def test_apply_patches_trims_prompts(run_in_subprocess): + out = run_in_subprocess( + """ + import skillspector + from skillspector.exaforce import apply_patches + apply_patches() + from skillspector.nodes.analyzers import semantic_developer_intent as d + from skillspector.nodes.analyzers import semantic_quality_policy as q + from skillspector.nodes import meta_analyzer as m + assert "Reference the L-prefixed line numbers" not in d.ANALYZER_PROMPT + assert "Reference the L-prefixed line numbers" not in q.ANALYZER_PROMPT + assert "What is the likely intent" not in m.PER_FILE_ANALYSIS_PROMPT + assert "What is the potential impact" not in m.PER_FILE_ANALYSIS_PROMPT + assert "Use the rule IDs exactly as listed." in d.ANALYZER_PROMPT + print("OK") + """ + ) + assert "OK" in out + + +def test_apply_patches_is_idempotent(run_in_subprocess): + out = run_in_subprocess( + """ + import skillspector + from skillspector.exaforce import apply_patches + apply_patches() + apply_patches() + apply_patches() + print("OK") + """ + ) + assert "OK" in out diff --git a/tests/exaforce/test_patchlib.py b/tests/exaforce/test_patchlib.py new file mode 100644 index 00000000..dda79dc2 --- /dev/null +++ b/tests/exaforce/test_patchlib.py @@ -0,0 +1,51 @@ +from types import ModuleType + +import pytest +from pydantic import BaseModel + +from skillspector.exaforce._patchlib import ( + PatchDriftError, + pop_field_validator, + remove_model_fields, + replace_module_str, +) + + +def test_remove_model_fields_removes_then_rebuild_drops_key(): + class M(BaseModel): + a: str + b: str = "" + + remove_model_fields(M, ["b"]) + M.model_rebuild(force=True) + assert "b" not in M.model_json_schema()["properties"] + assert "a" in M.model_json_schema()["properties"] + + +def test_remove_model_fields_raises_on_missing_field(): + class M(BaseModel): + a: str + + with pytest.raises(PatchDriftError): + remove_model_fields(M, ["nope"]) + + +def test_pop_field_validator_is_noop_when_absent(): + class M(BaseModel): + a: str + + pop_field_validator(M, "nonexistent") # must not raise + + +def test_replace_module_str_replaces_substring(): + mod = ModuleType("dummy") + mod.P = "hello world" + replace_module_str(mod, "P", "world", "there") + assert mod.P == "hello there" + + +def test_replace_module_str_raises_on_missing_substring(): + mod = ModuleType("dummy") + mod.P = "hello world" + with pytest.raises(PatchDriftError): + replace_module_str(mod, "P", "absent", "x")