From cd24c48f12fe8c943ec10d870a81455eec413360 Mon Sep 17 00:00:00 2001 From: Vitalii <87299468+toderian@users.noreply.github.com> Date: Wed, 15 Jul 2026 13:11:09 +0300 Subject: [PATCH 1/6] RM feat NIS2 review submission (#469) * feat: add rulebook assessment backend What changed: - Added NIS2 baseline rulebook assessment models, service, and plugin endpoints. - Persisted assessment CIDs in job metadata and mutable reviewer state/audit in CStore. - Added purge cleanup for rulebook artifacts and review rows. - Added focused backend tests for generation, redaction, review semantics, eligibility failures, and purge cleanup. Why: - Implements RM-029 Phase 1 backend contract for RedMesh evidence-readiness assessments. Checks: - python3 -m py_compile changed backend files: passed - python3 -m unittest extensions.business.cybersec.red_mesh.tests.test_rulebook_assessment -q: passed - python3 -m pytest extensions/business/cybersec/red_mesh/tests/test_rulebook_assessment.py -q: blocked, pytest is not installed in this environment * feat: add NIS2 assessment ensure contract What changed: - Added idempotent force=false ensure behavior for rulebook assessments. - Relaxed assessment eligibility to completed pass evidence and added richer run metadata. - Covered historical rulebook artifact purge and failure status behavior. Why: - RM-030 needs default-on NIS2 readiness generation without duplicate same-pass artifacts. Checks: - python3 -m unittest extensions.business.cybersec.red_mesh.tests.test_rulebook_assessment -v: passed - python3 -m py_compile extensions/business/cybersec/red_mesh/services/rulebook_assessment.py extensions/business/cybersec/red_mesh/services/control.py extensions/business/cybersec/red_mesh/pentester_api_01.py extensions/business/cybersec/red_mesh/tests/test_rulebook_assessment.py: passed - python3 -m pytest extensions/business/cybersec/red_mesh/tests/test_rulebook_assessment.py -q: not run, pytest is not installed * feat: run NIS2 ensure after completed passes What changed: - Added a best-effort NIS2 rulebook ensure hook after pass report persistence. - Kept finalization and continuous scheduling non-blocking when NIS2 ensure fails. - Added tests for singlepass, failed ensure, and continuous pass refresh behavior. Why: - RM-030 requires default-on NIS2 readiness generation after eligible scan evidence exists. Checks: - python3 -m unittest extensions.business.cybersec.red_mesh.tests.test_api.TestPhase2PassFinalization -v: passed - python3 -m py_compile extensions/business/cybersec/red_mesh/services/finalization.py extensions/business/cybersec/red_mesh/tests/test_api.py: passed * feat: add rulebook submission storage models What changed: - added review revisions and typed pending/reference/registry submission models - added the dedicated CStore submission registry helpers - bumped assessment schema to 1.1.0 with artifact_kind metadata - excluded mutable drafts from persisted generated assessments and invalidated unsafe legacy cache reuse Why: - establish the durable and privacy-safe storage boundary before submission operations Checks: - python3 -m py_compile ... - python3 -m unittest extensions.business.cybersec.red_mesh.tests.test_rulebook_assessment -v (12 passed) - git diff --check * feat: add versioned rulebook review submission What changed: - added revisioned draft save, idempotent submit, and guarded reopen operations - implemented recoverable pending transitions across CStore and R1FS - exposed native history, staleness, and legacy revision-zero compatibility - extended purge and force-purge to formal/pending submission CIDs with shared-reference safety - added plugin endpoints and failure/concurrency/redaction coverage Why: - make explicit NIS2 review submission immutable, retryable, and purge-safe Checks: - python3 -m py_compile on changed backend modules - python3 -m unittest extensions.business.cybersec.red_mesh.tests.test_rulebook_assessment (24 passed) - python3 -m unittest extensions.business.cybersec.red_mesh.tests.test_api (165 passed) - git diff --check * test: cover two-revision submission purge smoke * fix: purge R1FS artifacts beyond unpinning * fix: harden rulebook submission recovery and purge * fix: fence submission inputs during cleanup and retry * fix: preserve submission integrity in legacy compatibility * fix: align purge with relay deletion semantics Treat acknowledged R1FS unpin and garbage-collection requests as purge completion without immediate CID read-back. Keep force and orphan cleanup fail-closed, prevent report reads from pinning content, and cover the relay acknowledgement boundary. --- .../cybersec/red_mesh/models/__init__.py | 26 + .../cybersec/red_mesh/models/cstore.py | 4 + .../cybersec/red_mesh/models/rulebook.py | 283 +++ .../cybersec/red_mesh/pentester_api_01.py | 123 +- .../red_mesh/repositories/artifacts.py | 11 +- .../cybersec/red_mesh/repositories/cstore.py | 109 + .../cybersec/red_mesh/services/__init__.py | 22 + .../cybersec/red_mesh/services/control.py | 313 ++- .../red_mesh/services/finalization.py | 22 + .../red_mesh/services/rulebook_assessment.py | 2094 +++++++++++++++++ .../cybersec/red_mesh/services/triage.py | 21 + .../cybersec/red_mesh/tests/test_api.py | 59 + .../red_mesh/tests/test_integration.py | 182 +- .../red_mesh/tests/test_repositories.py | 15 + .../tests/test_rulebook_assessment.py | 1132 +++++++++ 15 files changed, 4397 insertions(+), 19 deletions(-) create mode 100644 extensions/business/cybersec/red_mesh/models/rulebook.py create mode 100644 extensions/business/cybersec/red_mesh/services/rulebook_assessment.py create mode 100644 extensions/business/cybersec/red_mesh/tests/test_rulebook_assessment.py diff --git a/extensions/business/cybersec/red_mesh/models/__init__.py b/extensions/business/cybersec/red_mesh/models/__init__.py index 75dd64ccf..b542ca7e0 100644 --- a/extensions/business/cybersec/red_mesh/models/__init__.py +++ b/extensions/business/cybersec/red_mesh/models/__init__.py @@ -52,6 +52,20 @@ FindingTriageState, VALID_TRIAGE_STATUSES, ) +from extensions.business.cybersec.red_mesh.models.rulebook import ( + RULEBOOK_ASSESSMENT_SCHEMA, + RULEBOOK_ASSESSMENT_SCHEMA_VERSION, + RULEBOOK_SUBMISSION_CONTRACT_VERSION, + RulebookPendingSubmission, + RulebookReviewAuditEntry, + RulebookReviewState, + RulebookSubmissionReference, + RulebookSubmissionRegistry, + VALID_RULEBOOK_ANSWER_VALUES, + VALID_RULEBOOK_CHECK_STATUSES, + VALID_RULEBOOK_REVIEW_STATES, + VALID_RULEBOOK_SUBMISSION_STATES, +) from extensions.business.cybersec.red_mesh.models.engagement import ( ASSET_EXPOSURES, AuthorizationRef, @@ -109,6 +123,18 @@ "FindingTriageState", "FindingTriageAuditEntry", "VALID_TRIAGE_STATUSES", + "RULEBOOK_ASSESSMENT_SCHEMA", + "RULEBOOK_ASSESSMENT_SCHEMA_VERSION", + "RULEBOOK_SUBMISSION_CONTRACT_VERSION", + "RulebookReviewState", + "RulebookReviewAuditEntry", + "RulebookSubmissionReference", + "RulebookPendingSubmission", + "RulebookSubmissionRegistry", + "VALID_RULEBOOK_ANSWER_VALUES", + "VALID_RULEBOOK_CHECK_STATUSES", + "VALID_RULEBOOK_REVIEW_STATES", + "VALID_RULEBOOK_SUBMISSION_STATES", # engagement "Contact", "EngagementContext", diff --git a/extensions/business/cybersec/red_mesh/models/cstore.py b/extensions/business/cybersec/red_mesh/models/cstore.py index f88b53c58..f3962b7a4 100644 --- a/extensions/business/cybersec/red_mesh/models/cstore.py +++ b/extensions/business/cybersec/red_mesh/models/cstore.py @@ -146,6 +146,7 @@ class CStoreJobRunning: stix_export: dict = None opencti_export: dict = None taxii_export: dict = None + rulebook_assessments: dict = None graybox_assignment_summary: dict = None blockchain_attestation_enabled: bool = False start_attestation_required: bool = False @@ -195,6 +196,7 @@ def from_dict(cls, d: dict) -> CStoreJobRunning: stix_export=d.get("stix_export"), opencti_export=d.get("opencti_export"), taxii_export=d.get("taxii_export"), + rulebook_assessments=d.get("rulebook_assessments"), graybox_assignment_summary=d.get("graybox_assignment_summary"), blockchain_attestation_enabled=d.get("blockchain_attestation_enabled", False), start_attestation_required=d.get("start_attestation_required", False), @@ -243,6 +245,7 @@ class CStoreJobFinalized: stix_export: dict = None opencti_export: dict = None taxii_export: dict = None + rulebook_assessments: dict = None graybox_assignment_summary: dict = None blockchain_attestation_enabled: bool = False start_attestation_required: bool = False @@ -290,6 +293,7 @@ def from_dict(cls, d: dict) -> CStoreJobFinalized: stix_export=d.get("stix_export"), opencti_export=d.get("opencti_export"), taxii_export=d.get("taxii_export"), + rulebook_assessments=d.get("rulebook_assessments"), graybox_assignment_summary=d.get("graybox_assignment_summary"), blockchain_attestation_enabled=d.get("blockchain_attestation_enabled", False), start_attestation_required=d.get("start_attestation_required", False), diff --git a/extensions/business/cybersec/red_mesh/models/rulebook.py b/extensions/business/cybersec/red_mesh/models/rulebook.py new file mode 100644 index 000000000..9bbd86fbd --- /dev/null +++ b/extensions/business/cybersec/red_mesh/models/rulebook.py @@ -0,0 +1,283 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass + +from extensions.business.cybersec.red_mesh.models.shared import _strip_none + + +RULEBOOK_ASSESSMENT_SCHEMA = "redmesh.rulebook_assessment.v1" +RULEBOOK_ASSESSMENT_SCHEMA_VERSION = "1.1.0" +RULEBOOK_SUBMISSION_CONTRACT_VERSION = "1.0.0" + +VALID_RULEBOOK_CHECK_STATUSES = frozenset({ + "supported", + "gap", + "needs_review", + "not_observable", + "not_applicable", +}) + +VALID_RULEBOOK_REVIEW_STATES = frozenset({ + "draft", + "submitted", + "reviewed", +}) + +VALID_RULEBOOK_SUBMISSION_STATES = frozenset({ + "prepared", + "artifact_written", + "reference_recorded", +}) + +VALID_RULEBOOK_ANSWER_VALUES = frozenset({ + "yes", + "no", + "unknown", + "not_applicable", +}) + + +def _coerce_answers(value): + if not isinstance(value, dict): + return {} + answers = {} + for question_id, raw_answer in value.items(): + if not isinstance(question_id, str) or not question_id.strip(): + continue + payload = raw_answer if isinstance(raw_answer, dict) else {"value": raw_answer} + answer_value = str(payload.get("value") or "unknown").strip().lower() + if answer_value not in VALID_RULEBOOK_ANSWER_VALUES: + answer_value = "unknown" + answers[question_id.strip()] = _strip_none({ + "value": answer_value, + "note": str(payload.get("note") or "")[:1000], + "reviewer": str(payload.get("reviewer") or "")[:200], + "updated_at": float(payload.get("updated_at", 0.0) or 0.0), + }) + return answers + + +@dataclass(frozen=True) +class RulebookReviewState: + job_id: str + profile_id: str + profile_version: str = "" + review_state: str = "draft" + reviewer: str = "" + note: str = "" + answers: dict = None + updated_at: float = 0.0 + review_revision: int = 0 + last_reopen_idempotency_key: str = "" + last_reopen_from_revision: int = 0 + last_reopen_actor: str = "" + + def to_dict(self) -> dict: + return _strip_none({ + **asdict(self), + "answers": _coerce_answers(self.answers), + }) + + @classmethod + def from_dict(cls, payload: dict) -> "RulebookReviewState": + review_state = str(payload.get("review_state") or "draft").strip().lower() + if review_state not in VALID_RULEBOOK_REVIEW_STATES: + raise ValueError(f"Unsupported rulebook review state: {review_state}") + return cls( + job_id=str(payload["job_id"]), + profile_id=str(payload["profile_id"]), + profile_version=str(payload.get("profile_version") or ""), + review_state=review_state, + reviewer=str(payload.get("reviewer") or "")[:200], + note=str(payload.get("note") or "")[:1000], + answers=_coerce_answers(payload.get("answers")), + updated_at=float(payload.get("updated_at", 0.0) or 0.0), + review_revision=max(0, int(payload.get("review_revision", 0) or 0)), + last_reopen_idempotency_key=str(payload.get("last_reopen_idempotency_key") or "")[:200], + last_reopen_from_revision=max(0, int(payload.get("last_reopen_from_revision", 0) or 0)), + last_reopen_actor=str(payload.get("last_reopen_actor") or "")[:200], + ) + + +@dataclass(frozen=True) +class RulebookReviewAuditEntry: + job_id: str + profile_id: str + profile_version: str + review_state: str + reviewer: str = "" + note: str = "" + changed_question_ids: list = None + previous_answers: dict = None + current_answers: dict = None + timestamp: float = 0.0 + review_revision: int = 0 + + def to_dict(self) -> dict: + return _strip_none({ + **asdict(self), + "changed_question_ids": list(self.changed_question_ids or []), + "previous_answers": _coerce_answers(self.previous_answers), + "current_answers": _coerce_answers(self.current_answers), + }) + + @classmethod + def from_dict(cls, payload: dict) -> "RulebookReviewAuditEntry": + review_state = str(payload.get("review_state") or "draft").strip().lower() + if review_state not in VALID_RULEBOOK_REVIEW_STATES: + raise ValueError(f"Unsupported rulebook review state: {review_state}") + return cls( + job_id=str(payload["job_id"]), + profile_id=str(payload["profile_id"]), + profile_version=str(payload.get("profile_version") or ""), + review_state=review_state, + reviewer=str(payload.get("reviewer") or "")[:200], + note=str(payload.get("note") or "")[:1000], + changed_question_ids=[ + str(item) + for item in (payload.get("changed_question_ids") or []) + if isinstance(item, str) and item + ], + previous_answers=_coerce_answers(payload.get("previous_answers")), + current_answers=_coerce_answers(payload.get("current_answers")), + timestamp=float(payload.get("timestamp", 0.0) or 0.0), + review_revision=max(0, int(payload.get("review_revision", 0) or 0)), + ) + + +@dataclass(frozen=True) +class RulebookSubmissionReference: + revision: int + cid: str + submitted_at: float + actor: str + pass_nr: int + profile_id: str + profile_version: str + schema_version: str + review_revision: int + idempotency_key: str = "" + fingerprint: str = "" + legacy: bool = False + + def to_dict(self) -> dict: + return _strip_none(asdict(self)) + + @classmethod + def from_dict(cls, payload: dict) -> "RulebookSubmissionReference": + revision = int(payload.get("revision", 0) or 0) + if revision < 0: + raise ValueError("Submission revision cannot be negative") + cid = str(payload.get("cid") or payload.get("artifact_cid") or "").strip() + if not cid: + raise ValueError("Submission reference requires a CID") + return cls( + revision=revision, + cid=cid, + submitted_at=float(payload.get("submitted_at", 0.0) or 0.0), + actor=str(payload.get("actor") or "")[:200], + pass_nr=max(0, int(payload.get("pass_nr", 0) or 0)), + profile_id=str(payload.get("profile_id") or ""), + profile_version=str(payload.get("profile_version") or ""), + schema_version=str(payload.get("schema_version") or ""), + review_revision=max(0, int(payload.get("review_revision", 0) or 0)), + idempotency_key=str(payload.get("idempotency_key") or "")[:200], + fingerprint=str(payload.get("fingerprint") or "")[:128], + legacy=bool(payload.get("legacy", False)), + ) + + +@dataclass(frozen=True) +class RulebookPendingSubmission: + target_revision: int + expected_review_revision: int + expected_pass_nr: int + expected_profile_version: str + actor: str + idempotency_key: str + fingerprint: str + state: str = "prepared" + created_at: float = 0.0 + updated_at: float = 0.0 + attempt_count: int = 1 + cid: str = "" + last_error: dict = None + + def to_dict(self) -> dict: + return _strip_none(asdict(self)) + + @classmethod + def from_dict(cls, payload: dict) -> "RulebookPendingSubmission": + state = str(payload.get("state") or "prepared") + if state not in VALID_RULEBOOK_SUBMISSION_STATES: + raise ValueError(f"Unsupported pending submission state: {state}") + target_revision = int(payload.get("target_revision", 0) or 0) + if target_revision < 1: + raise ValueError("Pending submission requires a positive target revision") + idempotency_key = str(payload.get("idempotency_key") or "").strip() + fingerprint = str(payload.get("fingerprint") or "").strip() + if not idempotency_key or not fingerprint: + raise ValueError("Pending submission requires idempotency key and fingerprint") + last_error = payload.get("last_error") + return cls( + target_revision=target_revision, + expected_review_revision=max(0, int(payload.get("expected_review_revision", 0) or 0)), + expected_pass_nr=max(0, int(payload.get("expected_pass_nr", 0) or 0)), + expected_profile_version=str(payload.get("expected_profile_version") or ""), + actor=str(payload.get("actor") or "")[:200], + idempotency_key=idempotency_key[:200], + fingerprint=fingerprint[:128], + state=state, + created_at=float(payload.get("created_at", 0.0) or 0.0), + updated_at=float(payload.get("updated_at", 0.0) or 0.0), + attempt_count=max(1, int(payload.get("attempt_count", 1) or 1)), + cid=str(payload.get("cid") or ""), + last_error=dict(last_error) if isinstance(last_error, dict) else None, + ) + + +@dataclass(frozen=True) +class RulebookSubmissionRegistry: + contract_version: str = RULEBOOK_SUBMISSION_CONTRACT_VERSION + latest_revision: int = 0 + submissions: list = None + pending: dict = None + + def to_dict(self) -> dict: + submissions = [ + item.to_dict() if isinstance(item, RulebookSubmissionReference) else RulebookSubmissionReference.from_dict(item).to_dict() + for item in (self.submissions or []) + ] + pending = self.pending + if isinstance(pending, RulebookPendingSubmission): + pending = pending.to_dict() + elif isinstance(pending, dict): + pending = RulebookPendingSubmission.from_dict(pending).to_dict() + return _strip_none({ + "contract_version": self.contract_version, + "latest_revision": max([self.latest_revision] + [item["revision"] for item in submissions]), + "submissions": submissions, + "pending": pending, + }) + + @classmethod + def from_dict(cls, payload: dict) -> "RulebookSubmissionRegistry": + contract_version = str(payload.get("contract_version") or RULEBOOK_SUBMISSION_CONTRACT_VERSION) + if contract_version != RULEBOOK_SUBMISSION_CONTRACT_VERSION: + raise ValueError(f"Unsupported submission contract version: {contract_version}") + submissions = [ + RulebookSubmissionReference.from_dict(item).to_dict() + for item in (payload.get("submissions") or []) + if isinstance(item, dict) + ] + pending_payload = payload.get("pending") + pending = RulebookPendingSubmission.from_dict(pending_payload).to_dict() if isinstance(pending_payload, dict) else None + latest_revision = max( + [max(0, int(payload.get("latest_revision", 0) or 0))] + [item["revision"] for item in submissions] + ) + return cls( + contract_version=contract_version, + latest_revision=latest_revision, + submissions=submissions, + pending=pending, + ) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index e8bb699f1..75cd7b698 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -91,8 +91,14 @@ get_api_operation_status, get_detection_correlation, get_opencti_export_status, + get_rulebook_assessment_status, + get_rulebook_review, + reopen_rulebook_review, get_stix_export_status, get_taxii_export_status, + generate_rulebook_assessment, + save_rulebook_review_draft, + submit_rulebook_review, push_to_opencti, publish_to_taxii, get_job_analysis, @@ -140,6 +146,7 @@ stop_monitoring, test_event_export, update_finding_triage, + update_rulebook_review, validation_error, ) from .model_testing import ( @@ -2920,6 +2927,7 @@ def _build_job_archive(self, job_key, job_specs): stix_export=job_specs.get("stix_export"), opencti_export=job_specs.get("opencti_export"), taxii_export=job_specs.get("taxii_export"), + rulebook_assessments=job_specs.get("rulebook_assessments"), blockchain_attestation_enabled=bool(job_specs.get("blockchain_attestation_enabled", False)), start_attestation_required=bool(job_specs.get("start_attestation_required", False)), end_attestation_required=bool(job_specs.get("end_attestation_required", False)), @@ -3962,6 +3970,119 @@ def get_stix_export_status(self, job_id: str): """Return STIX 2.1 manual export status for a job.""" return get_stix_export_status(self, job_id) + @BasePlugin.endpoint(method="post") + def generate_rulebook_assessment( + self, + job_id: str, + profile_id: str = None, + pass_nr: int = None, + persist: bool = True, + force: bool = True, + ): + """Build and optionally persist a RedMesh rulebook evidence assessment.""" + return generate_rulebook_assessment( + self, + job_id, + profile_id=profile_id, + pass_nr=pass_nr, + persist=persist, + force=force, + ) + + @BasePlugin.endpoint + def get_rulebook_assessment_status(self, job_id: str, profile_id: str = None): + """Return RedMesh rulebook assessment generation status for a job.""" + return get_rulebook_assessment_status(self, job_id, profile_id=profile_id) + + @BasePlugin.endpoint + def get_rulebook_review(self, job_id: str, profile_id: str = None): + """Return mutable reviewer state for a RedMesh rulebook profile.""" + return get_rulebook_review(self, job_id, profile_id=profile_id) + + @BasePlugin.endpoint(method="post") + def save_rulebook_review_draft( + self, + job_id: str, + profile_id: str = None, + answers: dict = None, + actor: str = "", + note: str = "", + expected_review_revision: int = None, + ): + """Save mutable rulebook review draft state without persisting an R1FS artifact.""" + return save_rulebook_review_draft( + self, + job_id, + profile_id=profile_id, + answers=answers, + actor=actor, + note=note, + expected_review_revision=expected_review_revision, + ) + + @BasePlugin.endpoint(method="post") + def submit_rulebook_review( + self, + job_id: str, + profile_id: str = None, + expected_review_revision: int = None, + expected_pass_nr: int = None, + expected_profile_version: str = None, + idempotency_key: str = "", + actor: str = "", + ): + """Persist and register one immutable rulebook review submission revision.""" + return submit_rulebook_review( + self, + job_id, + profile_id=profile_id, + expected_review_revision=expected_review_revision, + expected_pass_nr=expected_pass_nr, + expected_profile_version=expected_profile_version, + idempotency_key=idempotency_key, + actor=actor, + ) + + @BasePlugin.endpoint(method="post") + def reopen_rulebook_review( + self, + job_id: str, + profile_id: str = None, + expected_review_revision: int = None, + idempotency_key: str = "", + actor: str = "", + ): + """Create an editable draft while preserving every submitted revision.""" + return reopen_rulebook_review( + self, + job_id, + profile_id=profile_id, + expected_review_revision=expected_review_revision, + idempotency_key=idempotency_key, + actor=actor, + ) + + @BasePlugin.endpoint(method="post") + def update_rulebook_review( + self, + job_id: str, + profile_id: str = None, + answers: dict = None, + reviewer: str = "", + note: str = "", + review_state: str = "draft", + ): + """Save mutable reviewer answers for a RedMesh rulebook profile.""" + return update_rulebook_review( + self, + job_id, + profile_id=profile_id, + answers=answers, + reviewer=reviewer, + note=note, + review_state=review_state, + ) + @BasePlugin.endpoint(method="post") def dry_run_opencti_export(self, job_id: str, pass_nr: int = None): """Build and persist a redacted OpenCTI STIX bundle without pushing it.""" @@ -4042,7 +4163,7 @@ def get_report(self, cid: str): if not cid: return {"error": "No CID provided"} try: - report = self.r1fs.get_json(cid) + report = self.r1fs.get_json(cid, pin=False) if report is None: return {"error": "Report not found", "cid": cid} if is_restricted_raw_evidence_artifact(report): diff --git a/extensions/business/cybersec/red_mesh/repositories/artifacts.py b/extensions/business/cybersec/red_mesh/repositories/artifacts.py index 323f328d9..15ab21534 100644 --- a/extensions/business/cybersec/red_mesh/repositories/artifacts.py +++ b/extensions/business/cybersec/red_mesh/repositories/artifacts.py @@ -27,9 +27,18 @@ def put_json(self, payload, *, show_logs=False, secret=None): return self.owner.r1fs.add_json(payload, show_logs=show_logs, secret=secret) return self.owner.r1fs.add_json(payload, show_logs=show_logs) - def delete(self, cid, *, show_logs=False, raise_on_error=False): + def delete(self, cid, *, show_logs=False, raise_on_error=False, purge=False): if not cid: return False + if purge: + return self.owner.r1fs.delete_file( + cid, + unpin_remote=True, + run_gc=True, + cleanup_local_files=True, + show_logs=show_logs, + raise_on_error=raise_on_error, + ) return self.owner.r1fs.delete_file(cid, show_logs=show_logs, raise_on_error=raise_on_error) def get_job_config(self, job_specs): diff --git a/extensions/business/cybersec/red_mesh/repositories/cstore.py b/extensions/business/cybersec/red_mesh/repositories/cstore.py index a919ef093..15b01b0bd 100644 --- a/extensions/business/cybersec/red_mesh/repositories/cstore.py +++ b/extensions/business/cybersec/red_mesh/repositories/cstore.py @@ -3,6 +3,9 @@ CStoreJobRunning, FindingTriageAuditEntry, FindingTriageState, + RulebookReviewAuditEntry, + RulebookReviewState, + RulebookSubmissionRegistry, WorkerProgress, ) @@ -46,6 +49,18 @@ def _triage_audit_hkey(self): def _model_test_raw_evidence_hkey(self): return f"{self.owner.cfg_instance_id}:model_test_raw_evidence" + @property + def _rulebook_review_hkey(self): + return f"{self.owner.cfg_instance_id}:rulebook_review" + + @property + def _rulebook_review_audit_hkey(self): + return f"{self.owner.cfg_instance_id}:rulebook_review:audit" + + @property + def _rulebook_review_submissions_hkey(self): + return f"{self.owner.cfg_instance_id}:rulebook_review:submissions" + def get_job(self, job_id): return self.owner.chainstore_hget(hkey=self._jobs_hkey, key=job_id) @@ -304,3 +319,97 @@ def delete_job_triage(self, job_id): value=None, ) return + + @staticmethod + def rulebook_key(job_id, profile_id): + return f"{job_id}:{profile_id}" + + def get_rulebook_review(self, job_id, profile_id): + return self.owner.chainstore_hget( + hkey=self._rulebook_review_hkey, + key=self.rulebook_key(job_id, profile_id), + ) + + def get_rulebook_review_model(self, job_id, profile_id): + payload = self.get_rulebook_review(job_id, profile_id) + if not isinstance(payload, dict): + return None + return RulebookReviewState.from_dict(payload) + + def put_rulebook_review(self, review): + if isinstance(review, RulebookReviewState): + payload = review.to_dict() + else: + payload = RulebookReviewState.from_dict(review).to_dict() + self.owner.chainstore_hset( + hkey=self._rulebook_review_hkey, + key=self.rulebook_key(payload["job_id"], payload["profile_id"]), + value=payload, + ) + return payload + + def get_rulebook_review_audit(self, job_id, profile_id): + payload = self.owner.chainstore_hget( + hkey=self._rulebook_review_audit_hkey, + key=self.rulebook_key(job_id, profile_id), + ) + return payload if isinstance(payload, list) else [] + + def append_rulebook_review_audit(self, entry): + if isinstance(entry, RulebookReviewAuditEntry): + payload = entry.to_dict() + else: + payload = RulebookReviewAuditEntry.from_dict(entry).to_dict() + key = self.rulebook_key(payload["job_id"], payload["profile_id"]) + audit_log = list(self.get_rulebook_review_audit(payload["job_id"], payload["profile_id"])) + audit_log.append(payload) + self.owner.chainstore_hset(hkey=self._rulebook_review_audit_hkey, key=key, value=audit_log) + return audit_log + + def get_rulebook_submission_registry(self, job_id, profile_id): + return self.owner.chainstore_hget( + hkey=self._rulebook_review_submissions_hkey, + key=self.rulebook_key(job_id, profile_id), + ) + + def get_rulebook_submission_registry_model(self, job_id, profile_id): + payload = self.get_rulebook_submission_registry(job_id, profile_id) + if not isinstance(payload, dict): + return None + return RulebookSubmissionRegistry.from_dict(payload) + + def put_rulebook_submission_registry(self, job_id, profile_id, registry): + if isinstance(registry, RulebookSubmissionRegistry): + payload = registry.to_dict() + else: + payload = RulebookSubmissionRegistry.from_dict(registry).to_dict() + self.owner.chainstore_hset( + hkey=self._rulebook_review_submissions_hkey, + key=self.rulebook_key(job_id, profile_id), + value=payload, + ) + return payload + + def list_job_rulebook_submission_registries(self, job_id): + payload = self.owner.chainstore_hgetall(hkey=self._rulebook_review_submissions_hkey) or {} + prefix = f"{job_id}:" + return { + key[len(prefix):]: value + for key, value in payload.items() + if isinstance(key, str) and key.startswith(prefix) and isinstance(value, dict) + } + + def delete_job_rulebook_reviews(self, job_id): + prefix = f"{job_id}:" + for hkey in ( + self._rulebook_review_hkey, + self._rulebook_review_audit_hkey, + self._rulebook_review_submissions_hkey, + ): + payload = self.owner.chainstore_hgetall(hkey=hkey) or {} + if not isinstance(payload, dict): + continue + for key in list(payload): + if isinstance(key, str) and key.startswith(prefix): + self.owner.chainstore_hset(hkey=hkey, key=key, value=None) + return diff --git a/extensions/business/cybersec/red_mesh/services/__init__.py b/extensions/business/cybersec/red_mesh/services/__init__.py index 7abf84570..facdfb765 100644 --- a/extensions/business/cybersec/red_mesh/services/__init__.py +++ b/extensions/business/cybersec/red_mesh/services/__init__.py @@ -43,6 +43,18 @@ export_stix_bundle, get_stix_export_status, ) +from .rulebook_assessment import ( + DEFAULT_RULEBOOK_PROFILE_ID, + build_rulebook_assessment, + generate_rulebook_assessment, + get_rulebook_assessment_status, + get_rulebook_review, + list_rulebook_profiles, + reopen_rulebook_review, + save_rulebook_review_draft, + submit_rulebook_review, + update_rulebook_review, +) from .opencti_export import ( dry_run_opencti_export, get_opencti_export_status, @@ -183,6 +195,7 @@ "DEFAULT_EVENT_EXPORT_CONFIG", "DEFAULT_MODEL_TESTING_CONFIG", "DEFAULT_OPENCTI_EXPORT_CONFIG", + "DEFAULT_RULEBOOK_PROFILE_ID", "DEFAULT_STIX_EXPORT_CONFIG", "DEFAULT_SURICATA_CORRELATION_CONFIG", "DEFAULT_TAXII_EXPORT_CONFIG", @@ -213,6 +226,7 @@ "build_finding_event", "build_lifecycle_event", "build_misp_event", + "build_rulebook_assessment", "build_stix_bundle", "build_redmesh_event", "build_service_observed_event", @@ -223,8 +237,11 @@ "dry_run_taxii_export", "export_misp_json", "export_stix_bundle", + "generate_rulebook_assessment", "get_misp_export_status", "get_opencti_export_status", + "get_rulebook_assessment_status", + "get_rulebook_review", "get_stix_export_status", "get_taxii_export_status", "push_to_misp", @@ -268,6 +285,10 @@ "launch_webapp_scan", "list_local_jobs", "list_network_jobs", + "list_rulebook_profiles", + "reopen_rulebook_review", + "save_rulebook_review_draft", + "submit_rulebook_review", "maybe_finalize_pass", "normalize_common_launch_options", "parse_exceptions", @@ -292,6 +313,7 @@ "get_job_archive_with_triage", "get_job_triage", "update_finding_triage", + "update_rulebook_review", "validation_error", # engagement context (Phase 3) "AuthorizationUploadError", diff --git a/extensions/business/cybersec/red_mesh/services/control.py b/extensions/business/cybersec/red_mesh/services/control.py index 5412a70fe..4864d4d6f 100644 --- a/extensions/business/cybersec/red_mesh/services/control.py +++ b/extensions/business/cybersec/red_mesh/services/control.py @@ -1,3 +1,5 @@ +from contextlib import ExitStack + from ..constants import ( JOB_STATUS_FINALIZED, JOB_STATUS_RUNNING, @@ -101,6 +103,16 @@ def stop_and_delete_job(owner, job_id: str): def purge_job(owner, job_id: str): + """Serialize purge with every supported rulebook review mutation for the job.""" + from .rulebook_assessment import _submission_lock, list_rulebook_profiles + + with ExitStack() as stack: + for profile in sorted(list_rulebook_profiles(), key=lambda item: item["profile_id"]): + stack.enter_context(_submission_lock(owner, job_id, profile["profile_id"])) + return _purge_job_locked(owner, job_id) + + +def _purge_job_locked(owner, job_id: str): """ Purge a job: delete all R1FS artifacts, clean up live progress keys, then tombstone the CStore entry. @@ -151,6 +163,59 @@ def _track(cid, source): for addr, w in workers.items(): _track(w.get("report_cid"), f"workers[{addr}].report_cid") + rulebook_assessments = job_specs.get("rulebook_assessments") + if isinstance(rulebook_assessments, dict): + for profile_id, meta in rulebook_assessments.items(): + if isinstance(meta, dict): + _track(meta.get("artifact_cid"), f"rulebook_assessments[{profile_id}].artifact_cid") + for hi, historical in enumerate(meta.get("history") or []): + if isinstance(historical, dict): + _track(historical.get("artifact_cid"), f"rulebook_assessments[{profile_id}].history[{hi}].artifact_cid") + + submission_hkey = f"{owner.cfg_instance_id}:rulebook_review:submissions" + submission_key_prefix = f"{job_id}:" + all_submission_rows = owner.chainstore_hgetall(hkey=submission_hkey) or {} + formal_submission_cids = set() + if isinstance(all_submission_rows, dict): + for key, registry in all_submission_rows.items(): + if not isinstance(key, str) or not key.startswith(submission_key_prefix) or not isinstance(registry, dict): + continue + for reference in registry.get("submissions") or []: + if isinstance(reference, dict) and isinstance(reference.get("cid"), str) and reference.get("cid"): + formal_submission_cids.add(reference["cid"]) + _track(reference["cid"], f"rulebook_review_submissions[{key}].submissions") + pending = registry.get("pending") + if isinstance(pending, dict) and isinstance(pending.get("cid"), str) and pending.get("cid"): + formal_submission_cids.add(pending["cid"]) + _track(pending["cid"], f"rulebook_review_submissions[{key}].pending") + + other_job_cids = set() + for key, registry in all_submission_rows.items(): + if not isinstance(key, str) or key.startswith(submission_key_prefix) or not isinstance(registry, dict): + continue + for reference in registry.get("submissions") or []: + if isinstance(reference, dict) and isinstance(reference.get("cid"), str) and reference.get("cid"): + other_job_cids.add(reference["cid"]) + pending = registry.get("pending") + if isinstance(pending, dict) and isinstance(pending.get("cid"), str) and pending.get("cid"): + other_job_cids.add(pending["cid"]) + all_jobs = _job_repo(owner).list_jobs() or {} + if isinstance(all_jobs, dict): + for other_job_id, other_payload in all_jobs.items(): + if other_job_id != job_id and isinstance(other_payload, dict): + other_job_cids.update(_collect_cids_from_raw(other_payload)) + shared_cids = formal_submission_cids & other_job_cids + if shared_cids: + owner.P(f"[PURGE] Shared submission CIDs retained: {sorted(shared_cids)}", color='r') + return { + "status": "partial", + "job_id": job_id, + "cids_deleted": 0, + "cids_failed": len(shared_cids), + "cids_total": len(cids), + "message": "Submission artifacts are referenced by another job; CStore was kept for retry.", + } + for ri, ref in enumerate(job_specs.get("pass_reports", [])): report_cid = ref.get("report_cid") if report_cid: @@ -170,9 +235,11 @@ def _track(cid, source): owner.P(f"[PURGE] Total CIDs collected: {len(cids)}: {sorted(cids)}") deleted, failed = 0, 0 + # R1FS deletion acknowledges local/relay unpin and garbage-collection requests. + # The relay sweep can take up to 24 hours, and reading here can rehydrate the CID. for cid in cids: try: - success = artifacts.delete(cid, show_logs=True, raise_on_error=False) + success = artifacts.delete(cid, show_logs=True, raise_on_error=False, purge=True) if success: deleted += 1 owner.P(f"[PURGE] Deleted CID {cid}") @@ -202,6 +269,7 @@ def _track(cid, source): _job_repo(owner).delete_live_progress(key) _job_repo(owner).delete_job_triage(job_id) + _job_repo(owner).delete_job_rulebook_reviews(job_id) _delete_job_record(owner, job_id) owner.P(f"Purged job {job_id}: {deleted}/{len(cids)} CIDs deleted.") @@ -223,36 +291,122 @@ def _collect_cids_from_raw(payload): yield from _collect_cids_from_raw(item) +def _collect_rulebook_submission_cids(payload): + if not isinstance(payload, dict): + return set() + cids = { + reference.get("cid") + for reference in payload.get("submissions") or [] + if isinstance(reference, dict) and isinstance(reference.get("cid"), str) and reference.get("cid") + } + pending = payload.get("pending") + if isinstance(pending, dict) and isinstance(pending.get("cid"), str) and pending.get("cid"): + cids.add(pending["cid"]) + return cids + + def _force_purge_job(owner, job_id, raw_payload, errors): + from .rulebook_assessment import _submission_lock, list_rulebook_profiles + + with ExitStack() as stack: + for profile in sorted(list_rulebook_profiles(), key=lambda item: item["profile_id"]): + stack.enter_context(_submission_lock(owner, job_id, profile["profile_id"])) + return _force_purge_job_locked(owner, job_id, raw_payload, errors) + + +def _force_purge_job_locked(owner, job_id, raw_payload, errors): """ Best-effort wipe of a job whose record could not be parsed/purged by ``stop_and_delete_job``. Returns (cids_deleted, cids_failed). - Scans the raw payload for ``*_cid`` fields, attempts R1FS deletion, then - tombstones the CStore record and matching live/triage rows regardless. + Scans the raw payload for ``*_cid`` fields and attempts R1FS deletion. + Formal submission pointers remain retryable unless R1FS acknowledges deletion; + other legacy artifacts retain the existing best-effort force-wipe behavior. """ - cids = sorted({c for c in _collect_cids_from_raw(raw_payload) if isinstance(c, str)}) + cids = {c for c in _collect_cids_from_raw(raw_payload) if isinstance(c, str)} + submission_hkey = f"{owner.cfg_instance_id}:rulebook_review:submissions" + try: + submission_rows = owner.chainstore_hgetall(hkey=submission_hkey) or {} + except Exception as exc: + errors.append({"job_id": job_id, "scope": submission_hkey, "message": f"{type(exc).__name__}: {exc}"}) + owner.P(f"[PURGE_ALL_FORCE] Could not inspect submission registry for {job_id}; retaining rows.", color='r') + return 0, 1 + if not isinstance(submission_rows, dict): + errors.append({"job_id": job_id, "scope": submission_hkey, "message": "unexpected non-dict submission registry"}) + owner.P(f"[PURGE_ALL_FORCE] Invalid submission registry for {job_id}; retaining rows.", color='r') + return 0, 1 + shared_submission_cids = set() + job_submission_cids = set() + prefix = f"{job_id}:" + other_submission_cids = set() + for key, registry in submission_rows.items(): + if isinstance(key, str) and key.startswith(prefix): + job_submission_cids.update(_collect_rulebook_submission_cids(registry)) + elif isinstance(key, str): + other_submission_cids.update(_collect_rulebook_submission_cids(registry)) + shared_submission_cids = job_submission_cids & other_submission_cids + if job_submission_cids: + try: + all_jobs = _job_repo(owner).list_jobs() or {} + if not isinstance(all_jobs, dict): + raise TypeError("unexpected non-dict job registry") + for other_job_id, other_payload in all_jobs.items(): + if other_job_id != job_id and isinstance(other_payload, dict): + other_submission_cids.update(_collect_cids_from_raw(other_payload)) + shared_submission_cids = job_submission_cids & other_submission_cids + except Exception as exc: + errors.append({"job_id": job_id, "scope": owner.cfg_instance_id, "message": f"{type(exc).__name__}: {exc}"}) + owner.P(f"[PURGE_ALL_FORCE] Could not inspect shared CID references for {job_id}; retaining rows.", color='r') + return 0, len(job_submission_cids) + cids.update(job_submission_cids - shared_submission_cids) + cids.difference_update(shared_submission_cids) + cids = sorted(cids) cids_deleted = 0 - cids_failed = 0 + cids_failed = len(shared_submission_cids) + if shared_submission_cids: + errors.append({ + "job_id": job_id, + "scope": submission_hkey, + "message": f"retained shared submission CIDs: {sorted(shared_submission_cids)}", + }) artifacts = _artifact_repo(owner) + failed_submission_cids = set(shared_submission_cids) for cid in cids: try: - success = artifacts.delete(cid, show_logs=True, raise_on_error=False) + success = artifacts.delete(cid, show_logs=True, raise_on_error=False, purge=True) if success: cids_deleted += 1 owner.P(f"[PURGE_ALL_FORCE] Deleted CID {cid} for {job_id}") else: cids_failed += 1 + if cid in job_submission_cids: + failed_submission_cids.add(cid) owner.P(f"[PURGE_ALL_FORCE] delete returned False for CID {cid} ({job_id})", color='y') except Exception as exc: cids_failed += 1 + if cid in job_submission_cids: + failed_submission_cids.add(cid) owner.P(f"[PURGE_ALL_FORCE] Failed to delete CID {cid} ({job_id}): {exc}", color='r') errors.append({"job_id": job_id, "scope": "r1fs", "message": f"{type(exc).__name__}: {exc}"}) + if failed_submission_cids: + owner.P( + f"[PURGE_ALL_FORCE] Retaining CStore rows for {job_id}; formal submission CIDs require retry.", + color='r', + ) + return cids_deleted, cids_failed + cfg_instance_id = owner.cfg_instance_id prefix = f"{job_id}:" - for hkey in (f"{cfg_instance_id}:live", f"{cfg_instance_id}:triage", f"{cfg_instance_id}:triage:audit"): + for hkey in ( + f"{cfg_instance_id}:live", + f"{cfg_instance_id}:triage", + f"{cfg_instance_id}:triage:audit", + f"{cfg_instance_id}:rulebook_review", + f"{cfg_instance_id}:rulebook_review:audit", + f"{cfg_instance_id}:rulebook_review:submissions", + ): try: rows = owner.chainstore_hgetall(hkey=hkey) except Exception as exc: @@ -320,6 +474,8 @@ def purge_all_jobs(owner): cids_failed += fc_failed jobs_failed += 1 jobs_force_purged += 1 + if fc_failed: + failed_job_ids.add(job_id) continue if not isinstance(result, dict): @@ -329,6 +485,8 @@ def purge_all_jobs(owner): cids_failed += fc_failed jobs_failed += 1 jobs_force_purged += 1 + if fc_failed: + failed_job_ids.add(job_id) continue status = result.get("status") @@ -354,11 +512,16 @@ def purge_all_jobs(owner): cids_failed += fc_failed jobs_failed += 1 jobs_force_purged += 1 + if fc_failed: + failed_job_ids.add(job_id) cfg_instance_id = owner.cfg_instance_id live_hkey = f"{cfg_instance_id}:live" triage_hkey = f"{cfg_instance_id}:triage" triage_audit_hkey = f"{cfg_instance_id}:triage:audit" + rulebook_review_hkey = f"{cfg_instance_id}:rulebook_review" + rulebook_review_audit_hkey = f"{cfg_instance_id}:rulebook_review:audit" + rulebook_review_submissions_hkey = f"{cfg_instance_id}:rulebook_review:submissions" integrations_hkey = f"{cfg_instance_id}:integrations" def _job_id_from_compound_key(key): @@ -387,9 +550,145 @@ def _sweep_hash(hkey, expected_value_types): errors.append({"job_id": job_id_prefix or "", "scope": hkey, "message": f"{type(exc).__name__}: {exc}"}) return rows_deleted + def _sweep_submission_hash(): + nonlocal cids_deleted, cids_failed + + from .rulebook_assessment import _submission_lock, list_rulebook_profiles + + try: + initial_rows = owner.chainstore_hgetall(hkey=rulebook_review_submissions_hkey) + except Exception as exc: + cids_failed += 1 + failed_job_ids.update(raw_jobs) + errors.append({ + "job_id": "", + "scope": rulebook_review_submissions_hkey, + "message": f"{type(exc).__name__}: {exc}", + }) + return 0 + if not isinstance(initial_rows, dict): + cids_failed += 1 + failed_job_ids.update(raw_jobs) + errors.append({ + "job_id": "", + "scope": rulebook_review_submissions_hkey, + "message": "unexpected non-dict submission registry", + }) + return 0 + + initial_keys = list(initial_rows) + lock_targets = set() + for key in initial_keys: + job_id_prefix = _job_id_from_compound_key(key) + if job_id_prefix and isinstance(key, str) and ":" in key: + lock_targets.add((job_id_prefix, key.split(":", 1)[1])) + profile_ids = { + profile["profile_id"] + for profile in list_rulebook_profiles() + if isinstance(profile, dict) and isinstance(profile.get("profile_id"), str) + } + for failed_job_id in failed_job_ids: + for profile_id in profile_ids: + lock_targets.add((failed_job_id, profile_id)) + + rows_deleted = 0 + deletion_results = {} + artifacts = _artifact_repo(owner) + with ExitStack() as stack: + for lock_job_id, lock_profile_id in sorted(lock_targets): + stack.enter_context(_submission_lock(owner, lock_job_id, lock_profile_id)) + try: + rows = owner.chainstore_hgetall(hkey=rulebook_review_submissions_hkey) + except Exception as exc: + cids_failed += 1 + failed_job_ids.update(raw_jobs) + errors.append({ + "job_id": "", + "scope": rulebook_review_submissions_hkey, + "message": f"{type(exc).__name__}: {exc}", + }) + return 0 + if not isinstance(rows, dict): + cids_failed += 1 + failed_job_ids.update(raw_jobs) + errors.append({ + "job_id": "", + "scope": rulebook_review_submissions_hkey, + "message": "unexpected non-dict submission registry", + }) + return 0 + + protected_cids = set() + for failed_job_id in failed_job_ids: + raw_failed_job = raw_jobs.get(failed_job_id) + if isinstance(raw_failed_job, dict): + protected_cids.update(_collect_cids_from_raw(raw_failed_job)) + for key, value in rows.items(): + if _job_id_from_compound_key(key) in failed_job_ids: + protected_cids.update(_collect_rulebook_submission_cids(value)) + + for key in initial_keys: + value = rows.get(key) + job_id_prefix = _job_id_from_compound_key(key) + if not job_id_prefix or job_id_prefix in failed_job_ids or not isinstance(value, dict): + continue + row_cids = _collect_rulebook_submission_cids(value) + shared_cids = row_cids & protected_cids + if shared_cids: + failed_job_ids.add(job_id_prefix) + cids_failed += len(shared_cids) + errors.append({ + "job_id": job_id_prefix, + "scope": rulebook_review_submissions_hkey, + "message": f"retained orphan registry with shared CIDs: {sorted(shared_cids)}", + }) + continue + + row_failed = False + for cid in sorted(row_cids): + success = deletion_results.get(cid) + if success is None: + try: + success = artifacts.delete(cid, show_logs=True, raise_on_error=False, purge=True) + except Exception as exc: + success = False + errors.append({ + "job_id": job_id_prefix, + "scope": "r1fs", + "message": f"{type(exc).__name__}: {exc}", + }) + deletion_results[cid] = success + if success: + cids_deleted += 1 + else: + cids_failed += 1 + if not success: + row_failed = True + if row_failed: + failed_job_ids.add(job_id_prefix) + errors.append({ + "job_id": job_id_prefix, + "scope": rulebook_review_submissions_hkey, + "message": "orphan submission CID deletion was not acknowledged", + }) + continue + try: + owner.chainstore_hset(hkey=rulebook_review_submissions_hkey, key=key, value=None) + rows_deleted += 1 + except Exception as exc: + errors.append({ + "job_id": job_id_prefix, + "scope": rulebook_review_submissions_hkey, + "message": f"{type(exc).__name__}: {exc}", + }) + return rows_deleted + _sweep_hash(live_hkey, dict) _sweep_hash(triage_hkey, dict) _sweep_hash(triage_audit_hkey, list) + _sweep_hash(rulebook_review_hkey, dict) + _sweep_hash(rulebook_review_audit_hkey, list) + _sweep_submission_hash() integration_status_rows_deleted = 0 if jobs_failed == 0 and cids_failed == 0: integration_status_rows_deleted = _sweep_hash(integrations_hkey, dict) diff --git a/extensions/business/cybersec/red_mesh/services/finalization.py b/extensions/business/cybersec/red_mesh/services/finalization.py index c61d6e570..c000d3531 100644 --- a/extensions/business/cybersec/red_mesh/services/finalization.py +++ b/extensions/business/cybersec/red_mesh/services/finalization.py @@ -28,6 +28,7 @@ emit_finding_event, emit_lifecycle_event, ) +from .rulebook_assessment import ensure_rulebook_assessment from .scan_strategy import coerce_scan_type, get_scan_strategy from .state_machine import is_intermediate_job_status, is_terminal_job_status, set_job_status @@ -129,6 +130,26 @@ def _mark_attestation_failed(owner, job_key, job_specs, *, job_id, pass_nr, mess owner._clear_live_progress(job_id, list((job_specs.get("workers") or {}).keys())) +def _ensure_rulebook_assessment_after_pass(owner, job_specs, *, job_id, pass_nr): + try: + result = ensure_rulebook_assessment(owner, job_id, pass_nr=pass_nr) + except Exception as exc: + owner.P(f"[NIS2] Rulebook assessment ensure failed for job {job_id} pass {pass_nr}: {exc}", color='y') + return job_specs + + if not isinstance(result, dict) or result.get("status") != "ok": + error = result.get("error") if isinstance(result, dict) else "unknown_error" + owner.P(f"[NIS2] Rulebook assessment not generated for job {job_id} pass {pass_nr}: {error}", color='y') + return job_specs + + owner.P( + f"[NIS2] Rulebook assessment ready for job {job_id} pass {pass_nr}: " + f"{result.get('artifact_cid') or 'cached'}" + ) + refreshed = owner._get_job_from_cstore(job_id) + return refreshed if isinstance(refreshed, dict) else job_specs + + def maybe_finalize_pass(owner): """ Launcher finalizes completed passes and orchestrates continuous monitoring. @@ -421,6 +442,7 @@ def maybe_finalize_pass(owner): set_job_status(job_specs, JOB_STATUS_FINALIZING) job_specs = _write_job_record(owner, job_key, job_specs, context="finalize_finalizing") + job_specs = _ensure_rulebook_assessment_after_pass(owner, job_specs, job_id=job_id, pass_nr=job_pass) if required_attestation_failed: _mark_attestation_failed( diff --git a/extensions/business/cybersec/red_mesh/services/rulebook_assessment.py b/extensions/business/cybersec/red_mesh/services/rulebook_assessment.py new file mode 100644 index 000000000..f62893269 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/services/rulebook_assessment.py @@ -0,0 +1,2094 @@ +from __future__ import annotations + +import copy +import hashlib +import json +import re +import threading +import time as _time +from datetime import datetime, timezone +from urllib.parse import urlsplit + +from ..constants import JOB_STATUS_FINALIZED +from ..models import ( + RULEBOOK_ASSESSMENT_SCHEMA, + RULEBOOK_ASSESSMENT_SCHEMA_VERSION, + RULEBOOK_SUBMISSION_CONTRACT_VERSION, + RulebookPendingSubmission, + RulebookReviewAuditEntry, + RulebookReviewState, + RulebookSubmissionReference, + RulebookSubmissionRegistry, + VALID_RULEBOOK_ANSWER_VALUES, + VALID_RULEBOOK_CHECK_STATUSES, + VALID_RULEBOOK_REVIEW_STATES, +) +from ..repositories import ArtifactRepository, JobStateRepository +from .event_redaction import stable_hmac_pseudonym, strip_sensitive_fields +from .scan_guards import reject_model_test_for_scan_operation + + +DEFAULT_RULEBOOK_PROFILE_ID = "nis2.eu_baseline.v1" +RULEBOOK_PROFILE_VERSION = "1.0.0" + +_SUBMISSION_LOCKS = {} +_SUBMISSION_LOCKS_GUARD = threading.Lock() + +_IPV4_RE = re.compile( + r"(?" + + def _replace_ip(match): + return stable_hmac_pseudonym(match.group(0), hmac_secret, prefix="ip") + + text = _PEM_PRIVATE_KEY_RE.sub("", text) + text = _BEARER_TOKEN_RE.sub("Authorization: Bearer ", text) + text = _JWT_RE.sub("", text) + text = _SECRET_ASSIGNMENT_RE.sub(_replace_secret, text) + text = _PROVIDER_TOKEN_RE.sub("", text) + public_references = [] + + def _preserve_public_reference(match): + marker = f"publicrefmarker{len(public_references)}" + public_references.append((marker, match.group(0))) + return marker + + text = _PUBLIC_REFERENCE_RE.sub(_preserve_public_reference, text) + text = _UNLABELLED_HEX_RE.sub("", text) + text = _UNLABELLED_TOKEN_RE.sub("", text) + for marker, public_reference in public_references: + text = text.replace(marker, public_reference) + text = _IPV4_RE.sub(_replace_ip, text) + text = " ".join(text.split()) + return text[:max_len] + + +def _error(code, job_id, **extra): + return { + "status": "error", + "error": code, + "job_id": job_id, + **extra, + } + + +def _error_message(payload): + return str(payload.get("message") or payload.get("error") or "Rulebook assessment failed.") + + +def _sanitize_error(owner, payload): + hmac_secret = str(getattr(owner, "cfg_instance_id", "") or "redmesh-rulebook") + return { + "error": _safe_text(payload.get("error") or "rulebook_assessment_failed", hmac_secret=hmac_secret, max_len=120), + "message": _safe_text(_error_message(payload), hmac_secret=hmac_secret, max_len=280), + "retryable": bool(payload.get("retryable", True)), + "at": _utc_timestamp(), + } + + +def _profile(profile_id): + return _PROFILES.get(profile_id or DEFAULT_RULEBOOK_PROFILE_ID) + + +def list_rulebook_profiles(): + return [ + { + "profile_id": profile["profile_id"], + "profile_version": profile["profile_version"], + "title": profile["title"], + "jurisdiction_scope": profile["jurisdiction_scope"], + } + for profile in _PROFILES.values() + ] + + +def _resolve_scan_context(owner, job_id, pass_nr=None): + job_specs = owner._get_job_from_cstore(job_id) + if not isinstance(job_specs, dict): + return None, _error("job_not_found", job_id) + + unsupported = reject_model_test_for_scan_operation(job_specs, job_id, "rulebook_assessment") + if unsupported: + return None, { + **unsupported, + "error": "model_test_not_supported", + "error_class": unsupported.get("error_class") or unsupported.get("error"), + } + + artifacts = _artifact_repo(owner) + job_cid = job_specs.get("job_cid") + archive = {} + if job_cid: + archive = artifacts.get_archive(job_specs) + if not isinstance(archive, dict): + return None, _error("artifact_not_found", job_id, artifact="archive", artifact_cid=job_cid) + job_config = archive.get("job_config") or artifacts.get_job_config(job_specs) or {} + passes = archive.get("passes") or [] + if not passes: + return None, _error("no_completed_passes", job_id) + if pass_nr is None: + pass_data = passes[-1] + else: + pass_data = next((item for item in passes if item.get("pass_nr") == pass_nr), None) + if not isinstance(pass_data, dict): + return None, _error( + "pass_not_found", + job_id, + available_passes=[item.get("pass_nr") for item in passes if isinstance(item, dict)], + ) + agg_cid = pass_data.get("aggregated_report_cid") + aggregated = artifacts.get_json(agg_cid) if agg_cid else {} + else: + pass_reports = job_specs.get("pass_reports") or [] + if not pass_reports: + return None, _error("no_completed_passes", job_id) + if pass_nr is None: + pass_ref = pass_reports[-1] + else: + pass_ref = next((item for item in pass_reports if item.get("pass_nr") == pass_nr), None) + if not isinstance(pass_ref, dict): + return None, _error( + "pass_not_found", + job_id, + available_passes=[item.get("pass_nr") for item in pass_reports if isinstance(item, dict)], + ) + pass_data = artifacts.get_pass_report(pass_ref.get("report_cid")) + if not isinstance(pass_data, dict): + return None, _error("artifact_not_found", job_id, artifact="pass_report", artifact_cid=pass_ref.get("report_cid")) + agg_cid = pass_data.get("aggregated_report_cid") + aggregated = artifacts.get_json(agg_cid) if agg_cid else {} + job_config = artifacts.get_job_config(job_specs) or {} + + return { + "job_specs": job_specs, + "job_config": job_config if isinstance(job_config, dict) else {}, + "archive": archive if isinstance(archive, dict) else {}, + "pass_data": pass_data, + "aggregated": aggregated if isinstance(aggregated, dict) else {}, + }, None + + +def _finding_id(finding): + return str(finding.get("finding_id") or finding.get("id") or "").strip() + + +def _severity(finding): + return str(finding.get("severity") or "").strip().upper() + + +def _triage_status(finding, triage_map): + triage = triage_map.get(_finding_id(finding)) + if isinstance(triage, dict): + return str(triage.get("status") or "").strip().lower() + return str((finding.get("triage") or {}).get("status") or finding.get("triage_state") or "").strip().lower() + + +def _is_actionable_finding(finding, triage_map): + if not isinstance(finding, dict): + return False + finding_status = str(finding.get("status") or "").strip().lower() + if finding_status == "not_vulnerable": + return False + if _triage_status(finding, triage_map) in _CLOSED_TRIAGE_STATUSES: + return False + return bool(_finding_id(finding) or finding.get("title") or finding.get("description")) + + +def _search_text(finding): + parts = [ + finding.get("title"), + finding.get("description"), + finding.get("category"), + finding.get("probe"), + finding.get("cwe_id"), + finding.get("owasp_id"), + finding.get("cve"), + finding.get("cve_id"), + ] + return " ".join(str(part).lower() for part in parts if part) + + +def _matches_keywords(finding, keywords): + text = _search_text(finding) + return any(keyword in text for keyword in keywords) + + +def _safe_finding_ref(finding, *, hmac_secret, redaction_values, triage_map): + payload = { + "type": "finding", + "finding_id": _safe_text(_finding_id(finding), hmac_secret=hmac_secret, redaction_values=redaction_values, max_len=160), + "severity": _severity(finding), + "title": _safe_text(finding.get("title") or "Untitled finding", hmac_secret=hmac_secret, redaction_values=redaction_values, max_len=240), + "category": _safe_text(finding.get("category") or "", hmac_secret=hmac_secret, redaction_values=redaction_values, max_len=120), + "probe": _safe_text(finding.get("probe") or "", hmac_secret=hmac_secret, redaction_values=redaction_values, max_len=160), + "cwe_id": _safe_text(finding.get("cwe_id") or "", hmac_secret=hmac_secret, redaction_values=redaction_values, max_len=80), + "owasp_id": _safe_text(finding.get("owasp_id") or "", hmac_secret=hmac_secret, redaction_values=redaction_values, max_len=80), + "triage_status": _triage_status(finding, triage_map) or None, + } + return {key: value for key, value in payload.items() if value not in (None, "", [])} + + +def _source_refs(ctx, actual_pass_nr): + pass_data = ctx["pass_data"] + refs = [{"type": "job", "job_id": ctx["job_specs"].get("job_id")}] + refs.append({"type": "pass", "pass_nr": actual_pass_nr}) + if pass_data.get("aggregated_report_cid"): + refs.append({"type": "artifact", "artifact_kind": "aggregated_report", "artifact_cid": pass_data.get("aggregated_report_cid")}) + return refs + + +def _auto_from_findings(check, findings, *, hmac_secret, redaction_values, triage_map): + gap_findings = [finding for finding in findings if _severity(finding) in _GAP_SEVERITIES] + review_findings = [ + finding + for finding in findings + if _severity(finding) not in _GAP_SEVERITIES or _severity(finding) in _REVIEW_SEVERITIES + ] + refs = [ + _safe_finding_ref(finding, hmac_secret=hmac_secret, redaction_values=redaction_values, triage_map=triage_map) + for finding in gap_findings[:10] + ] + if gap_findings: + return { + "status": "gap", + "summary": f"{len(gap_findings)} unresolved medium-or-higher finding(s) match this check.", + "gap_reason": "Unresolved scan findings need treatment evidence.", + "evidence_refs": refs, + } + if review_findings: + return { + "status": "needs_review", + "summary": f"{len(review_findings)} lower-severity or inconclusive finding(s) need reviewer interpretation.", + "gap_reason": "", + "evidence_refs": [ + _safe_finding_ref(finding, hmac_secret=hmac_secret, redaction_values=redaction_values, triage_map=triage_map) + for finding in review_findings[:10] + ], + } + return { + "status": "supported", + "summary": "No unresolved matching findings were observed in the selected RedMesh pass.", + "gap_reason": "", + "evidence_refs": [], + } + + +def _automated_check_result(check, ctx, *, hmac_secret, redaction_values, triage_map): + pass_data = ctx["pass_data"] + findings = [ + finding + for finding in (pass_data.get("findings") or []) + if _is_actionable_finding(finding, triage_map) + ] + signal_kind = check.get("signal_kind") + + if signal_kind == "vulnerability_management": + result = _auto_from_findings(check, findings, hmac_secret=hmac_secret, redaction_values=redaction_values, triage_map=triage_map) + elif signal_kind == "auth_access_findings": + matched = [finding for finding in findings if _matches_keywords(finding, _ACCESS_KEYWORDS)] + result = _auto_from_findings(check, matched, hmac_secret=hmac_secret, redaction_values=redaction_values, triage_map=triage_map) + elif signal_kind == "crypto_transport_findings": + matched = [finding for finding in findings if _matches_keywords(finding, _CRYPTO_KEYWORDS)] + result = _auto_from_findings(check, matched, hmac_secret=hmac_secret, redaction_values=redaction_values, triage_map=triage_map) + elif signal_kind == "assessment_performed": + result = { + "status": "supported", + "summary": "A completed RedMesh scan pass is available as retained technical assessment evidence.", + "gap_reason": "", + "evidence_refs": _source_refs(ctx, pass_data.get("pass_nr") or 1), + } + elif signal_kind == "soc_incident_evidence": + soc = ctx["job_specs"].get("soc_event_status") or ctx["archive"].get("soc_event_status") + detection = ctx["job_specs"].get("detection_correlation") or ctx["archive"].get("detection_correlation") + status_text = " ".join( + str(item).lower() + for item in [ + (soc or {}).get("last_status") if isinstance(soc, dict) else "", + (soc or {}).get("status") if isinstance(soc, dict) else "", + (soc or {}).get("outcome") if isinstance(soc, dict) else "", + (detection or {}).get("status") if isinstance(detection, dict) else "", + ] + if item + ) + if any(token in status_text for token in ("sent", "completed", "success", "ok")): + result = { + "status": "supported", + "summary": "RedMesh has SOC export or detection-correlation metadata for this job.", + "gap_reason": "", + "evidence_refs": [{"type": "job_metadata", "fields": ["soc_event_status", "detection_correlation"]}], + } + elif isinstance(soc, dict) or isinstance(detection, dict): + result = { + "status": "needs_review", + "summary": "SOC or detection metadata exists but does not show a completed export/correlation outcome.", + "gap_reason": "", + "evidence_refs": [{"type": "job_metadata", "fields": ["soc_event_status", "detection_correlation"]}], + } + else: + result = { + "status": "needs_review", + "summary": "No SOC export or detection-correlation metadata was observed for this job.", + "gap_reason": "", + "evidence_refs": [], + } + else: + result = { + "status": "not_observable", + "summary": "This check needs reviewer evidence outside the RedMesh scan archive.", + "gap_reason": "", + "evidence_refs": [], + } + + return { + "check_id": check["check_id"], + "title": check["title"], + "theme": check["theme"], + "article_refs": list(check.get("article_refs") or []), + "status": result["status"], + "automated_status": result["status"], + "source": "automated" if result["status"] != "not_observable" else "not_observable", + "summary": result["summary"], + "gap_reason": result.get("gap_reason") or "", + "evidence_refs": result.get("evidence_refs") or [], + "review_question_id": check.get("review_question_id"), + "review_answer": None, + "limitations": [check.get("limitations")] if check.get("limitations") else [], + } + + +def _apply_review(check_result, review): + question_id = check_result.get("review_question_id") + answers = (review.to_dict().get("answers") if review else {}) or {} + answer = answers.get(question_id) + if not answer: + return check_result + + result = dict(check_result) + result["review_answer"] = answer + auto_status = result.get("automated_status") + value = answer.get("value") + if auto_status == "gap": + result["status"] = "gap" + result["source"] = "mixed" + return result + + if value == "yes": + result["status"] = "supported" + result["gap_reason"] = "" + elif value == "no": + result["status"] = "gap" + result["gap_reason"] = "Reviewer indicated evidence is missing or insufficient." + elif value == "not_applicable": + result["status"] = "not_applicable" + result["gap_reason"] = "" + else: + result["status"] = "needs_review" + result["source"] = "reviewer" + return result + + +def _status_counts(checks): + counts = {status: 0 for status in sorted(VALID_RULEBOOK_CHECK_STATUSES)} + for check in checks: + status = check.get("status") + if status in counts: + counts[status] += 1 + return counts + + +def _review_view(review, *, hmac_secret): + if review is None: + return {"review_state": "draft", "answers": {}} + payload = review.to_dict() + payload.pop("last_reopen_idempotency_key", None) + payload.pop("last_reopen_from_revision", None) + payload.pop("last_reopen_actor", None) + payload["note"] = _safe_text(payload.get("note") or "", hmac_secret=hmac_secret, max_len=1000) + for answer in (payload.get("answers") or {}).values(): + if isinstance(answer, dict): + answer["note"] = _safe_text(answer.get("note") or "", hmac_secret=hmac_secret, max_len=1000) + return payload + + +def _profile_meta(job_specs, profile_id): + assessments = job_specs.get("rulebook_assessments") or {} + if not isinstance(assessments, dict): + return {} + meta = assessments.get(profile_id) + return meta if isinstance(meta, dict) else {} + + +def _meta_pass_nr(meta): + value = meta.get("latest_pass_nr", meta.get("pass_nr")) + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _existing_same_pass_artifact(owner, job_id, profile_id, pass_nr): + job_specs = owner._get_job_from_cstore(job_id) + if not isinstance(job_specs, dict): + return None + meta = _profile_meta(job_specs, profile_id) + artifact_cid = meta.get("artifact_cid") + if not artifact_cid or _meta_pass_nr(meta) != int(pass_nr): + return None + if meta.get("run_state") not in (None, "succeeded"): + return None + assessment = _artifact_repo(owner).get_json(artifact_cid) + if not isinstance(assessment, dict): + return None + if ( + assessment.get("schema_version") != RULEBOOK_ASSESSMENT_SCHEMA_VERSION + or assessment.get("artifact_kind") != "generated_assessment" + ): + return None + return meta, assessment + + +def _history_with_previous(existing_meta, new_meta): + history = [] + for item in existing_meta.get("history") or []: + if isinstance(item, dict): + history.append(dict(item)) + previous_cid = existing_meta.get("artifact_cid") + if previous_cid and previous_cid != new_meta.get("artifact_cid"): + previous = { + "artifact_cid": previous_cid, + "pass_nr": existing_meta.get("latest_pass_nr", existing_meta.get("pass_nr")), + "profile_version": existing_meta.get("profile_version"), + "schema_version": existing_meta.get("schema_version"), + "artifact_kind": existing_meta.get("artifact_kind"), + "last_generated_at": existing_meta.get("last_generated_at"), + "status_counts": existing_meta.get("status_counts"), + "review_state": existing_meta.get("review_state"), + } + if not any(item.get("artifact_cid") == previous_cid for item in history): + history.append({key: value for key, value in previous.items() if value not in (None, "", [])}) + return history[-20:] + + +def _write_assessment_meta(owner, job_id, profile_id, meta, *, context): + job_specs = owner._get_job_from_cstore(job_id) + if not isinstance(job_specs, dict): + return None + assessments = dict(job_specs.get("rulebook_assessments") or {}) + existing_meta = assessments.get(profile_id) if isinstance(assessments.get(profile_id), dict) else {} + if meta.get("run_state") == "succeeded": + meta = dict(meta) + meta["history"] = _history_with_previous(existing_meta, meta) + assessments[profile_id] = meta + job_specs["rulebook_assessments"] = assessments + return _write_job_record(owner, job_id, job_specs, context=context) + + +def _success_meta(result, artifact_cid): + generated_at = result["assessment"]["generated_at"] + return { + "schema": RULEBOOK_ASSESSMENT_SCHEMA, + "schema_version": RULEBOOK_ASSESSMENT_SCHEMA_VERSION, + "artifact_kind": "generated_assessment", + "profile_id": result["profile_id"], + "profile_version": result["profile_version"], + "artifact_cid": artifact_cid, + "last_generated_at": generated_at, + "pass_nr": result["pass_nr"], + "latest_pass_nr": result["pass_nr"], + "status_counts": result["status_counts"], + "review_state": result["assessment"]["review_state"].get("review_state", "draft"), + "auto_enabled": True, + "run_state": "succeeded", + } + + +def _failed_meta(owner, job_id, profile_id, payload): + profile = _profile(profile_id) + existing = _profile_meta(owner._get_job_from_cstore(job_id) or {}, profile_id) + meta = dict(existing) + meta.update({ + "schema": RULEBOOK_ASSESSMENT_SCHEMA, + "schema_version": RULEBOOK_ASSESSMENT_SCHEMA_VERSION, + "profile_id": profile_id, + "profile_version": profile["profile_version"] if profile else existing.get("profile_version"), + "auto_enabled": True, + "run_state": "failed", + "last_error": _sanitize_error(owner, payload), + }) + return meta + + +def build_rulebook_assessment( + owner, + job_id, + profile_id=DEFAULT_RULEBOOK_PROFILE_ID, + pass_nr=None, + *, + include_review=True, + artifact_kind="generated_assessment", + submission=None, +): + profile = _profile(profile_id) + if not profile: + return _error("invalid_profile", job_id, profile_id=profile_id) + ctx, err = _resolve_scan_context(owner, job_id, pass_nr=pass_nr) + if err: + return err + + job_specs = ctx["job_specs"] + actual_pass_nr = ctx["pass_data"].get("pass_nr") or pass_nr or 1 + hmac_secret = str(getattr(owner, "cfg_instance_id", "") or "redmesh-rulebook") + redaction_values = _target_values(ctx["job_config"], job_specs) + target_value = (redaction_values or [job_id or "unknown"])[0] + target_pseudonym = stable_hmac_pseudonym(target_value, hmac_secret, prefix="target") + triage_map = _job_repo(owner).list_job_triage(job_id) + review = _job_repo(owner).get_rulebook_review_model(job_id, profile["profile_id"]) if include_review else None + + checks = [] + for check in profile["checks"]: + check_result = _automated_check_result( + check, + ctx, + hmac_secret=hmac_secret, + redaction_values=redaction_values, + triage_map=triage_map, + ) + checks.append(_apply_review(check_result, review)) + + counts = _status_counts(checks) + assessment = { + "schema": RULEBOOK_ASSESSMENT_SCHEMA, + "schema_version": RULEBOOK_ASSESSMENT_SCHEMA_VERSION, + "artifact_kind": artifact_kind, + "job_id": job_id, + "generated_at": _utc_timestamp(), + "profile": { + "profile_id": profile["profile_id"], + "profile_version": profile["profile_version"], + "title": profile["title"], + "jurisdiction_scope": profile["jurisdiction_scope"], + "source_refs": list(profile.get("source_refs") or []), + "legal_notice": profile["legal_notice"], + }, + "scan_context": { + "target_ref": target_pseudonym, + "scan_type": job_specs.get("scan_type"), + "job_status": job_specs.get("job_status"), + "pass_nr": actual_pass_nr, + "job_created_at": _utc_timestamp(job_specs.get("date_created")) if job_specs.get("date_created") else None, + "job_completed_at": _utc_timestamp(job_specs.get("date_completed")) if job_specs.get("date_completed") else None, + "job_archive_cid": job_specs.get("job_cid"), + "aggregated_report_cid": ctx["pass_data"].get("aggregated_report_cid"), + "finding_count": len(ctx["pass_data"].get("findings") or []), + "worker_count": job_specs.get("worker_count") or len(job_specs.get("workers") or {}), + }, + "status_counts": counts, + "checks": checks, + "review_state": _review_view(review, hmac_secret=hmac_secret), + "limitations": [ + "Evidence states are based on RedMesh scan data plus reviewer input.", + "National transposition, governance approval, contracts, and reporting operations need separate review.", + "A clean technical scan does not prove organization-wide control operation.", + ], + } + if isinstance(submission, dict): + assessment["submission"] = dict(submission) + return { + "status": "ok", + "job_id": job_id, + "profile_id": profile["profile_id"], + "profile_version": profile["profile_version"], + "pass_nr": actual_pass_nr, + "status_counts": counts, + "assessment": assessment, + } + + +def generate_rulebook_assessment(owner, job_id, profile_id=DEFAULT_RULEBOOK_PROFILE_ID, pass_nr=None, persist=True, force=True): + result = build_rulebook_assessment( + owner, + job_id, + profile_id=profile_id, + pass_nr=pass_nr, + include_review=not persist, + ) + if result.get("status") != "ok": + profile = _profile(profile_id) + if profile and persist: + _write_assessment_meta( + owner, + job_id, + profile["profile_id"], + _failed_meta(owner, job_id, profile["profile_id"], result), + context="rulebook_assessment_failed", + ) + return result + + artifact_cid = None + if persist: + existing = None if force else _existing_same_pass_artifact(owner, job_id, result["profile_id"], result["pass_nr"]) + if existing: + meta, assessment = existing + return { + **result, + "assessment": assessment, + "artifact_cid": meta.get("artifact_cid"), + "generated": True, + "cached": True, + } + + artifact_cid = _artifact_repo(owner).put_json(result["assessment"], show_logs=False) + if not artifact_cid: + failed = _error("artifact_write_failed", job_id, profile_id=profile_id) + _write_assessment_meta( + owner, + job_id, + result["profile_id"], + _failed_meta(owner, job_id, result["profile_id"], failed), + context="rulebook_assessment_failed", + ) + return failed + + _write_assessment_meta( + owner, + job_id, + result["profile_id"], + _success_meta(result, artifact_cid), + context="rulebook_assessment", + ) + + return { + **result, + "artifact_cid": artifact_cid, + "generated": bool(artifact_cid), + } + + +def ensure_rulebook_assessment(owner, job_id, profile_id=DEFAULT_RULEBOOK_PROFILE_ID, pass_nr=None): + return generate_rulebook_assessment( + owner, + job_id, + profile_id=profile_id, + pass_nr=pass_nr, + persist=True, + force=False, + ) + + +def get_rulebook_assessment_status(owner, job_id, profile_id=DEFAULT_RULEBOOK_PROFILE_ID): + profile = _profile(profile_id) + if not profile: + return _error("invalid_profile", job_id, profile_id=profile_id) + job_specs = owner._get_job_from_cstore(job_id) + if not isinstance(job_specs, dict): + return {"job_id": job_id, "found": False, "generated": False, "profile_id": profile_id} + unsupported = reject_model_test_for_scan_operation(job_specs, job_id, "rulebook_assessment_status") + if unsupported: + return { + **unsupported, + "error": "model_test_not_supported", + "error_class": unsupported.get("error_class") or unsupported.get("error"), + "found": True, + "generated": False, + } + meta = (job_specs.get("rulebook_assessments") or {}).get(profile["profile_id"]) + if not isinstance(meta, dict) or not meta: + result = { + "job_id": job_id, + "found": True, + "generated": False, + "profile_id": profile["profile_id"], + "profile_version": profile["profile_version"], + "schema": RULEBOOK_ASSESSMENT_SCHEMA, + "schema_version": RULEBOOK_ASSESSMENT_SCHEMA_VERSION, + } + else: + result = { + "job_id": job_id, + "found": True, + "generated": bool(meta.get("artifact_cid")) and meta.get("run_state") != "failed", + **meta, + } + result["submission_contract_version"] = RULEBOOK_SUBMISSION_CONTRACT_VERSION + try: + review = _job_repo(owner).get_rulebook_review_model(job_id, profile["profile_id"]) + result.update(_submission_public_state(owner, job_id, job_specs, profile, review)) + except ValueError: + result.update({ + "submission_contract_version": None, + "submission_contract_unsupported": True, + }) + return result + + +def _submission_lock(owner, job_id, profile_id): + key = f"{getattr(owner, 'cfg_instance_id', '')}:{job_id}:{profile_id}" + with _SUBMISSION_LOCKS_GUARD: + lock = _SUBMISSION_LOCKS.get(key) + if lock is None: + lock = threading.RLock() + _SUBMISSION_LOCKS[key] = lock + return lock + + +def _empty_submission_registry(): + return RulebookSubmissionRegistry() + + +def _submission_registry(repo, job_id, profile_id): + return repo.get_rulebook_submission_registry_model(job_id, profile_id) or _empty_submission_registry() + + +def _owner_time(owner): + return float(getattr(owner, "time", _time.time)()) + + +def _latest_pass_nr(owner, job_id): + ctx, err = _resolve_scan_context(owner, job_id) + if err: + return None, err + raw = ctx["pass_data"].get("pass_nr") or 1 + try: + return int(raw), None + except (TypeError, ValueError): + return None, _error("pass_not_found", job_id) + + +def _legacy_submission_reference(job_specs, profile, review): + if review is None or review.review_state != "reviewed": + return None + meta = _profile_meta(job_specs, profile["profile_id"]) + cid = str(meta.get("artifact_cid") or "").strip() + if not cid: + return None + try: + pass_nr = int(meta.get("latest_pass_nr", meta.get("pass_nr", 0)) or 0) + except (TypeError, ValueError): + pass_nr = 0 + return RulebookSubmissionReference( + revision=0, + cid=cid, + submitted_at=review.updated_at, + actor=review.reviewer, + pass_nr=pass_nr, + profile_id=profile["profile_id"], + profile_version=str(meta.get("profile_version") or review.profile_version or profile["profile_version"]), + schema_version=str(meta.get("schema_version") or "1.0.0"), + review_revision=review.review_revision, + legacy=True, + ).to_dict() + + +def _submission_reference_view(reference, *, latest_pass_nr, profile_version): + payload = reference.to_dict() if isinstance(reference, RulebookSubmissionReference) else dict(reference) + stale_reasons = [] + if latest_pass_nr is not None and int(payload.get("pass_nr", 0) or 0) != int(latest_pass_nr): + stale_reasons.append("newer_scan_pass") + if payload.get("profile_version") != profile_version: + stale_reasons.append("newer_profile_version") + return { + **payload, + "artifact_cid": payload.get("cid"), + "stale": bool(stale_reasons), + "stale_reasons": stale_reasons, + } + + +def _submission_public_state(owner, job_id, job_specs, profile, review): + repo = _job_repo(owner) + registry = _submission_registry(repo, job_id, profile["profile_id"]) + registry_payload = registry.to_dict() + latest_pass_nr, _ = _latest_pass_nr(owner, job_id) + references = list(registry_payload.get("submissions") or []) + legacy_reference = _legacy_submission_reference(job_specs, profile, review) + if legacy_reference and not any(item.get("cid") == legacy_reference["cid"] for item in references): + references.append(legacy_reference) + views = [ + _submission_reference_view( + reference, + latest_pass_nr=latest_pass_nr, + profile_version=profile["profile_version"], + ) + for reference in sorted(references, key=lambda item: int(item.get("revision", 0) or 0), reverse=True) + ] + pending = registry_payload.get("pending") + latest = views[0] if views else None + migration_required = bool(review and review.review_state == "reviewed" and legacy_reference is None) + effective_state = "draft" + if not pending and latest and review and review.review_state in {"submitted", "reviewed"}: + effective_state = "submitted" + operation_state = None + if pending: + operation_state = "failed" if pending.get("last_error") else "submitting" + return { + "submission_contract_version": RULEBOOK_SUBMISSION_CONTRACT_VERSION, + "effective_review_state": effective_state, + "review_revision": review.review_revision if review else 0, + "latest_submission": latest, + "submissions": views, + "submission_operation_state": operation_state, + "submission_error": (pending or {}).get("last_error") if isinstance(pending, dict) else None, + "migration_submission_required": migration_required, + } + + +def _submission_error(code, job_id, profile_id, message, *, retryable=False, **extra): + return _error( + code, + job_id, + profile_id=profile_id, + submission_contract_version=RULEBOOK_SUBMISSION_CONTRACT_VERSION, + message=message, + retryable=retryable, + **extra, + ) + + +def _unsupported_submission_registry_error(job_id, profile_id): + return _submission_error( + "submission_contract_unsupported", + job_id, + profile_id, + "Submission registry contract version is not supported by this backend.", + ) + + +def get_rulebook_review(owner, job_id, profile_id=DEFAULT_RULEBOOK_PROFILE_ID): + profile = _profile(profile_id) + if not profile: + return _error("invalid_profile", job_id, profile_id=profile_id) + job_specs = owner._get_job_from_cstore(job_id) + if not isinstance(job_specs, dict): + return _error("job_not_found", job_id, profile_id=profile_id) + unsupported = reject_model_test_for_scan_operation(job_specs, job_id, "rulebook_review") + if unsupported: + return { + **unsupported, + "error": "model_test_not_supported", + "error_class": unsupported.get("error_class") or unsupported.get("error"), + } + if job_specs.get("job_status") != JOB_STATUS_FINALIZED: + return _error( + "job_not_finalized", + job_id, + profile_id=profile["profile_id"], + job_status=job_specs.get("job_status"), + ) + hmac_secret = str(getattr(owner, "cfg_instance_id", "") or "redmesh-rulebook") + repo = _job_repo(owner) + review = repo.get_rulebook_review_model(job_id, profile["profile_id"]) + try: + submission_state = _submission_public_state(owner, job_id, job_specs, profile, review) + except ValueError: + return _submission_error( + "submission_contract_unsupported", + job_id, + profile["profile_id"], + "Submission registry contract version is not supported by this backend.", + ) + return { + "status": "ok", + "job_id": job_id, + "profile": { + "profile_id": profile["profile_id"], + "profile_version": profile["profile_version"], + "title": profile["title"], + "checks": [ + { + "check_id": check["check_id"], + "title": check["title"], + "theme": check["theme"], + "review_question_id": check.get("review_question_id"), + "control_intent": check.get("control_intent"), + "limitations": check.get("limitations"), + } + for check in profile["checks"] + ], + }, + "found": review is not None, + "review": _review_view(review, hmac_secret=hmac_secret), + "audit": repo.get_rulebook_review_audit(job_id, profile["profile_id"]), + **submission_state, + } + + +def _validate_review_answers(profile, answers): + if answers is None: + return {} + if not isinstance(answers, dict): + raise ValueError("answers must be an object") + known_questions = { + check.get("review_question_id") + for check in profile["checks"] + if check.get("review_question_id") + } + validated = {} + for question_id, raw_answer in answers.items(): + if question_id not in known_questions: + raise ValueError(f"Unknown review question: {question_id}") + payload = raw_answer if isinstance(raw_answer, dict) else {"value": raw_answer} + answer_value = str(payload.get("value") or "unknown").strip().lower() + if answer_value not in VALID_RULEBOOK_ANSWER_VALUES: + raise ValueError(f"Unsupported answer value for {question_id}: {answer_value}") + validated[question_id] = { + "value": answer_value, + "note": str(payload.get("note") or "")[:1000], + } + return validated + + +def _validated_expected_revision(value, job_id, profile_id): + if value is None: + return None, _submission_error( + "review_revision_conflict", + job_id, + profile_id, + "expected_review_revision is required.", + ) + try: + revision = int(value) + except (TypeError, ValueError): + return None, _submission_error( + "review_revision_conflict", + job_id, + profile_id, + "expected_review_revision must be a non-negative integer.", + ) + if revision < 0: + return None, _submission_error( + "review_revision_conflict", + job_id, + profile_id, + "expected_review_revision must be a non-negative integer.", + ) + return revision, None + + +def _sanitize_review_patch(owner, answers, actor, note): + hmac_secret = str(getattr(owner, "cfg_instance_id", "") or "redmesh-rulebook") + actor = _safe_text(actor or "", hmac_secret=hmac_secret, max_len=200) + note = _safe_text(note or "", hmac_secret=hmac_secret, max_len=1000) + now = _owner_time(owner) + sanitized_answers = {} + for question_id, answer in answers.items(): + sanitized_answers[question_id] = { + "value": answer["value"], + "note": _safe_text(answer.get("note") or "", hmac_secret=hmac_secret, max_len=1000), + "reviewer": actor, + "updated_at": now, + } + return sanitized_answers, actor, note, now + + +def _put_review_with_audit( + owner, + repo, + *, + job_id, + profile, + previous, + state, + changed_question_ids, + event_type, +): + previous_answers = (previous.to_dict().get("answers") if previous else {}) or {} + current_answers = (state.to_dict().get("answers") if state else {}) or {} + review_payload = repo.put_rulebook_review(state) + audit_payload = repo.append_rulebook_review_audit(RulebookReviewAuditEntry( + job_id=job_id, + profile_id=profile["profile_id"], + profile_version=profile["profile_version"], + review_state=state.review_state, + reviewer=state.reviewer, + note=state.note, + changed_question_ids=list(changed_question_ids or []), + previous_answers={question_id: previous_answers.get(question_id) for question_id in changed_question_ids or []}, + current_answers={question_id: current_answers.get(question_id) for question_id in changed_question_ids or []}, + timestamp=state.updated_at, + review_revision=state.review_revision, + )) + if hasattr(owner, "_log_audit_event"): + owner._log_audit_event(event_type, { + "job_id": job_id, + "profile_id": profile["profile_id"], + "review_state": state.review_state, + "review_revision": state.review_revision, + "changed_question_ids": list(changed_question_ids or []), + }) + return review_payload, audit_payload + + +def save_rulebook_review_draft( + owner, + job_id, + profile_id=DEFAULT_RULEBOOK_PROFILE_ID, + answers=None, + actor="", + note="", + expected_review_revision=None, +): + profile = _profile(profile_id) + if not profile: + return _error("invalid_profile", job_id, profile_id=profile_id) + expected_revision, err = _validated_expected_revision( + expected_review_revision, + job_id, + profile["profile_id"], + ) + if err: + return err + try: + validated_answers = _validate_review_answers(profile, answers) + except ValueError as exc: + return _error("invalid_review_answer", job_id, profile_id=profile["profile_id"], message=str(exc)) + + with _submission_lock(owner, job_id, profile["profile_id"]): + job_specs = owner._get_job_from_cstore(job_id) + if not isinstance(job_specs, dict): + return _error("job_not_found", job_id, profile_id=profile["profile_id"]) + if job_specs.get("job_status") != JOB_STATUS_FINALIZED: + return _error("job_not_finalized", job_id, profile_id=profile["profile_id"]) + unsupported = reject_model_test_for_scan_operation(job_specs, job_id, "rulebook_review") + if unsupported: + return {**unsupported, "error": "model_test_not_supported"} + repo = _job_repo(owner) + previous = repo.get_rulebook_review_model(job_id, profile["profile_id"]) + try: + registry = _submission_registry(repo, job_id, profile["profile_id"]).to_dict() + except ValueError: + return _unsupported_submission_registry_error(job_id, profile["profile_id"]) + if registry.get("pending"): + return _submission_error( + "submission_in_progress", + job_id, + profile["profile_id"], + "A submission retry must finish before the draft can change.", + retryable=True, + ) + current_revision = previous.review_revision if previous else 0 + if expected_revision != current_revision: + return _submission_error( + "review_revision_conflict", + job_id, + profile["profile_id"], + "The review changed. Reload before saving this draft.", + expected_review_revision=expected_revision, + current_review_revision=current_revision, + ) + legacy_reference = _legacy_submission_reference(job_specs, profile, previous) + if (previous and previous.review_state == "submitted") or legacy_reference: + return _submission_error( + "review_already_submitted", + job_id, + profile["profile_id"], + "Reopen the submitted review before editing it.", + ) + + sanitized_answers, actor, note, now = _sanitize_review_patch(owner, validated_answers, actor, note) + previous_answers = (previous.to_dict().get("answers") if previous else {}) or {} + current_answers = dict(previous_answers) + current_answers.update(sanitized_answers) + changed = sorted( + question_id + for question_id in set(previous_answers) | set(current_answers) + if previous_answers.get(question_id) != current_answers.get(question_id) + ) + state = RulebookReviewState( + job_id=job_id, + profile_id=profile["profile_id"], + profile_version=profile["profile_version"], + review_state="draft", + reviewer=actor, + note=note, + answers=current_answers, + updated_at=now, + review_revision=current_revision + 1, + ) + try: + _put_review_with_audit( + owner, + repo, + job_id=job_id, + profile=profile, + previous=previous, + state=state, + changed_question_ids=changed, + event_type="rulebook_review_draft_saved", + ) + except Exception: + return _submission_error( + "review_draft_save_failed", + job_id, + profile["profile_id"], + "Unable to save the review draft.", + retryable=True, + ) + return get_rulebook_review(owner, job_id, profile["profile_id"]) + + +def _validate_submission_answers(profile, review): + answers = (review.to_dict().get("answers") if review else {}) or {} + missing = [] + comments_required = [] + for check in profile["checks"]: + question_id = check.get("review_question_id") + if not question_id: + continue + answer = answers.get(question_id) + if not isinstance(answer, dict) or answer.get("value") not in VALID_RULEBOOK_ANSWER_VALUES: + missing.append(question_id) + continue + if answer.get("value") in {"no", "unknown", "not_applicable"} and not str(answer.get("note") or "").strip(): + comments_required.append(question_id) + return missing, comments_required + + +def _submission_fingerprint(assessment): + canonical = json.dumps(assessment, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def _build_submission_snapshot( + owner, + job_id, + profile, + *, + pass_nr, + revision, + submitted_at, + actor, + review_revision, +): + submission_meta = { + "contract_version": RULEBOOK_SUBMISSION_CONTRACT_VERSION, + "revision": revision, + "submitted_at": submitted_at, + "actor": actor, + "pass_nr": pass_nr, + "profile_id": profile["profile_id"], + "profile_version": profile["profile_version"], + "review_revision": review_revision, + } + built = build_rulebook_assessment( + owner, + job_id, + profile_id=profile["profile_id"], + pass_nr=pass_nr, + include_review=True, + artifact_kind="review_submission", + submission=submission_meta, + ) + if built.get("status") != "ok": + return built + assessment = built["assessment"] + assessment["generated_at"] = _utc_timestamp(submitted_at) + assessment["review_state"].update({ + "review_state": "submitted", + "reviewer": actor, + "updated_at": submitted_at, + "review_revision": review_revision, + }) + return { + "status": "ok", + "assessment": assessment, + "fingerprint": _submission_fingerprint(assessment), + } + + +def _registry_with(registry, *, submissions=None, pending=None, keep_pending=False): + payload = registry.to_dict() if isinstance(registry, RulebookSubmissionRegistry) else dict(registry) + if submissions is not None: + payload["submissions"] = submissions + if keep_pending or pending is not None: + payload["pending"] = pending + else: + payload.pop("pending", None) + return RulebookSubmissionRegistry.from_dict(payload) + + +def _persist_pending_error(repo, job_id, profile_id, registry, pending, payload): + try: + next_pending = dict(pending) + next_pending["last_error"] = payload + next_pending["updated_at"] = payload.get("at_epoch", next_pending.get("updated_at", 0.0)) + repo.put_rulebook_submission_registry( + job_id, + profile_id, + _registry_with(registry, pending=next_pending, keep_pending=True), + ) + except Exception: + pass + + +def submit_rulebook_review( + owner, + job_id, + profile_id=DEFAULT_RULEBOOK_PROFILE_ID, + expected_review_revision=None, + expected_pass_nr=None, + expected_profile_version=None, + idempotency_key="", + actor="", +): + profile = _profile(profile_id) + if not profile: + return _error("invalid_profile", job_id, profile_id=profile_id) + expected_revision, err = _validated_expected_revision( + expected_review_revision, + job_id, + profile["profile_id"], + ) + if err: + return err + try: + expected_pass = int(expected_pass_nr) + except (TypeError, ValueError): + return _submission_error( + "submission_pass_stale", + job_id, + profile["profile_id"], + "expected_pass_nr must identify the latest completed pass.", + ) + idempotency_key = str(idempotency_key or "").strip() + if not idempotency_key or len(idempotency_key) > 200: + return _submission_error( + "submission_idempotency_conflict", + job_id, + profile["profile_id"], + "A bounded idempotency key is required.", + ) + if str(expected_profile_version or "") != profile["profile_version"]: + return _submission_error( + "submission_profile_stale", + job_id, + profile["profile_id"], + "The rulebook profile changed. Reload before submitting.", + current_profile_version=profile["profile_version"], + ) + hmac_secret = str(getattr(owner, "cfg_instance_id", "") or "redmesh-rulebook") + actor = _safe_text(actor or "", hmac_secret=hmac_secret, max_len=200) + if not actor: + return _submission_error( + "invalid_review_actor", + job_id, + profile["profile_id"], + "A server-derived review actor is required.", + ) + + with _submission_lock(owner, job_id, profile["profile_id"]): + job_specs = owner._get_job_from_cstore(job_id) + if not isinstance(job_specs, dict): + return _error("job_not_found", job_id, profile_id=profile["profile_id"]) + if job_specs.get("job_status") != JOB_STATUS_FINALIZED: + return _error("job_not_finalized", job_id, profile_id=profile["profile_id"]) + unsupported = reject_model_test_for_scan_operation(job_specs, job_id, "rulebook_review") + if unsupported: + return {**unsupported, "error": "model_test_not_supported"} + repo = _job_repo(owner) + review = repo.get_rulebook_review_model(job_id, profile["profile_id"]) + try: + registry = _submission_registry(repo, job_id, profile["profile_id"]) + except ValueError: + return _unsupported_submission_registry_error(job_id, profile["profile_id"]) + registry_payload = registry.to_dict() + existing_pending = registry_payload.get("pending") + + for reference in registry_payload.get("submissions") or []: + if reference.get("idempotency_key") != idempotency_key: + continue + if existing_pending: + break + if ( + int(reference.get("review_revision", -1)) == expected_revision + and int(reference.get("pass_nr", -1)) == expected_pass + and reference.get("profile_version") == profile["profile_version"] + and reference.get("actor") == actor + ): + replay_snapshot = _build_submission_snapshot( + owner, + job_id, + profile, + pass_nr=int(reference["pass_nr"]), + revision=int(reference["revision"]), + submitted_at=float(reference["submitted_at"]), + actor=actor, + review_revision=expected_revision, + ) + if replay_snapshot.get("status") != "ok": + return replay_snapshot + if replay_snapshot["fingerprint"] != reference.get("fingerprint"): + return _submission_error( + "submission_idempotency_conflict", + job_id, + profile["profile_id"], + "The idempotency key was already used for different submission inputs.", + ) + result = get_rulebook_review(owner, job_id, profile["profile_id"]) + result.update({"submission": _submission_reference_view( + reference, + latest_pass_nr=expected_pass, + profile_version=profile["profile_version"], + ), "idempotent_replay": True}) + return result + return _submission_error( + "submission_idempotency_conflict", + job_id, + profile["profile_id"], + "The idempotency key was already used for different submission inputs.", + ) + + current_revision = review.review_revision if review else 0 + if expected_revision != current_revision: + return _submission_error( + "review_revision_conflict", + job_id, + profile["profile_id"], + "The review changed. Reload before submitting.", + expected_review_revision=expected_revision, + current_review_revision=current_revision, + ) + latest_pass, pass_error = _latest_pass_nr(owner, job_id) + if pass_error: + return pass_error + if expected_pass != latest_pass: + return _submission_error( + "submission_pass_stale", + job_id, + profile["profile_id"], + "Newer scan evidence exists. Reload before submitting.", + expected_pass_nr=expected_pass, + current_pass_nr=latest_pass, + ) + + pending = existing_pending + if pending and pending.get("idempotency_key") != idempotency_key: + return _submission_error( + "submission_in_progress", + job_id, + profile["profile_id"], + "Another submission is pending for this review.", + retryable=True, + ) + if review and review.review_state == "submitted" and not pending: + return _submission_error( + "review_already_submitted", + job_id, + profile["profile_id"], + "Reopen the review before creating another submission revision.", + ) + legacy_reference = _legacy_submission_reference(job_specs, profile, review) + if legacy_reference and not pending: + return _submission_error( + "review_already_submitted", + job_id, + profile["profile_id"], + "Reopen the legacy submitted review before creating a native revision.", + ) + + missing, comments_required = _validate_submission_answers(profile, review) + if missing or comments_required: + return _submission_error( + "submission_comments_required", + job_id, + profile["profile_id"], + "Every review question must be answered; No, Unknown, and Not applicable require comments.", + missing_question_ids=missing, + comment_required_question_ids=comments_required, + ) + + now = _owner_time(owner) + target_revision = int((pending or {}).get("target_revision") or (registry.latest_revision + 1)) + submitted_at = float((pending or {}).get("created_at") or now) + built = _build_submission_snapshot( + owner, + job_id, + profile, + pass_nr=latest_pass, + revision=target_revision, + submitted_at=submitted_at, + actor=actor, + review_revision=expected_revision, + ) + if built.get("status") != "ok": + return built + assessment = built["assessment"] + fingerprint = built["fingerprint"] + + if pending: + if pending.get("fingerprint") != fingerprint: + return _submission_error( + "submission_idempotency_conflict", + job_id, + profile["profile_id"], + "The pending submission no longer matches the canonical review snapshot.", + ) + pending = { + **pending, + "attempt_count": int(pending.get("attempt_count", 1) or 1) + 1, + "updated_at": now, + "last_error": None, + } + else: + pending = RulebookPendingSubmission( + target_revision=target_revision, + expected_review_revision=expected_revision, + expected_pass_nr=latest_pass, + expected_profile_version=profile["profile_version"], + actor=actor, + idempotency_key=idempotency_key, + fingerprint=fingerprint, + state="prepared", + created_at=submitted_at, + updated_at=now, + ).to_dict() + try: + registry = _registry_with(registry, pending=pending, keep_pending=True) + repo.put_rulebook_submission_registry(job_id, profile["profile_id"], registry) + except Exception: + return _submission_error( + "submission_record_failed", + job_id, + profile["profile_id"], + "Unable to record pending submission state.", + retryable=True, + ) + + state_order = {"prepared": 0, "artifact_written": 1, "reference_recorded": 2} + if state_order[pending["state"]] < state_order["artifact_written"]: + try: + cid = _artifact_repo(owner).put_json(assessment, show_logs=False) + except Exception: + cid = None + if not cid: + failure = { + "error": "submission_persist_failed", + "message": "Unable to persist the review submission artifact.", + "retryable": True, + "at": _utc_timestamp(), + "at_epoch": now, + } + _persist_pending_error(repo, job_id, profile["profile_id"], registry, pending, failure) + return _submission_error( + "submission_persist_failed", + job_id, + profile["profile_id"], + failure["message"], + retryable=True, + ) + pending = {**pending, "state": "artifact_written", "cid": cid, "updated_at": now} + try: + registry = _registry_with(registry, pending=pending, keep_pending=True) + repo.put_rulebook_submission_registry(job_id, profile["profile_id"], registry) + except Exception: + return _submission_error( + "submission_record_failed", + job_id, + profile["profile_id"], + "The artifact exists but its CID could not be recorded. Retry with the same key.", + retryable=True, + ) + + submissions = list(registry.to_dict().get("submissions") or []) + reference = next((item for item in submissions if int(item.get("revision", -1)) == target_revision), None) + if reference is None: + reference = RulebookSubmissionReference( + revision=target_revision, + cid=pending["cid"], + submitted_at=submitted_at, + actor=actor, + pass_nr=latest_pass, + profile_id=profile["profile_id"], + profile_version=profile["profile_version"], + schema_version=RULEBOOK_ASSESSMENT_SCHEMA_VERSION, + review_revision=expected_revision, + idempotency_key=idempotency_key, + fingerprint=fingerprint, + ).to_dict() + submissions.append(reference) + if state_order[pending["state"]] < state_order["reference_recorded"]: + pending = {**pending, "state": "reference_recorded", "updated_at": now} + try: + registry = _registry_with(registry, submissions=submissions, pending=pending, keep_pending=True) + repo.put_rulebook_submission_registry(job_id, profile["profile_id"], registry) + except Exception: + failure = { + "error": "submission_record_failed", + "message": "The submission reference could not be recorded.", + "retryable": True, + "at": _utc_timestamp(), + "at_epoch": now, + } + _persist_pending_error(repo, job_id, profile["profile_id"], registry, pending, failure) + return _submission_error( + "submission_record_failed", + job_id, + profile["profile_id"], + "The submission reference could not be recorded. Retry with the same key.", + retryable=True, + ) + + submitted_state = RulebookReviewState( + job_id=job_id, + profile_id=profile["profile_id"], + profile_version=profile["profile_version"], + review_state="submitted", + reviewer=actor, + note=review.note if review else "", + answers=(review.to_dict().get("answers") if review else {}) or {}, + updated_at=submitted_at, + review_revision=expected_revision, + ) + try: + if not review or review.review_state != "submitted": + _put_review_with_audit( + owner, + repo, + job_id=job_id, + profile=profile, + previous=review, + state=submitted_state, + changed_question_ids=[], + event_type="rulebook_review_submitted", + ) + committed_registry = RulebookSubmissionRegistry( + contract_version=RULEBOOK_SUBMISSION_CONTRACT_VERSION, + latest_revision=max(registry.latest_revision, target_revision), + submissions=submissions, + pending=None, + ) + repo.put_rulebook_submission_registry(job_id, profile["profile_id"], committed_registry) + except Exception: + failure = { + "error": "submission_record_failed", + "message": "The final submission state was not committed.", + "retryable": True, + "at": _utc_timestamp(), + "at_epoch": now, + } + _persist_pending_error(repo, job_id, profile["profile_id"], registry, pending, failure) + return _submission_error( + "submission_record_failed", + job_id, + profile["profile_id"], + "The submission is recoverable but its final state was not committed. Retry with the same key.", + retryable=True, + ) + + result = get_rulebook_review(owner, job_id, profile["profile_id"]) + result.update({ + "submission": _submission_reference_view( + reference, + latest_pass_nr=latest_pass, + profile_version=profile["profile_version"], + ), + "idempotent_replay": False, + }) + return result + + +def reopen_rulebook_review( + owner, + job_id, + profile_id=DEFAULT_RULEBOOK_PROFILE_ID, + expected_review_revision=None, + idempotency_key="", + actor="", +): + profile = _profile(profile_id) + if not profile: + return _error("invalid_profile", job_id, profile_id=profile_id) + expected_revision, err = _validated_expected_revision( + expected_review_revision, + job_id, + profile["profile_id"], + ) + if err: + return err + idempotency_key = str(idempotency_key or "").strip() + if not idempotency_key or len(idempotency_key) > 200: + return _submission_error( + "reopen_idempotency_conflict", + job_id, + profile["profile_id"], + "A bounded reopen idempotency key is required.", + ) + hmac_secret = str(getattr(owner, "cfg_instance_id", "") or "redmesh-rulebook") + actor = _safe_text(actor or "", hmac_secret=hmac_secret, max_len=200) + if not actor: + return _submission_error("invalid_review_actor", job_id, profile["profile_id"], "A server-derived actor is required.") + + with _submission_lock(owner, job_id, profile["profile_id"]): + job_specs = owner._get_job_from_cstore(job_id) + if not isinstance(job_specs, dict): + return _error("job_not_found", job_id, profile_id=profile["profile_id"]) + if job_specs.get("job_status") != JOB_STATUS_FINALIZED: + return _error("job_not_finalized", job_id, profile_id=profile["profile_id"]) + unsupported = reject_model_test_for_scan_operation(job_specs, job_id, "rulebook_review") + if unsupported: + return {**unsupported, "error": "model_test_not_supported"} + repo = _job_repo(owner) + review = repo.get_rulebook_review_model(job_id, profile["profile_id"]) + try: + registry = _submission_registry(repo, job_id, profile["profile_id"]) + except ValueError: + return _unsupported_submission_registry_error(job_id, profile["profile_id"]) + registry_payload = registry.to_dict() + if registry_payload.get("pending"): + return _submission_error( + "submission_in_progress", + job_id, + profile["profile_id"], + "Finish the pending submission before reopening.", + retryable=True, + ) + current_revision = review.review_revision if review else 0 + if review and review.last_reopen_idempotency_key == idempotency_key: + if ( + review.review_state == "draft" + and review.last_reopen_from_revision == expected_revision + and review.last_reopen_actor == actor + ): + result = get_rulebook_review(owner, job_id, profile["profile_id"]) + result["idempotent_replay"] = True + return result + return _submission_error( + "reopen_idempotency_conflict", + job_id, + profile["profile_id"], + "The reopen idempotency key was already used for different inputs.", + ) + if expected_revision != current_revision: + return _submission_error( + "review_revision_conflict", + job_id, + profile["profile_id"], + "The review changed. Reload before reopening.", + current_review_revision=current_revision, + ) + legacy_reference = _legacy_submission_reference(job_specs, profile, review) + if not registry_payload.get("submissions") and not legacy_reference: + return _submission_error( + "review_not_submitted", + job_id, + profile["profile_id"], + "Only a submitted review can be reopened.", + ) + if not review or review.review_state not in {"submitted", "reviewed"}: + return _submission_error( + "review_not_submitted", + job_id, + profile["profile_id"], + "Only a submitted review can be reopened.", + ) + now = _owner_time(owner) + reopened = RulebookReviewState( + job_id=job_id, + profile_id=profile["profile_id"], + profile_version=profile["profile_version"], + review_state="draft", + reviewer=actor, + note=review.note, + answers=review.to_dict().get("answers") or {}, + updated_at=now, + review_revision=current_revision + 1, + last_reopen_idempotency_key=idempotency_key, + last_reopen_from_revision=current_revision, + last_reopen_actor=actor, + ) + try: + _put_review_with_audit( + owner, + repo, + job_id=job_id, + profile=profile, + previous=review, + state=reopened, + changed_question_ids=[], + event_type="rulebook_review_reopened", + ) + except Exception: + return _submission_error( + "review_reopen_failed", + job_id, + profile["profile_id"], + "Unable to reopen the submitted review.", + retryable=True, + ) + result = get_rulebook_review(owner, job_id, profile["profile_id"]) + result["idempotent_replay"] = False + return result + + +def _update_rulebook_review_locked( + owner, + job_id, + profile, + job_specs, + validated_answers, + reviewer, + note, + review_state, +): + repo = _job_repo(owner) + previous = repo.get_rulebook_review_model(job_id, profile["profile_id"]) + try: + registry = _submission_registry(repo, job_id, profile["profile_id"]).to_dict() + except ValueError: + return _unsupported_submission_registry_error(job_id, profile["profile_id"]) + formal_history_blocks_legacy_write = bool( + registry.get("submissions") + and ( + previous is None + or previous.review_state != "draft" + or review_state != "draft" + ) + ) + if registry.get("pending") or formal_history_blocks_legacy_write or _legacy_submission_reference(job_specs, profile, previous): + return _submission_error( + "review_already_submitted", + job_id, + profile["profile_id"], + "Use the explicit reopen operation before changing a submitted review.", + ) + now = float(getattr(owner, "time", _time.time)()) + hmac_secret = str(getattr(owner, "cfg_instance_id", "") or "redmesh-rulebook") + reviewer = _safe_text(reviewer or "", hmac_secret=hmac_secret, max_len=200) + note = _safe_text(note or "", hmac_secret=hmac_secret, max_len=1000) + sanitized_answers = {} + for question_id, answer in validated_answers.items(): + sanitized_answers[question_id] = { + "value": answer["value"], + "note": _safe_text(answer.get("note") or "", hmac_secret=hmac_secret, max_len=1000), + "reviewer": reviewer, + "updated_at": now, + } + + previous_answers = (previous.to_dict().get("answers") if previous else {}) or {} + current_answers = dict(previous_answers) + current_answers.update(sanitized_answers) + changed = sorted( + question_id + for question_id in set(previous_answers) | set(current_answers) + if previous_answers.get(question_id) != current_answers.get(question_id) + ) + state = RulebookReviewState( + job_id=job_id, + profile_id=profile["profile_id"], + profile_version=profile["profile_version"], + review_state=review_state, + reviewer=reviewer, + note=note, + answers=current_answers, + updated_at=now, + review_revision=(previous.review_revision if previous else 0) + 1, + ) + review_payload = repo.put_rulebook_review(state) + audit_payload = repo.append_rulebook_review_audit(RulebookReviewAuditEntry( + job_id=job_id, + profile_id=profile["profile_id"], + profile_version=profile["profile_version"], + review_state=review_state, + reviewer=reviewer, + note=note, + changed_question_ids=changed, + previous_answers={question_id: previous_answers.get(question_id) for question_id in changed}, + current_answers={question_id: current_answers.get(question_id) for question_id in changed}, + timestamp=now, + review_revision=state.review_revision, + )) + if hasattr(owner, "_log_audit_event"): + owner._log_audit_event("rulebook_review_updated", { + "job_id": job_id, + "profile_id": profile["profile_id"], + "review_state": review_state, + "review_revision": state.review_revision, + "changed_question_ids": changed, + }) + return { + "status": "ok", + "job_id": job_id, + "profile_id": profile["profile_id"], + "review": review_payload, + "audit": audit_payload, + } + + +def update_rulebook_review( + owner, + job_id, + profile_id=DEFAULT_RULEBOOK_PROFILE_ID, + answers=None, + reviewer="", + note="", + review_state="draft", +): + profile = _profile(profile_id) + if not profile: + return _error("invalid_profile", job_id, profile_id=profile_id) + if review_state not in VALID_RULEBOOK_REVIEW_STATES: + return _error("invalid_review_answer", job_id, profile_id=profile_id, message="Unsupported review_state.") + try: + validated_answers = _validate_review_answers(profile, answers) + except ValueError as exc: + return _error("invalid_review_answer", job_id, profile_id=profile_id, message=str(exc)) + with _submission_lock(owner, job_id, profile["profile_id"]): + job_specs = owner._get_job_from_cstore(job_id) + if not isinstance(job_specs, dict): + return _error("job_not_found", job_id, profile_id=profile_id) + unsupported = reject_model_test_for_scan_operation(job_specs, job_id, "rulebook_review") + if unsupported: + return { + **unsupported, + "error": "model_test_not_supported", + "error_class": unsupported.get("error_class") or unsupported.get("error"), + } + if job_specs.get("job_status") != JOB_STATUS_FINALIZED: + return _error( + "job_not_finalized", + job_id, + profile_id=profile["profile_id"], + job_status=job_specs.get("job_status"), + ) + return _update_rulebook_review_locked( + owner, + job_id, + profile, + job_specs, + validated_answers, + reviewer, + note, + review_state, + ) diff --git a/extensions/business/cybersec/red_mesh/services/triage.py b/extensions/business/cybersec/red_mesh/services/triage.py index d4b8a1b77..addbff7d3 100644 --- a/extensions/business/cybersec/red_mesh/services/triage.py +++ b/extensions/business/cybersec/red_mesh/services/triage.py @@ -1,4 +1,5 @@ from copy import deepcopy +from contextlib import ExitStack from ..model_testing.artifacts import ModelTestArchive from ..model_testing.constants import is_model_test_job @@ -65,6 +66,26 @@ def get_job_triage(owner, job_id: str, finding_id: str = ""): def update_finding_triage(owner, job_id: str, finding_id: str, status: str, note: str = "", actor: str = "", review_at: float = 0): + from .rulebook_assessment import _submission_lock, list_rulebook_profiles + + with ExitStack() as stack: + profiles = sorted(list_rulebook_profiles(), key=lambda item: item["profile_id"]) + for profile in profiles: + stack.enter_context(_submission_lock(owner, job_id, profile["profile_id"])) + repo = _job_repo(owner) + for profile in profiles: + registry = repo.get_rulebook_submission_registry(job_id, profile["profile_id"]) + if isinstance(registry, dict) and registry.get("pending"): + return { + "error": "submission_in_progress", + "message": "Finding triage cannot change while a formal review submission is pending.", + "job_id": job_id, + "finding_id": finding_id, + } + return _update_finding_triage_locked(owner, job_id, finding_id, status, note, actor, review_at) + + +def _update_finding_triage_locked(owner, job_id: str, finding_id: str, status: str, note: str = "", actor: str = "", review_at: float = 0): if status not in VALID_TRIAGE_STATUSES: return { "error": "validation_error", diff --git a/extensions/business/cybersec/red_mesh/tests/test_api.py b/extensions/business/cybersec/red_mesh/tests/test_api.py index 15d9cfced..0082066a3 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_api.py +++ b/extensions/business/cybersec/red_mesh/tests/test_api.py @@ -2338,6 +2338,55 @@ def test_pass_reports_survive_typed_job_record_rewrites(self): archived_job_specs = plugin._build_job_archive.call_args[0][1] self.assertEqual(len(archived_job_specs["pass_reports"]), 1) + def test_finalization_runs_nis2_ensure_after_pass_report(self): + """Completed pass finalization triggers default-on NIS2 assessment ensure.""" + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin() + self._configure_successful_pass_finalization(plugin, job_specs) + + with patch( + "extensions.business.cybersec.red_mesh.services.finalization.ensure_rulebook_assessment", + return_value={"status": "ok", "artifact_cid": "QmRulebook", "pass_nr": 1}, + ) as ensure_mock: + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + ensure_mock.assert_called_once_with(plugin, job_specs["job_id"], pass_nr=1) + self.assertEqual(job_specs["job_status"], "FINALIZED") + plugin._build_job_archive.assert_called_once_with(job_specs["job_id"], job_specs) + + def test_nis2_ensure_failure_does_not_block_finalization(self): + """NIS2 generation is best-effort and must not fail the scan.""" + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin() + self._configure_successful_pass_finalization(plugin, job_specs) + + with patch( + "extensions.business.cybersec.red_mesh.services.finalization.ensure_rulebook_assessment", + return_value={"status": "error", "error": "artifact_write_failed"}, + ) as ensure_mock: + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + ensure_mock.assert_called_once_with(plugin, job_specs["job_id"], pass_nr=1) + self.assertEqual(job_specs["job_status"], "FINALIZED") + plugin._build_job_archive.assert_called_once_with(job_specs["job_id"], job_specs) + + def test_continuous_pass_runs_nis2_ensure_before_next_pass_schedule(self): + """Continuous jobs refresh NIS2 readiness against each completed pass.""" + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin(run_mode="CONTINUOUS_MONITORING") + self._configure_successful_pass_finalization(plugin, job_specs) + + with patch( + "extensions.business.cybersec.red_mesh.services.finalization.ensure_rulebook_assessment", + return_value={"status": "ok", "artifact_cid": "QmRulebook", "pass_nr": 1}, + ) as ensure_mock: + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + ensure_mock.assert_called_once_with(plugin, job_specs["job_id"], pass_nr=1) + self.assertEqual(job_specs["job_status"], "RUNNING") + self.assertIsNotNone(job_specs["next_pass_at"]) + plugin._build_job_archive.assert_not_called() + def test_aggregated_report_write_failure(self): """R1FS fails for aggregated → pass finalization skipped, no partial state.""" PentesterApi01Plugin = self._get_plugin_class() @@ -3404,6 +3453,16 @@ def _build_plugin(self, jobs_dict): plugin._get_job_from_cstore = lambda job_id: Plugin._get_job_from_cstore(plugin, job_id) return plugin + def test_get_report_does_not_pin_retrieved_cid(self): + Plugin = self._get_plugin_class() + plugin = self._build_plugin({}) + plugin.r1fs.get_json.return_value = {"artifact_kind": "review_submission"} + + result = Plugin.get_report(plugin, "QmReportCID") + + self.assertEqual(result["report"]["artifact_kind"], "review_submission") + plugin.r1fs.get_json.assert_called_once_with("QmReportCID", pin=False) + def test_get_job_archive_finalized(self): """get_job_archive for finalized job returns archive with matching job_id.""" Plugin = self._get_plugin_class() diff --git a/extensions/business/cybersec/red_mesh/tests/test_integration.py b/extensions/business/cybersec/red_mesh/tests/test_integration.py index 649ae7a5a..6d26cc1c5 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_integration.py +++ b/extensions/business/cybersec/red_mesh/tests/test_integration.py @@ -1610,11 +1610,14 @@ def test_purge_finalized_collects_all_cids(self): } plugin.r1fs.get_json.return_value = archive plugin.r1fs.delete_file.return_value = True - plugin.chainstore_hgetall.side_effect = [ - {}, - {"job-1:f-1": {"job_id": "job-1", "finding_id": "f-1", "status": "accepted_risk"}}, - {"job-1:f-1": [{"job_id": "job-1", "finding_id": "f-1", "status": "accepted_risk", "timestamp": 1.0}]}, - ] + plugin.chainstore_hgetall.side_effect = lambda *, hkey: { + "test-instance:triage": { + "job-1:f-1": {"job_id": "job-1", "finding_id": "f-1", "status": "accepted_risk"}, + }, + "test-instance:triage:audit": { + "job-1:f-1": [{"job_id": "job-1", "finding_id": "f-1", "status": "accepted_risk", "timestamp": 1.0}], + }, + }.get(hkey, {}) # Normalize returns the specs as-is plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) @@ -1654,11 +1657,14 @@ def test_purge_finalized_no_pass_report_cids(self): plugin.chainstore_hget.return_value = job_specs plugin.r1fs.get_json.return_value = {"passes": []} plugin.r1fs.delete_file.return_value = True - plugin.chainstore_hgetall.side_effect = [ - {}, - {"job-1:f-1": {"job_id": "job-1", "finding_id": "f-1", "status": "accepted_risk"}}, - {"job-1:f-1": [{"job_id": "job-1", "finding_id": "f-1", "status": "accepted_risk", "timestamp": 1.0}]}, - ] + plugin.chainstore_hgetall.side_effect = lambda *, hkey: { + "test-instance:triage": { + "job-1:f-1": {"job_id": "job-1", "finding_id": "f-1", "status": "accepted_risk"}, + }, + "test-instance:triage:audit": { + "job-1:f-1": [{"job_id": "job-1", "finding_id": "f-1", "status": "accepted_risk", "timestamp": 1.0}], + }, + }.get(hkey, {}) plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) result = Plugin.purge_job(plugin, "job-1") @@ -2130,6 +2136,162 @@ def test_partial_status_preserves_state_no_force_purge(self): # no force-purge R1FS calls plugin.r1fs.delete_file.assert_not_called() + def test_purge_all_deletes_orphan_submission_cids_before_sweeping_registry(self): + Plugin = self._get_plugin_class() + plugin = self._make_plugin({}) + submission_hkey = "test-instance:rulebook_review:submissions" + plugin._hashes[submission_hkey] = { + "orphan-job:nis2.eu_baseline.v1": { + "contract_version": "1.0.0", + "latest_revision": 1, + "submissions": [{"revision": 1, "cid": "cid-orphan-submission"}], + }, + } + plugin.r1fs = MagicMock() + plugin.r1fs.delete_file.return_value = True + plugin.r1fs.get_json.return_value = {"artifact_kind": "review_submission"} + + result = Plugin.purge_all_redmesh_data(plugin, confirm=True) + + self.assertEqual(result["status"], "success") + self.assertEqual(result["cids_deleted"], 1) + plugin.r1fs.delete_file.assert_called_once() + plugin.r1fs.get_json.assert_not_called() + self.assertEqual(plugin._hashes[submission_hkey], {}) + + def test_purge_all_protects_orphan_cid_referenced_by_failed_job_record(self): + Plugin = self._get_plugin_class() + jobs = { + "job-partial": { + "job_id": "job-partial", + "job_status": "RUNNING", + "legacy_submission_cid": "cid-shared-submission", + }, + } + plugin = self._make_plugin(jobs) + submission_hkey = "test-instance:rulebook_review:submissions" + orphan_key = "orphan-job:nis2.eu_baseline.v1" + plugin._hashes[submission_hkey] = { + orphan_key: {"submissions": [{"revision": 1, "cid": "cid-shared-submission"}]}, + } + plugin.stop_and_delete_job.return_value = { + "status": "partial", + "cids_deleted": 0, + "cids_failed": 1, + } + plugin.r1fs = MagicMock() + + result = Plugin.purge_all_redmesh_data(plugin, confirm=True) + + self.assertEqual(result["status"], "partial") + self.assertIn(orphan_key, plugin._hashes[submission_hkey]) + plugin.r1fs.delete_file.assert_not_called() + + def test_force_purge_retains_rows_when_submission_cid_is_shared(self): + Plugin = self._get_plugin_class() + jobs = {"job-bad": {"job_id": "job-bad", "job_status": "legacy"}} + plugin = self._make_plugin(jobs) + submission_hkey = "test-instance:rulebook_review:submissions" + plugin._hashes[submission_hkey] = { + "job-bad:nis2.eu_baseline.v1": { + "submissions": [{"revision": 1, "cid": "cid-shared-submission"}], + }, + "orphan-peer:nis2.eu_baseline.v1": { + "submissions": [{"revision": 1, "cid": "cid-shared-submission"}], + }, + } + plugin.r1fs = MagicMock() + plugin.r1fs.delete_file.return_value = True + plugin.stop_and_delete_job.side_effect = RuntimeError("legacy parse failure") + + result = Plugin.purge_all_redmesh_data(plugin, confirm=True) + + self.assertEqual(result["status"], "partial") + self.assertIn("job-bad", plugin._hashes["test-instance"]) + self.assertIn("job-bad:nis2.eu_baseline.v1", plugin._hashes[submission_hkey]) + self.assertIn("orphan-peer:nis2.eu_baseline.v1", plugin._hashes[submission_hkey]) + self.assertNotIn("cid-shared-submission", {call.args[0] for call in plugin.r1fs.delete_file.call_args_list}) + + def test_force_purge_retains_rows_when_submission_registry_read_fails(self): + from extensions.business.cybersec.red_mesh.services.control import _force_purge_job + + jobs = {"job-bad": {"job_id": "job-bad", "job_status": "legacy"}} + plugin = self._make_plugin(jobs) + submission_hkey = "test-instance:rulebook_review:submissions" + submission_key = "job-bad:nis2.eu_baseline.v1" + plugin._hashes[submission_hkey] = { + submission_key: {"submissions": [{"revision": 1, "cid": "cid-retained-submission"}]}, + } + original_hgetall = plugin.chainstore_hgetall.side_effect + + def _fail_submission_read(*, hkey): + if hkey == submission_hkey: + raise RuntimeError("submission registry unavailable") + return original_hgetall(hkey=hkey) + + plugin.chainstore_hgetall.side_effect = _fail_submission_read + plugin.r1fs = MagicMock() + errors = [] + + cids_deleted, cids_failed = _force_purge_job(plugin, "job-bad", jobs["job-bad"], errors) + + self.assertEqual((cids_deleted, cids_failed), (0, 1)) + self.assertIn("job-bad", plugin._hashes["test-instance"]) + self.assertIn(submission_key, plugin._hashes[submission_hkey]) + plugin.r1fs.delete_file.assert_not_called() + + def test_force_purge_retains_rows_when_shared_cid_discovery_fails(self): + from extensions.business.cybersec.red_mesh.services.control import _force_purge_job + + jobs = {"job-bad": {"job_id": "job-bad", "job_status": "legacy"}} + plugin = self._make_plugin(jobs) + submission_hkey = "test-instance:rulebook_review:submissions" + submission_key = "job-bad:nis2.eu_baseline.v1" + plugin._hashes[submission_hkey] = { + submission_key: {"submissions": [{"revision": 1, "cid": "cid-retained-submission"}]}, + } + original_hgetall = plugin.chainstore_hgetall.side_effect + + def _fail_job_registry_read(*, hkey): + if hkey == "test-instance": + raise RuntimeError("job registry unavailable") + return original_hgetall(hkey=hkey) + + plugin.chainstore_hgetall.side_effect = _fail_job_registry_read + plugin.r1fs = MagicMock() + errors = [] + + cids_deleted, cids_failed = _force_purge_job(plugin, "job-bad", jobs["job-bad"], errors) + + self.assertEqual((cids_deleted, cids_failed), (0, 1)) + self.assertIn("job-bad", plugin._hashes["test-instance"]) + self.assertIn(submission_key, plugin._hashes[submission_hkey]) + plugin.r1fs.delete_file.assert_not_called() + + def test_force_purge_clears_rows_after_relay_acknowledges_formal_cid_delete(self): + Plugin = self._get_plugin_class() + jobs = {"job-bad": {"job_id": "job-bad", "job_status": "legacy"}} + plugin = self._make_plugin(jobs) + submission_hkey = "test-instance:rulebook_review:submissions" + submission_key = "job-bad:nis2.eu_baseline.v1" + plugin._hashes[submission_hkey] = { + submission_key: { + "submissions": [{"revision": 1, "cid": "cid-retained-submission"}], + }, + } + plugin.r1fs = MagicMock() + plugin.r1fs.delete_file.return_value = True + plugin.r1fs.get_json.return_value = {"artifact_kind": "review_submission"} + plugin.stop_and_delete_job.side_effect = RuntimeError("legacy parse failure") + + result = Plugin.purge_all_redmesh_data(plugin, confirm=True) + + self.assertEqual(result["status"], "partial") + self.assertEqual(result["cids_failed"], 0) + self.assertNotIn("job-bad", plugin._hashes["test-instance"]) + self.assertNotIn(submission_key, plugin._hashes[submission_hkey]) + plugin.r1fs.get_json.assert_not_called() + def test_confirm_required(self): """Endpoint refuses to purge without confirm=True.""" Plugin = self._get_plugin_class() diff --git a/extensions/business/cybersec/red_mesh/tests/test_repositories.py b/extensions/business/cybersec/red_mesh/tests/test_repositories.py index fffb55b3b..5000e7741 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_repositories.py +++ b/extensions/business/cybersec/red_mesh/tests/test_repositories.py @@ -301,6 +301,21 @@ def test_artifact_repository_delete_is_guarded_on_empty_cid(self): self.assertFalse(repo.delete("")) owner.r1fs.delete_file.assert_not_called() + def test_artifact_repository_purge_removes_remote_pin_and_local_cache(self): + owner = self._make_owner() + repo = ArtifactRepository(owner) + + repo.delete("QmCID", show_logs=True, purge=True) + + owner.r1fs.delete_file.assert_called_once_with( + "QmCID", + unpin_remote=True, + run_gc=True, + cleanup_local_files=True, + show_logs=True, + raise_on_error=False, + ) + def test_artifact_repository_supports_typed_models(self): owner = self._make_owner() repo = ArtifactRepository(owner) diff --git a/extensions/business/cybersec/red_mesh/tests/test_rulebook_assessment.py b/extensions/business/cybersec/red_mesh/tests/test_rulebook_assessment.py new file mode 100644 index 000000000..438585bfa --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_rulebook_assessment.py @@ -0,0 +1,1132 @@ +import json +import sys +import time +import types +import unittest +from concurrent.futures import ThreadPoolExecutor +from unittest.mock import MagicMock + + +def _install_pymisp_stub(): + if "pymisp" in sys.modules: + return + pymisp_stub = types.ModuleType("pymisp") + pymisp_stub.MISPEvent = MagicMock + pymisp_stub.MISPObject = MagicMock + pymisp_stub.MISPAttribute = MagicMock + pymisp_stub.PyMISP = MagicMock + sys.modules["pymisp"] = pymisp_stub + + +_install_pymisp_stub() + +from extensions.business.cybersec.red_mesh.services.control import purge_job +from extensions.business.cybersec.red_mesh.models import ( + RULEBOOK_ASSESSMENT_SCHEMA_VERSION, + RULEBOOK_SUBMISSION_CONTRACT_VERSION, + RulebookPendingSubmission, + RulebookSubmissionReference, + RulebookSubmissionRegistry, +) +from extensions.business.cybersec.red_mesh.repositories import JobStateRepository +from extensions.business.cybersec.red_mesh.services.rulebook_assessment import ( + DEFAULT_RULEBOOK_PROFILE_ID, + build_rulebook_assessment, + ensure_rulebook_assessment, + generate_rulebook_assessment, + get_rulebook_assessment_status, + get_rulebook_review, + reopen_rulebook_review, + save_rulebook_review_draft, + submit_rulebook_review, + update_rulebook_review, +) +from extensions.business.cybersec.red_mesh.services.triage import update_finding_triage + + +def _sample_findings(): + return [ + { + "finding_id": "finding-auth-1", + "severity": "HIGH", + "title": "Authentication bypass on app.example.test", + "description": "The endpoint leaks token=supersecret for 10.0.0.4.", + "evidence": {"raw_response": "password=supersecret", "credential_ref": "secret://graybox"}, + "remediation": "Require authentication and validate sessions.", + "cwe_id": "CWE-306", + "owasp_id": "A01:2021", + "probe": "_web_test_auth_bypass", + "category": "auth", + "status": "vulnerable", + }, + { + "finding_id": "finding-crypto-1", + "severity": "LOW", + "title": "TLS certificate expires soon on app.example.test", + "description": "Certificate metadata needs review.", + "cwe_id": "CWE-295", + "probe": "_tls_certificate_check", + "category": "tls", + "status": "vulnerable", + }, + ] + + +def _sample_pass_report(): + return { + "pass_nr": 3, + "date_started": 1770000000.0, + "date_completed": 1770000300.0, + "aggregated_report_cid": "agg-cid", + "risk_score": 75, + "quick_summary": "Issues were observed on app.example.test.", + "findings": _sample_findings(), + } + + +def _sample_archive(): + return { + "job_id": "job-1", + "job_config": { + "target": "https://app.example.test/login", + "target_url": "https://app.example.test/login", + "scan_type": "webapp", + "task_name": "Application scan", + "start_port": 443, + "end_port": 443, + "secret_ref": "secret://graybox", + }, + "passes": [_sample_pass_report()], + "ui_aggregate": {}, + "duration": 300.0, + "date_created": 1770000000.0, + "date_completed": 1770000300.0, + "soc_event_status": {"last_status": "sent"}, + } + + +def _sample_job_specs(**overrides): + payload = { + "job_id": "job-1", + "job_status": "FINALIZED", + "scan_type": "webapp", + "target": "https://app.example.test/login", + "target_url": "https://app.example.test/login", + "job_cid": "archive-cid", + "job_config_cid": "config-cid", + "date_created": 1770000000.0, + "date_completed": 1770000300.0, + "run_mode": "SINGLEPASS", + "launcher": "operator", + "start_port": 443, + "end_port": 443, + } + payload.update(overrides) + return payload + + +def _complete_review_answers(): + return { + "nis2.risk.risk_treatment_reviewed": {"value": "yes"}, + "nis2.incident.incident_process": {"value": "yes"}, + "nis2.bcm.business_continuity": {"value": "yes"}, + "nis2.supply.supplier_risk_reviewed": {"value": "yes"}, + "nis2.access.access_controls_reviewed": {"value": "yes"}, + "nis2.crypto.crypto_policy_reviewed": {"value": "yes"}, + "nis2.effectiveness.assessment_reviewed": {"value": "yes"}, + "nis2.reporting.reporting_process": {"value": "yes"}, + } + + +class _FakeArtifactRepo: + def __init__(self, owner, archive=None, aggregated=None): + self.owner = owner + self.archive = archive or _sample_archive() + self.aggregated = aggregated or {"open_ports": [443]} + self.deleted = [] + + def get_archive(self, job_specs): + return self.archive + + def get_json(self, cid): + if cid == "archive-cid": + return self.archive + if cid == "agg-cid": + return self.aggregated + if cid == "config-cid": + return self.archive.get("job_config", {}) + return self.owner.artifacts.get(cid) + + def get_job_config(self, job_specs): + return self.archive.get("job_config", {}) + + def get_pass_report(self, report_cid): + return self.owner.artifacts.get(report_cid) + + def put_json(self, payload, *, show_logs=False): + return self.owner.r1fs.add_json(payload, show_logs=show_logs) + + def delete(self, cid, *, show_logs=False, raise_on_error=False, purge=False): + self.deleted.append(cid) + self.owner.artifacts.pop(cid, None) + return True + + +class _Owner: + cfg_instance_id = "tenant-a" + + def __init__(self, job_specs=None, archive=None): + self.job_specs = job_specs or _sample_job_specs() + self.archive = archive or _sample_archive() + self.artifacts = {} + self.records = {} + self.messages = [] + self.audit_events = [] + self.artifact_repo = _FakeArtifactRepo(self, archive=self.archive) + self.r1fs = MagicMock() + self.r1fs.add_json.side_effect = self._add_json + + def _add_json(self, payload, show_logs=False): + cid = f"QmRulebook{len(self.artifacts) + 1}" + self.artifacts[cid] = payload + return cid + + def P(self, msg, **kwargs): + self.messages.append(msg) + + def time(self): + return 1770000400.0 + + def _get_job_from_cstore(self, job_id): + return self.job_specs if job_id == "job-1" else None + + def _get_artifact_repository(self): + return self.artifact_repo + + def _write_job_record(self, job_id, updated, context=""): + self.job_specs = updated + self.records[(self.cfg_instance_id, job_id)] = updated + return updated + + def _normalize_job_record(self, job_id, raw): + return job_id, raw + + def _log_audit_event(self, event_type, payload): + self.audit_events.append((event_type, payload)) + + def chainstore_hget(self, hkey, key): + return self.records.get((hkey, key)) + + def chainstore_hset(self, hkey, key, value): + self.records[(hkey, key)] = value + return True + + def chainstore_hgetall(self, hkey): + return { + key: value + for (row_hkey, key), value in self.records.items() + if row_hkey == hkey and value is not None + } + + +class TestRulebookAssessment(unittest.TestCase): + + def test_generate_persists_artifact_metadata_and_redacts_sensitive_values(self): + owner = _Owner() + + result = generate_rulebook_assessment(owner, "job-1") + + self.assertEqual(result["status"], "ok") + self.assertEqual(result["artifact_cid"], "QmRulebook1") + self.assertEqual( + owner.job_specs["rulebook_assessments"][DEFAULT_RULEBOOK_PROFILE_ID]["artifact_cid"], + "QmRulebook1", + ) + meta = owner.job_specs["rulebook_assessments"][DEFAULT_RULEBOOK_PROFILE_ID] + self.assertTrue(meta["auto_enabled"]) + self.assertEqual(meta["run_state"], "succeeded") + self.assertEqual(meta["latest_pass_nr"], 3) + self.assertEqual(meta["history"], []) + assessment = owner.artifacts["QmRulebook1"] + self.assertEqual(assessment["schema"], "redmesh.rulebook_assessment.v1") + self.assertEqual(assessment["schema_version"], "1.1.0") + self.assertEqual(assessment["artifact_kind"], "generated_assessment") + self.assertEqual(assessment["profile"]["profile_id"], DEFAULT_RULEBOOK_PROFILE_ID) + self.assertGreater(assessment["status_counts"]["gap"], 0) + + serialized = json.dumps(assessment, sort_keys=True).lower() + self.assertNotIn("app.example.test", serialized) + self.assertNotIn("10.0.0.4", serialized) + self.assertNotIn("supersecret", serialized) + self.assertNotIn("credential_ref", serialized) + self.assertNotIn("secret://graybox", serialized) + self.assertNotIn("non_compliant", serialized) + self.assertNotIn("compliant", serialized) + self.assertIn("target:", serialized) + + status = get_rulebook_assessment_status(owner, "job-1") + self.assertTrue(status["found"]) + self.assertTrue(status["generated"]) + self.assertEqual(status["artifact_cid"], "QmRulebook1") + self.assertEqual(status["run_state"], "succeeded") + + def test_persisted_generated_assessment_excludes_mutable_draft_review(self): + owner = _Owner() + update_rulebook_review( + owner, + "job-1", + answers={"nis2.bcm.business_continuity": {"value": "yes", "note": "private draft comment"}}, + reviewer="draft-reviewer", + review_state="draft", + ) + + preview = generate_rulebook_assessment(owner, "job-1", persist=False) + persisted = generate_rulebook_assessment(owner, "job-1", persist=True) + + self.assertEqual( + preview["assessment"]["checks"][2]["review_answer"]["note"], + "private draft comment", + ) + artifact = owner.artifacts[persisted["artifact_cid"]] + self.assertEqual(artifact["review_state"], {"review_state": "draft", "answers": {}}) + serialized = json.dumps(artifact, sort_keys=True) + self.assertNotIn("private draft comment", serialized) + self.assertNotIn("draft-reviewer", serialized) + + def test_submission_models_and_registry_round_trip(self): + reference = RulebookSubmissionReference( + revision=1, + cid="QmSubmission1", + submitted_at=1770000400.0, + actor="alice", + pass_nr=3, + profile_id=DEFAULT_RULEBOOK_PROFILE_ID, + profile_version="1.0.0", + schema_version=RULEBOOK_ASSESSMENT_SCHEMA_VERSION, + review_revision=2, + idempotency_key="submission-1", + fingerprint="a" * 64, + ) + pending = RulebookPendingSubmission( + target_revision=2, + expected_review_revision=3, + expected_pass_nr=3, + expected_profile_version="1.0.0", + actor="alice", + idempotency_key="submission-2", + fingerprint="b" * 64, + ) + registry = RulebookSubmissionRegistry( + submissions=[reference], + pending=pending, + ) + + payload = registry.to_dict() + restored = RulebookSubmissionRegistry.from_dict(payload).to_dict() + + self.assertEqual(restored["contract_version"], RULEBOOK_SUBMISSION_CONTRACT_VERSION) + self.assertEqual(restored["latest_revision"], 1) + self.assertEqual(restored["submissions"][0]["cid"], "QmSubmission1") + self.assertEqual(restored["pending"]["target_revision"], 2) + + def test_submission_registry_uses_dedicated_cstore_hash(self): + owner = _Owner() + repo = JobStateRepository(owner) + registry = RulebookSubmissionRegistry(submissions=[]) + + stored = repo.put_rulebook_submission_registry("job-1", DEFAULT_RULEBOOK_PROFILE_ID, registry) + loaded = repo.get_rulebook_submission_registry_model("job-1", DEFAULT_RULEBOOK_PROFILE_ID) + + key = f"job-1:{DEFAULT_RULEBOOK_PROFILE_ID}" + self.assertEqual( + owner.records[(f"{owner.cfg_instance_id}:rulebook_review:submissions", key)], + stored, + ) + self.assertEqual(loaded.to_dict(), stored) + + def test_save_draft_is_revisioned_and_never_writes_r1fs(self): + owner = _Owner() + + saved = save_rulebook_review_draft( + owner, + "job-1", + answers={"nis2.bcm.business_continuity": {"value": "yes", "note": "Plan reviewed."}}, + actor="alice", + expected_review_revision=0, + ) + stale = save_rulebook_review_draft( + owner, + "job-1", + answers={}, + actor="alice", + expected_review_revision=0, + ) + + self.assertEqual(saved["status"], "ok") + self.assertEqual(saved["review"]["review_revision"], 1) + self.assertEqual(saved["effective_review_state"], "draft") + self.assertEqual(stale["error"], "review_revision_conflict") + self.assertEqual(owner.r1fs.add_json.call_count, 0) + + def test_submit_persists_complete_snapshot_and_replays_idempotently(self): + owner = _Owner() + saved = save_rulebook_review_draft( + owner, + "job-1", + answers=_complete_review_answers(), + actor="alice", + expected_review_revision=0, + ) + + submitted = submit_rulebook_review( + owner, + "job-1", + expected_review_revision=saved["review_revision"], + expected_pass_nr=3, + expected_profile_version="1.0.0", + idempotency_key="submission-1", + actor="alice", + ) + replay = submit_rulebook_review( + owner, + "job-1", + expected_review_revision=saved["review_revision"], + expected_pass_nr=3, + expected_profile_version="1.0.0", + idempotency_key="submission-1", + actor="alice", + ) + conflict = submit_rulebook_review( + owner, + "job-1", + expected_review_revision=saved["review_revision"], + expected_pass_nr=3, + expected_profile_version="1.0.0", + idempotency_key="submission-1", + actor="mallory", + ) + + self.assertEqual(submitted["effective_review_state"], "submitted") + self.assertEqual(submitted["submission"]["revision"], 1) + self.assertEqual(replay["submission"]["cid"], submitted["submission"]["cid"]) + self.assertTrue(replay["idempotent_replay"]) + self.assertEqual(conflict["error"], "submission_idempotency_conflict") + self.assertEqual(owner.r1fs.add_json.call_count, 1) + snapshot = owner.artifacts[submitted["submission"]["cid"]] + self.assertEqual(snapshot["schema_version"], "1.1.0") + self.assertEqual(snapshot["artifact_kind"], "review_submission") + self.assertEqual(snapshot["submission"]["revision"], 1) + self.assertEqual(snapshot["review_state"]["review_state"], "submitted") + + def test_committed_idempotency_replay_conflicts_when_assessment_inputs_change(self): + owner = _Owner() + saved = save_rulebook_review_draft( + owner, "job-1", answers=_complete_review_answers(), actor="alice", expected_review_revision=0, + ) + submitted = submit_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], expected_pass_nr=3, + expected_profile_version="1.0.0", idempotency_key="submission-triage", actor="alice", + ) + owner.records[(f"{owner.cfg_instance_id}:triage", "job-1:finding-auth-1")] = { + "job_id": "job-1", + "finding_id": "finding-auth-1", + "status": "false_positive", + "actor": "analyst", + "updated_at": owner.time(), + } + + replay = submit_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], expected_pass_nr=3, + expected_profile_version="1.0.0", idempotency_key="submission-triage", actor="alice", + ) + + self.assertEqual(submitted["status"], "ok") + self.assertEqual(replay["error"], "submission_idempotency_conflict") + self.assertEqual(owner.r1fs.add_json.call_count, 1) + + def test_submission_requires_complete_answers_and_comments(self): + owner = _Owner() + answers = _complete_review_answers() + answers["nis2.reporting.reporting_process"] = {"value": "unknown", "note": ""} + saved = save_rulebook_review_draft( + owner, + "job-1", + answers=answers, + actor="alice", + expected_review_revision=0, + ) + + result = submit_rulebook_review( + owner, + "job-1", + expected_review_revision=saved["review_revision"], + expected_pass_nr=3, + expected_profile_version="1.0.0", + idempotency_key="submission-comments", + actor="alice", + ) + + self.assertEqual(result["error"], "submission_comments_required") + self.assertEqual( + result["comment_required_question_ids"], + ["nis2.reporting.reporting_process"], + ) + self.assertEqual(owner.r1fs.add_json.call_count, 0) + + def test_r1fs_failure_keeps_draft_and_same_key_retry_recovers(self): + owner = _Owner() + saved = save_rulebook_review_draft( + owner, + "job-1", + answers=_complete_review_answers(), + actor="alice", + expected_review_revision=0, + ) + owner.r1fs.add_json.side_effect = None + owner.r1fs.add_json.return_value = None + + failed = submit_rulebook_review( + owner, + "job-1", + expected_review_revision=saved["review_revision"], + expected_pass_nr=3, + expected_profile_version="1.0.0", + idempotency_key="submission-retry", + actor="alice", + ) + visible = get_rulebook_review(owner, "job-1") + owner.r1fs.add_json.side_effect = owner._add_json + recovered = submit_rulebook_review( + owner, + "job-1", + expected_review_revision=saved["review_revision"], + expected_pass_nr=3, + expected_profile_version="1.0.0", + idempotency_key="submission-retry", + actor="alice", + ) + + self.assertEqual(failed["error"], "submission_persist_failed") + self.assertEqual(visible["effective_review_state"], "draft") + self.assertEqual(visible["submission_operation_state"], "failed") + self.assertEqual(recovered["effective_review_state"], "submitted") + self.assertEqual(recovered["submission"]["revision"], 1) + + def test_pending_submission_fences_finding_triage_until_retry(self): + owner = _Owner() + saved = save_rulebook_review_draft( + owner, "job-1", answers=_complete_review_answers(), actor="alice", expected_review_revision=0, + ) + owner.r1fs.add_json.side_effect = None + owner.r1fs.add_json.return_value = None + failed = submit_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], expected_pass_nr=3, + expected_profile_version="1.0.0", idempotency_key="pending-triage", actor="alice", + ) + + triage = update_finding_triage( + owner, "job-1", "finding-auth-1", "false_positive", actor="analyst", + ) + + self.assertEqual(failed["error"], "submission_persist_failed") + self.assertEqual(triage["error"], "submission_in_progress") + self.assertIsNone(owner.records.get((f"{owner.cfg_instance_id}:triage", "job-1:finding-auth-1"))) + + def test_partial_final_registry_write_recovers_without_second_r1fs_write(self): + owner = _Owner() + saved = save_rulebook_review_draft( + owner, + "job-1", + answers=_complete_review_answers(), + actor="alice", + expected_review_revision=0, + ) + original_hset = owner.chainstore_hset + failed_once = {"value": False} + + def fail_final_registry_write(hkey, key, value): + if ( + hkey.endswith(":rulebook_review:submissions") + and isinstance(value, dict) + and value.get("submissions") + and not value.get("pending") + and not failed_once["value"] + ): + failed_once["value"] = True + raise RuntimeError("simulated final registry failure") + return original_hset(hkey, key, value) + + owner.chainstore_hset = fail_final_registry_write + failed = submit_rulebook_review( + owner, + "job-1", + expected_review_revision=saved["review_revision"], + expected_pass_nr=3, + expected_profile_version="1.0.0", + idempotency_key="submission-partial", + actor="alice", + ) + owner.chainstore_hset = original_hset + recovered = submit_rulebook_review( + owner, + "job-1", + expected_review_revision=saved["review_revision"], + expected_pass_nr=3, + expected_profile_version="1.0.0", + idempotency_key="submission-partial", + actor="alice", + ) + + self.assertEqual(failed["error"], "submission_record_failed") + self.assertEqual(recovered["effective_review_state"], "submitted") + self.assertEqual(owner.r1fs.add_json.call_count, 1) + + def test_concurrent_duplicate_submission_allocates_one_revision(self): + owner = _Owner() + saved = save_rulebook_review_draft( + owner, + "job-1", + answers=_complete_review_answers(), + actor="alice", + expected_review_revision=0, + ) + + def submit(): + return submit_rulebook_review( + owner, + "job-1", + expected_review_revision=saved["review_revision"], + expected_pass_nr=3, + expected_profile_version="1.0.0", + idempotency_key="submission-concurrent", + actor="alice", + ) + + with ThreadPoolExecutor(max_workers=2) as executor: + results = list(executor.map(lambda _: submit(), range(2))) + + self.assertEqual([result["status"] for result in results], ["ok", "ok"]) + self.assertEqual(owner.r1fs.add_json.call_count, 1) + self.assertEqual(len(get_rulebook_review(owner, "job-1")["submissions"]), 1) + + def test_submit_rejects_stale_revision_pass_and_profile(self): + owner = _Owner() + saved = save_rulebook_review_draft( + owner, + "job-1", + answers=_complete_review_answers(), + actor="alice", + expected_review_revision=0, + ) + + stale_revision = submit_rulebook_review( + owner, "job-1", expected_review_revision=0, expected_pass_nr=3, + expected_profile_version="1.0.0", idempotency_key="stale-revision", actor="alice", + ) + stale_pass = submit_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], expected_pass_nr=2, + expected_profile_version="1.0.0", idempotency_key="stale-pass", actor="alice", + ) + stale_profile = submit_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], expected_pass_nr=3, + expected_profile_version="0.9.0", idempotency_key="stale-profile", actor="alice", + ) + + self.assertEqual(stale_revision["error"], "review_revision_conflict") + self.assertEqual(stale_pass["error"], "submission_pass_stale") + self.assertEqual(stale_profile["error"], "submission_profile_stale") + + def test_reopen_and_resubmit_preserve_both_revisions(self): + owner = _Owner() + saved = save_rulebook_review_draft( + owner, "job-1", answers=_complete_review_answers(), actor="alice", expected_review_revision=0, + ) + first = submit_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], expected_pass_nr=3, + expected_profile_version="1.0.0", idempotency_key="revision-1", actor="alice", + ) + reopened = reopen_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], + idempotency_key="reopen-revision-1", actor="alice", + ) + second = submit_rulebook_review( + owner, "job-1", expected_review_revision=reopened["review_revision"], expected_pass_nr=3, + expected_profile_version="1.0.0", idempotency_key="revision-2", actor="alice", + ) + + self.assertEqual(first["submission"]["revision"], 1) + self.assertEqual(reopened["effective_review_state"], "draft") + self.assertEqual(reopened["review_revision"], 2) + self.assertEqual(second["submission"]["revision"], 2) + history = get_rulebook_review(owner, "job-1")["submissions"] + self.assertEqual([item["revision"] for item in history], [2, 1]) + self.assertNotEqual(history[0]["cid"], history[1]["cid"]) + + def test_reopen_replays_same_operation_key_after_response_loss(self): + owner = _Owner() + saved = save_rulebook_review_draft( + owner, "job-1", answers=_complete_review_answers(), actor="alice", expected_review_revision=0, + ) + submit_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], expected_pass_nr=3, + expected_profile_version="1.0.0", idempotency_key="reopen-source", actor="alice", + ) + + first = reopen_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], + idempotency_key="reopen-operation", actor="alice", + ) + replay = reopen_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], + idempotency_key="reopen-operation", actor="alice", + ) + conflict = reopen_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], + idempotency_key="reopen-operation", actor="mallory", + ) + + self.assertEqual(first["review_revision"], 2) + self.assertTrue(replay["idempotent_replay"]) + self.assertEqual(replay["review_revision"], 2) + self.assertEqual(conflict["error"], "reopen_idempotency_conflict") + + def test_legacy_write_remains_compatible_after_native_review_is_reopened(self): + owner = _Owner() + saved = save_rulebook_review_draft( + owner, "job-1", answers=_complete_review_answers(), actor="alice", expected_review_revision=0, + ) + submit_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], expected_pass_nr=3, + expected_profile_version="1.0.0", idempotency_key="rollback-source", actor="alice", + ) + reopened = reopen_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], + idempotency_key="rollback-reopen", actor="alice", + ) + + legacy_saved = update_rulebook_review( + owner, + "job-1", + answers={"nis2.bcm.business_continuity": {"value": "yes", "note": "edited after rollback"}}, + reviewer="legacy-navigator", + review_state="draft", + ) + legacy_reviewed = update_rulebook_review( + owner, + "job-1", + answers={}, + reviewer="legacy-navigator", + review_state="reviewed", + ) + + self.assertEqual(reopened["effective_review_state"], "draft") + self.assertEqual(legacy_saved["status"], "ok") + self.assertEqual(legacy_saved["review"]["review_revision"], 3) + self.assertEqual(legacy_reviewed["error"], "review_already_submitted") + self.assertEqual(len(get_rulebook_review(owner, "job-1")["submissions"]), 1) + + def test_two_revision_submission_smoke_retrieves_then_purges_every_snapshot(self): + owner = _Owner(job_specs=_sample_job_specs()) + owner.records[(owner.cfg_instance_id, "job-1")] = owner.job_specs + review_key = f"job-1:{DEFAULT_RULEBOOK_PROFILE_ID}" + + saved = save_rulebook_review_draft( + owner, "job-1", answers=_complete_review_answers(), actor="alice", expected_review_revision=0, + ) + first = submit_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], expected_pass_nr=3, + expected_profile_version="1.0.0", idempotency_key="smoke-revision-1", actor="alice", + ) + reopened = reopen_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], + idempotency_key="smoke-reopen-revision-1", actor="alice", + ) + second = submit_rulebook_review( + owner, "job-1", expected_review_revision=reopened["review_revision"], expected_pass_nr=3, + expected_profile_version="1.0.0", idempotency_key="smoke-revision-2", actor="bob", + ) + + first_cid = first["submission"]["cid"] + second_cid = second["submission"]["cid"] + self.assertEqual(owner.artifact_repo.get_json(first_cid)["submission"]["revision"], 1) + self.assertEqual(owner.artifact_repo.get_json(second_cid)["submission"]["revision"], 2) + self.assertEqual( + [item["cid"] for item in get_rulebook_review(owner, "job-1")["submissions"]], + [second_cid, first_cid], + ) + + result = purge_job(owner, "job-1") + + self.assertEqual(result["status"], "success") + self.assertIsNone(owner.artifact_repo.get_json(first_cid)) + self.assertIsNone(owner.artifact_repo.get_json(second_cid)) + self.assertIn(first_cid, owner.artifact_repo.deleted) + self.assertIn(second_cid, owner.artifact_repo.deleted) + self.assertIsNone(owner.records[(f"{owner.cfg_instance_id}:rulebook_review", review_key)]) + self.assertIsNone(owner.records[(f"{owner.cfg_instance_id}:rulebook_review:audit", review_key)]) + self.assertIsNone(owner.records[(f"{owner.cfg_instance_id}:rulebook_review:submissions", review_key)]) + + def test_newer_scan_evidence_marks_prior_submission_stale(self): + owner = _Owner() + saved = save_rulebook_review_draft( + owner, "job-1", answers=_complete_review_answers(), actor="alice", expected_review_revision=0, + ) + submit_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], expected_pass_nr=3, + expected_profile_version="1.0.0", idempotency_key="stale-after-pass", actor="alice", + ) + owner.archive["passes"].append({**_sample_pass_report(), "pass_nr": 4}) + + current = get_rulebook_review(owner, "job-1") + + self.assertTrue(current["submissions"][0]["stale"]) + self.assertEqual(current["submissions"][0]["stale_reasons"], ["newer_scan_pass"]) + + def test_formal_submission_redacts_sensitive_review_comments(self): + owner = _Owner() + answers = _complete_review_answers() + answers["nis2.bcm.business_continuity"] = { + "value": "unknown", + "note": "password=supersecret observed at 10.0.0.4", + } + saved = save_rulebook_review_draft( + owner, "job-1", answers=answers, actor="alice", expected_review_revision=0, + ) + submitted = submit_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], expected_pass_nr=3, + expected_profile_version="1.0.0", idempotency_key="redacted-submission", actor="alice", + ) + + serialized = json.dumps(owner.artifacts[submitted["submission"]["cid"]], sort_keys=True).lower() + self.assertNotIn("supersecret", serialized) + self.assertNotIn("10.0.0.4", serialized) + self.assertIn("", serialized) + self.assertIn("ip:", serialized) + + def test_formal_submission_redacts_bearer_jwt_pem_and_unlabelled_tokens(self): + owner = _Owner() + evidence_cid = "QmbuqxraU9uNEYcwiKnMZacSNrRwaGpUXctewuiL5HNF94" + evidence_sha256 = "0123456789abcdef" * 4 + answers = _complete_review_answers() + answers["nis2.bcm.business_continuity"] = { + "value": "unknown", + "note": ( + "Authorization: Bearer sk_live_1234567890abcdef " + "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.signature1234 " + "-----BEGIN PRIVATE KEY-----\nprivatekeymaterial123456\n-----END PRIVATE KEY----- " + "a9b8c7d6e5f4a3b2c1d0e9f8a7b6c5d4 " + "AKIA1234567890ABCDEF glpat-1234567890abcdefghij " + f"evidence {evidence_cid} sha256:{evidence_sha256}" + ), + } + saved = save_rulebook_review_draft( + owner, "job-1", answers=answers, actor="alice", expected_review_revision=0, + ) + submitted = submit_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], expected_pass_nr=3, + expected_profile_version="1.0.0", idempotency_key="credential-redaction", actor="alice", + ) + + serialized = json.dumps(owner.artifacts[submitted["submission"]["cid"]], sort_keys=True) + self.assertNotIn("sk_live_1234567890abcdef", serialized) + self.assertNotIn("eyJhbGciOiJIUzI1NiJ9", serialized) + self.assertNotIn("privatekeymaterial123456", serialized) + self.assertNotIn("a9b8c7d6e5f4a3b2c1d0e9f8a7b6c5d4", serialized) + self.assertNotIn("AKIA1234567890ABCDEF", serialized) + self.assertNotIn("glpat-1234567890abcdefghij", serialized) + self.assertIn(evidence_cid, serialized) + self.assertIn(evidence_sha256, serialized) + self.assertIn("", serialized) + + def test_future_registry_contract_returns_stable_upgrade_error_for_writes(self): + owner = _Owner() + hkey = f"{owner.cfg_instance_id}:rulebook_review:submissions" + owner.records[(hkey, f"job-1:{DEFAULT_RULEBOOK_PROFILE_ID}")] = { + "contract_version": "2.0.0", + "latest_revision": 0, + "submissions": [], + } + + saved = save_rulebook_review_draft( + owner, "job-1", answers={}, actor="alice", expected_review_revision=0, + ) + submitted = submit_rulebook_review( + owner, "job-1", expected_review_revision=0, expected_pass_nr=3, + expected_profile_version="1.0.0", idempotency_key="future-submit", actor="alice", + ) + reopened = reopen_rulebook_review( + owner, "job-1", expected_review_revision=0, + idempotency_key="future-reopen", actor="alice", + ) + + self.assertEqual(saved["error"], "submission_contract_unsupported") + self.assertEqual(submitted["error"], "submission_contract_unsupported") + self.assertEqual(reopened["error"], "submission_contract_unsupported") + + def test_legacy_reviewed_records_surface_revision_zero_or_migration(self): + owner = _Owner() + updated = update_rulebook_review( + owner, + "job-1", + answers=_complete_review_answers(), + reviewer="legacy-reviewer", + review_state="reviewed", + ) + migration = get_rulebook_review(owner, "job-1") + owner.job_specs["rulebook_assessments"] = { + DEFAULT_RULEBOOK_PROFILE_ID: { + "artifact_cid": "QmLegacyReviewed", + "pass_nr": 3, + "profile_version": "1.0.0", + "schema_version": "1.0.0", + }, + } + referenced = get_rulebook_review(owner, "job-1") + + self.assertEqual(updated["review"]["review_state"], "reviewed") + self.assertTrue(migration["migration_submission_required"]) + self.assertEqual(migration["effective_review_state"], "draft") + self.assertEqual(referenced["effective_review_state"], "submitted") + self.assertEqual(referenced["submissions"][0]["revision"], 0) + self.assertTrue(referenced["submissions"][0]["legacy"]) + + def test_ensure_is_idempotent_for_existing_same_pass_and_force_regenerates(self): + owner = _Owner() + + first = ensure_rulebook_assessment(owner, "job-1") + second = ensure_rulebook_assessment(owner, "job-1") + forced = generate_rulebook_assessment(owner, "job-1", force=True) + + self.assertEqual(first["artifact_cid"], "QmRulebook1") + self.assertEqual(second["artifact_cid"], "QmRulebook1") + self.assertTrue(second["cached"]) + self.assertEqual(forced["artifact_cid"], "QmRulebook2") + self.assertEqual(owner.r1fs.add_json.call_count, 2) + + meta = owner.job_specs["rulebook_assessments"][DEFAULT_RULEBOOK_PROFILE_ID] + self.assertEqual(meta["artifact_cid"], "QmRulebook2") + self.assertEqual(meta["history"][0]["artifact_cid"], "QmRulebook1") + + def test_ensure_replaces_legacy_same_pass_artifact_before_reuse(self): + owner = _Owner(job_specs=_sample_job_specs(rulebook_assessments={ + DEFAULT_RULEBOOK_PROFILE_ID: { + "artifact_cid": "QmLegacyRulebook", + "pass_nr": 3, + "latest_pass_nr": 3, + "run_state": "succeeded", + "schema_version": "1.0.0", + }, + })) + owner.artifacts["QmLegacyRulebook"] = { + "schema": "redmesh.rulebook_assessment.v1", + "schema_version": "1.0.0", + "review_state": {"answers": {"q": {"note": "legacy draft"}}}, + } + + result = ensure_rulebook_assessment(owner, "job-1") + + self.assertFalse(result.get("cached", False)) + self.assertNotEqual(result["artifact_cid"], "QmLegacyRulebook") + self.assertEqual(owner.artifacts[result["artifact_cid"]]["artifact_kind"], "generated_assessment") + self.assertNotIn("legacy draft", json.dumps(owner.artifacts[result["artifact_cid"]])) + + def test_running_job_with_completed_pass_report_is_eligible(self): + owner = _Owner(job_specs=_sample_job_specs( + job_status="RUNNING", + job_cid="", + pass_reports=[{"pass_nr": 3, "report_cid": "pass-cid", "risk_score": 75}], + )) + owner.artifacts["pass-cid"] = _sample_pass_report() + + result = build_rulebook_assessment(owner, "job-1") + + self.assertEqual(result["status"], "ok") + self.assertEqual(result["pass_nr"], 3) + self.assertEqual(result["assessment"]["scan_context"]["job_status"], "RUNNING") + + def test_artifact_write_failure_persists_failed_status_without_generated_flag(self): + owner = _Owner() + owner.r1fs.add_json.side_effect = None + owner.r1fs.add_json.return_value = None + + result = generate_rulebook_assessment(owner, "job-1") + + self.assertEqual(result["status"], "error") + self.assertEqual(result["error"], "artifact_write_failed") + meta = owner.job_specs["rulebook_assessments"][DEFAULT_RULEBOOK_PROFILE_ID] + self.assertEqual(meta["run_state"], "failed") + self.assertTrue(meta["auto_enabled"]) + self.assertNotIn("artifact_cid", meta) + status = get_rulebook_assessment_status(owner, "job-1") + self.assertFalse(status["generated"]) + self.assertEqual(status["run_state"], "failed") + self.assertEqual(status["last_error"]["error"], "artifact_write_failed") + + def test_reviewer_answer_cannot_hide_automated_gap_but_can_support_manual_check(self): + owner = _Owner() + + review = update_rulebook_review( + owner, + "job-1", + answers={ + "nis2.access.access_controls_reviewed": {"value": "yes", "note": "Reviewed."}, + "nis2.bcm.business_continuity": {"value": "yes", "note": "Plan exists."}, + }, + reviewer="alice", + review_state="reviewed", + ) + self.assertEqual(review["status"], "ok") + self.assertEqual(review["audit"][-1]["changed_question_ids"], [ + "nis2.access.access_controls_reviewed", + "nis2.bcm.business_continuity", + ]) + + result = build_rulebook_assessment(owner, "job-1") + checks = {check["check_id"]: check for check in result["assessment"]["checks"]} + + self.assertEqual(checks["NIS2-21-ACCESS-001"]["automated_status"], "gap") + self.assertEqual(checks["NIS2-21-ACCESS-001"]["status"], "gap") + self.assertEqual(checks["NIS2-21-ACCESS-001"]["source"], "mixed") + self.assertEqual(checks["NIS2-21-BCM-001"]["automated_status"], "not_observable") + self.assertEqual(checks["NIS2-21-BCM-001"]["status"], "supported") + self.assertEqual(checks["NIS2-21-BCM-001"]["source"], "reviewer") + + review_payload = get_rulebook_review(owner, "job-1") + self.assertTrue(review_payload["found"]) + self.assertEqual(review_payload["review"]["review_state"], "reviewed") + + def test_invalid_review_answer_is_rejected(self): + owner = _Owner() + + result = update_rulebook_review( + owner, + "job-1", + answers={"nis2.unknown": {"value": "yes"}}, + ) + + self.assertEqual(result["status"], "error") + self.assertEqual(result["error"], "invalid_review_answer") + + def test_ineligible_jobs_fail_cleanly(self): + model_owner = _Owner(job_specs=_sample_job_specs(job_type="model_test", scan_type="model_test")) + running_owner = _Owner(job_specs=_sample_job_specs(job_status="RUNNING", job_cid="")) + missing_pass_owner = _Owner(archive={**_sample_archive(), "passes": []}) + + self.assertEqual( + build_rulebook_assessment(model_owner, "job-1")["error"], + "model_test_not_supported", + ) + self.assertEqual( + build_rulebook_assessment(running_owner, "job-1")["error"], + "no_completed_passes", + ) + self.assertEqual( + update_rulebook_review(running_owner, "job-1", answers={})["error"], + "job_not_finalized", + ) + self.assertEqual( + build_rulebook_assessment(missing_pass_owner, "job-1")["error"], + "no_completed_passes", + ) + + def test_purge_removes_rulebook_artifact_and_review_rows(self): + owner = _Owner(job_specs=_sample_job_specs( + job_cid="", + rulebook_assessments={ + DEFAULT_RULEBOOK_PROFILE_ID: { + "artifact_cid": "QmRulebookAssessment", + "history": [{"artifact_cid": "QmRulebookAssessmentOld"}], + }, + }, + )) + owner.records[(owner.cfg_instance_id, "job-1")] = owner.job_specs + review_key = f"job-1:{DEFAULT_RULEBOOK_PROFILE_ID}" + owner.records[(f"{owner.cfg_instance_id}:rulebook_review", review_key)] = {"job_id": "job-1"} + owner.records[(f"{owner.cfg_instance_id}:rulebook_review:audit", review_key)] = [{"job_id": "job-1"}] + owner.records[(f"{owner.cfg_instance_id}:rulebook_review:submissions", review_key)] = { + "contract_version": "1.0.0", + "latest_revision": 2, + "submissions": [ + {"revision": 1, "cid": "QmSubmission1"}, + {"revision": 2, "cid": "QmSubmission2"}, + ], + "pending": {"cid": "QmPendingSubmission"}, + } + + result = purge_job(owner, "job-1") + + self.assertEqual(result["status"], "success") + self.assertIn("QmRulebookAssessment", owner.artifact_repo.deleted) + self.assertIn("QmRulebookAssessmentOld", owner.artifact_repo.deleted) + self.assertIn("QmSubmission1", owner.artifact_repo.deleted) + self.assertIn("QmSubmission2", owner.artifact_repo.deleted) + self.assertIn("QmPendingSubmission", owner.artifact_repo.deleted) + self.assertIsNone(owner.records[(f"{owner.cfg_instance_id}:rulebook_review", review_key)]) + self.assertIsNone(owner.records[(f"{owner.cfg_instance_id}:rulebook_review:audit", review_key)]) + self.assertIsNone(owner.records[(f"{owner.cfg_instance_id}:rulebook_review:submissions", review_key)]) + + def test_purge_fails_closed_for_submission_cid_shared_with_another_job(self): + owner = _Owner(job_specs=_sample_job_specs(job_cid="")) + owner.records[(owner.cfg_instance_id, "job-1")] = owner.job_specs + hkey = f"{owner.cfg_instance_id}:rulebook_review:submissions" + owner.records[(hkey, f"job-1:{DEFAULT_RULEBOOK_PROFILE_ID}")] = { + "submissions": [{"revision": 1, "cid": "QmSharedSubmission"}], + } + owner.records[(hkey, f"job-2:{DEFAULT_RULEBOOK_PROFILE_ID}")] = { + "submissions": [{"revision": 1, "cid": "QmSharedSubmission"}], + } + + result = purge_job(owner, "job-1") + + self.assertEqual(result["status"], "partial") + self.assertEqual(result["cids_deleted"], 0) + self.assertNotIn("QmSharedSubmission", owner.artifact_repo.deleted) + self.assertIsNotNone(owner.records[(owner.cfg_instance_id, "job-1")]) + + def test_purge_clears_cstore_after_relay_acknowledges_formal_cid_delete(self): + owner = _Owner(job_specs=_sample_job_specs(job_cid="")) + owner.records[(owner.cfg_instance_id, "job-1")] = owner.job_specs + hkey = f"{owner.cfg_instance_id}:rulebook_review:submissions" + review_key = f"job-1:{DEFAULT_RULEBOOK_PROFILE_ID}" + owner.records[(hkey, review_key)] = { + "contract_version": "1.0.0", + "latest_revision": 1, + "submissions": [{"revision": 1, "cid": "QmRetainedSubmission"}], + } + owner.artifacts["QmRetainedSubmission"] = {"artifact_kind": "review_submission"} + owner.artifact_repo.delete = MagicMock(return_value=True) + owner.artifact_repo.get_json = MagicMock(return_value={"artifact_kind": "review_submission"}) + + result = purge_job(owner, "job-1") + + self.assertEqual(result["status"], "success") + owner.artifact_repo.delete.assert_any_call( + "QmRetainedSubmission", show_logs=True, raise_on_error=False, purge=True, + ) + self.assertIsNone(owner.records[(owner.cfg_instance_id, "job-1")]) + self.assertIsNone(owner.records[(hkey, review_key)]) + owner.artifact_repo.get_json.assert_not_called() + + +if __name__ == "__main__": + unittest.main() From d736de87ff4ff81bcbf45a3edd44e63082c59dd5 Mon Sep 17 00:00:00 2001 From: Vitalii <87299468+toderian@users.noreply.github.com> Date: Fri, 17 Jul 2026 13:38:06 +0300 Subject: [PATCH 2/6] fix: make deeploy updates request-authoritative (#472) * fix: make deeploy updates request-authoritative What changed: - deploy only the submitted plugin replacement set during updates - preserve plugin identity checks and emit bounded value-free replica drift warnings - cover omissions, drift, offline fallback, and identity safeguards Why: - update requests must be authoritative and live replica drift must not block safe replacement Checks: - focused Deeploy update tests: 47 passed - full Deeploy discovery: 204 passed - touched py_compile and git diff --check: passed * chore: inc ver --- .../business/deeploy/deeploy_manager_api.py | 33 +- extensions/business/deeploy/deeploy_mixin.py | 355 ++++++++--------- .../deeploy/tests/test_update_requests.py | 356 ++++++++++++++---- ver.py | 2 +- 4 files changed, 486 insertions(+), 260 deletions(-) diff --git a/extensions/business/deeploy/deeploy_manager_api.py b/extensions/business/deeploy/deeploy_manager_api.py index 283747e95..36cd18351 100644 --- a/extensions/business/deeploy/deeploy_manager_api.py +++ b/extensions/business/deeploy/deeploy_manager_api.py @@ -777,9 +777,8 @@ def _process_pipeline_request( current_nodes = pipeline_context["nodes"] deeploy_specs_for_update = pipeline_context["deeploy_specs"] self.P( - "Discovered plugin instances: {}".format( - self.json_dumps(self._redact_per_node_config_for_log(discovered_plugin_instances)) - ) + f"Discovered {len(discovered_plugin_instances)} live plugin instance record(s) " + f"for update job_id={job_id}, app_id={app_id}." ) requested_nodes = inputs.get(DEEPLOY_KEYS.TARGET_NODES, None) @@ -841,6 +840,15 @@ def _process_pipeline_request( app_id=app_id, job_id=job_id, ) + self._validate_update_plugin_identities( + inputs, + discovered_plugin_instances=discovered_plugin_instances, + ) + self._warn_on_live_plugin_config_drift( + discovered_plugin_instances, + job_id=job_id, + app_id=app_id, + ) if deeploy_specs_for_update is not None and not isinstance(deeploy_specs_for_update, dict): msg = ( @@ -873,15 +881,9 @@ def _process_pipeline_request( plugins_array = inputs.get(DEEPLOY_KEYS.PLUGINS) if isinstance(plugins_array, list): - materialized_plugins = self._materialize_update_plugins_for_redeploy( - inputs, - discovered_plugin_instances, - ) - inputs[DEEPLOY_KEYS.PLUGINS] = materialized_plugins - inputs.plugins = materialized_plugins - # Validate the exact replacement payload, including omitted live plugins - # that were materialized from discovery, before any payment/node/delete work. - self._validate_plugins_array(materialized_plugins) + # The submitted plugins array is the complete desired replacement. + # Discovery is used only for identity safety and drift diagnostics. + self._validate_plugins_array(plugins_array) if not has_request_job_app_type: replacement_job_app_type = deeploy_specs_payload.get(DEEPLOY_KEYS.JOB_APP_TYPE) @@ -896,7 +898,7 @@ def _process_pipeline_request( isinstance(plugin_entry, dict) and isinstance(plugin_entry.get(DEEPLOY_KEYS.PLUGIN_SIGNATURE), str) and plugin_entry.get(DEEPLOY_KEYS.PLUGIN_SIGNATURE).upper() in CONTAINERIZED_APPS_SIGNATURES - for plugin_entry in materialized_plugins + for plugin_entry in plugins_array ) if job_app_type == JOB_APP_TYPES.NATIVE and has_containerized_replacement: msg = ( @@ -1538,11 +1540,12 @@ def update_pipeline( **Plugin instances:** plugins : list - Array of plugin instance configurations. Each object represents ONE plugin instance: + Complete desired replacement set. Each object represents ONE plugin instance: - plugin_signature : str (required) - instance_id : str (required when updating an existing plugin instance) - **instance-specific parameters** (payload merged into the instance configuration) - Omit instance_id to attach a brand new plugin instance; supported for native apps only + - Omit a live plugin from this array to remove it from the replacement deployment **Legacy format:** plugin_signature : str @@ -1558,7 +1561,7 @@ def update_pipeline( ----- - Existing pipelines are stopped and redeployed in place; requests must reference the active node set. - Updates are applied to existing plugin instances on the same nodes - - For multi-plugin pipelines, all plugins are updated with new configurations + - The plugins array is a full replacement, not a partial patch; omitted live plugins are removed - Resource validation applies the same as create operations - The simplified plugins array format is the same as create_pipeline - New plugin instances can be introduced by omitting `instance_id` (native job type only) diff --git a/extensions/business/deeploy/deeploy_mixin.py b/extensions/business/deeploy/deeploy_mixin.py index 6959dd677..bbc4580f0 100644 --- a/extensions/business/deeploy/deeploy_mixin.py +++ b/extensions/business/deeploy/deeploy_mixin.py @@ -2586,206 +2586,211 @@ def _remove_consumed_new_plugin_config(new_plugin_configs, plugin_config): return True return False - def _materialize_update_plugins_for_redeploy(self, inputs, discovered_plugin_instances): + def _validate_update_plugin_identities(self, inputs, discovered_plugin_instances): """ - Build a full plugin request array for delete/redeploy updates. + Validate explicit requested plugin IDs against discovered live identities. - Update preflight can reconstruct omitted live plugin configs from discovery. - The post-delete create path only sees inputs, so partial update requests must - be expanded before deletion to keep the deployed replacement equivalent to - the validated update payload. + Plugins without an ID are new instances. Legacy existing plugins have their + IDs backfilled before this method is called. """ - requested_by_instance_id, requested_by_signature, new_plugin_configs = self._organize_requested_plugins(inputs) - materialized_plugins = [] + requested_by_instance_id, _, _ = self._organize_requested_plugins(inputs) + discovered_by_instance_id = self.defaultdict(list) + for plugin in discovered_plugin_instances or []: + instance_id = plugin.get(DEEPLOY_PLUGIN_DATA.INSTANCE_ID) + if instance_id: + discovered_by_instance_id[str(instance_id)].append(plugin) - instance_id_key = self.ct.BIZ_PLUGIN_DATA.INSTANCE_ID - chainstore_response_key = self.ct.BIZ_PLUGIN_DATA.CHAINSTORE_RESPONSE_KEY - chainstore_peers_key = self.ct.BIZ_PLUGIN_DATA.CHAINSTORE_PEERS - discovered_records = [] - discovered_by_key = {} - config_occurrences_by_node = {} - nameless_record_counts = self.defaultdict(int) - - def get_plugin_name_from_conf(discovered_plugin, extracted_config): - return ( - extracted_config.get(DEEPLOY_KEYS.PLUGIN_NAME) - or discovered_plugin.get(DEEPLOY_KEYS.PLUGIN_NAME) + for instance_id, requested_plugin in requested_by_instance_id.items(): + discovered = discovered_by_instance_id.get(instance_id, []) + if not discovered: + raise ValueError( + f"{DEEPLOY_ERRORS.PLUGINS3}: Unknown plugin instance_id(s) in update request: " + f"{[instance_id]}" + ) + + requested_signature = ( + requested_plugin.get(DEEPLOY_KEYS.PLUGIN_SIGNATURE) + or requested_plugin.get("signature") + ) + normalized_requested_signature = ( + requested_signature.upper() + if isinstance(requested_signature, str) + else requested_signature ) + observed_signatures = { + signature.upper() if isinstance(signature, str) else signature + for signature in ( + plugin.get(DEEPLOY_PLUGIN_DATA.PLUGIN_SIGNATURE) + for plugin in discovered + ) + if signature + } + if observed_signatures != {normalized_requested_signature}: + raise ValueError( + f"{DEEPLOY_ERRORS.PLUGINS3}: Plugin instance_id '{instance_id}' cannot be reused " + f"with signature '{requested_signature}'." + ) + return True - def get_discovered_materialization_key(discovered_plugin, signature, instance_id, extracted_config): - normalized_sig = signature.upper() if isinstance(signature, str) else signature - if instance_id: - return (normalized_sig, "instance_id", str(instance_id)) + @staticmethod + def _bounded_drift_log_values(values, max_items=8, max_length=80): + bounded = [] + for value in sorted({str(item) for item in values if item is not None}): + cleaned = "".join(char if char.isprintable() else "?" for char in value) + if len(cleaned) > max_length: + cleaned = cleaned[:max_length - 3] + "..." + bounded.append(cleaned) + if len(bounded) >= max_items: + break + return bounded - plugin_name = get_plugin_name_from_conf(discovered_plugin, extracted_config) - if plugin_name: - return (normalized_sig, "plugin_name", str(plugin_name)) + @staticmethod + def _bounded_drift_log_message(message, max_length=2000): + if len(message) <= max_length: + return message + suffix = "...[truncated]" + return message[:max_length - len(suffix)] + suffix + + @classmethod + def _differing_config_paths(cls, configs, max_paths=20, max_depth=8): + if len(configs) < 2: + return [] - config_hash = compact_canonical_sha256(extracted_config) - node = discovered_plugin.get(DEEPLOY_PLUGIN_DATA.NODE, "") - occurrence_bucket = (node, normalized_sig, config_hash) - occurrence_idx = config_occurrences_by_node.get(occurrence_bucket, 0) - config_occurrences_by_node[occurrence_bucket] = occurrence_idx + 1 - return (normalized_sig, "config", config_hash, occurrence_idx) + missing = object() + differing_paths = [] - for plugin in discovered_plugin_instances: - signature = plugin.get(DEEPLOY_PLUGIN_DATA.PLUGIN_SIGNATURE) - if not signature: - continue + def add_path(parts): + if len(differing_paths) >= max_paths: + return + if not parts: + differing_paths.append("$") + return + formatted = "$" + for part in parts: + if isinstance(part, int): + formatted += f"[{part}]" + continue + segment = "".join(char if char.isprintable() else "?" for char in str(part)) + if len(segment) > 64: + segment = segment[:61] + "..." + formatted += f".{segment}" + differing_paths.append(formatted) + + def walk(values, parts, depth): + if len(differing_paths) >= max_paths: + return + first = values[0] + if all(value == first for value in values[1:]): + return + if depth >= max_depth or missing in values: + add_path(parts) + return + if all(isinstance(value, dict) for value in values): + keys = sorted( + {key for value in values for key in value}, + key=lambda key: str(key), + ) + for key in keys: + walk([value.get(key, missing) for value in values], parts + [key], depth + 1) + return + if all(isinstance(value, list) for value in values): + lengths = {len(value) for value in values} + if len(lengths) != 1: + add_path(parts) + return + for idx in range(len(values[0])): + walk([value[idx] for value in values], parts + [idx], depth + 1) + return + add_path(parts) - normalized_signature = signature.upper() if isinstance(signature, str) else signature + walk(configs, [], 0) + return differing_paths + + def _warn_on_live_plugin_config_drift( + self, + discovered_plugin_instances, + job_id=None, + app_id=None, + max_groups=10, + ): + """ + Warn about differing live replica configs without logging config values. + + Discovery is an oracle signal only. Drift never changes or blocks the + request-authoritative replacement payload. + """ + instance_id_key = self.ct.BIZ_PLUGIN_DATA.INSTANCE_ID + chainstore_response_key = self.ct.BIZ_PLUGIN_DATA.CHAINSTORE_RESPONSE_KEY + chainstore_peers_key = self.ct.BIZ_PLUGIN_DATA.CHAINSTORE_PEERS + grouped = self.defaultdict(list) + + for plugin in discovered_plugin_instances or []: + signature = plugin.get(DEEPLOY_PLUGIN_DATA.PLUGIN_SIGNATURE) instance_id = plugin.get(DEEPLOY_PLUGIN_DATA.INSTANCE_ID) - if instance_id: - instance_id = str(instance_id) + if not instance_id: + continue extracted_config = self._extract_discovered_plugin_conf( plugin, instance_id_key=instance_id_key, chainstore_response_key=chainstore_response_key, chainstore_peers_key=chainstore_peers_key, ) - plugin_name = get_plugin_name_from_conf(plugin, extracted_config) - config_hash = compact_canonical_sha256(extracted_config) - materialization_key = get_discovered_materialization_key( - plugin, - signature, - instance_id, - extracted_config, - ) - - compatibility_key = (normalized_signature, str(plugin_name or ""), config_hash) - existing_record = discovered_by_key.get(materialization_key) - if existing_record is not None: - if existing_record["compatibility_key"] != compatibility_key: - raise ValueError( - f"{DEEPLOY_ERRORS.PLUGINS3}: Corrupt live discovery for plugin identity " - f"{materialization_key}. Incompatible plugin instances were reported." - ) - continue - - record = { - "key": materialization_key, - "compatibility_key": compatibility_key, + grouped[str(instance_id)].append({ + "node": plugin.get(DEEPLOY_PLUGIN_DATA.NODE), "signature": signature, - "normalized_signature": normalized_signature, - "instance_id": instance_id, - "plugin_name": str(plugin_name) if plugin_name else None, + "plugin_name": ( + extracted_config.get(DEEPLOY_KEYS.PLUGIN_NAME) + or plugin.get(DEEPLOY_KEYS.PLUGIN_NAME) + ), "config": extracted_config, - "config_hash": config_hash, - } - discovered_by_key[materialization_key] = record - discovered_records.append(record) + }) - for record in discovered_records: - if record["instance_id"] or record["plugin_name"]: + warned_groups = 0 + drift_group_count = 0 + for instance_id, replicas in sorted(grouped.items()): + differing_paths = self._differing_config_paths( + [replica["config"] for replica in replicas], + ) + if not differing_paths: + continue + drift_group_count += 1 + if warned_groups >= max_groups: continue - nameless_match_key = (record["normalized_signature"], record["config_hash"]) - nameless_record_counts[nameless_match_key] += 1 - - def candidate_has_instance_id(candidate): - return bool( - candidate.get(DEEPLOY_KEYS.PLUGIN_INSTANCE_ID) - or candidate.get("instance_id") - or candidate.get(instance_id_key) + warned_groups += 1 + nodes = self._bounded_drift_log_values(replica["node"] for replica in replicas) + signatures = self._bounded_drift_log_values(replica["signature"] for replica in replicas) + plugin_names = self._bounded_drift_log_values(replica["plugin_name"] for replica in replicas) + safe_instance_id = self._bounded_drift_log_values([instance_id], max_items=1)[0] + safe_job_id = self._bounded_drift_log_values([job_id], max_items=1) + safe_app_id = self._bounded_drift_log_values([app_id], max_items=1) + warning = ( + "Deeploy live replica config drift: " + f"job_id={safe_job_id[0] if safe_job_id else None}, " + f"app_id={safe_app_id[0] if safe_app_id else None}, " + f"plugin_instance_id={safe_instance_id}, " + f"nodes={nodes}, observed_signatures={signatures}, " + f"observed_plugin_names={plugin_names}, differing_paths={differing_paths}. " + "The submitted update remains authoritative." ) - - def consume_signature_candidate(record): - candidate_list = requested_by_signature.get(record["normalized_signature"], []) - if not candidate_list: - return None - - if record["plugin_name"]: - candidates = [ - candidate for candidate in candidate_list - if not candidate_has_instance_id(candidate) - and candidate.get(DEEPLOY_KEYS.PLUGIN_NAME) == record["plugin_name"] - ] - if len(candidates) > 1: - raise ValueError( - f"{DEEPLOY_ERRORS.PLUGINS3}: Ambiguous update request for plugin_name " - f"'{record['plugin_name']}'." - ) - else: - match_key = (record["normalized_signature"], record["config_hash"]) - candidates = [] - for candidate in candidate_list: - if candidate_has_instance_id(candidate) or candidate.get(DEEPLOY_KEYS.PLUGIN_NAME): - continue - requested_conf = self._extract_plugin_request_conf( - candidate, - instance_id_key=instance_id_key, - chainstore_response_key=chainstore_response_key, - chainstore_peers_key=chainstore_peers_key, - ) - if self._plugin_update_request_matches_identity( - requested_conf, - discovered_plugin_name=record["plugin_name"], - discovered_config_hash=record["config_hash"], - ): - candidates.append(candidate) - - if candidates and nameless_record_counts[match_key] > 1: - self._raise_ambiguous_plugin_update_match( - requested_name=None, - signature=record["signature"], - ) - - self._validate_single_plugin_update_match( - candidates, - requested_name=record["plugin_name"], - signature=record["signature"], + self.P( + self._bounded_drift_log_message(warning), + color='y', ) - if not candidates: - return None - - plugin_config = candidates[0] - for idx, candidate in enumerate(candidate_list): - if candidate is plugin_config: - candidate_list.pop(idx) - break - self._remove_consumed_new_plugin_config(new_plugin_configs, plugin_config) - return plugin_config - - for record in discovered_records: - instance_id = record["instance_id"] - plugin_config = None - - if instance_id: - plugin_config = requested_by_instance_id.pop(instance_id, None) - candidate_list = requested_by_signature.get(record["normalized_signature"], []) - if plugin_config and candidate_list: - for idx, candidate in enumerate(candidate_list): - if candidate is plugin_config: - candidate_list.pop(idx) - break - else: - plugin_config = consume_signature_candidate(record) - - if plugin_config: - plugin_entry = self.deepcopy(plugin_config) - plugin_entry.pop("signature", None) - else: - plugin_entry = self.deepcopy(record["config"]) - - plugin_entry[DEEPLOY_KEYS.PLUGIN_SIGNATURE] = record["signature"] - if instance_id: - plugin_entry[DEEPLOY_KEYS.PLUGIN_INSTANCE_ID] = instance_id - materialized_plugins.append(plugin_entry) - - if requested_by_instance_id: - missing_ids = list(requested_by_instance_id.keys()) - raise ValueError( - f"{DEEPLOY_ERRORS.PLUGINS3}: Unknown plugin instance_id(s) in update request: {missing_ids}" + if drift_group_count > max_groups: + safe_job_id = self._bounded_drift_log_values([job_id], max_items=1) + safe_app_id = self._bounded_drift_log_values([app_id], max_items=1) + summary = ( + "Additional Deeploy live replica drift groups omitted from logs: " + f"job_id={safe_job_id[0] if safe_job_id else None}, " + f"app_id={safe_app_id[0] if safe_app_id else None}, " + f"omitted_groups={drift_group_count - max_groups}." ) - - for plugin_config in new_plugin_configs: - plugin_entry = self.deepcopy(plugin_config) - if DEEPLOY_KEYS.PLUGIN_SIGNATURE not in plugin_entry and plugin_entry.get("signature"): - plugin_entry[DEEPLOY_KEYS.PLUGIN_SIGNATURE] = plugin_entry.get("signature") - plugin_entry.pop("signature", None) - materialized_plugins.append(plugin_entry) - - return materialized_plugins + self.P( + self._bounded_drift_log_message(summary), + color='y', + ) + return warned_groups def deeploy_check_payment_and_job_owner(self, inputs, owner, is_create, debug=False): """ diff --git a/extensions/business/deeploy/tests/test_update_requests.py b/extensions/business/deeploy/tests/test_update_requests.py index 36b7861a9..93f0a84aa 100644 --- a/extensions/business/deeploy/tests/test_update_requests.py +++ b/extensions/business/deeploy/tests/test_update_requests.py @@ -78,7 +78,6 @@ def _make_process_update_plugin(self, discovered_instances, nodes=None, deeploy_ "deeploy_specs": deeploy_specs or {"job_id": 11}, } plugin._get_pipeline_from_cstore = lambda job_id: None - plugin._ensure_plugin_instance_ids = lambda *args, **kwargs: None plugin._check_nodes_availability = lambda inputs: nodes or ["node-1"] called = {"delete": 0, "deploy": 0, "deploy_kwargs": None, "queued": 0, "bc_update": 0} @@ -821,7 +820,7 @@ def test_process_update_rejects_duplicate_instance_ids_before_delete(self): self.assertEqual(called["delete"], 0) self.assertEqual(called["deploy"], 0) - def test_process_update_materializes_omitted_live_plugins_before_redeploy(self): + def test_process_update_omits_unrequested_live_plugins_from_redeploy(self): plugin, called = self._make_process_update_plugin( discovered_instances=[ { @@ -878,17 +877,11 @@ def test_process_update_materializes_omitted_live_plugins_before_redeploy(self): self.assertEqual(called["deploy"], 1) redeploy_inputs = called["deploy_kwargs"]["inputs"] redeploy_plugins = redeploy_inputs[DEEPLOY_KEYS.PLUGINS] - self.assertEqual(len(redeploy_plugins), 2) + self.assertEqual(len(redeploy_plugins), 1) self.assertEqual( {entry[DEEPLOY_KEYS.PLUGIN_INSTANCE_ID] for entry in redeploy_plugins}, - {"api-instance", "worker-instance"}, - ) - worker = next( - entry for entry in redeploy_plugins - if entry[DEEPLOY_KEYS.PLUGIN_INSTANCE_ID] == "worker-instance" + {"api-instance"}, ) - self.assertEqual(worker["IMAGE"], "repo/worker:1.0") - self.assertEqual(worker[DEEPLOY_KEYS.PLUGIN_NAME], "worker") prepared_plan = called["deploy_kwargs"]["prepared_create_deploy_plan"] self.assertIsNotNone(prepared_plan) prepared_instances = [ @@ -898,7 +891,7 @@ def test_process_update_materializes_omitted_live_plugins_before_redeploy(self): ] self.assertEqual( {instance[plugin.ct.CONFIG_INSTANCE.K_INSTANCE_ID] for instance in prepared_instances}, - {"api-instance", "worker-instance"}, + {"api-instance"}, ) prepared_api = next( instance for instance in prepared_instances @@ -906,7 +899,7 @@ def test_process_update_materializes_omitted_live_plugins_before_redeploy(self): ) self.assertEqual(prepared_api["PROCESS_DELAY"], 10) - def test_process_update_validates_materialized_replacement_payload_before_delete(self): + def test_process_update_validates_requested_replacement_payload_before_delete(self): plugin, called = self._make_process_update_plugin( discovered_instances=[ { @@ -948,7 +941,7 @@ def assert_full_replacement_payload(inputs, context): for entry in plugins ] validation_calls.append((context, plugin_ids, inputs.get(DEEPLOY_KEYS.JOB_APP_TYPE))) - self.assertEqual(set(plugin_ids), {"api-instance", "worker-instance"}) + self.assertEqual(set(plugin_ids), {"api-instance"}) self.assertEqual(inputs.get(DEEPLOY_KEYS.JOB_APP_TYPE), "stack") def check_payment_and_owner(inputs, *args, **kwargs): @@ -1026,6 +1019,16 @@ def test_process_update_uses_persisted_pipeline_when_all_old_nodes_are_offline(s }, ], }, + { + plugin.ct.CONFIG_PLUGIN.K_SIGNATURE: "CONTAINER_APP_RUNNER", + plugin.ct.CONFIG_PLUGIN.K_INSTANCES: [ + { + plugin.ct.CONFIG_INSTANCE.K_INSTANCE_ID: "omitted-invalid-instance", + DEEPLOY_KEYS.PLUGIN_NAME: "omitted-invalid", + "CONTAINER_RESOURCES": {"cpu": 1, "memory": "256m"}, + }, + ], + }, ], } plugin._check_nodes_availability = lambda inputs: ["new-node-1"] @@ -1062,6 +1065,11 @@ def test_process_update_uses_persisted_pipeline_when_all_old_nodes_are_offline(s self.assertEqual(called["bc_update"], 1) self.assertEqual(called["deploy_kwargs"]["new_nodes"], ["new-node-1"]) redeploy_plugins = called["deploy_kwargs"]["inputs"][DEEPLOY_KEYS.PLUGINS] + self.assertEqual(len(redeploy_plugins), 1) + self.assertEqual( + redeploy_plugins[0][DEEPLOY_KEYS.PLUGIN_INSTANCE_ID], + "current-instance", + ) self.assertEqual(redeploy_plugins[0]["IMAGE"], "repo/app:2.0") def test_process_update_rejects_job_app_type_change_before_payment_or_delete(self): @@ -1122,7 +1130,7 @@ def test_process_update_rejects_job_app_type_change_before_payment_or_delete(sel self.assertEqual(called["deploy"], 0) self.assertEqual(called["queued"], 0) - def test_process_update_rejects_invalid_materialized_omitted_plugin_before_payment_and_delete(self): + def test_process_update_ignores_invalid_omitted_live_plugin_config(self): plugin, called = self._make_process_update_plugin( discovered_instances=[ { @@ -1181,13 +1189,20 @@ def test_process_update_rejects_invalid_materialized_omitted_plugin_before_payme async_mode=True, ) - self.assertIn("'IMAGE' field is required", response[DEEPLOY_KEYS.ERROR]) - self.assertEqual(payment_calls, []) - self.assertEqual(node_calls, []) - self.assertEqual(called["delete"], 0) - self.assertEqual(called["deploy"], 0) + self.assertEqual(response[DEEPLOY_KEYS.STATUS], "command_delivered") + self.assertEqual(len(payment_calls), 1) + self.assertEqual(len(node_calls), 1) + self.assertEqual(called["delete"], 1) + self.assertEqual(called["deploy"], 1) + self.assertEqual( + [ + entry[DEEPLOY_KEYS.PLUGIN_INSTANCE_ID] + for entry in called["deploy_kwargs"]["inputs"][DEEPLOY_KEYS.PLUGINS] + ], + ["api-instance"], + ) - def test_process_update_rejects_ambiguous_containerized_replacement_type_before_delete(self): + def test_process_update_detects_type_from_requested_replacement_only(self): plugin, called = self._make_process_update_plugin( discovered_instances=[ { @@ -1241,10 +1256,13 @@ def test_process_update_rejects_ambiguous_containerized_replacement_type_before_ async_mode=True, ) - self.assertIn("omitted job_app_type", response[DEEPLOY_KEYS.ERROR]) - self.assertIn("containerized replacement payload", response[DEEPLOY_KEYS.ERROR]) - self.assertEqual(called["delete"], 0) - self.assertEqual(called["deploy"], 0) + self.assertEqual(response[DEEPLOY_KEYS.STATUS], "command_delivered") + self.assertEqual(called["delete"], 1) + self.assertEqual(called["deploy"], 1) + self.assertEqual( + called["deploy_kwargs"]["inputs"][DEEPLOY_KEYS.PLUGINS][0]["IMAGE"], + "repo/api:2.0", + ) def test_process_update_does_not_append_consumed_no_id_update_as_new_plugin(self): plugin, called = self._make_process_update_plugin( @@ -1293,7 +1311,7 @@ def test_process_update_does_not_append_consumed_no_id_update_as_new_plugin(self self.assertEqual(redeploy_plugins[0][DEEPLOY_KEYS.PLUGIN_NAME], "legacy") self.assertEqual(redeploy_plugins[0]["PROCESS_DELAY"], 10) - def test_process_update_dedupes_nameless_no_id_legacy_plugin_across_nodes(self): + def test_process_update_does_not_restore_nameless_legacy_plugins_across_nodes(self): discovered_instances = [ { DEEPLOY_PLUGIN_DATA.PLUGIN_SIGNATURE: "A_SIMPLE_PLUGIN", @@ -1369,21 +1387,11 @@ def test_process_update_dedupes_nameless_no_id_legacy_plugin_across_nodes(self): self.assertEqual(called["deploy"], 1) redeploy_plugins = called["deploy_kwargs"]["inputs"][DEEPLOY_KEYS.PLUGINS] - self.assertEqual(len(redeploy_plugins), 2) - api = next( - entry for entry in redeploy_plugins - if entry.get(DEEPLOY_KEYS.PLUGIN_INSTANCE_ID) == "api-instance" - ) - legacy = next( - entry for entry in redeploy_plugins - if entry.get(DEEPLOY_KEYS.PLUGIN_INSTANCE_ID) != "api-instance" - ) + self.assertEqual(len(redeploy_plugins), 1) + api = redeploy_plugins[0] self.assertEqual(api["PROCESS_DELAY"], 10) - self.assertEqual(legacy["PROCESS_DELAY"], 5) - self.assertNotIn(DEEPLOY_KEYS.PLUGIN_INSTANCE_ID, legacy) - self.assertNotIn(DEEPLOY_KEYS.PLUGIN_NAME, legacy) - def test_process_update_preserves_same_node_nameless_no_id_legacy_multiplicity(self): + def test_process_update_does_not_restore_same_node_nameless_legacy_plugins(self): discovered_instances = [ { DEEPLOY_PLUGIN_DATA.PLUGIN_SIGNATURE: "A_SIMPLE_PLUGIN", @@ -1448,18 +1456,11 @@ def test_process_update_preserves_same_node_nameless_no_id_legacy_multiplicity(s self.assertEqual(called["deploy"], 1) redeploy_plugins = called["deploy_kwargs"]["inputs"][DEEPLOY_KEYS.PLUGINS] - self.assertEqual(len(redeploy_plugins), 3) - legacy_plugins = [ - entry for entry in redeploy_plugins - if entry.get(DEEPLOY_KEYS.PLUGIN_INSTANCE_ID) != "api-instance" - ] - self.assertEqual(len(legacy_plugins), 2) - for legacy in legacy_plugins: - self.assertEqual(legacy["PROCESS_DELAY"], 5) - self.assertNotIn(DEEPLOY_KEYS.PLUGIN_INSTANCE_ID, legacy) - self.assertNotIn(DEEPLOY_KEYS.PLUGIN_NAME, legacy) + self.assertEqual(len(redeploy_plugins), 1) + self.assertEqual(redeploy_plugins[0][DEEPLOY_KEYS.PLUGIN_INSTANCE_ID], "api-instance") + self.assertEqual(redeploy_plugins[0]["PROCESS_DELAY"], 10) - def test_process_update_materializes_one_logical_plugin_set_for_multinode_redeploy(self): + def test_process_update_uses_requested_plugin_set_for_multinode_redeploy(self): discovered_instances = [ { DEEPLOY_PLUGIN_DATA.INSTANCE_ID: "api-instance", @@ -1543,52 +1544,129 @@ def test_process_update_materializes_one_logical_plugin_set_for_multinode_redepl redeploy_inputs = called["deploy_kwargs"]["inputs"] redeploy_plugins = redeploy_inputs[DEEPLOY_KEYS.PLUGINS] - self.assertEqual(len(redeploy_plugins), 2) + self.assertEqual(len(redeploy_plugins), 1) self.assertEqual( {entry[DEEPLOY_KEYS.PLUGIN_INSTANCE_ID] for entry in redeploy_plugins}, - {"api-instance", "worker-instance"}, + {"api-instance"}, ) - api = next( - entry for entry in redeploy_plugins - if entry[DEEPLOY_KEYS.PLUGIN_INSTANCE_ID] == "api-instance" - ) - worker = next( - entry for entry in redeploy_plugins - if entry[DEEPLOY_KEYS.PLUGIN_INSTANCE_ID] == "worker-instance" - ) + api = redeploy_plugins[0] self.assertEqual(api["PROCESS_DELAY"], 10) - self.assertEqual(worker["IMAGE"], "repo/worker:1.0") - self.assertEqual(worker[DEEPLOY_KEYS.PLUGIN_NAME], "worker") - def test_process_update_rejects_corrupt_duplicate_live_instance_id_before_delete(self): + def test_process_update_warns_on_live_replica_drift_and_uses_requested_config(self): plugin, called = self._make_process_update_plugin( discovered_instances=[ { DEEPLOY_PLUGIN_DATA.INSTANCE_ID: "shared-instance", - DEEPLOY_PLUGIN_DATA.PLUGIN_SIGNATURE: "A_SIMPLE_PLUGIN", + DEEPLOY_PLUGIN_DATA.PLUGIN_SIGNATURE: "WORKER_APP_RUNNER", DEEPLOY_PLUGIN_DATA.NODE: "node-1", DEEPLOY_PLUGIN_DATA.PLUGIN_INSTANCE: { "instance_conf": { - DEEPLOY_KEYS.PLUGIN_NAME: "api", - "PROCESS_DELAY": 5, + DEEPLOY_KEYS.PLUGIN_NAME: "worker", + "IMAGE": "node:22", + "CONTAINER_RESOURCES": {"cpu": 1, "memory": "256m"}, + "ENV": {"API_TOKEN": "live-secret-a", "MODE": "replica-a"}, }, }, }, { DEEPLOY_PLUGIN_DATA.INSTANCE_ID: "shared-instance", - DEEPLOY_PLUGIN_DATA.PLUGIN_SIGNATURE: "A_SIMPLE_PLUGIN", + DEEPLOY_PLUGIN_DATA.PLUGIN_SIGNATURE: "WORKER_APP_RUNNER", DEEPLOY_PLUGIN_DATA.NODE: "node-2", DEEPLOY_PLUGIN_DATA.PLUGIN_INSTANCE: { "instance_conf": { DEEPLOY_KEYS.PLUGIN_NAME: "worker", - "PROCESS_DELAY": 10, + "IMAGE": "node:22", + "CONTAINER_RESOURCES": {"cpu": 1, "memory": "256m"}, + "ENV": {"API_TOKEN": "live-secret-b", "MODE": "replica-b"}, }, }, }, ], nodes=["node-1", "node-2"], ) + log_lines = [] + plugin.P = lambda message, **kwargs: log_lines.append(message) + + response = plugin._process_pipeline_request( + { + DEEPLOY_KEYS.APP_ID: "app-123", + DEEPLOY_KEYS.APP_ALIAS: "app", + DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.JOB_APP_TYPE: "generic", + DEEPLOY_KEYS.PIPELINE_INPUT_TYPE: "void", + DEEPLOY_KEYS.CHAINSTORE_RESPONSE: False, + DEEPLOY_KEYS.TARGET_NODES: ["node-1", "node-2"], + DEEPLOY_KEYS.TARGET_NODES_COUNT: 2, + DEEPLOY_KEYS.PLUGINS: [ + { + DEEPLOY_KEYS.PLUGIN_SIGNATURE: "WORKER_APP_RUNNER", + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID: "shared-instance", + DEEPLOY_KEYS.PLUGIN_NAME: "worker", + "IMAGE": "node:22", + "CONTAINER_RESOURCES": {"cpu": 1, "memory": "256m"}, + "ENV": {"API_TOKEN": "requested-secret", "MODE": "requested"}, + }, + ], + }, + is_create=False, + async_mode=True, + ) + + self.assertEqual(response[DEEPLOY_KEYS.STATUS], "command_delivered") + self.assertEqual(called["delete"], 1) + self.assertEqual(called["deploy"], 1) + prepared_plan = called["deploy_kwargs"]["prepared_create_deploy_plan"] + for node_plugins in prepared_plan["node_plugins_by_addr"].values(): + instance = node_plugins[0][plugin.ct.CONFIG_PLUGIN.K_INSTANCES][0] + self.assertEqual(instance["ENV"]["API_TOKEN"], "requested-secret") + self.assertEqual(instance["ENV"]["MODE"], "requested") + self.assertNotIn(instance["ENV"]["API_TOKEN"], {"live-secret-a", "live-secret-b"}) + self.assertNotIn(instance["ENV"]["MODE"], {"replica-a", "replica-b"}) + + drift_logs = [line for line in log_lines if "live replica config drift" in line] + self.assertEqual(len(drift_logs), 1) + warning = drift_logs[0] + self.assertIn("job_id=11", warning) + self.assertIn("app_id=app-123", warning) + self.assertIn("plugin_instance_id=shared-instance", warning) + self.assertIn("node-1", warning) + self.assertIn("node-2", warning) + self.assertIn("WORKER_APP_RUNNER", warning) + self.assertIn("worker", warning) + self.assertIn("$.ENV.API_TOKEN", warning) + self.assertIn("$.ENV.MODE", warning) + for value in ( + "live-secret-a", + "live-secret-b", + "requested-secret", + "replica-a", + "replica-b", + "requested", + ): + self.assertNotIn(value, warning) + + def test_process_update_emits_no_drift_warning_for_identical_replicas(self): + discovered_instances = [ + { + DEEPLOY_PLUGIN_DATA.INSTANCE_ID: "shared-instance", + DEEPLOY_PLUGIN_DATA.PLUGIN_SIGNATURE: "A_SIMPLE_PLUGIN", + DEEPLOY_PLUGIN_DATA.NODE: node, + DEEPLOY_PLUGIN_DATA.PLUGIN_INSTANCE: { + "instance_conf": { + DEEPLOY_KEYS.PLUGIN_NAME: "api", + "PROCESS_DELAY": 5, + }, + }, + } + for node in ("node-1", "node-2") + ] + plugin, called = self._make_process_update_plugin( + discovered_instances=discovered_instances, + nodes=["node-1", "node-2"], + ) + log_lines = [] + plugin.P = lambda message, **kwargs: log_lines.append(message) response = plugin._process_pipeline_request( { @@ -1605,7 +1683,7 @@ def test_process_update_rejects_corrupt_duplicate_live_instance_id_before_delete DEEPLOY_KEYS.PLUGIN_SIGNATURE: "A_SIMPLE_PLUGIN", DEEPLOY_KEYS.PLUGIN_INSTANCE_ID: "shared-instance", DEEPLOY_KEYS.PLUGIN_NAME: "api", - "PROCESS_DELAY": 5, + "PROCESS_DELAY": 10, }, ], }, @@ -1613,14 +1691,153 @@ def test_process_update_rejects_corrupt_duplicate_live_instance_id_before_delete async_mode=True, ) - self.assertIn("Corrupt live discovery", response[DEEPLOY_KEYS.ERROR]) + self.assertEqual(response[DEEPLOY_KEYS.STATUS], "command_delivered") + self.assertEqual(called["delete"], 1) + self.assertFalse(any("live replica config drift" in line for line in log_lines)) + + def test_process_update_rejects_instance_id_signature_mismatch_before_delete(self): + plugin, called = self._make_process_update_plugin( + discovered_instances=[ + { + DEEPLOY_PLUGIN_DATA.INSTANCE_ID: "shared-instance", + DEEPLOY_PLUGIN_DATA.PLUGIN_SIGNATURE: "A_SIMPLE_PLUGIN", + DEEPLOY_PLUGIN_DATA.NODE: "node-1", + DEEPLOY_PLUGIN_DATA.PLUGIN_INSTANCE: { + "instance_conf": { + DEEPLOY_KEYS.PLUGIN_NAME: "api", + "PROCESS_DELAY": 5, + }, + }, + }, + ], + ) + + response = plugin._process_pipeline_request( + { + DEEPLOY_KEYS.APP_ID: "app-123", + DEEPLOY_KEYS.APP_ALIAS: "app", + DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.JOB_APP_TYPE: "native", + DEEPLOY_KEYS.PIPELINE_INPUT_TYPE: "void", + DEEPLOY_KEYS.CHAINSTORE_RESPONSE: False, + DEEPLOY_KEYS.TARGET_NODES: ["node-1"], + DEEPLOY_KEYS.TARGET_NODES_COUNT: 1, + DEEPLOY_KEYS.PLUGINS: [ + { + DEEPLOY_KEYS.PLUGIN_SIGNATURE: "ANOTHER_SIMPLE_PLUGIN", + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID: "shared-instance", + DEEPLOY_KEYS.PLUGIN_NAME: "api", + "PROCESS_DELAY": 10, + }, + ], + }, + is_create=False, + async_mode=True, + ) + + self.assertIn("cannot be reused with signature", response[DEEPLOY_KEYS.ERROR]) self.assertEqual(called["delete"], 0) self.assertEqual(called["deploy"], 0) + def test_process_update_generates_id_for_new_plugin_without_id(self): + plugin, called = self._make_process_update_plugin( + discovered_instances=[ + { + DEEPLOY_PLUGIN_DATA.INSTANCE_ID: "existing-instance", + DEEPLOY_PLUGIN_DATA.PLUGIN_SIGNATURE: "A_SIMPLE_PLUGIN", + DEEPLOY_PLUGIN_DATA.NODE: "node-1", + DEEPLOY_PLUGIN_DATA.PLUGIN_INSTANCE: { + "instance_conf": { + DEEPLOY_KEYS.PLUGIN_NAME: "existing", + "PROCESS_DELAY": 5, + }, + }, + }, + ], + ) + + response = plugin._process_pipeline_request( + { + DEEPLOY_KEYS.APP_ID: "app-123", + DEEPLOY_KEYS.APP_ALIAS: "app", + DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.JOB_APP_TYPE: "native", + DEEPLOY_KEYS.PIPELINE_INPUT_TYPE: "void", + DEEPLOY_KEYS.CHAINSTORE_RESPONSE: False, + DEEPLOY_KEYS.TARGET_NODES: ["node-1"], + DEEPLOY_KEYS.TARGET_NODES_COUNT: 1, + DEEPLOY_KEYS.PLUGINS: [ + { + DEEPLOY_KEYS.PLUGIN_SIGNATURE: "A_SIMPLE_PLUGIN", + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID: "existing-instance", + DEEPLOY_KEYS.PLUGIN_NAME: "existing", + "PROCESS_DELAY": 10, + }, + { + DEEPLOY_KEYS.PLUGIN_SIGNATURE: "A_SIMPLE_PLUGIN", + DEEPLOY_KEYS.PLUGIN_NAME: "new-plugin", + "PROCESS_DELAY": 15, + }, + ], + }, + is_create=False, + async_mode=True, + ) + + self.assertEqual(response[DEEPLOY_KEYS.STATUS], "command_delivered") + prepared_plan = called["deploy_kwargs"]["prepared_create_deploy_plan"] + instances = prepared_plan["node_plugins_by_addr"]["node-1"][0][plugin.ct.CONFIG_PLUGIN.K_INSTANCES] + by_name = {instance[DEEPLOY_KEYS.PLUGIN_NAME]: instance for instance in instances} + self.assertEqual( + by_name["existing"][plugin.ct.CONFIG_INSTANCE.K_INSTANCE_ID], + "existing-instance", + ) + self.assertEqual( + by_name["new-plugin"][plugin.ct.CONFIG_INSTANCE.K_INSTANCE_ID], + "A_SIMPLE_PLUG_xxxxxx", + ) + + def test_live_replica_drift_warning_is_bounded_and_value_free(self): + plugin = make_deeploy_plugin() + log_lines = [] + plugin.P = lambda message, **kwargs: log_lines.append(message) + long_segment = "X" * 200 + discovered_instances = [] + for idx in range(12): + for node_idx in range(2): + discovered_instances.append({ + DEEPLOY_PLUGIN_DATA.INSTANCE_ID: f"instance-{idx}", + DEEPLOY_PLUGIN_DATA.PLUGIN_SIGNATURE: "A_SIMPLE_PLUGIN", + DEEPLOY_PLUGIN_DATA.NODE: f"node-{idx}-{node_idx}\nforged", + DEEPLOY_PLUGIN_DATA.PLUGIN_INSTANCE: { + "instance_conf": { + DEEPLOY_KEYS.PLUGIN_NAME: f"plugin-{idx}", + f"{long_segment}-{idx}": f"secret-value-{idx}-{node_idx}", + }, + }, + }) + + warned = plugin._warn_on_live_plugin_config_drift( + discovered_instances, + job_id=11, + app_id="app-123", + ) + + self.assertEqual(warned, 10) + self.assertEqual(len(log_lines), 11) + self.assertTrue(all(len(line) <= 2000 for line in log_lines)) + self.assertTrue(all("\n" not in line for line in log_lines)) + self.assertIn("omitted_groups=2", log_lines[-1]) + joined = "\n".join(log_lines) + for idx in range(12): + self.assertNotIn(f"secret-value-{idx}-0", joined) + self.assertNotIn(f"secret-value-{idx}-1", joined) + def test_process_update_rejects_ambiguous_nameless_no_id_update_before_delete(self): plugin, called = self._make_process_update_plugin( discovered_instances=[ { + DEEPLOY_PLUGIN_DATA.INSTANCE_ID: "legacy-instance-a", DEEPLOY_PLUGIN_DATA.PLUGIN_SIGNATURE: "A_SIMPLE_PLUGIN", DEEPLOY_PLUGIN_DATA.NODE: "node-1", DEEPLOY_PLUGIN_DATA.PLUGIN_INSTANCE: { @@ -1630,6 +1847,7 @@ def test_process_update_rejects_ambiguous_nameless_no_id_update_before_delete(se }, }, { + DEEPLOY_PLUGIN_DATA.INSTANCE_ID: "legacy-instance-b", DEEPLOY_PLUGIN_DATA.PLUGIN_SIGNATURE: "A_SIMPLE_PLUGIN", DEEPLOY_PLUGIN_DATA.NODE: "node-1", DEEPLOY_PLUGIN_DATA.PLUGIN_INSTANCE: { diff --git a/ver.py b/ver.py index 18ac0b5b5..a529ab885 100644 --- a/ver.py +++ b/ver.py @@ -1 +1 @@ -__VER__ = '2.10.390' +__VER__ = '2.10.391' From 43cc3d8a0ef453a3ccbf34ef4cf8223fbce2b46e Mon Sep 17 00:00:00 2001 From: Vitalii <87299468+toderian@users.noreply.github.com> Date: Tue, 21 Jul 2026 22:38:43 +0300 Subject: [PATCH 3/6] RM postponed analyze job (#473) * chore(redmesh): remove API operation queue What changed: - removed the CStore-backed API operation service, config, endpoints, worker, and queue tests - kept synchronous analyze_job and unrelated model-testing/SOC behavior - added a regression for the reduced public surface Why: - roll back the rejected operation-ledger architecture before replacing analyze_job with PostponedRequest Checks: - Python compilation: passed - API/removal unittest suites: 167 passed - model-testing unittest suite: 88 passed * feat(redmesh): postpone manual analysis Move analyze_job provider work into one bounded executor and use the native PostponedRequest lifecycle while keeping persistence on the plugin loop. Add fail-closed bearer admission, explicit input/time bounds, sanitized failure contracts, busy-while-draining cleanup, stale-state checks, and native IPC responsiveness coverage. * fix(redmesh): harden postponed analysis boundaries Enforce monotonic total deadlines with a bounded aiohttp transport, contain bearer values across native error/debug paths, and reject detected stale or deleted job writes. Add real trickle and oversized-response transport tests, late-outcome and deletion-race regressions, plus a portable native-runtime fixture gate. * fix(redmesh): keep automatic analysis responsive What changed: - removed REDMESH_ANALYZE_TOKEN and restored unauthenticated analyze_job - moved automatic structured analysis onto the existing single-worker executor - resumed finalization from minimal job/pass/report future state on later turns - covered manual, automatic, cleanup, failure, and native IPC behavior Why: - prevent automatic model analysis from blocking the serialized plugin process loop Checks: - focused RedMesh suites: 220 passed, 2 skipped, 2 subtests - native postponed IPC: 2 passed - py_compile and git diff --check: pass * fix(redmesh): preserve soft stop during analysis Allow continuous jobs to schedule a soft stop while automatic model work is pending. Preserve that newer status when the future resumes so finalization ends the pass as STOPPED, with regression coverage for the responsive control window. --- .../cybersec/red_mesh/mixins/report.py | 18 +- .../cybersec/red_mesh/pentester_api_01.py | 894 +++++++++-- .../cybersec/red_mesh/services/__init__.py | 24 - .../red_mesh/services/api_operations.py | 1363 ----------------- .../cybersec/red_mesh/services/config.py | 102 -- .../red_mesh/services/finalization.py | 164 +- .../red_mesh/services/llm_structured.py | 23 +- .../red_mesh/services/state_machine.py | 1 + .../cybersec/red_mesh/tests/conftest.py | 13 + .../cybersec/red_mesh/tests/test_api.py | 215 ++- .../tests/test_api_operation_queue.py | 729 --------- .../tests/test_api_operation_removal.py | 32 + .../red_mesh/tests/test_model_testing.py | 5 - .../red_mesh/tests/test_postponed_analyze.py | 714 +++++++++ .../test_postponed_analyze_native_ipc.py | 526 +++++++ .../red_mesh/tests/test_state_machine.py | 7 + requirements.txt | 1 + 17 files changed, 2429 insertions(+), 2402 deletions(-) delete mode 100644 extensions/business/cybersec/red_mesh/services/api_operations.py delete mode 100644 extensions/business/cybersec/red_mesh/tests/test_api_operation_queue.py create mode 100644 extensions/business/cybersec/red_mesh/tests/test_api_operation_removal.py create mode 100644 extensions/business/cybersec/red_mesh/tests/test_postponed_analyze.py create mode 100644 extensions/business/cybersec/red_mesh/tests/test_postponed_analyze_native_ipc.py diff --git a/extensions/business/cybersec/red_mesh/mixins/report.py b/extensions/business/cybersec/red_mesh/mixins/report.py index daed3d58d..67b7ca3f1 100644 --- a/extensions/business/cybersec/red_mesh/mixins/report.py +++ b/extensions/business/cybersec/red_mesh/mixins/report.py @@ -355,10 +355,13 @@ def _stamp_worker_source(self, local_job_status, worker_id, node_addr): self._stamp_finding_list(local_job_status.get("findings"), worker_id, node_addr) - def _get_aggregated_report(self, local_jobs, worker_cls=None): + def _get_aggregated_report(self, local_jobs, worker_cls=None, log_details=True): """ Aggregate results from multiple local workers. + ``log_details=False`` is used at request trust boundaries where aggregate + content must not be copied into runtime logs on failure. + Parameters ---------- local_jobs : dict @@ -429,11 +432,14 @@ def _get_aggregated_report(self, local_jobs, worker_cls=None): self.P(f"Report aggregation done.") # endif we have local jobs except Exception as exc: - self.P("Error during report aggregation: {}:\n{}\n{}\ntype_or_func={}, field={}".format( - exc, self.trace_info(), - self.json_dumps(dct_aggregated_report, indent=2), - type_or_func, field - )) + if log_details: + self.P("Error during report aggregation: {}:\n{}\n{}\ntype_or_func={}, field={}".format( + exc, self.trace_info(), + self.json_dumps(dct_aggregated_report, indent=2), + type_or_func, field + )) + else: + self.P("Manual report aggregation failed", color='y') # Phase 0 dedup pass: collapse findings duplicated across workers # because each worker stamps its own _source_worker_id / # _source_node_addr before merge. The JSON-key fallback in diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index 75cd7b698..b2cc934a9 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -30,11 +30,18 @@ """ +import asyncio +import json import random +import time from collections import deque +from concurrent.futures import CancelledError, ThreadPoolExecutor from copy import deepcopy +from dataclasses import dataclass +from uuid import uuid4 from naeural_core.business.default.web_app.fast_api_web_app import FastApiWebAppPlugin as BasePlugin +from .llm_input_builder import LlmInput, build_llm_input from .mixins import ( _RedMeshLlmAgentMixin, _AttestationMixin, _RiskScoringMixin, _ReportMixin, _LiveProgressMixin, _MispExportMixin, @@ -77,18 +84,13 @@ AuthorizationUploadError, build_network_workers, build_webapp_workers, - cancel_api_operation, collect_engagement_document_cids, - create_analyze_job_operation, delete_engagement_data, DeleteEngagementError, correlate_suricata_eve, dry_run_opencti_export, dry_run_taxii_export, - DEFAULT_API_OPERATIONS_CONFIG, export_stix_bundle, - get_api_operation_result, - get_api_operation_status, get_detection_correlation, get_opencti_export_status, get_rulebook_assessment_status, @@ -119,7 +121,6 @@ list_local_jobs, list_network_jobs, maybe_finalize_pass, - maybe_start_api_operation_worker, normalize_common_launch_options, parse_exceptions, purge_all_jobs, @@ -183,6 +184,15 @@ from .model_testing.secrets import sanitize_model_test_job_config_for_archive from .model_testing.worker import ModelTestWorker from .repositories import ArtifactRepository, JobStateRepository +from .services.llm_structured import ( + PROMPT_PROFILE_AUTO, + PROVIDER_PATH_LOCAL, + PROVIDER_PATH_REMOTE, + build_response_format_for_prompt_profile, + generate_exec_summary, + infer_provider_path, + resolve_prompt_profile, +) from .graybox.scenario_runtime import ( GrayboxWorkerAssignment, rehash_worker_assignment_dict, @@ -279,6 +289,262 @@ def _sanitize_model_test_status_job(job_specs: dict) -> dict: "weak_auth": "Testing credentials", } +_ANALYZE_TOTAL_TIMEOUT_SECONDS = 90.0 +_ANALYZE_PROVIDER_TIMEOUT_SECONDS = 30.0 +_ANALYZE_POLL_INTERVAL_SECONDS = 0.1 +_ANALYZE_MAX_WORKERS = 64 +_ANALYZE_MAX_REPORT_BYTES = 8 * 1024 * 1024 +_ANALYZE_MAX_PROVIDER_RESPONSE_BYTES = 2 * 1024 * 1024 +_ANALYZE_MAX_FOCUS_AREAS = 8 +_ANALYZE_MAX_TEXT_CHARS = 64 + + +async def _bounded_provider_post_async( + url: str, + payload: dict, + timeout_seconds: float, + max_response_bytes: int, +) -> bytes: + """POST JSON with a hard total deadline and bounded streamed response.""" + import aiohttp + + timeout = aiohttp.ClientTimeout( + total=timeout_seconds, + connect=min(10.0, timeout_seconds), + ) + try: + async with aiohttp.ClientSession(timeout=timeout) as session: + async with session.post( + url, + json=payload, + headers={"Content-Type": "application/json"}, + ) as response: + if response.status != 200: + raise RuntimeError("manual analysis provider rejected the request") + if ( + response.content_length is not None + and response.content_length > max_response_bytes + ): + raise RuntimeError("manual analysis provider response is too large") + + raw_response = bytearray() + async for chunk in response.content.iter_chunked(64 * 1024): + raw_response.extend(chunk) + if len(raw_response) > max_response_bytes: + raise RuntimeError("manual analysis provider response is too large") + return bytes(raw_response) + except asyncio.TimeoutError as exc: + raise TimeoutError("manual analysis provider timed out") from exc + except RuntimeError: + raise + except Exception as exc: + raise RuntimeError("manual analysis provider unavailable") from exc + + +def _bounded_provider_post( + url: str, + payload: dict, + timeout_seconds: float, + max_response_bytes: int, +) -> bytes: + """Run the bounded async transport from the isolated worker thread.""" + return asyncio.run(_bounded_provider_post_async( + url=url, + payload=payload, + timeout_seconds=timeout_seconds, + max_response_bytes=max_response_bytes, + )) + + +@dataclass(frozen=True) +class _ManualAnalysisWork: + """Immutable input owned exclusively by the manual-analysis worker.""" + + llm_input: LlmInput + llm_config: dict + api_host: str + api_port: int + deadline_monotonic: float + + +@dataclass(frozen=True) +class _ManualAnalysisOutcome: + """Sanitized worker result returned to the serialized plugin loop.""" + + sections: dict | None + failed: bool + deadline_exceeded: bool = False + + +class _ManualAnalysisWorker: + """Run structured generation without a reference to the RedMesh plugin.""" + + def __init__(self, work: _ManualAnalysisWork): + self._work = work + + @staticmethod + def _safe_validation(validation) -> dict: + issues = tuple(getattr(validation, "issues", ()) or ()) + + def _items(severity): + return [ + { + "code": str(getattr(issue, "code", "") or "")[:80], + "field": str(getattr(issue, "field", "") or "")[:120], + } + for issue in issues + if getattr(issue, "severity", "") == severity + ] + + errors = _items("error") + return { + "ok": not errors, + "errors": errors, + "warnings": _items("warning"), + } + + @staticmethod + def _safe_attempt_logs(result) -> list: + public_logs = [] + for item in tuple(getattr(result, "attempt_logs", ()) or ()): + if not isinstance(item, dict): + continue + public_logs.append({ + "attempt": item.get("attempt"), + "chunk": item.get("chunk") or None, + "elapsed_seconds": item.get("elapsed_seconds"), + "raw_len": item.get("raw_len"), + "validation_codes": [ + str(code)[:80] for code in list(item.get("validation_codes") or [])[:16] + ], + }) + return public_logs + + def _remaining_seconds(self) -> float: + return self._work.deadline_monotonic - time.monotonic() + + def _chat(self, messages: list, max_tokens: int, temperature: float, response_format=None) -> str: + remaining = self._remaining_seconds() + if remaining <= 0: + raise TimeoutError("manual analysis deadline exceeded") + + request_timeout = min(_ANALYZE_PROVIDER_TIMEOUT_SECONDS, remaining) + payload = { + "messages": messages, + "max_tokens": max_tokens, + "temperature": temperature, + } + if response_format is not None: + payload["response_format"] = response_format + + try: + raw_response = _bounded_provider_post( + f"http://{self._work.api_host}:{self._work.api_port}/chat", + payload, + request_timeout, + _ANALYZE_MAX_PROVIDER_RESPONSE_BYTES, + ) + except TimeoutError: + raise + except Exception as exc: + raise RuntimeError("manual analysis provider unavailable") from exc + + try: + response_data = json.loads(raw_response) + except Exception as exc: + raise RuntimeError("manual analysis provider returned invalid JSON") from exc + if isinstance(response_data, dict) and "result" in response_data: + response_data = response_data["result"] + if not isinstance(response_data, dict) or "error" in response_data: + raise RuntimeError("manual analysis provider returned an invalid response") + + choices = response_data.get("choices") + if not isinstance(choices, list) or not choices: + raise RuntimeError("manual analysis provider returned no choices") + choice = choices[0] if isinstance(choices[0], dict) else {} + message = choice.get("message") if isinstance(choice, dict) else {} + content = message.get("content") if isinstance(message, dict) else None + if not isinstance(content, str) or not content.strip(): + raise RuntimeError("manual analysis provider returned empty content") + return content + + def run(self) -> _ManualAnalysisOutcome: + config = self._work.llm_config + model_name = str(config.get("MODEL") or "CyberSecQwen-4B.Q4_K_M.gguf") + provider_path = str(config.get("PROVIDER") or "local") + requested_profile = str(config.get("PROMPT_PROFILE") or PROMPT_PROFILE_AUTO) + if requested_profile == PROMPT_PROFILE_AUTO: + effective_path = infer_provider_path( + provider_path=provider_path, + model_name=model_name, + ) + requested_profile = ( + config.get("REMOTE_PROMPT_PROFILE") + if effective_path == PROVIDER_PATH_REMOTE + else config.get("LOCAL_PROMPT_PROFILE") + ) + profile = resolve_prompt_profile( + requested_profile, + provider_path=provider_path, + model_name=model_name, + ) + response_format = None + if profile.provider_path != PROVIDER_PATH_LOCAL: + response_format = build_response_format_for_prompt_profile(profile) + + def _chat(messages, max_tokens, temperature): + return self._chat( + messages, + max_tokens, + temperature, + response_format=response_format, + ) + + try: + result = generate_exec_summary( + llm_call=_chat, + prepared_input=self._work.llm_input, + model_name=model_name, + provider_path=provider_path, + prompt_profile=profile.id, + max_findings=config.get("STRUCTURED_MAX_FINDINGS", 1), + max_tokens=config.get("STRUCTURED_MAX_TOKENS", 1024), + temperature=config.get("STRUCTURED_TEMPERATURE"), + ) + except Exception: + return _ManualAnalysisOutcome( + sections=None, + failed=True, + deadline_exceeded=self._remaining_seconds() <= 0, + ) + + if self._remaining_seconds() <= 0: + return _ManualAnalysisOutcome( + sections=None, + failed=True, + deadline_exceeded=True, + ) + + sections = result.sections.to_dict() + sections["prompt_profile"] = result.prompt_profile + sections["provider_path"] = result.provider_path + if result.error: + sections["validation"] = self._safe_validation(result.validation) + sections["error"] = True + sections["attempts"] = result.attempts + sections["diagnostics"] = { + "attempt_logs": self._safe_attempt_logs(result), + } + return _ManualAnalysisOutcome( + sections=sections, + failed=bool(result.error), + ) + + +def _run_manual_analysis_worker(work: _ManualAnalysisWork) -> _ManualAnalysisOutcome: + return _ManualAnalysisWorker(work).run() + + __VER__ = '0.9.0' @@ -350,8 +616,6 @@ def _sanitize_model_test_status_job(job_specs: dict) -> dict: }, "LLM_AGENT_API_HOST": "127.0.0.1", # Host where LLM Agent API is running "LLM_AGENT_API_PORT": None, # Port for LLM Agent API (required if enabled) - "API_OPERATIONS": dict(DEFAULT_API_OPERATIONS_CONFIG), - # Security hardening controls "REDACT_CREDENTIALS": True, # Strip passwords from persisted reports "ICS_SAFE_MODE": True, # Halt probing when ICS/SCADA indicators detected @@ -430,6 +694,12 @@ def on_init(self): None """ super(PentesterApi01Plugin, self).on_init() + self._manual_analysis_executor = ThreadPoolExecutor( + max_workers=1, + thread_name_prefix="redmesh-manual-analysis", + ) + self._manual_analysis_state = None + self._automatic_analysis_state = None self.__features = self._get_all_features() self._validate_feature_catalog() # Track active and completed jobs by target @@ -494,6 +764,20 @@ def _setup_semaphore_env(self): def on_close(self): + for state_name in ("_manual_analysis_state", "_automatic_analysis_state"): + state = self.__dict__.get(state_name) + if isinstance(state, dict): + future = state.get("future") + if future is not None: + future.cancel() + setattr(self, state_name, None) + executor = self.__dict__.get("_manual_analysis_executor") + if executor is not None: + try: + executor.shutdown(wait=False, cancel_futures=True) + except TypeError: + executor.shutdown(wait=False) + self._manual_analysis_executor = None super(PentesterApi01Plugin, self).on_close() return @@ -2441,11 +2725,20 @@ def _get_job_write_guarantees(self): "job_revision": True, } - def _write_job_record(self, job_id, job_specs, expected_revision=None, context=""): + def _write_job_record( + self, + job_id, + job_specs, + expected_revision=None, + context="", + reject_stale=False, + ): """ Persist mutable job state with revision bump and stale-write detection. - This is observability only; it does not provide compare-and-swap semantics. + This does not provide compare-and-swap semantics. ``reject_stale`` closes + the detectable read-before-write window for sensitive completions, but a + distributed writer can still race the underlying plain hset. """ current = PentesterApi01Plugin._get_job_state_repository(self).get_job(job_id) current_revision = PentesterApi01Plugin._get_job_revision(self, current) @@ -2453,6 +2746,21 @@ def _write_job_record(self, job_id, job_specs, expected_revision=None, context=" if expected_revision is None: expected_revision = incoming_revision + if reject_stale and not isinstance(current, dict): + self.P( + f"[CSTORE] Guarded write rejected for missing job {job_id}: " + f"context={context or 'unspecified'}", + color='y', + ) + self._log_audit_event("stale_write_detected", { + "job_id": job_id, + "expected_revision": expected_revision, + "current_revision": None, + "context": context or "", + "write_mode": PentesterApi01Plugin._get_job_write_guarantees(self)["mode"], + }) + return None + if isinstance(current, dict) and current_revision != expected_revision: self.P( f"[CSTORE] Stale write detected for job {job_id}: " @@ -2466,6 +2774,8 @@ def _write_job_record(self, job_id, job_specs, expected_revision=None, context=" "context": context or "", "write_mode": PentesterApi01Plugin._get_job_write_guarantees(self)["mode"], }) + if reject_stale: + return None persisted = job_specs if isinstance(job_specs, dict) else dict(job_specs) persisted["job_revision"] = current_revision + 1 @@ -4213,159 +4523,486 @@ def stop_monitoring(self, job_id: str, stop_type: str = "SOFT"): return stop_monitoring(self, job_id, stop_type=stop_type) - @BasePlugin.endpoint(method="post") - def analyze_job( - self, - job_id: str, - analysis_type: str = "", - focus_areas: list[str] = None - ): - """ - Manually trigger LLM analysis for a completed job. + @staticmethod + def _manual_analysis_error(code, message, status_code, *, retryable=False, job_id=""): + result = { + "error": code, + "message": message, + "status_code": status_code, + "retryable": bool(retryable), + } + if job_id: + result["job_id"] = job_id + return result - Aggregates reports from all workers and runs analysis on the combined data. + def _discard_drained_manual_analysis(self): + state = self.__dict__.get("_manual_analysis_state") + if not isinstance(state, dict) or not state.get("discard_result"): + return + future = state.get("future") + if future is not None and future.done(): + try: + future.result() + except Exception: + pass + self._manual_analysis_state = None + return - Parameters - ---------- - job_id : str - Identifier of the job to analyze. - analysis_type : str, optional - Type of analysis: "security_assessment", "vulnerability_summary", "remediation_plan". - focus_areas : list[str], optional - Areas to focus on: ["web", "network", "databases", "authentication"]. + def _get_manual_analysis_executor(self): + """Return the single model worker shared by manual and automatic analysis.""" + executor = self.__dict__.get("_manual_analysis_executor") + if executor is None: + executor = ThreadPoolExecutor( + max_workers=1, + thread_name_prefix="redmesh-manual-analysis", + ) + self._manual_analysis_executor = executor + return executor - Returns - ------- - dict - LLM analysis result or error message. - """ + @staticmethod + def _validate_manual_analysis_request(job_id, analysis_type, focus_areas): + if not isinstance(job_id, str) or not job_id.strip() or len(job_id) > 128: + return PentesterApi01Plugin._manual_analysis_error( + "invalid_job_id", + "A valid job_id is required", + 400, + ) + if not isinstance(analysis_type, str) or len(analysis_type) > _ANALYZE_MAX_TEXT_CHARS: + return PentesterApi01Plugin._manual_analysis_error( + "invalid_analysis_type", + "analysis_type is invalid", + 400, + job_id=job_id, + ) + if focus_areas is None: + return None + if ( + not isinstance(focus_areas, list) + or len(focus_areas) > _ANALYZE_MAX_FOCUS_AREAS + or any( + not isinstance(item, str) or not item.strip() or len(item) > _ANALYZE_MAX_TEXT_CHARS + for item in focus_areas + ) + ): + return PentesterApi01Plugin._manual_analysis_error( + "invalid_focus_areas", + "focus_areas is invalid", + 400, + job_id=job_id, + ) + return None + + def _collect_bounded_manual_analysis_reports(self, workers): + reports = {} + total_bytes = 0 + for worker_addr, worker_entry in workers.items(): + if not isinstance(worker_entry, dict): + continue + report = None + report_cid = worker_entry.get("report_cid") + if report_cid: + try: + report = self.r1fs.get_json(report_cid) + except Exception: + self.P("Manual analysis could not read one worker report", color='y') + if not report: + report = worker_entry.get("result") + if not isinstance(report, dict): + continue + report = deepcopy(report) + try: + total_bytes += len( + json.dumps(report, separators=(",", ":"), default=str).encode("utf-8") + ) + except Exception: + return None, PentesterApi01Plugin._manual_analysis_error( + "analysis_input_invalid", + "Manual analysis report data is invalid", + 422, + ) + if total_bytes > _ANALYZE_MAX_REPORT_BYTES: + return None, PentesterApi01Plugin._manual_analysis_error( + "analysis_input_too_large", + "Manual analysis report data exceeds the supported limit", + 413, + ) + reports[str(worker_addr)] = report + return reports, None + + def _prepare_manual_analysis(self, job_id): llm_cfg = get_llm_agent_config(self) if not llm_cfg["ENABLED"]: - return {"error": "LLM Agent API is not enabled", "job_id": job_id} - + return None, {"error": "LLM Agent API is not enabled", "job_id": job_id} if not self.cfg_llm_agent_api_port: - return {"error": "LLM Agent API port not configured", "job_id": job_id} + return None, {"error": "LLM Agent API port not configured", "job_id": job_id} + native_timeout = getattr(self, "cfg_request_timeout", 120) + if not isinstance(native_timeout, (int, float)) or native_timeout < 120: + return None, PentesterApi01Plugin._manual_analysis_error( + "analysis_timeout_unavailable", + "Manual analysis requires a native request timeout of at least 120 seconds", + 503, + job_id=job_id, + ) - # Get job from CStore job_specs = self._get_job_from_cstore(job_id) - if not job_specs: - return {"error": "Job not found", "job_id": job_id} - - workers = job_specs.get("workers", {}) - if not workers: - return {"error": "No workers found for this job", "job_id": job_id} - - # Check if all workers have finished - all_finished = all(w.get("finished") for w in workers.values()) - if not all_finished: - return {"error": "Job not yet complete, some workers still running", "job_id": job_id} - - # Collect and aggregate reports from all workers - node_reports = self._collect_node_reports(workers) - aggregated_report = self._get_aggregated_report(node_reports) if node_reports else {} + if not isinstance(job_specs, dict): + return None, {"error": "Job not found", "job_id": job_id} + + workers = job_specs.get("workers") + if not isinstance(workers, dict) or not workers: + return None, {"error": "No workers found for this job", "job_id": job_id} + if len(workers) > _ANALYZE_MAX_WORKERS: + return None, PentesterApi01Plugin._manual_analysis_error( + "analysis_input_too_large", + "Manual analysis worker count exceeds the supported limit", + 413, + job_id=job_id, + ) + if not all(isinstance(worker, dict) and worker.get("finished") for worker in workers.values()): + return None, { + "error": "Job not yet complete, some workers still running", + "job_id": job_id, + } + pass_reports = job_specs.get("pass_reports") + if not isinstance(pass_reports, list) or not pass_reports or not isinstance(pass_reports[-1], dict): + return None, {"error": "No report data available for this job", "job_id": job_id} + latest_ref = pass_reports[-1] + expected_report_cid = str(latest_ref.get("report_cid") or "") + if not expected_report_cid: + return None, {"error": "No report data available for this job", "job_id": job_id} + + node_reports, report_error = self._collect_bounded_manual_analysis_reports(workers) + if report_error is not None: + report_error["job_id"] = job_id + return None, report_error + aggregated_report = ( + self._get_aggregated_report(node_reports, log_details=False) + if node_reports else {} + ) if not aggregated_report: - return {"error": "No report data available for this job", "job_id": job_id} + return None, {"error": "No report data available for this job", "job_id": job_id} - target = job_specs.get("target", "unknown") job_config = self._get_job_config(job_specs) _risk_result, flat_findings = self._compute_risk_and_findings(aggregated_report) - - llm_report_sections = self._run_structured_report_sections( - job_id=job_id, + prepared_input = build_llm_input( findings=flat_findings, aggregated_report=aggregated_report, engagement=job_config.get("engagement") if isinstance(job_config, dict) else None, + max_findings=llm_cfg.get("STRUCTURED_MAX_FINDINGS", 6), ) - structured_failed = bool(getattr(self, "_last_structured_llm_failed", False)) - if llm_report_sections is None: + work = _ManualAnalysisWork( + llm_input=deepcopy(prepared_input), + llm_config=deepcopy(llm_cfg), + api_host=str(self.cfg_llm_agent_api_host or "127.0.0.1"), + api_port=int(self.cfg_llm_agent_api_port), + deadline_monotonic=time.monotonic() + _ANALYZE_TOTAL_TIMEOUT_SECONDS, + ) + current_pass = job_specs.get("job_pass", 1) + state = { + "pending_id": uuid4().hex, + "job_id": job_id, + "job_revision": PentesterApi01Plugin._get_job_revision(self, job_specs), + "pass_nr": latest_ref.get("pass_nr", current_pass), + "report_cid": expected_report_cid, + "target": job_specs.get("target", "unknown"), + "num_workers": len(workers), + "deadline_monotonic": time.monotonic() + _ANALYZE_TOTAL_TIMEOUT_SECONDS, + "next_check_monotonic": 0.0, + "discard_result": False, + "work": work, + } + return state, None + + def _postpone_manual_analysis(self, pending_id): + return self.create_postponed_request( + solver_method=self.solve_postponed_analyze_job, + method_kwargs={"pending_id": pending_id}, + ) + + def _manual_analysis_state_matches(self, job_specs, state): + if not isinstance(job_specs, dict): + return False + if PentesterApi01Plugin._get_job_revision(self, job_specs) != state.get("job_revision"): + return False + pass_reports = job_specs.get("pass_reports") + if not isinstance(pass_reports, list) or not pass_reports: + return False + latest_ref = pass_reports[-1] + if not isinstance(latest_ref, dict): + return False + return ( + latest_ref.get("pass_nr", job_specs.get("job_pass", 1)) == state.get("pass_nr") + and str(latest_ref.get("report_cid") or "") == state.get("report_cid") + ) + + def _finalize_manual_analysis(self, state, outcome): + job_id = state["job_id"] + if not isinstance(outcome, _ManualAnalysisOutcome) or outcome.sections is None: return { "error": "Structured LLM report generation failed", "status": "structured_llm_failed", "job_id": job_id, } - # Update the latest pass report with manual structured sections. - pass_reports = job_specs.get("pass_reports", []) - current_pass = job_specs.get("job_pass", 1) + current_job = self._get_job_from_cstore(job_id) + if not PentesterApi01Plugin._manual_analysis_state_matches(self, current_job, state): + return PentesterApi01Plugin._manual_analysis_error( + "analysis_state_changed", + "The job changed while manual analysis was running", + 409, + retryable=True, + job_id=job_id, + ) - if pass_reports: - latest_ref = pass_reports[-1] - try: - pass_data = self.r1fs.get_json(latest_ref["report_cid"]) - if pass_data: - pass_data["llm_report_sections"] = llm_report_sections - if not structured_failed: - llm_text, summary_text = render_legacy_llm_fields(llm_report_sections) - if llm_text: - pass_data["llm_analysis"] = llm_text - if summary_text: - pass_data["quick_summary"] = summary_text - if structured_failed: - pass_data["llm_failed"] = True - else: - pass_data.pop("llm_failed", None) - updated_cid = self.r1fs.add_json(pass_data, show_logs=False) - if updated_cid: - latest_ref["report_cid"] = updated_cid - self._emit_timeline_event( - job_specs, "llm_analysis", - "Manual structured LLM report sections completed", - actor_type="user", - meta={"report_cid": updated_cid, "pass_nr": latest_ref.get("pass_nr", current_pass)} - ) - PentesterApi01Plugin._write_job_record(self, job_id, job_specs, context="manual_llm_update") - self.P(f"Manual structured LLM sections saved for job {job_id}, updated pass report CID: {updated_cid}") - except Exception as e: - self.P(f"Failed to update pass report with structured LLM sections: {e}", color='y') + job_specs = deepcopy(current_job) + latest_ref = job_specs["pass_reports"][-1] + try: + pass_data = self.r1fs.get_json(state["report_cid"]) + if not isinstance(pass_data, dict): + raise ValueError("pass report unavailable") + pass_data = deepcopy(pass_data) + pass_data["llm_report_sections"] = deepcopy(outcome.sections) + if not outcome.failed: + llm_text, summary_text = render_legacy_llm_fields(outcome.sections) + if llm_text: + pass_data["llm_analysis"] = llm_text + if summary_text: + pass_data["quick_summary"] = summary_text + pass_data.pop("llm_failed", None) + else: + pass_data["llm_failed"] = True + updated_cid = self.r1fs.add_json(pass_data, show_logs=False) + if not updated_cid: + raise RuntimeError("pass report write failed") + + latest_job = self._get_job_from_cstore(job_id) + if not PentesterApi01Plugin._manual_analysis_state_matches(self, latest_job, state): + return PentesterApi01Plugin._manual_analysis_error( + "analysis_state_changed", + "The job changed while manual analysis was running", + 409, + retryable=True, + job_id=job_id, + ) + job_specs = deepcopy(latest_job) + latest_ref = job_specs["pass_reports"][-1] + latest_ref["report_cid"] = updated_cid + self._emit_timeline_event( + job_specs, + "llm_analysis", + "Manual structured LLM report sections completed", + actor_type="user", + meta={"report_cid": updated_cid, "pass_nr": state["pass_nr"]}, + ) + persisted = PentesterApi01Plugin._write_job_record( + self, + job_id, + job_specs, + expected_revision=state["job_revision"], + context="manual_llm_update", + reject_stale=True, + ) + if not isinstance(persisted, dict): + return PentesterApi01Plugin._manual_analysis_error( + "analysis_state_changed", + "The job changed while manual analysis was running", + 409, + retryable=True, + job_id=job_id, + ) + persisted_job = self._get_job_from_cstore(job_id) + persisted_refs = ( + persisted_job.get("pass_reports") + if isinstance(persisted_job, dict) else None + ) + if ( + not isinstance(persisted_refs, list) + or not persisted_refs + or not isinstance(persisted_refs[-1], dict) + or str(persisted_refs[-1].get("report_cid") or "") != str(updated_cid) + ): + raise RuntimeError("manual analysis job update was not confirmed") + except Exception: + self.P("Manual structured LLM persistence failed", color='y') + return PentesterApi01Plugin._manual_analysis_error( + "analysis_persistence_failed", + "Manual analysis could not be persisted", + 503, + retryable=True, + job_id=job_id, + ) + self.P(f"Manual structured LLM sections saved for job {job_id}") return { "job_id": job_id, - "target": target, - "num_workers": len(workers), - "pass_nr": pass_reports[-1].get("pass_nr", current_pass) if pass_reports else current_pass, + "target": state["target"], + "num_workers": state["num_workers"], + "pass_nr": state["pass_nr"], "analysis_type": "structured_report_sections", - "llm_failed": structured_failed, - "llm_report_sections": llm_report_sections, + "llm_failed": outcome.failed, + "llm_report_sections": outcome.sections, } - @BasePlugin.endpoint(method="post", require_token=True) - def create_analyze_job_operation( + def solve_postponed_analyze_job(self, pending_id: str): + try: + state = self.__dict__.get("_manual_analysis_state") + if not isinstance(state, dict) or state.get("pending_id") != pending_id: + return PentesterApi01Plugin._manual_analysis_error( + "analysis_request_unavailable", + "The manual analysis request is no longer available", + 410, + retryable=True, + ) + future = state.get("future") + if future is None: + self._manual_analysis_state = None + return PentesterApi01Plugin._manual_analysis_error( + "analysis_executor_failed", + "Manual analysis could not be completed", + 503, + retryable=True, + job_id=state.get("job_id", ""), + ) + + now = time.monotonic() + if now < state.get("next_check_monotonic", 0): + return PentesterApi01Plugin._postpone_manual_analysis(self, pending_id) + state["next_check_monotonic"] = now + _ANALYZE_POLL_INTERVAL_SECONDS + + if future.done(): + try: + outcome = future.result() + except CancelledError: + result = PentesterApi01Plugin._manual_analysis_error( + "analysis_canceled", + "Manual analysis was canceled", + 503, + retryable=True, + job_id=state["job_id"], + ) + except Exception: + result = PentesterApi01Plugin._manual_analysis_error( + "analysis_executor_failed", + "Manual analysis could not be completed", + 503, + retryable=True, + job_id=state["job_id"], + ) + else: + if ( + isinstance(outcome, _ManualAnalysisOutcome) + and outcome.deadline_exceeded + ): + result = PentesterApi01Plugin._manual_analysis_error( + "analysis_timeout", + "Manual analysis exceeded its deadline", + 504, + retryable=True, + job_id=state["job_id"], + ) + else: + result = PentesterApi01Plugin._finalize_manual_analysis( + self, + state, + outcome, + ) + self._manual_analysis_state = None + return result + + if now >= state["deadline_monotonic"]: + state["discard_result"] = True + future.cancel() + return PentesterApi01Plugin._manual_analysis_error( + "analysis_timeout", + "Manual analysis exceeded its deadline", + 504, + retryable=True, + job_id=state["job_id"], + ) + return PentesterApi01Plugin._postpone_manual_analysis(self, pending_id) + except Exception: + state = self.__dict__.get("_manual_analysis_state") + if isinstance(state, dict): + state["discard_result"] = True + return PentesterApi01Plugin._manual_analysis_error( + "analysis_executor_failed", + "Manual analysis could not be completed", + 503, + retryable=True, + ) + + @BasePlugin.endpoint(method="post") + def analyze_job( self, - token: str, job_id: str, analysis_type: str = "", focus_areas: list[str] = None, - idempotency_key: str = "", ): - """Create an async operation for manual structured LLM job analysis.""" - return create_analyze_job_operation( - self, - token=token, - job_id=job_id, - analysis_type=analysis_type, - focus_areas=focus_areas, - idempotency_key=idempotency_key, - ) - - - @BasePlugin.endpoint(require_token=True) - def get_api_operation_status(self, token: str, operation_id: str): - """Return a sanitized RedMesh API operation status.""" - return get_api_operation_status(self, token=token, operation_id=operation_id) - - - @BasePlugin.endpoint(method="post", require_token=True) - def cancel_api_operation(self, token: str, operation_id: str, reason: str = ""): - """Request cancellation of a RedMesh API operation.""" - return cancel_api_operation(self, token=token, operation_id=operation_id, reason=reason) - - - @BasePlugin.endpoint(require_token=True) - def get_api_operation_result(self, token: str, result_handle: str): - """Resolve an opaque RedMesh API operation result handle.""" - return get_api_operation_result(self, token=token, result_handle=result_handle) + """Run one manual analysis through a native postponed request.""" + try: + request_error = PentesterApi01Plugin._validate_manual_analysis_request( + job_id, + analysis_type, + focus_areas, + ) + if request_error is not None: + return request_error + + PentesterApi01Plugin._discard_drained_manual_analysis(self) + if isinstance(self.__dict__.get("_manual_analysis_state"), dict): + return PentesterApi01Plugin._manual_analysis_error( + "analysis_busy", + "Another manual analysis is already running", + 409, + retryable=True, + job_id=job_id, + ) + state, error = PentesterApi01Plugin._prepare_manual_analysis(self, job_id) + if error is not None: + return error + self._manual_analysis_state = state + try: + executor = PentesterApi01Plugin._get_manual_analysis_executor(self) + state["future"] = executor.submit(_run_manual_analysis_worker, state.pop("work")) + return PentesterApi01Plugin._postpone_manual_analysis(self, state["pending_id"]) + except Exception: + future = state.get("future") + canceled = False + if future is not None: + canceled = future.cancel() + if future is None or canceled or future.done(): + self._manual_analysis_state = None + else: + state["discard_result"] = True + return PentesterApi01Plugin._manual_analysis_error( + "analysis_executor_failed", + "Manual analysis could not be started", + 503, + retryable=True, + job_id=job_id, + ) + except Exception: + # Keep admission failures on the typed public error path. + state = self.__dict__.get("_manual_analysis_state") + if isinstance(state, dict): + future = state.get("future") + try: + canceled = future is None or future.cancel() + except Exception: + canceled = False + if canceled or (future is not None and future.done()): + self._manual_analysis_state = None + else: + state["discard_result"] = True + return PentesterApi01Plugin._manual_analysis_error( + "analysis_executor_failed", + "Manual analysis could not be started", + 503, + retryable=True, + job_id=job_id, + ) @BasePlugin.endpoint def get_analysis(self, job_id: str = "", cid: str = "", pass_nr: int = None): @@ -4413,6 +5050,7 @@ def process(self): ------- None """ + PentesterApi01Plugin._discard_drained_manual_analysis(self) super(PentesterApi01Plugin, self).process() # Wait for semaphore dependencies before signaling own readiness @@ -4447,6 +5085,4 @@ def process(self): self._maybe_reannounce_worker_assignments() # Finalize completed passes and handle continuous monitoring (launcher only) self._maybe_finalize_pass() - # Start at most one background API operation worker without blocking request handling. - maybe_start_api_operation_worker(self) return diff --git a/extensions/business/cybersec/red_mesh/services/__init__.py b/extensions/business/cybersec/red_mesh/services/__init__.py index facdfb765..bdd77b61e 100644 --- a/extensions/business/cybersec/red_mesh/services/__init__.py +++ b/extensions/business/cybersec/red_mesh/services/__init__.py @@ -1,5 +1,4 @@ from .config import ( - DEFAULT_API_OPERATIONS_CONFIG, DEFAULT_EVENT_EXPORT_CONFIG, DEFAULT_MODEL_TESTING_CONFIG, DEFAULT_OPENCTI_EXPORT_CONFIG, @@ -8,7 +7,6 @@ DEFAULT_TAXII_EXPORT_CONFIG, DEFAULT_WAZUH_EXPORT_CONFIG, get_attestation_config, - get_api_operations_config, get_event_export_config, get_graybox_budgets_config, get_llm_agent_config, @@ -20,17 +18,6 @@ get_wazuh_export_config, resolve_config_block, ) -from .api_operations import ( - ApiOperationRepository, - cancel_api_operation, - create_analyze_job_operation, - derive_operation_auth_context, - execute_api_operation_worker, - get_api_operation_result, - get_api_operation_status, - maybe_start_api_operation_worker, - public_operation_view, -) from .misp_config import get_misp_export_config from .misp_export import ( build_misp_event, @@ -191,7 +178,6 @@ "INTERMEDIATE_JOB_STATUSES", "ScanStrategy", "TERMINAL_JOB_STATUSES", - "DEFAULT_API_OPERATIONS_CONFIG", "DEFAULT_EVENT_EXPORT_CONFIG", "DEFAULT_MODEL_TESTING_CONFIG", "DEFAULT_OPENCTI_EXPORT_CONFIG", @@ -200,17 +186,9 @@ "DEFAULT_SURICATA_CORRELATION_CONFIG", "DEFAULT_TAXII_EXPORT_CONFIG", "DEFAULT_WAZUH_EXPORT_CONFIG", - "ApiOperationRepository", "can_transition_job_status", - "cancel_api_operation", "coerce_scan_type", - "create_analyze_job_operation", - "derive_operation_auth_context", - "execute_api_operation_worker", "get_attestation_config", - "get_api_operation_result", - "get_api_operation_status", - "get_api_operations_config", "get_event_export_config", "get_graybox_budgets_config", "get_llm_agent_config", @@ -247,8 +225,6 @@ "push_to_misp", "push_to_opencti", "publish_to_taxii", - "maybe_start_api_operation_worker", - "public_operation_view", "redact_event_payload", "resolve_config_block", "stable_hmac_pseudonym", diff --git a/extensions/business/cybersec/red_mesh/services/api_operations.py b/extensions/business/cybersec/red_mesh/services/api_operations.py deleted file mode 100644 index 699aa64fd..000000000 --- a/extensions/business/cybersec/red_mesh/services/api_operations.py +++ /dev/null @@ -1,1363 +0,0 @@ -from __future__ import annotations - -import copy -import hashlib -import hmac -import json -import os -import re -import threading -import uuid -from datetime import datetime, timedelta, timezone - -from ..models import render_legacy_llm_fields -from .config import get_api_operations_config, get_llm_agent_config -from .scan_strategy import coerce_scan_type, get_scan_strategy - - -OPERATION_SCHEMA_VERSION = "redmesh_api_operation_v1" -OPERATION_TYPE_ANALYZE_JOB = "analyze_job" - -STATE_QUEUED = "queued" -STATE_RUNNING = "running" -STATE_SUCCEEDED = "succeeded" -STATE_FAILED = "failed" -STATE_CANCEL_REQUESTED = "cancel_requested" -STATE_CANCELED = "canceled" -STATE_EXPIRED = "expired" - -ACTIVE_OPERATION_STATES = { - STATE_QUEUED, - STATE_RUNNING, - STATE_CANCEL_REQUESTED, -} -TERMINAL_OPERATION_STATES = { - STATE_SUCCEEDED, - STATE_FAILED, - STATE_CANCELED, - STATE_EXPIRED, -} - -PUBLIC_FAILURE_KEYS = { - "failure_class", - "retryable", - "phase", - "short_message", - "attempt_count", -} -RESULT_HANDLE_PREFIX = "opres_" - -ALLOWED_ANALYSIS_TYPES = { - "structured_report_sections", -} -ALLOWED_FOCUS_AREAS = { - "api", - "authentication", - "authorization", - "databases", - "network", - "services", - "tls", - "web", -} -_CID_RE = re.compile(r"(?i)\b(?:Qm[1-9A-HJ-NP-Za-km-z]{20,}|bafy[a-z2-7]{20,}|r1fs:[^\s,;]+)\b") -_URL_RE = re.compile(r"(?i)\bhttps?://[^\s,;]+") -_SENSITIVE_RE = re.compile(r"(?i)(api[_-]?key|authorization|bearer|password|secret|token|credential|prompt)") -_TOKEN_VALUE_RE = re.compile( - r"(?i)\b(?:sk|pk|rk|xox[baprs]|gh[pousr]|eyJ)[A-Za-z0-9._=-]{8,}\b" -) - - -class ApiOperationError(Exception): - def __init__(self, code: str, message: str = "", *, retryable: bool = False): - super().__init__(message or code) - self.code = code - self.message = message or code - self.retryable = bool(retryable) - - -def _utc_timestamp(): - return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _utc_timestamp_after(seconds: int): - delta = timedelta(seconds=max(_safe_int(seconds, 0), 0)) - return (datetime.now(timezone.utc) + delta).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _safe_int(value, default=0): - try: - return int(value) - except (TypeError, ValueError): - return default - - -def _sha256(value: str) -> str: - return hashlib.sha256(str(value or "").encode("utf-8")).hexdigest() - - -def _hmac_sha256(secret: str, value: str) -> str: - return hmac.new( - str(secret or "").encode("utf-8"), - str(value or "").encode("utf-8"), - hashlib.sha256, - ).hexdigest() - - -def _constant_time_in(value: str, candidates: list[str]) -> bool: - found = False - for candidate in candidates: - found = hmac.compare_digest(value, str(candidate or "")) or found - return found - - -def _error(code: str, message: str = "", *, retryable: bool = False, **extra): - payload = { - "error": code, - "message": message or code, - "retryable": bool(retryable), - } - payload.update(extra) - return payload - - -def _not_found(): - return _error("operation_not_found", "Operation not found") - - -def _operation_store_lock(owner): - lock = getattr(owner, "_api_operation_store_lock", None) - if lock is None: - lock = threading.RLock() - setattr(owner, "_api_operation_store_lock", lock) - return lock - - -def _job_operation_lock(owner, job_id: str): - locks = getattr(owner, "_api_operation_job_locks", None) - if not isinstance(locks, dict): - locks = {} - setattr(owner, "_api_operation_job_locks", locks) - key = str(job_id or "") - lock = locks.get(key) - if lock is None: - lock = threading.RLock() - locks[key] = lock - return lock - - -def _safe_log(owner, message: str, color=None): - logger = getattr(owner, "P", None) - if callable(logger): - if color is None: - logger(message) - else: - logger(message, color=color) - - -def _resolve_operation_hmac_secret(config: dict) -> str: - inline = str(config.get("HMAC_SECRET") or "").strip() - if inline: - return inline - env_name = str(config.get("HMAC_SECRET_ENV") or "").strip() - if env_name: - return str(os.environ.get(env_name, "") or "").strip() - return "" - - -def _valid_token_hashes(config: dict) -> list[str]: - hashes = [str(item or "").strip().lower() for item in config.get("TOKEN_HASHES") or [] if item] - env_name = str(config.get("TOKEN_ENV") or "").strip() - if env_name: - env_token = str(os.environ.get(env_name, "") or "").strip() - if env_token: - hashes.append(_sha256(env_token)) - return hashes - - -def derive_operation_auth_context(owner, token: str) -> dict: - config = get_api_operations_config(owner) - if not config.get("ENABLED"): - raise ApiOperationError("operation_auth_disabled", "API operations are disabled") - - token = str(token or "").strip() - if not token: - raise ApiOperationError("operation_auth_required", "Bearer token is required") - - token_hash = _sha256(token) - valid_hashes = _valid_token_hashes(config) - if not valid_hashes: - raise ApiOperationError("operation_auth_not_configured", "API operation auth is not configured") - if not _constant_time_in(token_hash, valid_hashes): - raise ApiOperationError("operation_auth_denied", "Bearer token is not authorized") - - secret = _resolve_operation_hmac_secret(config) - if not secret: - raise ApiOperationError("operation_auth_not_configured", "API operation HMAC secret is not configured") - - scope_id = str(getattr(owner, "cfg_instance_id", "") or "redmesh").strip() or "redmesh" - actor_digest = _hmac_sha256(secret, f"token:{token}")[:32] - actor_id = f"token:{actor_digest}" - return { - "tenant_id": scope_id, - "scope_id": scope_id, - "scope_hash": _hmac_sha256(secret, f"scope:{scope_id}"), - "actor_id": actor_id, - "actor_hash": _hmac_sha256(secret, f"actor:{actor_id}"), - "auth_source": "redmesh_api_operations", - "hmac_secret": secret, - "config": config, - } - - -class ApiOperationRepository: - """Repository for RedMesh API operation rows stored in CStore.""" - - def __init__(self, owner): - self.owner = owner - - @property - def operations_hkey(self): - return f"{getattr(self.owner, 'cfg_instance_id', 'redmesh')}:api_operations" - - @property - def idempotency_hkey(self): - return f"{getattr(self.owner, 'cfg_instance_id', 'redmesh')}:api_operations:idempotency" - - def get_operation(self, operation_id: str): - return self.owner.chainstore_hget(hkey=self.operations_hkey, key=operation_id) - - def list_operations(self): - payload = self.owner.chainstore_hgetall(hkey=self.operations_hkey) - return payload if isinstance(payload, dict) else {} - - def put_operation(self, operation: dict, *, expected_revision=None, context: str = "", allow_stale=False) -> dict: - operation_id = str((operation or {}).get("operation_id") or "") - current = self.get_operation(operation_id) - current_revision = _safe_int((current or {}).get("revision"), 0) if isinstance(current, dict) else 0 - incoming_revision = _safe_int((operation or {}).get("revision"), 0) - if expected_revision is None: - expected_revision = incoming_revision - - if isinstance(current, dict) and current_revision != expected_revision: - audit = getattr(self.owner, "_log_audit_event", None) - if callable(audit): - audit("api_operation_stale_write_detected", { - "operation_id": operation_id, - "expected_revision": expected_revision, - "current_revision": current_revision, - "context": context or "", - "write_mode": "detection_only", - }) - if not allow_stale: - return current - - payload = dict(operation or {}) - payload["revision"] = current_revision + 1 - payload["updated_at"] = _utc_timestamp() - self.owner.chainstore_hset(hkey=self.operations_hkey, key=operation_id, value=payload) - return payload - - def get_idempotency(self, key: str): - return self.owner.chainstore_hget(hkey=self.idempotency_hkey, key=key) - - def put_idempotency(self, key: str, value: dict): - self.owner.chainstore_hset(hkey=self.idempotency_hkey, key=key, value=dict(value or {})) - return value - - -def _normalize_focus_areas(value, config: dict) -> list[str]: - if value is None: - raw_values = [] - elif isinstance(value, (list, tuple, set)): - raw_values = list(value) - else: - raw_values = [value] - - max_items = _safe_int(config.get("MAX_FOCUS_AREAS"), 8) - max_len = _safe_int(config.get("MAX_FOCUS_AREA_LENGTH"), 80) - normalized = [] - seen = set() - for item in raw_values: - text = str(item or "").strip() - if not text: - continue - text = text[:max_len] - key = text.lower().replace("-", "_").replace(" ", "_") - if key not in ALLOWED_FOCUS_AREAS: - raise ApiOperationError("invalid_focus_area", "focus_areas contains an unsupported value") - if key in seen: - continue - seen.add(key) - normalized.append(key) - if len(normalized) >= max_items: - break - return sorted(normalized, key=lambda item: item.lower()) - - -def _normalize_analysis_type(value: str) -> str: - normalized = str(value or "structured_report_sections").strip().lower() - if not normalized: - normalized = "structured_report_sections" - if normalized not in ALLOWED_ANALYSIS_TYPES: - raise ApiOperationError("invalid_analysis_type", "analysis_type is not supported for async analyze operations") - return normalized - - -def _redact_public_text(value: str, limit=160) -> str: - text = str(value or "") - text = _CID_RE.sub("[redacted-cid]", text) - text = _URL_RE.sub("[redacted-url]", text) - text = _TOKEN_VALUE_RE.sub("[redacted-token]", text) - if _SENSITIVE_RE.search(text): - text = _SENSITIVE_RE.sub("[redacted]", text) - if len(text) > limit: - text = text[:limit].rstrip() + "..." - return text - - -def _sanitize_public_value(value, *, depth=0): - if depth > 3: - return None - if isinstance(value, str): - return _redact_public_text(value) - if isinstance(value, bool) or value is None: - return value - if isinstance(value, (int, float)): - return value - if isinstance(value, list): - return [ - item for item in ( - _sanitize_public_value(item, depth=depth + 1) - for item in value[:16] - ) - if item is not None - ] - if isinstance(value, dict): - public = {} - for key, item in value.items(): - key_text = str(key or "") - if _SENSITIVE_RE.search(key_text) or key_text.endswith("_cid") or key_text in {"cid", "url", "details"}: - continue - sanitized = _sanitize_public_value(item, depth=depth + 1) - if sanitized is not None: - public[key_text[:64]] = sanitized - return public - return _redact_public_text(value) - - -def _canonical_fingerprint(secret: str, payload: dict) -> str: - canonical = json.dumps(payload, sort_keys=True, separators=(",", ":")) - return _hmac_sha256(secret, canonical) - - -def _idempotency_hash(context: dict, idempotency_key: str) -> str: - return _hmac_sha256(context["hmac_secret"], f"idempotency:{idempotency_key}") - - -def _idempotency_index_key(context: dict, operation_type: str, idempotency_key_hash: str) -> str: - return ":".join([ - context["scope_hash"], - context["actor_hash"], - operation_type, - idempotency_key_hash, - ]) - - -def _operation_visible_to_context(operation: dict, context: dict) -> bool: - return ( - isinstance(operation, dict) - and operation.get("actor_hash") == context.get("actor_hash") - and operation.get("scope_hash") == context.get("scope_hash") - ) - - -def _parse_utc_timestamp(value: str): - try: - return datetime.fromisoformat(str(value or "").replace("Z", "+00:00")) - except ValueError: - return None - - -def _operation_expired(operation: dict) -> bool: - expires_at = _parse_utc_timestamp((operation or {}).get("expires_at")) - if expires_at is None: - return False - return datetime.now(timezone.utc) >= expires_at - - -def _lease_expired(operation: dict) -> bool: - lease = (operation or {}).get("lease") - if not isinstance(lease, dict): - return False - expires_at = _parse_utc_timestamp(lease.get("expires_at")) - if expires_at is None: - return False - return datetime.now(timezone.utc) >= expires_at - - -def _expire_operation_if_needed(repo: ApiOperationRepository, operation: dict) -> dict: - if not isinstance(operation, dict) or not _operation_expired(operation): - return operation - if operation.get("state") in TERMINAL_OPERATION_STATES: - return operation - updated = dict(operation) - updated["state"] = STATE_EXPIRED - updated["phase"] = "expired" - updated["finished_at"] = updated.get("finished_at") or _utc_timestamp() - return repo.put_operation( - updated, - expected_revision=operation.get("revision"), - context="expire_api_operation", - ) - - -def _recover_stale_operation_if_needed(repo: ApiOperationRepository, operation: dict) -> dict: - operation = _expire_operation_if_needed(repo, operation) - if not isinstance(operation, dict): - return operation - if operation.get("operation_type") != OPERATION_TYPE_ANALYZE_JOB: - return operation - if not _operation_owned_by_node(repo.owner, operation): - return operation - if not _operation_job_visible(repo.owner, operation): - return operation - if operation.get("state") in TERMINAL_OPERATION_STATES: - return operation - if operation.get("state") not in {STATE_RUNNING, STATE_CANCEL_REQUESTED}: - return operation - if not _lease_expired(operation): - return operation - - updated = dict(operation) - now = _utc_timestamp() - if operation.get("state") == STATE_CANCEL_REQUESTED: - cancel = dict(updated.get("cancel") or {}) - cancel["requested"] = True - cancel["observed_at"] = now - cancel["side_effects"] = "unknown_after_restart" - updated.update({ - "state": STATE_CANCELED, - "phase": "canceled", - "finished_at": now, - "lease": {}, - "cancel": cancel, - }) - elif _safe_int(operation.get("attempt"), 0) < _safe_int(operation.get("max_attempts"), 1): - updated.update({ - "state": STATE_QUEUED, - "phase": "recovered", - "lease": {}, - "retryable": True, - "recovered_at": now, - }) - else: - updated.update({ - "state": STATE_FAILED, - "phase": "lease_expired", - "finished_at": now, - "lease": {}, - "retryable": False, - "failure": _failure_payload( - "operation_failed", - "lease_expired", - "Operation worker lease expired", - retryable=False, - attempt_count=operation.get("attempt"), - ), - }) - - return repo.put_operation( - updated, - expected_revision=operation.get("revision"), - context="recover_stale_api_operation", - ) - - -def _operation_job_visible(owner, operation: dict) -> bool: - job_id = str((operation or {}).get("related_job_id") or "").strip() - if not job_id: - return False - return isinstance(_get_job(owner, job_id), dict) - - -def _operation_owned_by_node(owner, operation: dict) -> bool: - owner_node = str((operation or {}).get("owner_node") or "").strip() - local_node = str(getattr(owner, "ee_addr", "") or "").strip() - return not owner_node or not local_node or owner_node == local_node - - -def _operation_cancel_requested(operation: dict) -> bool: - cancel = (operation or {}).get("cancel") - return ( - (operation or {}).get("state") == STATE_CANCEL_REQUESTED - or isinstance(cancel, dict) and bool(cancel.get("requested")) - ) - - -def _failure_payload(failure_class: str, phase: str, message: str, *, retryable=False, attempt_count=0): - public_class = failure_class if failure_class in { - "job_changed", - "job_has_no_report_data", - "job_not_found", - "llm_disabled", - "llm_config_error", - "result_persist_failed", - "structured_llm_failed", - "unsupported_operation_type", - "worker_start_failed", - } else "operation_failed" - public_message = message if isinstance(message, str) and len(message) <= 240 else public_class - if public_class == "operation_failed": - public_message = "Operation failed" - return { - "failure_class": str(public_class or "operation_failed"), - "phase": str(phase or "failed"), - "short_message": _redact_public_text(public_message or public_class or "Operation failed"), - "retryable": bool(retryable), - "attempt_count": _safe_int(attempt_count, 0), - } - - -def _worker_thread_alive(owner) -> bool: - worker = getattr(owner, "_api_operation_worker_thread", None) - return bool(worker and worker.is_alive()) - - -def _operation_lease_valid(operation: dict, lease_token: str) -> bool: - lease = (operation or {}).get("lease") - return isinstance(lease, dict) and hmac.compare_digest( - str(lease.get("token") or ""), - str(lease_token or ""), - ) - - -def _result_handle(owner, operation_id: str, job_id: str, pass_nr) -> str: - secret = _resolve_operation_hmac_secret(get_api_operations_config(owner)) - if not secret: - return f"{RESULT_HANDLE_PREFIX}{uuid.uuid4().hex}" - digest = _hmac_sha256(secret, f"{operation_id}:{job_id}:{pass_nr}")[:32] - return f"{RESULT_HANDLE_PREFIX}{digest}" - - -def _write_job_record(owner, job_id: str, job_specs: dict, *, expected_revision=None, context=""): - writer = getattr(type(owner), "_write_job_record", None) - if callable(writer): - return writer(owner, job_id, job_specs, expected_revision=expected_revision, context=context) - writer = getattr(owner, "_write_job_record", None) - if callable(writer): - return writer(job_id, job_specs, expected_revision=expected_revision, context=context) - owner.chainstore_hset(hkey=getattr(owner, "cfg_instance_id", "redmesh"), key=job_id, value=job_specs) - return job_specs - - -def _job_revision(job_specs: dict) -> int: - return _safe_int((job_specs or {}).get("job_revision"), 0) - - -def _latest_pass_matches(job_specs: dict, *, expected_revision: int, report_cid: str, pass_nr) -> bool: - if not isinstance(job_specs, dict): - return False - if _job_revision(job_specs) != _safe_int(expected_revision, 0): - return False - pass_reports = job_specs.get("pass_reports") or [] - if not pass_reports: - return False - latest_ref = pass_reports[-1] - if latest_ref.get("report_cid") != report_cid: - return False - if pass_nr is not None and latest_ref.get("pass_nr") != pass_nr: - return False - return True - - -def _operation_target_matches_job(operation: dict, job_specs: dict) -> bool: - pass_reports = (job_specs or {}).get("pass_reports") or [] - latest_ref = pass_reports[-1] if pass_reports else {} - target_revision = _safe_int((operation or {}).get("target_job_revision_at_create"), 0) - return _latest_pass_matches( - job_specs, - expected_revision=target_revision, - report_cid=(operation or {}).get("target_report_cid_at_create"), - pass_nr=(operation or {}).get("target_pass_nr_at_create"), - ) and bool(latest_ref) - - -def _fail_job_changed(owner, operation_id: str, lease_token: str, *, artifact_written=False): - failure = _failure_payload( - "job_changed", - "job_changed", - "Job pass report changed before the operation could update it", - retryable=False, - ) - if artifact_written: - failure["side_effects"] = "result_artifact_written" - return _finish_worker_operation( - owner, - operation_id, - lease_token, - state=STATE_FAILED, - phase="job_changed", - failure=failure, - retryable=False, - ) - - -def _emit_operation_timeline(owner, job_specs: dict, event_type: str, message: str, *, pass_nr=None): - emitter = getattr(owner, "_emit_timeline_event", None) - if not callable(emitter): - return - meta = {} - if pass_nr is not None: - meta["pass_nr"] = pass_nr - emitter(job_specs, event_type, message, actor_type="user", meta=meta) - - -def _load_worker_operation(repo: ApiOperationRepository, operation_id: str, lease_token: str): - operation = repo.get_operation(operation_id) - operation = _expire_operation_if_needed(repo, operation) - if not isinstance(operation, dict): - return None - if operation.get("state") in TERMINAL_OPERATION_STATES: - return None - if not _operation_lease_valid(operation, lease_token): - return None - return operation - - -def _update_worker_operation(owner, operation_id: str, lease_token: str, updates: dict, *, context: str): - with _operation_store_lock(owner): - repo = ApiOperationRepository(owner) - operation = _load_worker_operation(repo, operation_id, lease_token) - if not isinstance(operation, dict) or operation.get("state") in TERMINAL_OPERATION_STATES: - return operation - - updated = dict(operation) - updated.update(dict(updates or {})) - lease = dict(updated.get("lease") or {}) - lease["heartbeat_at"] = _utc_timestamp() - lease["expires_at"] = _utc_timestamp_after(context_config(owner).get("LEASE_SECONDS")) - updated["lease"] = lease - return repo.put_operation( - updated, - expected_revision=operation.get("revision"), - context=context, - ) - - -def context_config(owner) -> dict: - return get_api_operations_config(owner) - - -def _finish_worker_operation( - owner, - operation_id: str, - lease_token: str, - *, - state: str, - phase: str, - failure=None, - result_public=None, - retryable=False, - cancel_side_effects=None, -): - with _operation_store_lock(owner): - repo = ApiOperationRepository(owner) - operation = _load_worker_operation(repo, operation_id, lease_token) - if not isinstance(operation, dict): - return None - if operation.get("state") in TERMINAL_OPERATION_STATES: - return operation - - updated = dict(operation) - updated["state"] = state - updated["phase"] = phase - updated["finished_at"] = _utc_timestamp() - updated["retryable"] = bool(retryable) - if failure: - updated["failure"] = failure - if result_public: - updated["result_public"] = result_public - if cancel_side_effects is not None: - cancel = dict(updated.get("cancel") or {}) - cancel["requested"] = True - cancel["observed_at"] = updated["finished_at"] - cancel["side_effects"] = cancel_side_effects - updated["cancel"] = cancel - return repo.put_operation( - updated, - expected_revision=operation.get("revision"), - context=f"finish_{phase}", - ) - - -def _fail_worker_operation(owner, operation_id: str, lease_token: str, phase: str, exc, *, retryable=False): - failure = _failure_payload( - exc.__class__.__name__ if not isinstance(exc, ApiOperationError) else exc.code, - phase, - getattr(exc, "message", str(exc)), - retryable=retryable or bool(getattr(exc, "retryable", False)), - attempt_count=0, - ) - return _finish_worker_operation( - owner, - operation_id, - lease_token, - state=STATE_FAILED, - phase=phase, - failure=failure, - retryable=failure["retryable"], - ) - - -def _cancel_worker_operation(owner, operation_id: str, lease_token: str, *, side_effects: str): - return _finish_worker_operation( - owner, - operation_id, - lease_token, - state=STATE_CANCELED, - phase="canceled", - cancel_side_effects=side_effects, - ) - - -def _public_failure(failure): - if not isinstance(failure, dict): - return None - return { - key: _sanitize_public_value(failure[key]) - for key in PUBLIC_FAILURE_KEYS - if key in failure - } - - -def _public_result(result): - if not isinstance(result, dict): - return None - public = {} - if result.get("kind"): - public["kind"] = result.get("kind") - if result.get("handle"): - public["handle"] = result.get("handle") - if result.get("pass_nr") is not None: - public["pass_nr"] = result.get("pass_nr") - summary = result.get("summary") - if isinstance(summary, dict): - public["summary"] = _sanitize_public_value(summary) - return public or None - - -def public_operation_view(operation: dict) -> dict: - if not isinstance(operation, dict): - return {} - view = { - "operation_id": operation.get("operation_id"), - "operation_type": operation.get("operation_type"), - "state": operation.get("state"), - "phase": operation.get("phase"), - "related_job_id": operation.get("related_job_id"), - "created_at": operation.get("created_at"), - "updated_at": operation.get("updated_at"), - "started_at": operation.get("started_at"), - "finished_at": operation.get("finished_at"), - "expires_at": operation.get("expires_at"), - "retryable": bool(operation.get("retryable", False)), - "attempt": operation.get("attempt"), - "max_attempts": operation.get("max_attempts"), - } - poll_after_ms = operation.get("poll_after_ms") - if poll_after_ms is not None: - view["poll_after_ms"] = poll_after_ms - failure = _public_failure(operation.get("failure")) - if failure: - view["failure"] = failure - result = _public_result(operation.get("result_public")) - if result: - view["result"] = result - cancel = operation.get("cancel") - if isinstance(cancel, dict): - view["cancel"] = { - key: cancel.get(key) - for key in ("requested", "requested_at", "observed_at", "side_effects") - if key in cancel - } - return {key: value for key, value in view.items() if value is not None} - - -def _active_operations(repo: ApiOperationRepository): - return [ - operation for operation in (repo.list_operations() or {}).values() - if ( - isinstance(operation, dict) - and operation.get("state") in ACTIVE_OPERATION_STATES - and not _operation_expired(operation) - and not _lease_expired(operation) - ) - ] - - -def _check_backpressure(repo: ApiOperationRepository, context: dict, job_id: str): - config = context["config"] - active = _active_operations(repo) - if len(active) >= _safe_int(config.get("MAX_QUEUE_GLOBAL"), 32): - raise ApiOperationError("operation_backpressure", "API operation queue is full", retryable=True) - - actor_count = sum(1 for operation in active if operation.get("actor_hash") == context["actor_hash"]) - if actor_count >= _safe_int(config.get("MAX_QUEUE_PER_ACTOR"), 8): - raise ApiOperationError("operation_backpressure", "API operation actor quota is full", retryable=True) - - job_count = sum( - 1 for operation in active - if operation.get("operation_type") == OPERATION_TYPE_ANALYZE_JOB - and operation.get("related_job_id") == job_id - ) - if job_count >= _safe_int(config.get("MAX_QUEUE_PER_JOB"), 1): - raise ApiOperationError("operation_backpressure", "API operation job quota is full", retryable=True) - - -def _get_job(owner, job_id: str): - getter = getattr(owner, "_get_job_from_cstore", None) - if callable(getter): - return getter(job_id) - return owner.chainstore_hget(hkey=getattr(owner, "cfg_instance_id", "redmesh"), key=job_id) - - -def _has_report_data(job_specs: dict) -> bool: - pass_reports = job_specs.get("pass_reports") - if isinstance(pass_reports, list) and pass_reports: - return True - workers = job_specs.get("workers") or {} - if not isinstance(workers, dict): - return False - for worker in workers.values(): - if not isinstance(worker, dict): - continue - if worker.get("report_cid") or worker.get("result"): - return True - return False - - -def _validate_analyze_job_admission(owner, job_id: str) -> dict: - llm_cfg = get_llm_agent_config(owner) - if not llm_cfg.get("ENABLED"): - raise ApiOperationError("llm_disabled", "LLM Agent API is not enabled") - if not getattr(owner, "cfg_llm_agent_api_port", None): - raise ApiOperationError("llm_config_error", "LLM Agent API port not configured") - - job_specs = _get_job(owner, job_id) - if not isinstance(job_specs, dict): - raise ApiOperationError("job_not_found", "Job not found") - - workers = job_specs.get("workers") or {} - if not isinstance(workers, dict) or not workers: - raise ApiOperationError("job_has_no_workers", "No workers found for this job") - if not all(isinstance(worker, dict) and worker.get("finished") for worker in workers.values()): - raise ApiOperationError("job_not_complete", "Job not yet complete") - if not _has_report_data(job_specs): - raise ApiOperationError("job_has_no_report_data", "No report data available for this job") - return job_specs - - -def _aggregate_node_reports(owner, job_specs: dict, node_reports: dict) -> dict: - worker_cls = None - try: - strategy = get_scan_strategy(coerce_scan_type((job_specs or {}).get("scan_type"))) - worker_cls = getattr(strategy, "worker_cls", None) - except Exception: - worker_cls = None - - if worker_cls is not None: - try: - return owner._get_aggregated_report(node_reports, worker_cls=worker_cls) - except TypeError: - pass - return owner._get_aggregated_report(node_reports) - - -def create_analyze_job_operation( - owner, - token: str, - job_id: str, - analysis_type: str = "", - focus_areas=None, - idempotency_key: str = "", -): - try: - context = derive_operation_auth_context(owner, token) - config = context["config"] - job_id = str(job_id or "").strip() - if not job_id: - raise ApiOperationError("invalid_job_id", "job_id is required") - - idempotency_key = str(idempotency_key or "").strip() - max_key_len = _safe_int(config.get("MAX_IDEMPOTENCY_KEY_LENGTH"), 128) - if idempotency_key and len(idempotency_key) > max_key_len: - raise ApiOperationError("invalid_idempotency_key", "Idempotency key is too long") - - normalized_focus = _normalize_focus_areas(focus_areas, config) - normalized_analysis = _normalize_analysis_type(analysis_type) - job_specs = _validate_analyze_job_admission(owner, job_id) - request_public = { - "analysis_type": normalized_analysis, - "focus_areas": normalized_focus, - } - request_fingerprint = _canonical_fingerprint(context["hmac_secret"], { - "operation_type": OPERATION_TYPE_ANALYZE_JOB, - "job_id": job_id, - "analysis_type": normalized_analysis, - "focus_areas": normalized_focus, - "scope_hash": context["scope_hash"], - "actor_hash": context["actor_hash"], - }) - - with _operation_store_lock(owner): - repo = ApiOperationRepository(owner) - idempotency_key_hash = "" - index_key = "" - if idempotency_key: - idempotency_key_hash = _idempotency_hash(context, idempotency_key) - index_key = _idempotency_index_key(context, OPERATION_TYPE_ANALYZE_JOB, idempotency_key_hash) - existing = repo.get_idempotency(index_key) - if isinstance(existing, dict): - if existing.get("request_fingerprint") != request_fingerprint: - return _error("idempotency_conflict", "Idempotency key was used with a different request") - operation = repo.get_operation(existing.get("operation_id", "")) - operation = _expire_operation_if_needed(repo, operation) - if _operation_visible_to_context(operation, context): - return { - "status": "accepted", - "idempotent_replay": True, - "operation": public_operation_view(operation), - } - - _check_backpressure(repo, context, job_id) - - pass_reports = job_specs.get("pass_reports") or [] - latest_ref = pass_reports[-1] if pass_reports else {} - now = _utc_timestamp() - operation_id = f"op_{uuid.uuid4().hex}" - operation = { - "schema_version": OPERATION_SCHEMA_VERSION, - "operation_id": operation_id, - "operation_type": OPERATION_TYPE_ANALYZE_JOB, - "owner_node": str(getattr(owner, "ee_addr", "") or ""), - "state": STATE_QUEUED, - "phase": "queued", - "actor": { - "actor_id": context["actor_id"], - "auth_source": context["auth_source"], - }, - "scope": { - "tenant_id": context["tenant_id"], - "scope_id": context["scope_id"], - }, - "actor_hash": context["actor_hash"], - "scope_hash": context["scope_hash"], - "related_job_id": job_id, - "target_job_revision_at_create": _safe_int(job_specs.get("job_revision"), 0), - "target_pass_nr_at_create": latest_ref.get("pass_nr"), - "target_report_cid_at_create": latest_ref.get("report_cid"), - "request_public": request_public, - "request_fingerprint": request_fingerprint, - "idempotency_key_hash": idempotency_key_hash, - "attempt": 0, - "max_attempts": max(_safe_int(getattr(owner, "cfg_llm_api_retries", 1), 1), 1), - "revision": 0, - "lease": {}, - "retryable": False, - "result_public": None, - "cancel": {}, - "created_at": now, - "updated_at": now, - "expires_at": _utc_timestamp_after(config.get("OPERATION_TTL_SECONDS")), - "poll_after_ms": _safe_int(config.get("POLL_AFTER_MS"), 1000), - } - operation = repo.put_operation(operation, expected_revision=0, context="create_analyze_job_operation") - if index_key: - repo.put_idempotency(index_key, { - "operation_id": operation_id, - "request_fingerprint": request_fingerprint, - "created_at": now, - "expires_at": operation.get("expires_at"), - }) - - return { - "status": "accepted", - "operation": public_operation_view(operation), - } - except ApiOperationError as exc: - extra = {} - if exc.code == "operation_backpressure": - extra["retry_after_ms"] = _safe_int(get_api_operations_config(owner).get("POLL_AFTER_MS"), 1000) - return _error(exc.code, exc.message, retryable=exc.retryable, **extra) - - -def _claim_next_api_operation_locked(owner): - repo = ApiOperationRepository(owner) - lease_seconds = context_config(owner).get("LEASE_SECONDS") - now = _utc_timestamp() - - operations = sorted( - (repo.list_operations() or {}).values(), - key=lambda item: str((item or {}).get("created_at") or ""), - ) - for operation in operations: - operation = _recover_stale_operation_if_needed(repo, operation) - if not isinstance(operation, dict): - continue - if operation.get("state") != STATE_QUEUED: - continue - if operation.get("operation_type") != OPERATION_TYPE_ANALYZE_JOB: - continue - if not _operation_owned_by_node(owner, operation): - continue - if not _operation_job_visible(owner, operation): - continue - - lease_token = uuid.uuid4().hex - claimed = dict(operation) - attempt = _safe_int(claimed.get("attempt"), 0) + 1 - claimed.update({ - "state": STATE_RUNNING, - "phase": "claimed", - "started_at": claimed.get("started_at") or now, - "attempt": attempt, - "lease": { - "owner_node": str(getattr(owner, "ee_addr", "") or ""), - "token": lease_token, - "acquired_at": now, - "heartbeat_at": now, - "expires_at": _utc_timestamp_after(lease_seconds), - }, - }) - updated = repo.put_operation( - claimed, - expected_revision=operation.get("revision"), - context="claim_api_operation", - ) - if _operation_lease_valid(updated, lease_token) and updated.get("state") == STATE_RUNNING: - return updated["operation_id"], lease_token - return None - - -def maybe_start_api_operation_worker(owner) -> bool: - """Start one bounded background API operation worker if queued work exists.""" - config = get_api_operations_config(owner) - if not config.get("ENABLED"): - return False - - with _operation_store_lock(owner): - if _worker_thread_alive(owner): - return False - claimed = _claim_next_api_operation_locked(owner) - if not claimed: - return False - operation_id, lease_token = claimed - thread = threading.Thread( - target=execute_api_operation_worker, - args=(owner, operation_id, lease_token), - name=f"redmesh-api-operation-{operation_id[:12]}", - daemon=True, - ) - setattr(owner, "_api_operation_worker_thread", thread) - - try: - thread.start() - except Exception as exc: - _safe_log(owner, f"Failed to start API operation worker for {operation_id}: {exc}", color="y") - _fail_worker_operation(owner, operation_id, lease_token, "worker_start_failed", exc, retryable=True) - return False - return True - - -def _execute_analyze_job_operation(owner, operation: dict, lease_token: str): - operation_id = operation["operation_id"] - job_id = str(operation.get("related_job_id") or "") - - _update_worker_operation( - owner, - operation_id, - lease_token, - {"phase": "collecting_reports"}, - context="api_operation_collecting_reports", - ) - - job_specs = _validate_analyze_job_admission(owner, job_id) - if not _operation_target_matches_job(operation, job_specs): - _fail_job_changed(owner, operation_id, lease_token) - return - - workers = job_specs.get("workers") or {} - node_reports = owner._collect_node_reports(workers) - aggregated_report = _aggregate_node_reports(owner, job_specs, node_reports) if node_reports else {} - if not aggregated_report: - raise ApiOperationError("job_has_no_report_data", "No report data available for this job") - - job_config = owner._get_job_config(job_specs) - _risk_result, flat_findings = owner._compute_risk_and_findings(aggregated_report) - - _update_worker_operation( - owner, - operation_id, - lease_token, - {"phase": "llm_pending"}, - context="api_operation_llm_pending", - ) - - llm_report_sections = owner._run_structured_report_sections( - job_id=job_id, - findings=flat_findings, - aggregated_report=aggregated_report, - engagement=job_config.get("engagement") if isinstance(job_config, dict) else None, - ) - structured_failed = bool(getattr(owner, "_last_structured_llm_failed", False)) - if llm_report_sections is None: - raise ApiOperationError( - "structured_llm_failed", - "Structured LLM report generation failed", - retryable=True, - ) - - with _operation_store_lock(owner): - repo = ApiOperationRepository(owner) - current = _load_worker_operation(repo, operation_id, lease_token) - if not isinstance(current, dict): - return - if _operation_cancel_requested(current): - _cancel_worker_operation(owner, operation_id, lease_token, side_effects="none") - return - - _update_worker_operation( - owner, - operation_id, - lease_token, - {"phase": "persisting_result"}, - context="api_operation_persisting_result", - ) - - target_report_cid = operation.get("target_report_cid_at_create") - target_pass_nr = operation.get("target_pass_nr_at_create") - with _job_operation_lock(owner, job_id): - latest_job_specs = _get_job(owner, job_id) - if not isinstance(latest_job_specs, dict): - raise ApiOperationError("job_not_found", "Job not found") - latest_job_specs = copy.deepcopy(latest_job_specs) - expected_job_revision = _safe_int(operation.get("target_job_revision_at_create"), 0) - - pass_reports = latest_job_specs.get("pass_reports") or [] - if not pass_reports: - raise ApiOperationError("job_has_no_report_data", "No pass report data available for this job") - latest_ref = pass_reports[-1] - current_report_cid = latest_ref.get("report_cid") - current_pass_nr = latest_ref.get("pass_nr", latest_job_specs.get("job_pass", 1)) - if target_report_cid and current_report_cid != target_report_cid: - _fail_job_changed(owner, operation_id, lease_token) - return - if target_pass_nr is not None and current_pass_nr != target_pass_nr: - _fail_job_changed(owner, operation_id, lease_token) - return - - pass_data = owner.r1fs.get_json(current_report_cid) - if not isinstance(pass_data, dict): - raise ApiOperationError("job_has_no_report_data", "Pass report data is not available") - - fresh_job_specs = _get_job(owner, job_id) - if not _latest_pass_matches( - fresh_job_specs, - expected_revision=expected_job_revision, - report_cid=current_report_cid, - pass_nr=current_pass_nr, - ): - _fail_job_changed(owner, operation_id, lease_token) - return - - pass_data = dict(pass_data) - pass_data["llm_report_sections"] = llm_report_sections - pass_data["llm_operation_id"] = operation_id - if structured_failed: - pass_data["llm_failed"] = True - else: - pass_data.pop("llm_failed", None) - llm_text, summary_text = render_legacy_llm_fields(llm_report_sections) - if llm_text: - pass_data["llm_analysis"] = llm_text - if summary_text: - pass_data["quick_summary"] = summary_text - - with _operation_store_lock(owner): - repo = ApiOperationRepository(owner) - current = _load_worker_operation(repo, operation_id, lease_token) - if not isinstance(current, dict): - return - if _operation_cancel_requested(current): - _cancel_worker_operation(owner, operation_id, lease_token, side_effects="none") - return - - updated_cid = owner.r1fs.add_json(pass_data, show_logs=False) - if not updated_cid: - raise ApiOperationError("result_persist_failed", "Failed to persist updated pass report", retryable=True) - - fresh_job_specs = _get_job(owner, job_id) - if not _latest_pass_matches( - fresh_job_specs, - expected_revision=expected_job_revision, - report_cid=current_report_cid, - pass_nr=current_pass_nr, - ): - _fail_job_changed(owner, operation_id, lease_token, artifact_written=True) - return - - with _operation_store_lock(owner): - repo = ApiOperationRepository(owner) - current = _load_worker_operation(repo, operation_id, lease_token) - if not isinstance(current, dict): - return - if _operation_cancel_requested(current): - _cancel_worker_operation(owner, operation_id, lease_token, side_effects="result_artifact_written") - return - - latest_job_specs = copy.deepcopy(fresh_job_specs) - pass_reports = latest_job_specs.get("pass_reports") or [] - latest_ref = pass_reports[-1] - latest_ref["report_cid"] = updated_cid - _emit_operation_timeline( - owner, - latest_job_specs, - "llm_analysis", - "Manual structured LLM report sections completed", - pass_nr=current_pass_nr, - ) - _write_job_record( - owner, - job_id, - latest_job_specs, - expected_revision=_safe_int(latest_job_specs.get("job_revision"), 0), - context="manual_llm_operation_update", - ) - - handle = _result_handle(owner, operation_id, job_id, current_pass_nr) - result_public = { - "kind": "redmesh_analyze_job_operation_result", - "handle": handle, - "pass_nr": current_pass_nr, - "summary": { - "llm_report_sections_available": True, - "llm_failed": structured_failed, - "job_updated": True, - }, - } - _finish_worker_operation( - owner, - operation_id, - lease_token, - state=STATE_SUCCEEDED, - phase="succeeded", - result_public=result_public, - ) - - -def execute_api_operation_worker(owner, operation_id: str, lease_token: str): - try: - repo = ApiOperationRepository(owner) - with _operation_store_lock(owner): - operation = _load_worker_operation(repo, operation_id, lease_token) - if not isinstance(operation, dict) or operation.get("state") in TERMINAL_OPERATION_STATES: - return - if operation.get("operation_type") != OPERATION_TYPE_ANALYZE_JOB: - raise ApiOperationError("unsupported_operation_type", "Operation type is not supported") - _execute_analyze_job_operation(owner, operation, lease_token) - except Exception as exc: - _safe_log(owner, f"API operation {operation_id} failed: {exc}", color="y") - _fail_worker_operation(owner, operation_id, lease_token, "failed", exc, retryable=getattr(exc, "retryable", False)) - finally: - with _operation_store_lock(owner): - worker = getattr(owner, "_api_operation_worker_thread", None) - if worker is threading.current_thread(): - setattr(owner, "_api_operation_worker_thread", None) - - -def get_api_operation_status(owner, token: str, operation_id: str): - try: - context = derive_operation_auth_context(owner, token) - except ApiOperationError as exc: - return _error(exc.code, exc.message, retryable=exc.retryable) - - with _operation_store_lock(owner): - repo = ApiOperationRepository(owner) - operation = repo.get_operation(str(operation_id or "").strip()) - operation = _expire_operation_if_needed(repo, operation) - if not _operation_visible_to_context(operation, context): - return _not_found() - if not _operation_job_visible(owner, operation): - return _not_found() - return {"operation": public_operation_view(operation)} - - -def cancel_api_operation(owner, token: str, operation_id: str, reason: str = ""): - try: - context = derive_operation_auth_context(owner, token) - except ApiOperationError as exc: - return _error(exc.code, exc.message, retryable=exc.retryable) - - with _operation_store_lock(owner): - repo = ApiOperationRepository(owner) - operation = repo.get_operation(str(operation_id or "").strip()) - operation = _expire_operation_if_needed(repo, operation) - if not _operation_visible_to_context(operation, context): - return _not_found() - if not _operation_job_visible(owner, operation): - return _not_found() - - state = operation.get("state") - if state in TERMINAL_OPERATION_STATES: - return {"operation": public_operation_view(operation)} - - cancel = dict(operation.get("cancel") or {}) - cancel.update({ - "requested": True, - "requested_at": cancel.get("requested_at") or _utc_timestamp(), - "reason": _redact_public_text(str(reason or ""), limit=160), - "side_effects": cancel.get("side_effects") or "none", - }) - operation["cancel"] = cancel - if state == STATE_QUEUED: - operation["state"] = STATE_CANCELED - operation["phase"] = "canceled" - operation["finished_at"] = _utc_timestamp() - cancel["observed_at"] = operation["finished_at"] - else: - operation["state"] = STATE_CANCEL_REQUESTED - operation["phase"] = "cancel_requested" - updated = repo.put_operation(operation, expected_revision=operation.get("revision"), context="cancel_api_operation") - return {"operation": public_operation_view(updated)} - - -def get_api_operation_result(owner, token: str, result_handle: str): - try: - context = derive_operation_auth_context(owner, token) - except ApiOperationError as exc: - return _error(exc.code, exc.message, retryable=exc.retryable) - - handle = str(result_handle or "").strip() - if _CID_RE.search(handle): - return _error("invalid_result_handle", "Operation result handles are opaque; raw CIDs are not accepted") - if not handle.startswith(RESULT_HANDLE_PREFIX): - return _not_found() - - with _operation_store_lock(owner): - repo = ApiOperationRepository(owner) - for operation in (repo.list_operations() or {}).values(): - operation = _expire_operation_if_needed(repo, operation) - if not _operation_visible_to_context(operation, context): - continue - if not _operation_job_visible(owner, operation): - continue - if operation.get("state") != STATE_SUCCEEDED: - continue - result = _public_result(operation.get("result_public")) - if isinstance(result, dict) and result.get("handle") == handle: - return { - "operation_id": operation.get("operation_id"), - "result": result, - } - return _not_found() diff --git a/extensions/business/cybersec/red_mesh/services/config.py b/extensions/business/cybersec/red_mesh/services/config.py index adb820249..f2f7c94e7 100644 --- a/extensions/business/cybersec/red_mesh/services/config.py +++ b/extensions/business/cybersec/red_mesh/services/config.py @@ -160,23 +160,6 @@ def resolve_config_block(owner, block_name, defaults, normalizer=None): }, } -DEFAULT_API_OPERATIONS_CONFIG = { - "ENABLED": False, - "TOKEN_HASHES": [], - "TOKEN_ENV": "REDMESH_API_OPERATION_TOKEN", - "HMAC_SECRET": "", - "HMAC_SECRET_ENV": "REDMESH_API_OPERATION_HMAC_SECRET", - "MAX_IDEMPOTENCY_KEY_LENGTH": 128, - "MAX_FOCUS_AREAS": 8, - "MAX_FOCUS_AREA_LENGTH": 80, - "MAX_QUEUE_GLOBAL": 32, - "MAX_QUEUE_PER_ACTOR": 8, - "MAX_QUEUE_PER_JOB": 1, - "OPERATION_TTL_SECONDS": 86400, - "LEASE_SECONDS": 300, - "POLL_AFTER_MS": 1000, -} - _WAZUH_AUTH_MODES = {"static", "wazuh_jwt"} _TAXII_AUTH_MODES = {"static", "basic"} _OPENCTI_AUTH_MODES = {"static"} @@ -555,91 +538,6 @@ def _normalize(merged, defaults): ) -def get_api_operations_config(owner): - """Return normalized RedMesh async API operation config.""" - def _normalize_hashes(value): - if isinstance(value, str): - values = [item.strip() for item in value.split(",")] - elif isinstance(value, (list, tuple, set)): - values = [str(item or "").strip() for item in value] - else: - values = [] - return [item.lower() for item in values if item] - - def _normalize(merged, defaults): - return { - "ENABLED": bool(merged.get("ENABLED", defaults["ENABLED"])), - "TOKEN_HASHES": _normalize_hashes(merged.get("TOKEN_HASHES", defaults["TOKEN_HASHES"])), - "TOKEN_ENV": _safe_secret_env(merged.get("TOKEN_ENV"), defaults["TOKEN_ENV"]), - "HMAC_SECRET": str(merged.get("HMAC_SECRET") or defaults["HMAC_SECRET"]), - "HMAC_SECRET_ENV": _safe_secret_env( - merged.get("HMAC_SECRET_ENV"), - defaults["HMAC_SECRET_ENV"], - ), - "MAX_IDEMPOTENCY_KEY_LENGTH": _bounded_int( - merged.get("MAX_IDEMPOTENCY_KEY_LENGTH"), - defaults["MAX_IDEMPOTENCY_KEY_LENGTH"], - minimum=16, - maximum=512, - ), - "MAX_FOCUS_AREAS": _bounded_int( - merged.get("MAX_FOCUS_AREAS"), - defaults["MAX_FOCUS_AREAS"], - minimum=0, - maximum=32, - ), - "MAX_FOCUS_AREA_LENGTH": _bounded_int( - merged.get("MAX_FOCUS_AREA_LENGTH"), - defaults["MAX_FOCUS_AREA_LENGTH"], - minimum=8, - maximum=256, - ), - "MAX_QUEUE_GLOBAL": _bounded_int( - merged.get("MAX_QUEUE_GLOBAL"), - defaults["MAX_QUEUE_GLOBAL"], - minimum=1, - maximum=1024, - ), - "MAX_QUEUE_PER_ACTOR": _bounded_int( - merged.get("MAX_QUEUE_PER_ACTOR"), - defaults["MAX_QUEUE_PER_ACTOR"], - minimum=1, - maximum=256, - ), - "MAX_QUEUE_PER_JOB": _bounded_int( - merged.get("MAX_QUEUE_PER_JOB"), - defaults["MAX_QUEUE_PER_JOB"], - minimum=1, - maximum=16, - ), - "OPERATION_TTL_SECONDS": _bounded_int( - merged.get("OPERATION_TTL_SECONDS"), - defaults["OPERATION_TTL_SECONDS"], - minimum=60, - maximum=30 * 86400, - ), - "LEASE_SECONDS": _bounded_int( - merged.get("LEASE_SECONDS"), - defaults["LEASE_SECONDS"], - minimum=10, - maximum=3600, - ), - "POLL_AFTER_MS": _bounded_int( - merged.get("POLL_AFTER_MS"), - defaults["POLL_AFTER_MS"], - minimum=250, - maximum=60000, - ), - } - - return resolve_config_block( - owner, - "API_OPERATIONS", - DEFAULT_API_OPERATIONS_CONFIG, - normalizer=_normalize, - ) - - def get_event_export_config(owner): """Return normalized canonical RedMesh event export config.""" def _normalize(merged, defaults): diff --git a/extensions/business/cybersec/red_mesh/services/finalization.py b/extensions/business/cybersec/red_mesh/services/finalization.py index c000d3531..ec67d5b6f 100644 --- a/extensions/business/cybersec/red_mesh/services/finalization.py +++ b/extensions/business/cybersec/red_mesh/services/finalization.py @@ -98,6 +98,82 @@ def _record_stale_intermediate_recovery(owner, job_specs, previous_status, worke }) +def _automatic_analysis_report_identity(workers): + return tuple(sorted( + (str(address), str(worker.get("report_cid") or "")) + for address, worker in (workers or {}).items() + if isinstance(worker, dict) + )) + + +def _automatic_analysis_state_matches(job_specs, state): + if not isinstance(job_specs, dict) or not isinstance(state, dict): + return False + return ( + job_specs.get("job_id") == state.get("job_id") + and job_specs.get("job_pass", 1) == state.get("pass_nr") + and _automatic_analysis_report_identity(job_specs.get("workers")) + == state.get("report_identity") + ) + + +def _poll_automatic_analysis(owner, all_jobs): + """Yield while automatic model work runs, then return its single result.""" + state = owner.__dict__.get("_automatic_analysis_state") + if not isinstance(state, dict): + return None, False + + current_job = None + for job_key, job_specs in all_jobs.items(): + normalized_key, normalized = owner._normalize_job_record(job_key, job_specs) + if normalized_key is None: + continue + if isinstance(normalized, dict) and normalized.get("job_id") == state.get("job_id"): + current_job = normalized + break + + future = state.get("future") + invalidated = ( + not _automatic_analysis_state_matches(current_job, state) + or is_terminal_job_status((current_job or {}).get("job_status")) + ) + if invalidated: + state["discard_result"] = True + if future is not None: + future.cancel() + + if future is None: + owner._automatic_analysis_state = None + return None, False + if not future.done(): + return None, True + if state.get("discard_result"): + try: + future.result() + except Exception: + pass + owner._automatic_analysis_state = None + return None, False + + try: + sections = future.result() + except Exception as exc: + owner.P( + f"Structured LLM call raised for job {state.get('job_id')}: {exc}", + color='y', + ) + sections = None + failed = True + else: + failed = bool(getattr(owner, "_last_structured_llm_failed", None)) + owner._automatic_analysis_state = None + return { + "state": state, + "sections": sections, + "failed": failed, + }, False + + def _attestation_required(job_specs, job_config) -> bool: if isinstance(job_specs, dict) and "blockchain_attestation_enabled" in job_specs: return bool(job_specs.get("blockchain_attestation_enabled")) @@ -156,6 +232,12 @@ def maybe_finalize_pass(owner): """ all_jobs = _job_repo(owner).list_jobs() artifacts = _artifact_repo(owner) + automatic_completion, should_yield = _poll_automatic_analysis( + owner, + all_jobs, + ) + if should_yield: + return for job_key, job_specs in all_jobs.items(): normalized_key, job_specs = owner._normalize_job_record(job_key, job_specs) @@ -178,12 +260,24 @@ def maybe_finalize_pass(owner): next_pass_at = job_specs.get("next_pass_at") job_pass = job_specs.get("job_pass", 1) job_id = job_specs.get("job_id") + resumed_automatic_analysis = ( + isinstance(automatic_completion, dict) + and _automatic_analysis_state_matches( + job_specs, + automatic_completion.get("state"), + ) + ) + if resumed_automatic_analysis: + if job_status != JOB_STATUS_SCHEDULED_FOR_STOP: + job_status = automatic_completion["state"]["job_status"] + elif isinstance(automatic_completion, dict): + continue if is_terminal_job_status(job_status): if not job_specs.get("job_cid") and job_specs.get("pass_reports"): owner.P(f"[STUCK RECOVERY] {job_id} is {job_status} but has no job_cid — retrying archive build", color='y') owner._build_job_archive(job_id, job_specs) continue - if is_intermediate_job_status(job_status): + if is_intermediate_job_status(job_status) and not resumed_automatic_analysis: if not _is_stale_intermediate_recovery_candidate(job_specs, workers, job_status, next_pass_at): continue _record_stale_intermediate_recovery(owner, job_specs, job_status, len(workers)) @@ -192,7 +286,10 @@ def maybe_finalize_pass(owner): if all_finished and next_pass_at is None: pass_date_started = owner._get_timeline_date(job_specs, "pass_started") or owner._get_timeline_date(job_specs, "created") - pass_date_completed = owner.time() + pass_date_completed = ( + automatic_completion["state"]["pass_date_completed"] + if resumed_automatic_analysis else owner.time() + ) now_ts = pass_date_completed set_job_status(job_specs, JOB_STATUS_COLLECTING) @@ -238,28 +335,47 @@ def maybe_finalize_pass(owner): llm_report_sections = None structured_llm_failed = None if llm_cfg["ENABLED"] and aggregated: - set_job_status(job_specs, JOB_STATUS_ANALYZING) - job_specs = _write_job_record(owner, job_key, job_specs, context="finalize_analyzing") - # PTES report narrative uses only the structured LLM path. - # Legacy aggregate/quick-summary calls accepted raw scan-shaped - # payloads and are intentionally bypassed for report finalization. - try: - llm_report_sections = owner._run_structured_report_sections( - job_id=job_id, - findings=flat_findings, - aggregated_report=aggregated, - engagement=job_config.get("engagement") if isinstance(job_config, dict) else None, - ) - structured_llm_failed = getattr(owner, "_last_structured_llm_failed", None) - if llm_report_sections and not structured_llm_failed: - llm_text, summary_text = render_legacy_llm_fields(llm_report_sections) - except Exception as exc: - owner.P( - f"Structured LLM call raised for job {job_id}: {exc}", - color='y', - ) - llm_report_sections = None - structured_llm_failed = True + if resumed_automatic_analysis: + llm_report_sections = automatic_completion["sections"] + structured_llm_failed = automatic_completion["failed"] + else: + set_job_status(job_specs, JOB_STATUS_ANALYZING) + job_specs = _write_job_record(owner, job_key, job_specs, context="finalize_analyzing") + try: + executor = owner._get_manual_analysis_executor() + # HTTP work yields with PostponedRequest. Automatic work has no + # request to postpone, so process() yields by checking this future + # on later turns. + future = executor.submit( + owner._run_structured_report_sections, + job_id=job_id, + findings=flat_findings, + aggregated_report=aggregated, + engagement=( + job_config.get("engagement") + if isinstance(job_config, dict) else None + ), + ) + owner._automatic_analysis_state = { + "job_id": job_id, + "pass_nr": job_pass, + "report_identity": _automatic_analysis_report_identity(workers), + "job_status": job_status, + "pass_date_completed": pass_date_completed, + "future": future, + "discard_result": False, + } + return + except Exception as exc: + owner._automatic_analysis_state = None + owner.P( + f"Structured LLM call raised for job {job_id}: {exc}", + color='y', + ) + structured_llm_failed = True + + if llm_report_sections and not structured_llm_failed: + llm_text, summary_text = render_legacy_llm_fields(llm_report_sections) llm_failed = True if (llm_cfg["ENABLED"] and structured_llm_failed) else None if llm_failed: diff --git a/extensions/business/cybersec/red_mesh/services/llm_structured.py b/extensions/business/cybersec/red_mesh/services/llm_structured.py index 4cb7c6132..c169adaed 100644 --- a/extensions/business/cybersec/red_mesh/services/llm_structured.py +++ b/extensions/business/cybersec/red_mesh/services/llm_structured.py @@ -502,9 +502,10 @@ def build_response_format_for_prompt_profile( def generate_exec_summary( *, llm_call: LlmCall, - findings: list[dict] | None, + findings: list[dict] | None = None, aggregated_report: dict | None = None, engagement: dict | None = None, + prepared_input: LlmInput | None = None, model_name: str = "", provider_path: str | None = None, prompt_profile: str | None = None, @@ -530,6 +531,9 @@ def generate_exec_summary( fields ONLY; raw blobs are dropped by build_llm_input. engagement : dict | None EngagementContext.to_dict() output. + prepared_input : LlmInput | None + Sanitized trust-boundary output prepared by the caller. When supplied, + raw findings/report/engagement inputs are ignored. model_name : str Stamped onto the resulting LlmReportSections.model. max_findings : int | None @@ -555,12 +559,17 @@ def generate_exec_summary( temperature = profile.default_temperature # --- Trust boundary: scrub inputs through build_llm_input. --- - llm_input = build_llm_input( - findings=findings, - aggregated_report=aggregated_report, - engagement=engagement, - max_findings=max_findings, - ) + if prepared_input is None: + llm_input = build_llm_input( + findings=findings, + aggregated_report=aggregated_report, + engagement=engagement, + max_findings=max_findings, + ) + elif isinstance(prepared_input, LlmInput): + llm_input = prepared_input + else: + raise TypeError("prepared_input must be an LlmInput") compact_findings = _compact_findings_for_structured_prompt(llm_input.findings) messages = _build_messages_for_profile( diff --git a/extensions/business/cybersec/red_mesh/services/state_machine.py b/extensions/business/cybersec/red_mesh/services/state_machine.py index 00f4e28d6..bbf06b1b8 100644 --- a/extensions/business/cybersec/red_mesh/services/state_machine.py +++ b/extensions/business/cybersec/red_mesh/services/state_machine.py @@ -31,6 +31,7 @@ JOB_STATUS_ANALYZING: { JOB_STATUS_COLLECTING, JOB_STATUS_FINALIZING, + JOB_STATUS_SCHEDULED_FOR_STOP, JOB_STATUS_STOPPED, JOB_STATUS_FAILED, }, diff --git a/extensions/business/cybersec/red_mesh/tests/conftest.py b/extensions/business/cybersec/red_mesh/tests/conftest.py index cfa44f92f..4e0a90b8c 100644 --- a/extensions/business/cybersec/red_mesh/tests/conftest.py +++ b/extensions/business/cybersec/red_mesh/tests/conftest.py @@ -155,6 +155,19 @@ class FakeBasePlugin: CONFIG = {'VALIDATION_RULES': {}} endpoint = staticmethod(endpoint_decorator) + def create_postponed_request(self, solver_method, method_kwargs=None): + return { + "solver_method": solver_method, + "method_kwargs": dict(method_kwargs or {}), + } + + def process(self): + return None + + def on_close(self): + self._base_closed = True + return None + mock_module = MagicMock() mock_module.FastApiWebAppPlugin = FakeBasePlugin diff --git a/extensions/business/cybersec/red_mesh/tests/test_api.py b/extensions/business/cybersec/red_mesh/tests/test_api.py index 0082066a3..0b6cbf642 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_api.py +++ b/extensions/business/cybersec/red_mesh/tests/test_api.py @@ -1,7 +1,10 @@ import json import sys import struct +import time import unittest +from concurrent.futures import Future +from copy import deepcopy from unittest.mock import MagicMock, patch from extensions.business.cybersec.red_mesh.constants import JOB_ARCHIVE_VERSION, MAX_CONTINUOUS_PASSES @@ -1735,6 +1738,20 @@ def _build_finalize_plugin(self, job_id="test-job", job_pass=1, run_mode="SINGLE plugin.cfg_attestation = {"ENABLED": True, "PRIVATE_KEY": "", "MIN_SECONDS_BETWEEN_SUBMITS": 300, "RETRIES": 2} plugin.time.return_value = 1000100.0 plugin.json_dumps.return_value = "{}" + plugin._automatic_analysis_state = None + if llm_enabled: + executor = MagicMock() + + def submit_immediately(fn, *args, **kwargs): + future = Future() + try: + future.set_result(fn(*args, **kwargs)) + except Exception as exc: + future.set_exception(exc) + return future + + executor.submit.side_effect = submit_immediately + plugin._get_manual_analysis_executor.return_value = executor # R1FS mock plugin.r1fs = MagicMock() @@ -2247,6 +2264,7 @@ def _structured_failure(*_args, **_kwargs): plugin._run_aggregated_llm_analysis = MagicMock(side_effect=AssertionError("legacy raw LLM path must not run")) plugin._run_quick_summary_analysis = MagicMock(side_effect=AssertionError("legacy quick summary path must not run")) + PentesterApi01Plugin._maybe_finalize_pass(plugin) PentesterApi01Plugin._maybe_finalize_pass(plugin) # Check PassReport has llm_failed=True @@ -2298,6 +2316,7 @@ def _structured_success(*_args, **_kwargs): plugin._run_aggregated_llm_analysis = MagicMock(side_effect=AssertionError("legacy raw LLM path must not run")) plugin._run_quick_summary_analysis = MagicMock(side_effect=AssertionError("legacy quick summary path must not run")) + PentesterApi01Plugin._maybe_finalize_pass(plugin) PentesterApi01Plugin._maybe_finalize_pass(plugin) plugin._run_quick_summary_analysis.assert_not_called() @@ -2308,6 +2327,155 @@ def _structured_success(*_args, **_kwargs): self.assertIn("## Overall Posture", pass_report_dict["llm_analysis"]) self.assertIn("Structured posture", pass_report_dict["llm_analysis"]) + def test_automatic_analysis_yields_and_resumes_without_stale_recovery(self): + """Pending model work returns promptly and is consumed once on a later turn.""" + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin(llm_enabled=True) + self._configure_successful_pass_finalization(plugin, job_specs) + future = Future() + executor = MagicMock() + executor.submit.return_value = future + plugin._get_manual_analysis_executor.return_value = executor + sections = { + "executive_headline": "Structured headline", + "overall_posture": "Structured posture", + "recommendation_summary": ["Patch exposed services"], + "conclusion": "Structured conclusion", + } + + started = time.monotonic() + PentesterApi01Plugin._maybe_finalize_pass(plugin) + self.assertLess(time.monotonic() - started, 0.1) + + self.assertEqual(job_specs["job_status"], "ANALYZING") + self.assertEqual(job_specs["pass_reports"], []) + self.assertIsNotNone(plugin._automatic_analysis_state) + executor.submit.assert_called_once() + + started = time.monotonic() + PentesterApi01Plugin._maybe_finalize_pass(plugin) + self.assertLess(time.monotonic() - started, 0.1) + + executor.submit.assert_called_once() + plugin._collect_node_reports.assert_called_once() + plugin._log_audit_event.assert_not_called() + + plugin._last_structured_llm_failed = False + future.set_result(sections) + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + self.assertEqual(job_specs["job_status"], "FINALIZED") + self.assertEqual(len(job_specs["pass_reports"]), 1) + self.assertIsNone(plugin._automatic_analysis_state) + executor.submit.assert_called_once() + plugin._log_audit_event.assert_not_called() + + def test_soft_stop_during_automatic_analysis_is_preserved_on_resume(self): + """A responsive soft stop requested while analysis runs must end the pass.""" + PentesterApi01Plugin = self._get_plugin_class() + from extensions.business.cybersec.red_mesh.services.control import stop_monitoring + + plugin, job_specs = self._build_finalize_plugin( + run_mode="CONTINUOUS_MONITORING", + llm_enabled=True, + ) + self._configure_successful_pass_finalization(plugin, job_specs) + future = Future() + executor = MagicMock() + executor.submit.return_value = future + plugin._get_manual_analysis_executor.return_value = executor + plugin.chainstore_hget.return_value = job_specs + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + self.assertEqual(job_specs["job_status"], "ANALYZING") + + stop_result = stop_monitoring(plugin, job_specs["job_id"], stop_type="SOFT") + self.assertEqual(stop_result["job_status"], "SCHEDULED_FOR_STOP") + + plugin._last_structured_llm_failed = False + future.set_result({"executive_headline": "Analysis complete"}) + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + self.assertEqual(job_specs["job_status"], "STOPPED") + self.assertEqual(len(job_specs["pass_reports"]), 1) + self.assertIsNone(plugin._automatic_analysis_state) + executor.submit.assert_called_once() + + def test_automatic_analysis_future_failure_keeps_existing_llm_failure_path(self): + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin(llm_enabled=True) + self._configure_successful_pass_finalization(plugin, job_specs) + future = Future() + executor = MagicMock() + executor.submit.return_value = future + plugin._get_manual_analysis_executor.return_value = executor + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + future.set_exception(RuntimeError("provider failed")) + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + pass_report_dict = plugin.r1fs.add_json.call_args_list[1][0][0] + self.assertTrue(pass_report_dict["llm_failed"]) + self.assertIsNone(pass_report_dict.get("llm_report_sections")) + self.assertEqual(job_specs["job_status"], "FINALIZED") + self.assertIsNone(plugin._automatic_analysis_state) + + def test_automatic_analysis_submit_failure_keeps_existing_llm_failure_path(self): + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin(llm_enabled=True) + self._configure_successful_pass_finalization(plugin, job_specs) + executor = MagicMock() + executor.submit.side_effect = RuntimeError("executor unavailable") + plugin._get_manual_analysis_executor.return_value = executor + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + pass_report_dict = plugin.r1fs.add_json.call_args_list[1][0][0] + self.assertTrue(pass_report_dict["llm_failed"]) + self.assertEqual(job_specs["job_status"], "FINALIZED") + self.assertIsNone(plugin._automatic_analysis_state) + + def test_terminal_job_cancels_pending_automatic_analysis(self): + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin(llm_enabled=True) + self._configure_successful_pass_finalization(plugin, job_specs) + future = Future() + executor = MagicMock() + executor.submit.return_value = future + plugin._get_manual_analysis_executor.return_value = executor + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + job_specs["job_status"] = "FINALIZED" + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + self.assertTrue(future.cancelled()) + self.assertIsNone(plugin._automatic_analysis_state) + executor.submit.assert_called_once() + self.assertEqual(job_specs["pass_reports"], []) + + def test_changed_report_identity_cancels_pending_automatic_analysis(self): + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin(llm_enabled=True) + self._configure_successful_pass_finalization(plugin, job_specs) + future = Future() + replacement_future = Future() + executor = MagicMock() + executor.submit.side_effect = [future, replacement_future] + plugin._get_manual_analysis_executor.return_value = executor + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + job_specs["workers"]["worker-A"]["report_cid"] = "QmChanged" + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + self.assertTrue(future.cancelled()) + self.assertIsNotNone(plugin._automatic_analysis_state) + self.assertIn( + ("worker-A", "QmChanged"), + plugin._automatic_analysis_state["report_identity"], + ) + self.assertEqual(executor.submit.call_count, 2) + self.assertEqual(job_specs["pass_reports"], []) + def test_pass_reports_survive_typed_job_record_rewrites(self): """Pass reports must stay attached after typed repository rewrites the job dict.""" PentesterApi01Plugin = self._get_plugin_class() @@ -3569,8 +3737,9 @@ def test_get_job_archive_running(self): self.assertEqual(result["error"], "not_available") def test_manual_structured_analysis_backfills_legacy_fields(self): - """Manual structured analysis updates the pass report for get_analysis compatibility.""" + """Postponed manual analysis updates the pass report for compatibility.""" Plugin = self._get_plugin_class() + from extensions.business.cybersec.red_mesh.pentester_api_01 import _ManualAnalysisOutcome job_specs = self._build_running_job("job-llm", pass_count=1) for worker in job_specs["workers"].values(): worker["finished"] = True @@ -3584,6 +3753,8 @@ def test_manual_structured_analysis_backfills_legacy_fields(self): "AUTO_ANALYSIS_TYPE": "security_assessment", } plugin.cfg_llm_agent_api_port = 8080 + plugin.cfg_llm_agent_api_host = "127.0.0.1" + plugin.cfg_request_timeout = 120 plugin.r1fs = MagicMock() plugin.r1fs.get_json.return_value = { "pass_nr": 1, @@ -3605,23 +3776,41 @@ def test_manual_structured_analysis_backfills_legacy_fields(self): }) plugin._get_job_config = MagicMock(return_value={"target": "example.com"}) plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 0, "breakdown": {}}, [])) + plugin._collect_bounded_manual_analysis_reports = ( + lambda workers: Plugin._collect_bounded_manual_analysis_reports(plugin, workers) + ) + sections = { + "executive_headline": "Manual structured headline", + "overall_posture": "Manual structured posture", + "recommendation_summary": ["Review internet exposure"], + "conclusion": "Manual structured conclusion", + } + future = MagicMock() + future.done.return_value = True + future.result.return_value = _ManualAnalysisOutcome(sections=sections, failed=False) + plugin._manual_analysis_executor = MagicMock() + plugin._manual_analysis_executor.submit.return_value = future + plugin._manual_analysis_state = None + plugin.time.return_value = 100.0 + plugin.create_postponed_request.return_value = "postponed" - def _structured_success(*_args, **_kwargs): - plugin._last_structured_llm_failed = False - return { - "executive_headline": "Manual structured headline", - "overall_posture": "Manual structured posture", - "recommendation_summary": ["Review internet exposure"], - "conclusion": "Manual structured conclusion", - } - - plugin._run_structured_report_sections = MagicMock(side_effect=_structured_success) + def _write_job(_owner, _job_id, persisted, **_kwargs): + job_specs["pass_reports"] = deepcopy(persisted["pass_reports"]) + return persisted - result = Plugin.analyze_job(plugin, job_id="job-llm") + with patch.object(Plugin, "_write_job_record", side_effect=_write_job) as write_job: + postponed = Plugin.analyze_job( + plugin, + job_id="job-llm", + ) + self.assertEqual(postponed, "postponed") + pending_id = plugin._manual_analysis_state["pending_id"] + result = Plugin.solve_postponed_analyze_job(plugin, pending_id) updated_pass = plugin.r1fs.add_json.call_args[0][0] + persisted_job = write_job.call_args[0][2] self.assertEqual(result["analysis_type"], "structured_report_sections") - self.assertEqual(job_specs["pass_reports"][-1]["report_cid"], "QmUpdatedPass") + self.assertEqual(persisted_job["pass_reports"][-1]["report_cid"], "QmUpdatedPass") self.assertEqual(updated_pass["quick_summary"], "Manual structured headline") self.assertIn("Manual structured posture", updated_pass["llm_analysis"]) self.assertIn("Review internet exposure", updated_pass["llm_analysis"]) diff --git a/extensions/business/cybersec/red_mesh/tests/test_api_operation_queue.py b/extensions/business/cybersec/red_mesh/tests/test_api_operation_queue.py deleted file mode 100644 index fb10938af..000000000 --- a/extensions/business/cybersec/red_mesh/tests/test_api_operation_queue.py +++ /dev/null @@ -1,729 +0,0 @@ -import hashlib -import copy -import json -import sys -import types -import unittest - -if "pymisp" not in sys.modules: - pymisp_stub = types.ModuleType("pymisp") - pymisp_stub.MISPEvent = object - pymisp_stub.MISPObject = object - pymisp_stub.MISPAttribute = object - pymisp_stub.PyMISP = object - sys.modules["pymisp"] = pymisp_stub - -from extensions.business.cybersec.red_mesh.services.api_operations import ( - ApiOperationRepository, - cancel_api_operation, - create_analyze_job_operation, - get_api_operation_result, - get_api_operation_status, - maybe_start_api_operation_worker, -) -from extensions.business.cybersec.red_mesh.services.config import ( - DEFAULT_API_OPERATIONS_CONFIG, - get_api_operations_config, -) - - -def _token_hash(token: str) -> str: - return hashlib.sha256(token.encode("utf-8")).hexdigest() - - -def _join_worker(owner): - worker = getattr(owner, "_api_operation_worker_thread", None) - if worker: - worker.join(timeout=2) - - -def _structured_sections(owner): - owner.llm_calls += 1 - owner._last_structured_llm_failed = False - return { - "executive_headline": "High priority HTTPS exposure", - "overall_posture": "The exposed service should be reviewed.", - "recommendation_summary": ["Review TLS and authentication controls."], - "strategic_roadmap": {}, - "attack_chain_narratives": [], - "coverage_gaps": [], - "conclusion": "Remediation is practical.", - } - - -class _Owner: - def __init__(self, *, token_hashes=None, max_queue_global=32, max_queue_per_job=1): - self.cfg_instance_id = "test-instance" - self.ee_addr = "node-a" - self.cfg_llm_agent = {"ENABLED": True, "TIMEOUT": 5} - self.cfg_llm_agent_api_port = 8080 - self.cfg_llm_api_retries = 2 - self.cfg_api_operations = { - "ENABLED": True, - "TOKEN_HASHES": token_hashes if token_hashes is not None else [_token_hash("good-token")], - "TOKEN_ENV": "", - "HMAC_SECRET": "operation-hmac-secret", - "HMAC_SECRET_ENV": "", - "MAX_QUEUE_GLOBAL": max_queue_global, - "MAX_QUEUE_PER_JOB": max_queue_per_job, - "MAX_QUEUE_PER_ACTOR": 8, - "POLL_AFTER_MS": 500, - } - self._store = {} - self._jobs = {} - self.r1fs = _R1FS() - self.audit_events = [] - self.timeline_events = [] - self.llm_calls = 0 - - def add_job(self, job_id="job-1"): - self.r1fs.objects["QmPassReport"] = { - "pass_nr": 1, - "aggregated_report_cid": "QmAggregatedReport", - "worker_reports": {}, - "risk_score": 12, - } - self._jobs[job_id] = { - "job_id": job_id, - "job_revision": 3, - "workers": { - "node-a": { - "finished": True, - "report_cid": "QmWorkerReport", - }, - }, - "pass_reports": [ - {"pass_nr": 1, "report_cid": "QmPassReport"}, - ], - } - return self._jobs[job_id] - - def _get_job_from_cstore(self, job_id): - return self._jobs.get(job_id) - - def chainstore_hget(self, *, hkey, key): - return self._store.get(hkey, {}).get(key) - - def chainstore_hgetall(self, *, hkey): - return dict(self._store.get(hkey, {})) - - def chainstore_hset(self, *, hkey, key, value): - self._store.setdefault(hkey, {}) - if value is None: - self._store[hkey].pop(key, None) - else: - self._store[hkey][key] = value - - def _log_audit_event(self, event, payload): - self.audit_events.append((event, payload)) - - def _collect_node_reports(self, workers): - return {"node-a": {"open_ports": [443], "service_info": {"443": {"name": "https"}}}} - - def _get_aggregated_report(self, node_reports): - return {"open_ports": [443], "service_info": {"443": {"name": "https"}}} - - def _compute_risk_and_findings(self, aggregated_report): - return {"score": 12, "breakdown": {"findings_score": 12}}, [ - {"severity": "HIGH", "port": 443, "category": "tls"} - ] - - def _get_job_config(self, job_specs): - return {"engagement": {"name": "test engagement"}} - - def _run_structured_report_sections(self, **kwargs): - return _structured_sections(self) - - def _emit_timeline_event(self, job_specs, event_type, message, actor_type="system", meta=None): - event = { - "event_type": event_type, - "message": message, - "actor_type": actor_type, - "meta": dict(meta or {}), - } - self.timeline_events.append(event) - job_specs.setdefault("timeline", []).append(event) - - def _write_job_record(self, job_id, job_specs, expected_revision=None, context=""): - current = self._jobs.get(job_id, {}) - job_specs["job_revision"] = int(current.get("job_revision", 0) or 0) + 1 - job_specs["write_context"] = context - self._jobs[job_id] = job_specs - return job_specs - - def P(self, *args, **kwargs): - return None - - -class _R1FS: - def __init__(self): - self.objects = {} - self.added_payloads = [] - - def get_json(self, cid): - payload = self.objects.get(cid) - return copy.deepcopy(payload) if isinstance(payload, dict) else payload - - def add_json(self, payload, show_logs=False): - self.added_payloads.append(copy.deepcopy(payload)) - cid = f"QmGeneratedPassReport{len(self.added_payloads)}" - self.objects[cid] = copy.deepcopy(payload) - return cid - - -class TestApiOperationConfig(unittest.TestCase): - - def test_default_api_operations_are_disabled(self): - owner = _Owner() - owner.cfg_api_operations = None - - cfg = get_api_operations_config(owner) - - self.assertFalse(cfg["ENABLED"]) - self.assertEqual(DEFAULT_API_OPERATIONS_CONFIG["TOKEN_ENV"], "REDMESH_API_OPERATION_TOKEN") - - def test_api_operation_config_bounds_values(self): - owner = _Owner() - owner.cfg_api_operations.update({ - "MAX_QUEUE_GLOBAL": 0, - "MAX_QUEUE_PER_ACTOR": -1, - "MAX_IDEMPOTENCY_KEY_LENGTH": 2, - "POLL_AFTER_MS": 1, - }) - - cfg = get_api_operations_config(owner) - - self.assertEqual(cfg["MAX_QUEUE_GLOBAL"], DEFAULT_API_OPERATIONS_CONFIG["MAX_QUEUE_GLOBAL"]) - self.assertEqual(cfg["MAX_QUEUE_PER_ACTOR"], DEFAULT_API_OPERATIONS_CONFIG["MAX_QUEUE_PER_ACTOR"]) - self.assertEqual( - cfg["MAX_IDEMPOTENCY_KEY_LENGTH"], - DEFAULT_API_OPERATIONS_CONFIG["MAX_IDEMPOTENCY_KEY_LENGTH"], - ) - self.assertEqual(cfg["POLL_AFTER_MS"], DEFAULT_API_OPERATIONS_CONFIG["POLL_AFTER_MS"]) - - -class TestApiOperationAdmission(unittest.TestCase): - - def test_create_requires_configured_server_actor(self): - owner = _Owner(token_hashes=[]) - owner.add_job() - - result = create_analyze_job_operation(owner, "good-token", "job-1") - - self.assertEqual(result["error"], "operation_auth_not_configured") - - def test_create_rejects_unauthorized_token(self): - owner = _Owner() - owner.add_job() - - result = create_analyze_job_operation(owner, "bad-token", "job-1") - - self.assertEqual(result["error"], "operation_auth_denied") - - def test_create_returns_sanitized_queued_operation(self): - owner = _Owner() - owner.add_job() - - result = create_analyze_job_operation( - owner, - "good-token", - "job-1", - focus_areas=["web", "web", "network"], - idempotency_key="idem-secret", - ) - - self.assertEqual(result["status"], "accepted") - operation = result["operation"] - self.assertEqual(operation["state"], "queued") - self.assertEqual(operation["operation_type"], "analyze_job") - self.assertNotIn("actor_hash", operation) - self.assertNotIn("request_fingerprint", operation) - - serialized_store = json.dumps(owner._store, sort_keys=True) - self.assertNotIn("good-token", serialized_store) - self.assertNotIn("idem-secret", serialized_store) - - def test_same_idempotency_key_replays_same_operation(self): - owner = _Owner() - owner.add_job() - - first = create_analyze_job_operation(owner, "good-token", "job-1", idempotency_key="idem-1") - second = create_analyze_job_operation(owner, "good-token", "job-1", idempotency_key="idem-1") - - self.assertTrue(second["idempotent_replay"]) - self.assertEqual(first["operation"]["operation_id"], second["operation"]["operation_id"]) - - def test_same_idempotency_key_different_request_conflicts_without_enqueue(self): - owner = _Owner(max_queue_per_job=2) - owner.add_job() - - first = create_analyze_job_operation(owner, "good-token", "job-1", idempotency_key="idem-1") - before_count = len(owner._store["test-instance:api_operations"]) - conflict = create_analyze_job_operation( - owner, - "good-token", - "job-1", - focus_areas=["web"], - idempotency_key="idem-1", - ) - - self.assertEqual(first["status"], "accepted") - self.assertEqual(conflict["error"], "idempotency_conflict") - self.assertEqual(len(owner._store["test-instance:api_operations"]), before_count) - - def test_create_rejects_invalid_analysis_type_and_focus_area(self): - owner = _Owner() - owner.add_job() - - invalid_type = create_analyze_job_operation( - owner, - "good-token", - "job-1", - analysis_type="legacy_raw_summary", - ) - invalid_focus = create_analyze_job_operation( - owner, - "good-token", - "job-1", - focus_areas=["custom prompt"], - ) - - self.assertEqual(invalid_type["error"], "invalid_analysis_type") - self.assertEqual(invalid_focus["error"], "invalid_focus_area") - - def test_same_idempotency_key_different_actor_does_not_replay_or_conflict(self): - owner = _Owner(token_hashes=[_token_hash("good-token"), _token_hash("other-token")]) - owner.add_job("job-1") - owner.add_job("job-2") - - first = create_analyze_job_operation(owner, "good-token", "job-1", idempotency_key="idem-1") - second = create_analyze_job_operation(owner, "other-token", "job-2", idempotency_key="idem-1") - - self.assertEqual(first["status"], "accepted") - self.assertEqual(second["status"], "accepted") - self.assertNotEqual(first["operation"]["operation_id"], second["operation"]["operation_id"]) - - def test_queue_full_returns_backpressure_without_operation_record(self): - owner = _Owner(max_queue_global=1) - owner.add_job("job-1") - owner.add_job("job-2") - - first = create_analyze_job_operation(owner, "good-token", "job-1") - before_count = len(owner._store["test-instance:api_operations"]) - second = create_analyze_job_operation(owner, "good-token", "job-2") - - self.assertEqual(first["status"], "accepted") - self.assertEqual(second["error"], "operation_backpressure") - self.assertTrue(second["retryable"]) - self.assertEqual(len(owner._store["test-instance:api_operations"]), before_count) - - def test_expired_queued_operation_does_not_consume_backpressure(self): - owner = _Owner(max_queue_global=1) - owner.add_job("job-1") - owner.add_job("job-2") - first = create_analyze_job_operation(owner, "good-token", "job-1") - repo = ApiOperationRepository(owner) - raw = repo.get_operation(first["operation"]["operation_id"]) - raw["expires_at"] = "2000-01-01T00:00:00Z" - repo.put_operation(raw, expected_revision=raw["revision"], context="test_expire") - - second = create_analyze_job_operation(owner, "good-token", "job-2") - status = get_api_operation_status(owner, "good-token", first["operation"]["operation_id"]) - - self.assertEqual(second["status"], "accepted") - self.assertEqual(status["operation"]["state"], "expired") - - -class TestApiOperationAccessAndState(unittest.TestCase): - - def test_cross_actor_status_and_unknown_status_are_indistinguishable(self): - owner = _Owner(token_hashes=[_token_hash("good-token"), _token_hash("other-token")]) - owner.add_job() - created = create_analyze_job_operation(owner, "good-token", "job-1") - operation_id = created["operation"]["operation_id"] - - foreign = get_api_operation_status(owner, "other-token", operation_id) - unknown = get_api_operation_status(owner, "other-token", "op_missing") - - self.assertEqual(foreign, unknown) - self.assertEqual(foreign["error"], "operation_not_found") - - def test_status_requires_continued_job_visibility(self): - owner = _Owner() - owner.add_job() - created = create_analyze_job_operation(owner, "good-token", "job-1") - owner._jobs.pop("job-1") - - status = get_api_operation_status(owner, "good-token", created["operation"]["operation_id"]) - - self.assertEqual(status["error"], "operation_not_found") - - def test_cancel_queued_operation_prevents_worker_start(self): - owner = _Owner() - owner.add_job() - created = create_analyze_job_operation(owner, "good-token", "job-1") - - canceled = cancel_api_operation(owner, "good-token", created["operation"]["operation_id"]) - - self.assertEqual(canceled["operation"]["state"], "canceled") - self.assertEqual(canceled["operation"]["phase"], "canceled") - self.assertEqual(canceled["operation"]["cancel"]["side_effects"], "none") - - def test_status_response_does_not_expose_raw_cids_or_diagnostics(self): - owner = _Owner() - owner.add_job() - created = create_analyze_job_operation(owner, "good-token", "job-1") - repo = ApiOperationRepository(owner) - raw = repo.get_operation(created["operation"]["operation_id"]) - raw["state"] = "succeeded" - raw["result_ref"] = "QmInternalResult" - raw["result_public"] = { - "kind": "redmesh_analyze_job_result", - "handle": "opaque-handle", - "pass_nr": 1, - "summary": { - "llm_report_sections_available": True, - "nested": { - "url": "https://provider.example/internal", - "note": "token abc", - }, - }, - "report_cid": "QmShouldNotLeak", - } - raw["failure"] = { - "failure_class": "llm_provider_error", - "retryable": False, - "short_message": "Provider failed", - "details": "secret diagnostic", - "provider_url": "https://provider.example", - } - repo.put_operation(raw, expected_revision=raw["revision"], context="test") - - status = get_api_operation_status(owner, "good-token", raw["operation_id"]) - serialized = json.dumps(status, sort_keys=True) - - self.assertIn("opaque-handle", serialized) - self.assertNotIn("QmInternalResult", serialized) - self.assertNotIn("QmShouldNotLeak", serialized) - self.assertNotIn("secret diagnostic", serialized) - self.assertNotIn("provider.example", serialized) - self.assertNotIn("token abc", serialized) - - def test_raw_cid_is_rejected_by_operation_result_endpoint(self): - owner = _Owner() - - result = get_api_operation_result(owner, "good-token", "Qmabcdefghijklmnopqrstuvwx") - - self.assertEqual(result["error"], "invalid_result_handle") - - def test_worker_executes_analyze_job_and_exposes_opaque_result(self): - owner = _Owner() - owner.add_job() - created = create_analyze_job_operation(owner, "good-token", "job-1") - - started = maybe_start_api_operation_worker(owner) - _join_worker(owner) - - self.assertTrue(started) - self.assertEqual(owner.llm_calls, 1) - status = get_api_operation_status(owner, "good-token", created["operation"]["operation_id"]) - self.assertEqual(status["operation"]["state"], "succeeded") - self.assertEqual(status["operation"]["phase"], "succeeded") - handle = status["operation"]["result"]["handle"] - self.assertTrue(handle.startswith("opres_")) - - latest_ref = owner._jobs["job-1"]["pass_reports"][-1] - self.assertNotEqual(latest_ref["report_cid"], "QmPassReport") - updated_pass = owner.r1fs.get_json(latest_ref["report_cid"]) - self.assertEqual(updated_pass["llm_operation_id"], created["operation"]["operation_id"]) - self.assertIn("llm_report_sections", updated_pass) - self.assertIn("llm_analysis", updated_pass) - self.assertEqual(owner.timeline_events[0]["meta"], {"pass_nr": 1}) - - result = get_api_operation_result(owner, "good-token", handle) - serialized_status = json.dumps(status, sort_keys=True) - serialized_result = json.dumps(result, sort_keys=True) - self.assertEqual(result["operation_id"], created["operation"]["operation_id"]) - self.assertEqual(result["result"]["handle"], handle) - self.assertNotIn("QmPassReport", serialized_status) - self.assertNotIn(latest_ref["report_cid"], serialized_status) - self.assertNotIn(latest_ref["report_cid"], serialized_result) - - def test_worker_aborts_if_job_changes_during_llm_wait(self): - owner = _Owner() - owner.add_job() - created = create_analyze_job_operation(owner, "good-token", "job-1") - - def _llm_changes_job(**kwargs): - owner._jobs["job-1"]["job_revision"] += 1 - owner._jobs["job-1"]["pass_reports"][-1]["report_cid"] = "QmConcurrentPassReport" - owner.r1fs.objects["QmConcurrentPassReport"] = {"pass_nr": 1, "aggregated_report_cid": "QmOther"} - return _structured_sections(owner) - - owner._run_structured_report_sections = _llm_changes_job - started = maybe_start_api_operation_worker(owner) - _join_worker(owner) - - status = get_api_operation_status(owner, "good-token", created["operation"]["operation_id"]) - self.assertTrue(started) - self.assertEqual(status["operation"]["state"], "failed") - self.assertEqual(status["operation"]["phase"], "job_changed") - self.assertEqual(status["operation"]["failure"]["failure_class"], "job_changed") - self.assertEqual(owner._jobs["job-1"]["pass_reports"][-1]["report_cid"], "QmConcurrentPassReport") - self.assertEqual(owner.r1fs.added_payloads, []) - - def test_worker_aborts_if_job_revision_changes_during_llm_wait(self): - owner = _Owner() - owner.add_job() - created = create_analyze_job_operation(owner, "good-token", "job-1") - - def _llm_changes_job_revision_only(**kwargs): - owner._jobs["job-1"]["job_revision"] += 1 - owner._jobs["job-1"]["unrelated_update"] = "export status changed" - return _structured_sections(owner) - - owner._run_structured_report_sections = _llm_changes_job_revision_only - started = maybe_start_api_operation_worker(owner) - _join_worker(owner) - - status = get_api_operation_status(owner, "good-token", created["operation"]["operation_id"]) - self.assertTrue(started) - self.assertEqual(status["operation"]["state"], "failed") - self.assertEqual(status["operation"]["phase"], "job_changed") - self.assertEqual(owner._jobs["job-1"]["pass_reports"][-1]["report_cid"], "QmPassReport") - self.assertEqual(owner._jobs["job-1"]["unrelated_update"], "export status changed") - self.assertEqual(owner.r1fs.added_payloads, []) - - def test_worker_cancel_during_llm_wait_avoids_side_effects(self): - owner = _Owner() - owner.add_job() - created = create_analyze_job_operation(owner, "good-token", "job-1") - operation_id = created["operation"]["operation_id"] - - def _llm_cancels(**kwargs): - cancel_api_operation(owner, "good-token", operation_id, reason="operator stop") - return _structured_sections(owner) - - owner._run_structured_report_sections = _llm_cancels - started = maybe_start_api_operation_worker(owner) - _join_worker(owner) - - status = get_api_operation_status(owner, "good-token", operation_id) - self.assertTrue(started) - self.assertEqual(status["operation"]["state"], "canceled") - self.assertEqual(status["operation"]["cancel"]["side_effects"], "none") - self.assertEqual(owner._jobs["job-1"]["pass_reports"][-1]["report_cid"], "QmPassReport") - self.assertEqual(owner.r1fs.added_payloads, []) - - def test_worker_expired_during_llm_wait_avoids_side_effects(self): - owner = _Owner() - owner.add_job() - created = create_analyze_job_operation(owner, "good-token", "job-1") - operation_id = created["operation"]["operation_id"] - - def _llm_expires_operation(**kwargs): - repo = ApiOperationRepository(owner) - raw = repo.get_operation(operation_id) - raw["expires_at"] = "2000-01-01T00:00:00Z" - repo.put_operation(raw, expected_revision=raw["revision"], context="test_expire_running") - return _structured_sections(owner) - - owner._run_structured_report_sections = _llm_expires_operation - started = maybe_start_api_operation_worker(owner) - _join_worker(owner) - - status = get_api_operation_status(owner, "good-token", operation_id) - self.assertTrue(started) - self.assertEqual(status["operation"]["state"], "expired") - self.assertEqual(status["operation"]["phase"], "expired") - self.assertEqual(owner._jobs["job-1"]["pass_reports"][-1]["report_cid"], "QmPassReport") - self.assertEqual(owner.r1fs.added_payloads, []) - - def test_worker_recovers_expired_running_lease_after_restart(self): - owner = _Owner() - owner.add_job() - created = create_analyze_job_operation(owner, "good-token", "job-1") - operation_id = created["operation"]["operation_id"] - repo = ApiOperationRepository(owner) - raw = repo.get_operation(operation_id) - raw.update({ - "state": "running", - "phase": "llm_pending", - "attempt": 1, - "lease": { - "owner_node": owner.ee_addr, - "token": "stale-worker-token", - "acquired_at": "2000-01-01T00:00:00Z", - "heartbeat_at": "2000-01-01T00:00:00Z", - "expires_at": "2000-01-01T00:00:01Z", - }, - }) - repo.put_operation(raw, expected_revision=raw["revision"], context="test_stale_running") - - started = maybe_start_api_operation_worker(owner) - _join_worker(owner) - - status = get_api_operation_status(owner, "good-token", operation_id) - self.assertTrue(started) - self.assertEqual(status["operation"]["state"], "succeeded") - self.assertEqual(status["operation"]["attempt"], 2) - self.assertEqual(owner.llm_calls, 1) - - def test_worker_finalizes_expired_cancel_requested_lease_after_restart(self): - owner = _Owner() - owner.add_job() - created = create_analyze_job_operation(owner, "good-token", "job-1") - operation_id = created["operation"]["operation_id"] - repo = ApiOperationRepository(owner) - raw = repo.get_operation(operation_id) - raw.update({ - "state": "cancel_requested", - "phase": "cancel_requested", - "attempt": 1, - "cancel": { - "requested": True, - "requested_at": "2000-01-01T00:00:00Z", - "side_effects": "none", - }, - "lease": { - "owner_node": owner.ee_addr, - "token": "stale-worker-token", - "acquired_at": "2000-01-01T00:00:00Z", - "heartbeat_at": "2000-01-01T00:00:00Z", - "expires_at": "2000-01-01T00:00:01Z", - }, - }) - repo.put_operation(raw, expected_revision=raw["revision"], context="test_stale_cancel") - - started = maybe_start_api_operation_worker(owner) - status = get_api_operation_status(owner, "good-token", operation_id) - - self.assertFalse(started) - self.assertEqual(status["operation"]["state"], "canceled") - self.assertEqual(status["operation"]["cancel"]["side_effects"], "unknown_after_restart") - - def test_operation_ttl_takes_precedence_over_stale_running_lease_recovery(self): - owner = _Owner() - owner.add_job() - created = create_analyze_job_operation(owner, "good-token", "job-1") - operation_id = created["operation"]["operation_id"] - repo = ApiOperationRepository(owner) - raw = repo.get_operation(operation_id) - raw.update({ - "state": "running", - "phase": "llm_pending", - "attempt": 1, - "expires_at": "2000-01-01T00:00:00Z", - "lease": { - "owner_node": owner.ee_addr, - "token": "stale-worker-token", - "acquired_at": "2000-01-01T00:00:00Z", - "heartbeat_at": "2000-01-01T00:00:00Z", - "expires_at": "2000-01-01T00:00:01Z", - }, - }) - repo.put_operation(raw, expected_revision=raw["revision"], context="test_expired_running") - - started = maybe_start_api_operation_worker(owner) - status = get_api_operation_status(owner, "good-token", operation_id) - - self.assertFalse(started) - self.assertEqual(status["operation"]["state"], "expired") - self.assertEqual(owner.llm_calls, 0) - - def test_expired_running_lease_does_not_consume_backpressure(self): - owner = _Owner(max_queue_global=1) - owner.add_job("job-1") - owner.add_job("job-2") - created = create_analyze_job_operation(owner, "good-token", "job-1") - repo = ApiOperationRepository(owner) - raw = repo.get_operation(created["operation"]["operation_id"]) - raw.update({ - "state": "running", - "phase": "llm_pending", - "attempt": raw["max_attempts"], - "lease": { - "owner_node": owner.ee_addr, - "token": "stale-worker-token", - "acquired_at": "2000-01-01T00:00:00Z", - "heartbeat_at": "2000-01-01T00:00:00Z", - "expires_at": "2000-01-01T00:00:01Z", - }, - }) - repo.put_operation(raw, expected_revision=raw["revision"], context="test_stale_backpressure") - - second = create_analyze_job_operation(owner, "good-token", "job-2") - - self.assertEqual(second["status"], "accepted") - - def test_worker_cancel_after_r1fs_write_does_not_update_job_record(self): - owner = _Owner() - owner.add_job() - created = create_analyze_job_operation(owner, "good-token", "job-1") - operation_id = created["operation"]["operation_id"] - original_add_json = owner.r1fs.add_json - - def _cancel_after_add_json(payload, show_logs=False): - cid = original_add_json(payload, show_logs=show_logs) - cancel_api_operation(owner, "good-token", operation_id, reason="late stop") - return cid - - owner.r1fs.add_json = _cancel_after_add_json - started = maybe_start_api_operation_worker(owner) - _join_worker(owner) - - status = get_api_operation_status(owner, "good-token", operation_id) - self.assertTrue(started) - self.assertEqual(status["operation"]["state"], "canceled") - self.assertEqual(status["operation"]["cancel"]["side_effects"], "result_artifact_written") - self.assertEqual(owner._jobs["job-1"]["pass_reports"][-1]["report_cid"], "QmPassReport") - self.assertEqual(len(owner.r1fs.added_payloads), 1) - - def test_worker_failure_does_not_expose_provider_diagnostics_or_token_values(self): - owner = _Owner() - owner.add_job() - created = create_analyze_job_operation(owner, "good-token", "job-1") - operation_id = created["operation"]["operation_id"] - - def _llm_raises(**kwargs): - raise RuntimeError("OpenAI provider returned sk-liveverysecretvalue via https://provider.example") - - owner._run_structured_report_sections = _llm_raises - started = maybe_start_api_operation_worker(owner) - _join_worker(owner) - - status = get_api_operation_status(owner, "good-token", operation_id) - serialized = json.dumps(status, sort_keys=True) - self.assertTrue(started) - self.assertEqual(status["operation"]["state"], "failed") - self.assertEqual(status["operation"]["failure"]["failure_class"], "operation_failed") - self.assertEqual(status["operation"]["failure"]["short_message"], "Operation failed") - self.assertNotIn("OpenAI", serialized) - self.assertNotIn("provider.example", serialized) - self.assertNotIn("sk-liveverysecretvalue", serialized) - - def test_operation_row_stale_revision_is_audit_logged(self): - owner = _Owner() - repo = ApiOperationRepository(owner) - operation = { - "operation_id": "op_1", - "state": "queued", - "phase": "queued", - "revision": 0, - } - stored = repo.put_operation(operation, expected_revision=0, context="first") - stale = dict(stored) - stale["revision"] = 0 - stale["state"] = "running" - - repo.put_operation(stale, expected_revision=0, context="stale") - - current = repo.get_operation("op_1") - self.assertEqual(owner.audit_events[0][0], "api_operation_stale_write_detected") - self.assertEqual(owner.audit_events[0][1]["operation_id"], "op_1") - self.assertEqual(owner.audit_events[0][1]["write_mode"], "detection_only") - self.assertEqual(current["state"], "queued") - - -if __name__ == "__main__": - unittest.main() diff --git a/extensions/business/cybersec/red_mesh/tests/test_api_operation_removal.py b/extensions/business/cybersec/red_mesh/tests/test_api_operation_removal.py new file mode 100644 index 000000000..3ba5e2297 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_api_operation_removal.py @@ -0,0 +1,32 @@ +import unittest +from pathlib import Path + +from .conftest import mock_plugin_modules + + +class TestApiOperationRemoval(unittest.TestCase): + + def test_operation_surface_is_absent_and_supported_endpoints_remain(self): + mock_plugin_modules() + from extensions.business.cybersec.red_mesh import services + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + + redmesh_root = Path(__file__).resolve().parents[1] + self.assertFalse((redmesh_root / "services" / "api_operations.py").exists()) + self.assertNotIn("API_OPERATIONS", PentesterApi01Plugin.CONFIG) + + for name in ( + "DEFAULT_API_OPERATIONS_CONFIG", + "get_api_operations_config", + "create_analyze_job_operation", + "get_api_operation_status", + "cancel_api_operation", + "get_api_operation_result", + "maybe_start_api_operation_worker", + ): + self.assertFalse(hasattr(services, name), name) + self.assertFalse(hasattr(PentesterApi01Plugin, name), name) + + self.assertTrue(callable(PentesterApi01Plugin.analyze_job)) + self.assertTrue(callable(PentesterApi01Plugin.solve_postponed_analyze_job)) + self.assertTrue(callable(PentesterApi01Plugin.preflight_model_test_provider)) diff --git a/extensions/business/cybersec/red_mesh/tests/test_model_testing.py b/extensions/business/cybersec/red_mesh/tests/test_model_testing.py index 97f68218c..4f728c618 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_model_testing.py +++ b/extensions/business/cybersec/red_mesh/tests/test_model_testing.py @@ -1870,11 +1870,6 @@ def _raw_evidence_endpoint_plugin(self): plugin = MagicMock() plugin.cfg_instance_id = "instance" - plugin.cfg_api_operations = { - "ENABLED": True, - "TOKEN_HASHES": [hashlib.sha256(b"backend-token").hexdigest()], - "HMAC_SECRET": "unit-test-hmac-secret", - } job_specs = { "job_id": "job-raw", "job_type": "model_test", diff --git a/extensions/business/cybersec/red_mesh/tests/test_postponed_analyze.py b/extensions/business/cybersec/red_mesh/tests/test_postponed_analyze.py new file mode 100644 index 000000000..d9f2d5c89 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_postponed_analyze.py @@ -0,0 +1,714 @@ +import importlib.util +import json +import threading +import time +import unittest +from concurrent.futures import ThreadPoolExecutor +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from unittest.mock import MagicMock, patch + +from .conftest import mock_plugin_modules + + +mock_plugin_modules() + +from extensions.business.cybersec.red_mesh.llm_input_builder import build_llm_input +from extensions.business.cybersec.red_mesh.pentester_api_01 import ( + PentesterApi01Plugin, + _ManualAnalysisOutcome, + _ManualAnalysisWork, + _ManualAnalysisWorker, + _bounded_provider_post, +) + + +SECRET_SENTINEL = "credential-sentinel-private-value" + + +def _valid_remote_sections(): + return { + "executive_headline": "Material exposure requires executive attention.", + "background_draft": "Authorized external assessment.", + "overall_posture": "The engagement identified high-severity issues requiring remediation.", + "recommendation_summary": ["Prioritize the verified high-severity findings."], + "strategic_roadmap": { + "near_term": ["Address verified exposure."], + "mid_term": ["Add regression coverage."], + "long_term": ["Adopt continuous assurance."], + }, + "attack_chain_narratives": ["Initial exposure could enable privilege escalation."], + "coverage_gaps": ["Social engineering was outside scope."], + "conclusion": "Retest after remediation.", + } + + +class TestPostponedAnalyze(unittest.TestCase): + + def test_busy_request_does_not_prepare_or_queue_work(self): + plugin = MagicMock() + plugin._manual_analysis_state = { + "future": MagicMock(), + "discard_result": False, + } + result = PentesterApi01Plugin.analyze_job(plugin, job_id="job-1") + self.assertEqual(result["error"], "analysis_busy") + self.assertEqual(result["status_code"], 409) + self.assertTrue(result["retryable"]) + plugin._get_job_from_cstore.assert_not_called() + + def test_pending_solver_uses_only_opaque_key(self): + plugin = MagicMock() + future = MagicMock() + future.done.return_value = False + plugin._manual_analysis_state = { + "pending_id": "opaque-key", + "future": future, + "job_id": "job-1", + "deadline_monotonic": 200.0, + "next_check_monotonic": 0.0, + "discard_result": False, + } + plugin.create_postponed_request.return_value = "postponed" + + with patch( + "extensions.business.cybersec.red_mesh.pentester_api_01.time.monotonic", + return_value=100.0, + ): + result = PentesterApi01Plugin.solve_postponed_analyze_job(plugin, "opaque-key") + + self.assertEqual(result, "postponed") + kwargs = plugin.create_postponed_request.call_args.kwargs + self.assertEqual(kwargs["method_kwargs"], {"pending_id": "opaque-key"}) + self.assertNotIn("job-1", str(kwargs["method_kwargs"])) + + def test_solver_does_not_poll_future_before_next_check(self): + plugin = MagicMock() + future = MagicMock() + plugin._manual_analysis_state = { + "pending_id": "opaque-key", + "future": future, + "job_id": "job-1", + "deadline_monotonic": 200.0, + "next_check_monotonic": 100.1, + "discard_result": False, + } + plugin.create_postponed_request.return_value = "postponed" + + with patch( + "extensions.business.cybersec.red_mesh.pentester_api_01.time.monotonic", + return_value=100.0, + ): + result = PentesterApi01Plugin.solve_postponed_analyze_job(plugin, "opaque-key") + + self.assertEqual(result, "postponed") + future.done.assert_not_called() + + def test_deadline_returns_timeout_and_holds_slot_until_worker_drains(self): + plugin = MagicMock() + future = MagicMock() + future.done.return_value = False + plugin._manual_analysis_state = { + "pending_id": "opaque-key", + "future": future, + "job_id": "job-1", + "deadline_monotonic": 99.0, + "next_check_monotonic": 0.0, + "discard_result": False, + } + + with patch( + "extensions.business.cybersec.red_mesh.pentester_api_01.time.monotonic", + return_value=100.0, + ): + result = PentesterApi01Plugin.solve_postponed_analyze_job(plugin, "opaque-key") + + self.assertEqual(result["error"], "analysis_timeout") + self.assertEqual(result["status_code"], 504) + self.assertTrue(plugin._manual_analysis_state["discard_result"]) + self.assertIsNotNone(plugin._manual_analysis_state) + + future.done.return_value = True + future.result.return_value = _ManualAnalysisOutcome(sections={}, failed=False) + PentesterApi01Plugin._discard_drained_manual_analysis(plugin) + self.assertIsNone(plugin._manual_analysis_state) + + def test_late_completed_outcome_is_not_finalized(self): + plugin = MagicMock() + future = MagicMock() + future.done.return_value = True + future.result.return_value = _ManualAnalysisOutcome( + sections=_valid_remote_sections(), + failed=False, + deadline_exceeded=True, + ) + plugin._manual_analysis_state = { + "pending_id": "opaque-key", + "future": future, + "job_id": "job-1", + "deadline_monotonic": 99.0, + "next_check_monotonic": 0.0, + "discard_result": False, + } + + with patch( + "extensions.business.cybersec.red_mesh.pentester_api_01.time.monotonic", + return_value=100.0, + ), patch.object( + PentesterApi01Plugin, + "_finalize_manual_analysis", + ) as finalize: + result = PentesterApi01Plugin.solve_postponed_analyze_job( + plugin, + "opaque-key", + ) + + self.assertEqual(result["error"], "analysis_timeout") + self.assertEqual(result["status_code"], 504) + finalize.assert_not_called() + self.assertIsNone(plugin._manual_analysis_state) + + def test_request_shape_bounds_are_explicit(self): + invalid_type = PentesterApi01Plugin._validate_manual_analysis_request( + "job-1", + "x" * 65, + [], + ) + invalid_focus_count = PentesterApi01Plugin._validate_manual_analysis_request( + "job-1", + "", + ["web"] * 9, + ) + invalid_focus_value = PentesterApi01Plugin._validate_manual_analysis_request( + "job-1", + "", + ["x" * 65], + ) + + self.assertEqual(invalid_type["error"], "invalid_analysis_type") + self.assertEqual(invalid_focus_count["error"], "invalid_focus_areas") + self.assertEqual(invalid_focus_value["error"], "invalid_focus_areas") + + def test_executor_start_failure_releases_unused_slot(self): + plugin = PentesterApi01Plugin.__new__(PentesterApi01Plugin) + plugin._manual_analysis_state = None + plugin._manual_analysis_executor = MagicMock() + plugin._manual_analysis_executor.submit.side_effect = RuntimeError( + "credential-sentinel start failure" + ) + state = { + "pending_id": "opaque-key", + "job_id": "job-1", + "work": object(), + "discard_result": False, + } + + with patch.object( + PentesterApi01Plugin, + "_prepare_manual_analysis", + return_value=(state, None), + ): + result = PentesterApi01Plugin.analyze_job( + plugin, + job_id="job-1", + ) + + self.assertEqual(result["error"], "analysis_executor_failed") + self.assertIsNone(plugin._manual_analysis_state) + self.assertNotIn("credential-sentinel", str(result)) + + def test_manual_and_automatic_analysis_share_one_bounded_executor(self): + plugin = PentesterApi01Plugin.__new__(PentesterApi01Plugin) + executor = ThreadPoolExecutor(max_workers=1) + plugin._manual_analysis_executor = executor + plugin._manual_analysis_state = None + plugin.create_postponed_request = MagicMock(return_value="postponed") + automatic_started = threading.Event() + release_automatic = threading.Event() + + def _automatic_work(): + automatic_started.set() + release_automatic.wait(timeout=2) + return {"executive_headline": "automatic"} + + automatic_future = executor.submit(_automatic_work) + plugin._automatic_analysis_state = {"future": automatic_future} + self.assertTrue(automatic_started.wait(timeout=1)) + state = { + "pending_id": "opaque-key", + "job_id": "job-1", + "work": object(), + "discard_result": False, + } + + try: + with patch.object( + PentesterApi01Plugin, + "_prepare_manual_analysis", + return_value=(state, None), + ), patch( + "extensions.business.cybersec.red_mesh.pentester_api_01._run_manual_analysis_worker", + return_value=_ManualAnalysisOutcome(sections={}, failed=False), + ): + result = PentesterApi01Plugin.analyze_job(plugin, job_id="job-1") + manual_future = plugin._manual_analysis_state["future"] + self.assertEqual(result, "postponed") + self.assertFalse(manual_future.done()) + release_automatic.set() + automatic_future.result(timeout=1) + self.assertIsInstance(manual_future.result(timeout=1), _ManualAnalysisOutcome) + finally: + release_automatic.set() + executor.shutdown(wait=True, cancel_futures=True) + + def test_completion_releases_slot_for_retry(self): + plugin = MagicMock() + future = MagicMock() + future.done.return_value = True + future.result.return_value = _ManualAnalysisOutcome( + sections=_valid_remote_sections(), + failed=False, + ) + plugin._manual_analysis_state = { + "pending_id": "opaque-key", + "future": future, + "job_id": "job-1", + "deadline_monotonic": 200.0, + "next_check_monotonic": 0.0, + "discard_result": False, + } + + with patch( + "extensions.business.cybersec.red_mesh.pentester_api_01.time.monotonic", + return_value=100.0, + ), patch.object( + PentesterApi01Plugin, + "_finalize_manual_analysis", + return_value={"job_id": "job-1"}, + ): + result = PentesterApi01Plugin.solve_postponed_analyze_job( + plugin, + "opaque-key", + ) + + self.assertEqual(result, {"job_id": "job-1"}) + self.assertIsNone(plugin._manual_analysis_state) + + def test_stale_completion_does_not_write_artifacts(self): + plugin = MagicMock() + plugin._get_job_from_cstore.return_value = { + "job_id": "job-1", + "job_revision": 8, + "pass_reports": [{"pass_nr": 1, "report_cid": "QmCurrent"}], + } + state = { + "job_id": "job-1", + "job_revision": 7, + "pass_nr": 1, + "report_cid": "QmExpected", + "target": "example.test", + "num_workers": 1, + } + + result = PentesterApi01Plugin._finalize_manual_analysis( + plugin, + state, + _ManualAnalysisOutcome(sections=_valid_remote_sections(), failed=False), + ) + + self.assertEqual(result["error"], "analysis_state_changed") + self.assertEqual(result["status_code"], 409) + plugin.r1fs.get_json.assert_not_called() + plugin.r1fs.add_json.assert_not_called() + + def test_persistence_failure_is_typed_and_does_not_expose_artifact(self): + current_job = { + "job_id": "job-1", + "job_revision": 7, + "pass_reports": [{"pass_nr": 1, "report_cid": "QmExpected"}], + } + plugin = MagicMock() + plugin._get_job_from_cstore.return_value = current_job + plugin.r1fs.get_json.return_value = {"pass_nr": 1} + plugin.r1fs.add_json.return_value = None + state = { + "job_id": "job-1", + "job_revision": 7, + "pass_nr": 1, + "report_cid": "QmExpected", + "target": "example.test", + "num_workers": 1, + } + + result = PentesterApi01Plugin._finalize_manual_analysis( + plugin, + state, + _ManualAnalysisOutcome(sections=_valid_remote_sections(), failed=False), + ) + + self.assertEqual(result["error"], "analysis_persistence_failed") + self.assertEqual(result["status_code"], 503) + self.assertNotIn("QmExpected", str(result)) + self.assertNotIn("QmExpected", str(plugin.P.call_args_list)) + + def test_manual_finalization_rejects_detected_prewrite_race(self): + current_job = { + "job_id": "job-1", + "job_revision": 7, + "pass_reports": [{"pass_nr": 1, "report_cid": "QmExpected"}], + } + plugin = MagicMock() + plugin._get_job_from_cstore.return_value = current_job + plugin.r1fs.get_json.return_value = {"pass_nr": 1} + plugin.r1fs.add_json.return_value = "QmUnreferenced" + state = { + "job_id": "job-1", + "job_revision": 7, + "pass_nr": 1, + "report_cid": "QmExpected", + "target": "example.test", + "num_workers": 1, + } + + with patch.object( + PentesterApi01Plugin, + "_write_job_record", + return_value=None, + ) as write: + result = PentesterApi01Plugin._finalize_manual_analysis( + plugin, + state, + _ManualAnalysisOutcome(sections=_valid_remote_sections(), failed=False), + ) + + self.assertEqual(result["error"], "analysis_state_changed") + self.assertEqual(result["status_code"], 409) + self.assertNotIn("QmUnreferenced", str(result)) + self.assertTrue(write.call_args.kwargs["reject_stale"]) + + def test_guarded_job_write_refuses_detected_stale_revision(self): + plugin = MagicMock() + repository = MagicMock() + repository.get_job.return_value = { + "job_id": "job-1", + "job_revision": 8, + } + with patch.object( + PentesterApi01Plugin, + "_get_job_state_repository", + return_value=repository, + ): + result = PentesterApi01Plugin._write_job_record( + plugin, + "job-1", + {"job_id": "job-1", "job_revision": 7}, + expected_revision=7, + context="manual_llm_update", + reject_stale=True, + ) + + self.assertIsNone(result) + repository.put_job.assert_not_called() + + def test_guarded_job_write_does_not_resurrect_deleted_job(self): + plugin = MagicMock() + repository = MagicMock() + repository.get_job.return_value = None + with patch.object( + PentesterApi01Plugin, + "_get_job_state_repository", + return_value=repository, + ): + result = PentesterApi01Plugin._write_job_record( + plugin, + "job-1", + {"job_id": "job-1", "job_revision": 7}, + expected_revision=7, + context="manual_llm_update", + reject_stale=True, + ) + + self.assertIsNone(result) + repository.put_job.assert_not_called() + plugin._log_audit_event.assert_called_once() + + def test_admission_exception_is_sanitized(self): + plugin = PentesterApi01Plugin.__new__(PentesterApi01Plugin) + plugin._manual_analysis_state = None + with patch.object( + PentesterApi01Plugin, + "_prepare_manual_analysis", + side_effect=RuntimeError(f"provider exploded with {SECRET_SENTINEL}"), + ): + result = PentesterApi01Plugin.analyze_job( + plugin, + job_id="job-1", + ) + + self.assertEqual(result["error"], "analysis_executor_failed") + self.assertNotIn(SECRET_SENTINEL, str(result)) + + def test_shutdown_cancels_pending_work_before_base_close(self): + plugin = PentesterApi01Plugin.__new__(PentesterApi01Plugin) + future = MagicMock() + automatic_future = MagicMock() + executor = MagicMock() + plugin._manual_analysis_state = {"future": future} + plugin._automatic_analysis_state = {"future": automatic_future} + plugin._manual_analysis_executor = executor + + PentesterApi01Plugin.on_close(plugin) + + future.cancel.assert_called_once_with() + automatic_future.cancel.assert_called_once_with() + executor.shutdown.assert_called_once_with(wait=False, cancel_futures=True) + self.assertIsNone(plugin._manual_analysis_state) + self.assertIsNone(plugin._automatic_analysis_state) + self.assertTrue(plugin._base_closed) + + def test_worker_uses_prepared_input_without_plugin_or_secret_state(self): + prepared = build_llm_input( + findings=[{ + "severity": "HIGH", + "title": "Authorization bypass", + "description": "Structured description", + "raw_response": "credential-sentinel-private-body", + }], + aggregated_report={"open_ports": [443], "service_info": {"443": {}}}, + engagement={"client_name": "Example"}, + ) + work = _ManualAnalysisWork( + llm_input=prepared, + llm_config={ + "MODEL": "deepseek-chat", + "PROVIDER": "remote", + "PROMPT_PROFILE": "remote_rich_v1", + "LOCAL_PROMPT_PROFILE": "local_cybersecqwen_quota_v1", + "REMOTE_PROMPT_PROFILE": "remote_rich_v1", + "STRUCTURED_MAX_FINDINGS": 6, + "STRUCTURED_MAX_TOKENS": 1024, + "STRUCTURED_TEMPERATURE": 0, + }, + api_host="127.0.0.1", + api_port=8080, + deadline_monotonic=time.monotonic() + 5, + ) + response_payload = { + "result": { + "choices": [{ + "message": {"content": json.dumps(_valid_remote_sections())}, + }], + }, + } + + with patch( + "extensions.business.cybersec.red_mesh.pentester_api_01._bounded_provider_post", + return_value=json.dumps(response_payload).encode("utf-8"), + ) as post: + outcome = _ManualAnalysisWorker(work).run() + + self.assertFalse(outcome.failed) + self.assertIsNotNone(outcome.sections) + serialized_payload = json.dumps(post.call_args.args[1]) + self.assertNotIn("credential-sentinel-private-body", serialized_payload) + self.assertNotIn(SECRET_SENTINEL, serialized_payload) + self.assertFalse(any("plugin" in name for name in work.__dataclass_fields__)) + provider_timeout = post.call_args.args[2] + self.assertGreater(provider_timeout, 0) + self.assertLessEqual(provider_timeout, 30) + self.assertEqual(post.call_args.args[3], 2 * 1024 * 1024) + + def test_provider_body_and_exception_details_are_not_returned(self): + prepared = build_llm_input(findings=[], aggregated_report={}, engagement={}) + work = _ManualAnalysisWork( + llm_input=prepared, + llm_config={ + "MODEL": "deepseek-chat", + "PROVIDER": "remote", + "PROMPT_PROFILE": "remote_rich_v1", + "LOCAL_PROMPT_PROFILE": "local_cybersecqwen_quota_v1", + "REMOTE_PROMPT_PROFILE": "remote_rich_v1", + "STRUCTURED_MAX_FINDINGS": 6, + "STRUCTURED_MAX_TOKENS": 1024, + "STRUCTURED_TEMPERATURE": 0, + }, + api_host="private.provider.internal", + api_port=8080, + deadline_monotonic=time.monotonic() + 5, + ) + with patch( + "extensions.business.cybersec.red_mesh.pentester_api_01._bounded_provider_post", + side_effect=RuntimeError( + "credential-sentinel provider body https://private.provider.internal" + ), + ): + outcome = _ManualAnalysisWorker(work).run() + + public = str(outcome) + self.assertTrue(outcome.failed) + self.assertNotIn("credential-sentinel", public) + self.assertNotIn("private.provider.internal", public) + self.assertNotIn("provider body", public) + + def test_invalid_structured_result_returns_sanitized_failed_sections(self): + prepared = build_llm_input(findings=[], aggregated_report={}, engagement={}) + work = _ManualAnalysisWork( + llm_input=prepared, + llm_config={ + "MODEL": "deepseek-chat", + "PROVIDER": "remote", + "PROMPT_PROFILE": "remote_rich_v1", + "LOCAL_PROMPT_PROFILE": "local_cybersecqwen_quota_v1", + "REMOTE_PROMPT_PROFILE": "remote_rich_v1", + "STRUCTURED_MAX_FINDINGS": 6, + "STRUCTURED_MAX_TOKENS": 1024, + "STRUCTURED_TEMPERATURE": 0, + }, + api_host="private.provider.internal", + api_port=8080, + deadline_monotonic=time.monotonic() + 5, + ) + response_payload = { + "result": { + "choices": [{ + "message": {"content": "not-json credential-sentinel provider body"}, + }], + }, + } + + with patch( + "extensions.business.cybersec.red_mesh.pentester_api_01._bounded_provider_post", + return_value=json.dumps(response_payload).encode("utf-8"), + ): + outcome = _ManualAnalysisWorker(work).run() + + self.assertTrue(outcome.failed) + self.assertTrue(outcome.sections["error"]) + self.assertNotIn("credential-sentinel", str(outcome)) + self.assertNotIn("private.provider.internal", str(outcome)) + + def test_oversized_provider_response_failure_is_sanitized(self): + prepared = build_llm_input(findings=[], aggregated_report={}, engagement={}) + work = _ManualAnalysisWork( + llm_input=prepared, + llm_config={ + "MODEL": "deepseek-chat", + "PROVIDER": "remote", + "PROMPT_PROFILE": "remote_rich_v1", + "REMOTE_PROMPT_PROFILE": "remote_rich_v1", + "STRUCTURED_MAX_FINDINGS": 6, + "STRUCTURED_MAX_TOKENS": 1024, + }, + api_host="private.provider.internal", + api_port=8080, + deadline_monotonic=time.monotonic() + 5, + ) + with patch( + "extensions.business.cybersec.red_mesh.pentester_api_01._bounded_provider_post", + side_effect=RuntimeError( + "private.provider.internal returned an oversized credential-sentinel body" + ), + ): + outcome = _ManualAnalysisWorker(work).run() + + self.assertTrue(outcome.failed) + self.assertNotIn("private.provider.internal", str(outcome)) + self.assertNotIn("credential-sentinel", str(outcome)) + + @unittest.skipUnless( + importlib.util.find_spec("aiohttp") is not None, + "aiohttp is unavailable in the host test environment", + ) + def test_provider_transport_enforces_total_wall_timeout_while_body_trickles(self): + class _TrickleHandler(BaseHTTPRequestHandler): + def log_message(self, *_args): + return + + def do_POST(self): + body_length = int(self.headers.get("Content-Length", "0")) + self.rfile.read(body_length) + self.send_response(200) + self.send_header("Content-Length", "100") + self.end_headers() + try: + for _ in range(100): + self.wfile.write(b"x") + self.wfile.flush() + time.sleep(0.05) + except (BrokenPipeError, ConnectionResetError): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), _TrickleHandler) + server.daemon_threads = True + server_thread = threading.Thread(target=server.serve_forever, daemon=True) + server_thread.start() + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + + started = time.monotonic() + with self.assertRaises(TimeoutError): + _bounded_provider_post( + f"http://127.0.0.1:{server.server_port}/chat", + {"messages": []}, + 0.2, + 1024, + ) + elapsed = time.monotonic() - started + + # Keep a wide scheduler margin while still proving the 0.2-second client + # deadline ends well before the server's five-second trickle completes. + self.assertLess(elapsed, 2.0) + + @unittest.skipUnless( + importlib.util.find_spec("aiohttp") is not None, + "aiohttp is unavailable in the host test environment", + ) + def test_provider_transport_rejects_declared_and_streamed_oversize_bodies(self): + class _OversizeHandler(BaseHTTPRequestHandler): + def log_message(self, *_args): + return + + def do_POST(self): + body_length = int(self.headers.get("Content-Length", "0")) + self.rfile.read(body_length) + self.send_response(200) + if self.path == "/declared": + self.send_header("Content-Length", "2048") + else: + self.send_header("Connection", "close") + self.close_connection = True + self.end_headers() + if self.path != "/declared": + self.wfile.write(b"x" * 2048) + self.wfile.flush() + + server = ThreadingHTTPServer(("127.0.0.1", 0), _OversizeHandler) + server.daemon_threads = True + server_thread = threading.Thread(target=server.serve_forever, daemon=True) + server_thread.start() + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + + for path in ("declared", "streamed"): + with self.subTest(path=path), self.assertRaisesRegex( + RuntimeError, + "response is too large", + ): + _bounded_provider_post( + f"http://127.0.0.1:{server.server_port}/{path}", + {"messages": []}, + 2.0, + 1024, + ) + + def test_prepared_input_rejects_wrong_type(self): + from extensions.business.cybersec.red_mesh.services.llm_structured import generate_exec_summary + + with self.assertRaises(TypeError): + generate_exec_summary( + llm_call=lambda *_args: "", + prepared_input={"not": "trusted"}, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/extensions/business/cybersec/red_mesh/tests/test_postponed_analyze_native_ipc.py b/extensions/business/cybersec/red_mesh/tests/test_postponed_analyze_native_ipc.py new file mode 100644 index 000000000..409d7a73a --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_postponed_analyze_native_ipc.py @@ -0,0 +1,526 @@ +import asyncio +import importlib.util +import inspect +import json +import os +import queue +import shutil +import socket +import subprocess +import sys +import tempfile +import threading +import time +import types +import unittest +import urllib.error +import urllib.request +from collections import deque +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from unittest.mock import patch + +REPO_ROOT = Path(__file__).resolve().parents[5] +FRAMEWORK_PACKAGE = REPO_ROOT / "naeural_core" / "naeural_core" +IPC_MANAGER_PATH = FRAMEWORK_PACKAGE / "utils" / "uvicorn_fast_api_ipc_manager.py" +FASTAPI_PLUGIN_PATH = ( + FRAMEWORK_PACKAGE / "business" / "default" / "web_app" / "fast_api_web_app.py" +) +FASTAPI_UTILS_PATH = FRAMEWORK_PACKAGE / "utils" / "fastapi_utils.py" +NATIVE_RUNTIME_SOURCE_AVAILABLE = all( + path.is_file() + for path in ( + IPC_MANAGER_PATH, + FASTAPI_PLUGIN_PATH, + FASTAPI_UTILS_PATH, + ) +) + + +def _load_module(name, path): + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Could not load {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _load_native_runtime(): + """Load the real scheduler with only its unrelated base dependencies stubbed.""" + module_names = ( + "naeural_core", + "naeural_core.business", + "naeural_core.business.base", + "naeural_core.business.base.web_app", + "naeural_core.business.base.web_app.base_web_app_plugin", + "naeural_core.utils", + "naeural_core.utils.fastapi_utils", + "naeural_core.utils.uvicorn_fast_api_ipc_manager", + ) + previous = {name: sys.modules.get(name) for name in module_names} + try: + for name in module_names: + if name not in { + "naeural_core.business.base.web_app.base_web_app_plugin", + "naeural_core.utils.fastapi_utils", + "naeural_core.utils.uvicorn_fast_api_ipc_manager", + }: + sys.modules[name] = types.ModuleType(name) + + class _BaseWebAppPlugin: + CONFIG = {"VALIDATION_RULES": {}} + + def _process(self): + return None + + def on_close(self): + return None + + base_module = types.ModuleType( + "naeural_core.business.base.web_app.base_web_app_plugin" + ) + base_module.BaseWebAppPlugin = _BaseWebAppPlugin + sys.modules[base_module.__name__] = base_module + + fastapi_utils = _load_module("_rm040_fastapi_utils", FASTAPI_UTILS_PATH) + sys.modules["naeural_core.utils.fastapi_utils"] = fastapi_utils + ipc_stub = types.ModuleType("naeural_core.utils.uvicorn_fast_api_ipc_manager") + ipc_stub.get_server_manager = lambda _auth: None + sys.modules[ipc_stub.__name__] = ipc_stub + runtime = _load_module("_rm040_fastapi_runtime", FASTAPI_PLUGIN_PATH) + return runtime.FastApiWebAppPlugin, fastapi_utils.PostponedRequest + finally: + for name, module in previous.items(): + if module is None: + sys.modules.pop(name, None) + else: + sys.modules[name] = module + + +if NATIVE_RUNTIME_SOURCE_AVAILABLE: + NativeFastApiPlugin, NativePostponedRequest = _load_native_runtime() + ipc_manager = _load_module("_rm040_ipc_manager", IPC_MANAGER_PATH) +else: + # Edge source checkouts do not vendor the Ratio1 runtime. The integration + # fixture is exercised when a sibling runtime source checkout is available. + NativeFastApiPlugin, NativePostponedRequest = object, None + ipc_manager = None + +from .conftest import mock_plugin_modules + + +mock_plugin_modules() + +from extensions.business.cybersec.red_mesh.pentester_api_01 import ( + PentesterApi01Plugin, + _ManualAnalysisOutcome, +) + + +class _SchedulerHarness(NativeFastApiPlugin): + def P(self, *args, **_kwargs): + if hasattr(self, "_test_messages") and args: + self._test_messages.append(str(args[0])) + return None + + def on_response(self, _method, _response): + return None + + def get_additional_fastapi_data(self): + return {} + + def get_process_budget_s(self): + return None + + def get_max_incoming_per_loop(self): + return 10 + + def get_max_postponed_per_loop(self): + return 10 + + def _maybe_log_profile_stats(self): + return None + + +class _Owner: + def __init__(self, worker): + self._manual_analysis_state = None + self._manual_analysis_executor = ThreadPoolExecutor( + max_workers=1, + thread_name_prefix="rm040-native-test", + ) + self._worker = worker + self.solve_postponed_analyze_job = ( + lambda pending_id: PentesterApi01Plugin.solve_postponed_analyze_job( + self, + pending_id, + ) + ) + + def time(self): + return time.monotonic() + + def create_postponed_request(self, solver_method, method_kwargs=None): + return NativePostponedRequest(solver_method, dict(method_kwargs or {})) + + +@unittest.skipUnless( + NATIVE_RUNTIME_SOURCE_AVAILABLE, + "native Ratio1 runtime source fixture is unavailable", +) +class TestPostponedAnalyzeNativeIpc(unittest.TestCase): + @staticmethod + def _find_free_port(): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + @staticmethod + def _descriptor(name, method, parameters, *, require_token=False): + return { + "name": name, + "method": method, + "args": [str(parameter) for parameter in parameters], + "params": [parameter.name for parameter in parameters], + "endpoint_doc": "", + "require_token": require_token, + "has_kwargs": False, + "streaming_type": None, + "chunk_size": 1024 * 1024, + } + + def _render_server(self, destination, manager_port, manager_auth): + from jinja2 import Environment, FileSystemLoader + + analyze_parameters = list( + inspect.signature(PentesterApi01Plugin.analyze_job).parameters.values() + )[1:] + status_parameters = [ + inspect.Parameter("job_id", inspect.Parameter.POSITIONAL_OR_KEYWORD), + ] + endpoints = [ + self._descriptor( + "analyze_job", + "post", + analyze_parameters, + ), + self._descriptor("get_job_status", "get", status_parameters), + ] + template_dir = FRAMEWORK_PACKAGE / "business" / "base" / "uvicorn_templates" + rendered = Environment(loader=FileSystemLoader(str(template_dir))).get_template( + "basic_server.j2" + ).render( + additional_fastapi_data={}, + manager_port=manager_port, + manager_auth=repr(manager_auth), + request_timeout=120, + api_title=repr("RM-041 native postponed test"), + api_summary=repr("RM-041"), + api_description=repr("RM-041"), + api_version=repr("0.0.0-test"), + static_directory="assets", + debug_web_app=False, + debug_timings=False, + debug_timings_steps=False, + default_route=None, + profile_rate=0, + profile_log_per_request=False, + node_comm_params=endpoints, + html_files=[], + ) + destination.mkdir(parents=True, exist_ok=True) + (destination / "assets").mkdir() + (destination / "main.py").write_text(rendered, encoding="utf-8") + temp_utils = destination / "naeural_core" / "utils" + temp_utils.mkdir(parents=True) + (temp_utils.parent / "__init__.py").write_text("", encoding="utf-8") + (temp_utils / "__init__.py").write_text("", encoding="utf-8") + shutil.copy2(IPC_MANAGER_PATH, temp_utils / IPC_MANAGER_PATH.name) + + @staticmethod + def _request(port, method, path, *, token="", payload=None, timeout=5): + headers = {} + if token: + headers["Authorization"] = f"Bearer {token}" + body = None + if payload is not None: + headers["Content-Type"] = "application/json" + body = json.dumps(payload).encode("utf-8") + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=body, + headers=headers, + method=method, + ) + started = time.monotonic() + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + return ( + response.status, + json.loads(response.read().decode("utf-8")), + time.monotonic() - started, + ) + except urllib.error.HTTPError as exc: + return ( + exc.code, + json.loads(exc.read().decode("utf-8")), + time.monotonic() - started, + ) + + def test_ipc_timeout_drops_waiter_without_canceling_plugin_work(self): + manager_auth = b"rm040-native-timeout" + manager = ipc_manager.get_server_manager(manager_auth) + self.addCleanup(manager.shutdown) + _, manager_port = manager.address + server_queue = manager.get_server_queue() + client_queue = manager.get_client_queue() + comms = ipc_manager.UvicornPluginComms( + port=manager_port, + auth=manager_auth, + timeout_s=0.05, + additional_fastapi_data={}, + ) + + loop = asyncio.new_event_loop() + try: + result = loop.run_until_complete( + comms.call_plugin("analyze_job", "job-1") + ) + + self.assertEqual(result["status_code"], 504) + self.assertEqual(comms._commands, {}) + request = server_queue.get(timeout=1) + self.assertEqual(request["value"][:2], ("analyze_job", "job-1")) + with self.assertRaises(queue.Empty): + server_queue.get(timeout=0.05) + + client_queue.put({ + "id": request["id"], + "value": {"result": {"job_id": "job-1"}}, + }) + loop.run_until_complete(asyncio.sleep(0.1)) + self.assertEqual(comms._commands, {}) + finally: + comms._stop_reader.set() + if comms._reader_thread is not None: + comms._reader_thread.join(timeout=1) + loop.close() + + def test_real_postponed_scheduler_keeps_status_responsive(self): + manager_auth = b"rm040-native-ipc" + manager = ipc_manager.get_server_manager(manager_auth) + self.addCleanup(manager.shutdown) + _, manager_port = manager.address + server_queue = manager.get_server_queue() + client_queue = manager.get_client_queue() + + worker_entered = threading.Event() + release_worker = threading.Event() + + def _blocking_worker(_work): + worker_entered.set() + if not release_worker.wait(timeout=10): + raise TimeoutError("native test worker was not released") + return _ManualAnalysisOutcome( + sections={"executive_headline": "done"}, + failed=False, + ) + + owner = _Owner(_blocking_worker) + self.addCleanup(owner._manual_analysis_executor.shutdown, wait=False) + harness = _SchedulerHarness.__new__(_SchedulerHarness) + harness._endpoints = { + "analyze_job": lambda job_id, analysis_type="", focus_areas=None: ( + PentesterApi01Plugin.analyze_job( + owner, + job_id, + analysis_type, + focus_areas, + ) + ), + "get_job_status": lambda job_id: {"status": "ok", "job_id": job_id}, + } + harness._incoming_requests = deque() + harness.postponed_requests = deque() + harness._incoming_lock = threading.Lock() + harness._client_queue = client_queue + harness._stop_request_monitor = threading.Event() + harness._stop_request_monitor.set() + harness._request_monitor_thread = None + harness._profile_stats = {} + harness.cfg_log_requests = False + harness.cfg_response_format = "WRAPPED" + harness.cfg_fair_scheduling = True + harness._test_messages = [] + + stop_dispatcher = threading.Event() + + def _dispatch(): + while not stop_dispatcher.is_set(): + try: + request = server_queue.get_nowait() + except queue.Empty: + request = None + if request is not None: + with harness._incoming_lock: + harness._incoming_requests.append(request) + harness._process() + time.sleep(0.005) + + dispatcher = threading.Thread(target=_dispatch, daemon=True) + dispatcher.start() + + def _stop_dispatcher(): + stop_dispatcher.set() + release_worker.set() + dispatcher.join(timeout=2) + + self.addCleanup(_stop_dispatcher) + + state = { + "pending_id": "unused", + "job_id": "job-1", + "job_revision": 1, + "pass_nr": 1, + "report_cid": "QmExpected", + "target": "example.test", + "num_workers": 1, + "deadline_monotonic": time.monotonic() + 4, + "next_check_monotonic": 0.0, + "discard_result": False, + "work": object(), + } + final_response = { + "job_id": "job-1", + "target": "example.test", + "num_workers": 1, + "pass_nr": 1, + "analysis_type": "structured_report_sections", + "llm_failed": False, + "llm_report_sections": {"executive_headline": "done"}, + } + + with tempfile.TemporaryDirectory(prefix="rm040-native-") as temp_dir: + app_dir = Path(temp_dir) + self._render_server(app_dir, manager_port, manager_auth) + port = self._find_free_port() + process = subprocess.Popen( + [ + sys.executable, + "-m", + "uvicorn", + "--app-dir", + str(app_dir), + "main:app", + "--host", + "127.0.0.1", + "--port", + str(port), + ], + cwd=app_dir, + env={**os.environ, "PYTHONPATH": str(app_dir)}, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + try: + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + try: + with socket.create_connection(("127.0.0.1", port), timeout=0.2): + break + except OSError: + time.sleep(0.05) + else: + stderr = (process.stderr.read() or b"").decode("utf-8", errors="replace") + self.fail(f"Generated Uvicorn server did not start:\n{stderr}") + + analysis_result = [] + + def _request_analysis(): + analysis_result.append(self._request( + port, + "POST", + "/analyze_job", + payload={"job_id": "job-1"}, + )) + + def _prepare(_plugin, job_id): + if job_id == "explode": + raise RuntimeError("native admission failure") + return dict(state), None + + with patch.object( + PentesterApi01Plugin, + "_prepare_manual_analysis", + side_effect=_prepare, + ), patch.object( + PentesterApi01Plugin, + "_finalize_manual_analysis", + return_value=final_response, + ), patch( + "extensions.business.cybersec.red_mesh.pentester_api_01._run_manual_analysis_worker", + side_effect=_blocking_worker, + ): + failed_status, failed_body, failed_elapsed = self._request( + port, + "POST", + "/analyze_job", + payload={"job_id": "explode"}, + ) + self.assertEqual(failed_status, 503, failed_body) + self.assertLess(failed_elapsed, 1.0) + self.assertEqual( + failed_body["detail"]["error"], + "analysis_executor_failed", + ) + analysis_thread = threading.Thread(target=_request_analysis, daemon=True) + analysis_thread.start() + self.assertTrue(worker_entered.wait(timeout=2)) + + status, body, elapsed = self._request( + port, + "GET", + "/get_job_status?job_id=job-1", + ) + self.assertEqual(status, 200, body) + self.assertLess(elapsed, 1.0) + self.assertEqual(body["result"]["status"], "ok") + + busy_status, busy_body, busy_elapsed = self._request( + port, + "POST", + "/analyze_job", + payload={"job_id": "job-1"}, + ) + self.assertEqual(busy_status, 409, busy_body) + self.assertLess(busy_elapsed, 1.0) + self.assertEqual(busy_body["detail"]["error"], "analysis_busy") + + release_worker.set() + analysis_thread.join(timeout=3) + self.assertFalse(analysis_thread.is_alive()) + + self.assertEqual(analysis_result[0][0], 200, analysis_result) + result_body = analysis_result[0][1] + self.assertEqual( + result_body["result"]["analysis_type"], + "structured_report_sections", + ) + self.assertNotIn("operation_id", json.dumps(result_body)) + finally: + release_worker.set() + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + if process.stdout: + process.stdout.close() + if process.stderr: + process.stderr.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/extensions/business/cybersec/red_mesh/tests/test_state_machine.py b/extensions/business/cybersec/red_mesh/tests/test_state_machine.py index 3c392b59d..374b83f43 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_state_machine.py +++ b/extensions/business/cybersec/red_mesh/tests/test_state_machine.py @@ -47,6 +47,13 @@ def test_allows_analyzing_retry_to_collecting(self): self.assertEqual(job_specs["job_status"], JOB_STATUS_COLLECTING) + def test_allows_soft_stop_to_be_scheduled_during_analysis(self): + job_specs = {"job_status": JOB_STATUS_ANALYZING} + + set_job_status(job_specs, JOB_STATUS_SCHEDULED_FOR_STOP) + + self.assertEqual(job_specs["job_status"], JOB_STATUS_SCHEDULED_FOR_STOP) + def test_allows_finalizing_retry_to_collecting(self): job_specs = {"job_status": JOB_STATUS_FINALIZING} diff --git a/requirements.txt b/requirements.txt index 2150c324a..18686e4e5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,7 @@ python-docx pdfplumber docker aiofiles +aiohttp paramiko pymisp # This has been moved to device.py additional_packages list for better compatibility with different devices. From b55a1c345fb90e03003d644b2580c10e05158aec Mon Sep 17 00:00:00 2001 From: Vitalii <87299468+toderian@users.noreply.github.com> Date: Wed, 22 Jul 2026 16:00:35 +0300 Subject: [PATCH 4/6] Fix/deeploy service update identity (#474) * fix: require identity for service updates What changed: - reject unresolved service plugin IDs after legacy backfill and job-type reconciliation - add four-replica CockroachDB identity and storage regressions - prove rejection occurs before payment, response-key reset, delete, or deploy Why: - prevent no-ID service edits from redeploying as a new logical container Checks: - focused Deeploy update/resource suite: 62 tests pass - full Deeploy suite: 206 tests pass - py_compile and git diff check: pass * chore: inc ver * fix: require explicit Deeploy update identity What changed: - require and normalize job_app_type on update requests - require canonical service instance IDs before backfill - add legacy CockroachDB and compatibility regressions Why: - prevent legacy service updates from being inferred as generic and redeployed under a new plugin identity Checks: - focused Deeploy update/resource suite: 65 passed - full Deeploy suite: 209 passed - py_compile and git diff --check: passed * docs: document explicit update contract Document mandatory update job_app_type and request-authoritative service identity on the public endpoint. Checks: - focused Deeploy update/resource suite: 65 passed - py_compile and git diff --check: passed * fix: preserve legacy service update identity What changed: - copy explicit top-level legacy update identity into the normalized plugin - reject conflicting legacy identity fields before discovery - document and test update-only compatibility without changing creates Why: - keep documented legacy service updates compatible with the fail-closed identity contract Checks: - focused update/create/stack suite: 127 passed - full Deeploy suite: 217 passed - py_compile and git diff --check: passed --- .../business/deeploy/deeploy_manager_api.py | 147 +++-- extensions/business/deeploy/deeploy_mixin.py | 20 +- .../deeploy/tests/test_update_requests.py | 615 +++++++++++++++++- ver.py | 2 +- 4 files changed, 719 insertions(+), 65 deletions(-) diff --git a/extensions/business/deeploy/deeploy_manager_api.py b/extensions/business/deeploy/deeploy_manager_api.py index 36cd18351..5dc1eef2b 100644 --- a/extensions/business/deeploy/deeploy_manager_api.py +++ b/extensions/business/deeploy/deeploy_manager_api.py @@ -21,7 +21,6 @@ DEEPLOY_APP_COMMAND_REQUEST, DEEPLOY_GET_ORACLE_JOB_DETAILS_REQUEST, DEEPLOY_GET_R1FS_JOB_PIPELINE_REQUEST, DEEPLOY_NODE_SPECS_REQUEST, DEEPLOY_PLUGIN_DATA, JOB_APP_TYPES, JOB_APP_TYPES_ALL, DEEPLOY_GET_PREFERRED_NODES_REQUEST, DEEPLOY_SAVE_PREFERRED_NODES_REQUEST, - CONTAINERIZED_APPS_SIGNATURES, ) @@ -656,6 +655,33 @@ def save_preferred_nodes( return response + def _validate_service_update_plugin_instance_ids(self, inputs, job_app_type): + """ + Require stable plugin identity for every managed-service update entry. + + This runs before legacy ID backfill and after request/persisted job-type + reconciliation. Native updates remain free to submit new no-ID plugins. + """ + if job_app_type != JOB_APP_TYPES.SERVICE: + return True + + plugins_array = inputs.get(DEEPLOY_KEYS.PLUGINS) + missing_indexes = [] + for index, plugin_entry in enumerate(plugins_array or []): + if not isinstance(plugin_entry, dict): + continue + instance_id = plugin_entry.get(DEEPLOY_KEYS.PLUGIN_INSTANCE_ID) + if not instance_id or not str(instance_id).strip(): + missing_indexes.append(index) + + if missing_indexes: + raise ValueError( + f"{DEEPLOY_ERRORS.PLUGINS3}: Service update plugins must include instance_id " + f"after identity resolution. Missing for plugin indexes {missing_indexes}." + ) + return True + + def _process_pipeline_request( self, request: dict, @@ -668,7 +694,8 @@ def _process_pipeline_request( Parameters ---------- request : dict - The request dictionary + The request dictionary. Updates must include a valid `job_app_type`; + service updates must also include `instance_id` for every plugin. is_create : bool True for create operations, False for update operations async_mode : bool @@ -683,8 +710,25 @@ def _process_pipeline_request( self.__ensure_eth_balance() request_type = "create pipeline" if is_create else "update pipeline" sender, inputs = self.deeploy_verify_and_get_inputs(request, request_type=request_type) - normalized_request = self._normalize_plugins_input(self.deepcopy(request)) + normalized_request = self._normalize_plugins_input( + self.deepcopy(request), + preserve_legacy_instance_id=not is_create, + ) self._sync_normalized_plugins_input(inputs, normalized_request) + submitted_job_app_type = inputs.get(DEEPLOY_KEYS.JOB_APP_TYPE, None) + if not is_create: + if not isinstance(submitted_job_app_type, str) or not submitted_job_app_type.strip(): + raise ValueError( + f"{DEEPLOY_ERRORS.REQUEST3}. job_app_type is required for update requests." + ) + job_app_type = submitted_job_app_type.strip().lower() + if job_app_type not in JOB_APP_TYPES_ALL: + raise ValueError( + f"{DEEPLOY_ERRORS.REQUEST3}. Invalid job_app_type '{submitted_job_app_type}'. " + f"Expected one of {JOB_APP_TYPES_ALL}." + ) + else: + job_app_type = submitted_job_app_type auth_result = self.deeploy_get_auth_result(inputs) job_id = inputs.get(DEEPLOY_KEYS.JOB_ID, None) is_confirmable_job = inputs.chainstore_response @@ -702,18 +746,17 @@ def _process_pipeline_request( app_alias = inputs.app_alias app_type = inputs.pipeline_input_type - job_app_type = inputs.get(DEEPLOY_KEYS.JOB_APP_TYPE, None) - has_request_job_app_type = bool(job_app_type) - if job_app_type: - job_app_type = str(job_app_type).lower() - if job_app_type not in JOB_APP_TYPES_ALL: - raise ValueError(f"Invalid job_app_type '{job_app_type}'. Expected one of {JOB_APP_TYPES_ALL}.") - else: - plugins_for_detection = self.deeploy_prepare_plugins(inputs) - job_app_type = self.deeploy_detect_job_app_type(plugins_for_detection) - if job_app_type not in JOB_APP_TYPES_ALL: - job_app_type = JOB_APP_TYPES.NATIVE - self.P(f"Detected job app type: {job_app_type}") + if is_create: + if job_app_type: + job_app_type = str(job_app_type).lower() + if job_app_type not in JOB_APP_TYPES_ALL: + raise ValueError(f"Invalid job_app_type '{job_app_type}'. Expected one of {JOB_APP_TYPES_ALL}.") + else: + plugins_for_detection = self.deeploy_prepare_plugins(inputs) + job_app_type = self.deeploy_detect_job_app_type(plugins_for_detection) + if job_app_type not in JOB_APP_TYPES_ALL: + job_app_type = JOB_APP_TYPES.NATIVE + self.P(f"Resolved job app type: {job_app_type}") # persist job type so downstream mixins can adjust validations (e.g. native app resource checks) inputs[DEEPLOY_KEYS.JOB_APP_TYPE] = job_app_type inputs.job_app_type = job_app_type @@ -832,24 +875,6 @@ def _process_pipeline_request( except Exception as exc: self.Pd(f"Unable to read previous pipeline CID for job {job_id}: {exc}", color='y') - # Ensure plugin IDs are preserved for existing instances before any destructive action. - self._ensure_plugin_instance_ids( - inputs, - discovered_plugin_instances=discovered_plugin_instances, - owner=auth_result[DEEPLOY_KEYS.ESCROW_OWNER], - app_id=app_id, - job_id=job_id, - ) - self._validate_update_plugin_identities( - inputs, - discovered_plugin_instances=discovered_plugin_instances, - ) - self._warn_on_live_plugin_config_drift( - discovered_plugin_instances, - job_id=job_id, - app_id=app_id, - ) - if deeploy_specs_for_update is not None and not isinstance(deeploy_specs_for_update, dict): msg = ( f"{DEEPLOY_ERRORS.REQUEST3}. Unexpected 'deeploy_specs' payload type " @@ -870,7 +895,6 @@ def _process_pipeline_request( existing_job_app_type = existing_job_app_type.lower() if ( existing_job_app_type in JOB_APP_TYPES_ALL - and has_request_job_app_type and job_app_type != existing_job_app_type ): msg = ( @@ -885,33 +909,26 @@ def _process_pipeline_request( # Discovery is used only for identity safety and drift diagnostics. self._validate_plugins_array(plugins_array) - if not has_request_job_app_type: - replacement_job_app_type = deeploy_specs_payload.get(DEEPLOY_KEYS.JOB_APP_TYPE) - if isinstance(replacement_job_app_type, str): - replacement_job_app_type = replacement_job_app_type.lower() - if replacement_job_app_type in JOB_APP_TYPES_ALL: - job_app_type = replacement_job_app_type - else: - plugins_for_detection = self.deeploy_prepare_plugins(inputs) - job_app_type = self.deeploy_detect_job_app_type(plugins_for_detection) - has_containerized_replacement = any( - isinstance(plugin_entry, dict) and - isinstance(plugin_entry.get(DEEPLOY_KEYS.PLUGIN_SIGNATURE), str) and - plugin_entry.get(DEEPLOY_KEYS.PLUGIN_SIGNATURE).upper() in CONTAINERIZED_APPS_SIGNATURES - for plugin_entry in plugins_array - ) - if job_app_type == JOB_APP_TYPES.NATIVE and has_containerized_replacement: - msg = ( - f"{DEEPLOY_ERRORS.REQUEST3}. Update request omitted job_app_type and the live " - "deeploy_specs do not identify the replacement job_app_type for a containerized " - "replacement payload. Provide job_app_type explicitly." - ) - raise ValueError(msg) - if job_app_type not in JOB_APP_TYPES_ALL: - job_app_type = JOB_APP_TYPES.NATIVE - inputs[DEEPLOY_KEYS.JOB_APP_TYPE] = job_app_type - inputs.job_app_type = job_app_type - self.P(f"Detected replacement job app type: {job_app_type}") + self._validate_service_update_plugin_instance_ids(inputs, job_app_type) + if job_app_type != JOB_APP_TYPES.SERVICE: + # Legacy non-service updates may still identify an existing plugin by + # exact identity matching. Service identity is request-authoritative. + self._ensure_plugin_instance_ids( + inputs, + discovered_plugin_instances=discovered_plugin_instances, + owner=auth_result[DEEPLOY_KEYS.ESCROW_OWNER], + app_id=app_id, + job_id=job_id, + ) + self._validate_update_plugin_identities( + inputs, + discovered_plugin_instances=discovered_plugin_instances, + ) + self._warn_on_live_plugin_config_drift( + discovered_plugin_instances, + job_id=job_id, + app_id=app_id, + ) is_valid = self.deeploy_check_payment_and_job_owner(inputs, auth_result[DEEPLOY_KEYS.ESCROW_OWNER], is_create=is_create, debug=self.cfg_deeploy_verbose > 1) if not is_valid: @@ -1525,6 +1542,10 @@ def update_pipeline( job_id : int The job ID from blockchain + job_app_type : str + Required for every update. Must be one of generic, native, service, or stack. + The submitted value is authoritative and is not inferred from plugin configuration. + pipeline_params : dict, optional Additional pipeline-level parameters forwarded to the data capture thread. `null` falls back to `{}`. The provided keys are merged into the pipeline configuration at the top level. @@ -1543,6 +1564,7 @@ def update_pipeline( Complete desired replacement set. Each object represents ONE plugin instance: - plugin_signature : str (required) - instance_id : str (required when updating an existing plugin instance) + - Required on every submitted service plugin and never inferred by the backend - **instance-specific parameters** (payload merged into the instance configuration) - Omit instance_id to attach a brand new plugin instance; supported for native apps only - Omit a live plugin from this array to remove it from the replacement deployment @@ -1551,6 +1573,9 @@ def update_pipeline( plugin_signature : str The signature of the single plugin. Legacy payloads without the plugins array are normalized internally; any deprecated `app_params` field is ignored in responses. + instance_id : str + Required at the top level for legacy service updates. The value is copied into the + normalized plugin and validated against the running instance. Returns ------- diff --git a/extensions/business/deeploy/deeploy_mixin.py b/extensions/business/deeploy/deeploy_mixin.py index bbc4580f0..b66728a0a 100644 --- a/extensions/business/deeploy/deeploy_mixin.py +++ b/extensions/business/deeploy/deeploy_mixin.py @@ -1894,13 +1894,20 @@ def _validate_send_instance_command_request(self, inputs): return - def _normalize_plugins_input(self, request: dict): + def _normalize_plugins_input( + self, + request: dict, + *, + preserve_legacy_instance_id: bool = False, + ): """ Normalize plugin input to always use the plugins array format. Converts legacy single-plugin format (plugin_signature + app_params) to new multi-plugin format. Args: request (dict): The request dictionary + preserve_legacy_instance_id (bool): Copy a canonical top-level identity into a + synthesized legacy plugin. Update processing enables this; create processing does not. Returns: dict: Request with normalized plugins array (simple format: each object is a plugin instance) @@ -1948,6 +1955,17 @@ def _normalize_plugins_input(self, request: dict): DEEPLOY_KEYS.PLUGIN_SIGNATURE: plugin_signature, **app_params } + if preserve_legacy_instance_id and DEEPLOY_KEYS.PLUGIN_INSTANCE_ID in request: + top_level_instance_id = request.get(DEEPLOY_KEYS.PLUGIN_INSTANCE_ID) + if ( + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID in app_params + and app_params.get(DEEPLOY_KEYS.PLUGIN_INSTANCE_ID) != top_level_instance_id + ): + raise ValueError( + f"{DEEPLOY_ERRORS.REQUEST3}. Conflicting legacy instance_id values were provided " + "at the top level and in app_params." + ) + plugin_instance[DEEPLOY_KEYS.PLUGIN_INSTANCE_ID] = top_level_instance_id request_key, request_config = self._get_request_per_node_config(request) if request_key is not None: existing_keys = [ diff --git a/extensions/business/deeploy/tests/test_update_requests.py b/extensions/business/deeploy/tests/test_update_requests.py index 93f0a84aa..14d0ac29d 100644 --- a/extensions/business/deeploy/tests/test_update_requests.py +++ b/extensions/business/deeploy/tests/test_update_requests.py @@ -63,7 +63,7 @@ def _make_process_update_plugin(self, discovered_instances, nodes=None, deeploy_ DEEPLOY_KEYS.ERROR: str(exc), } plugin.deeploy_verify_and_get_inputs = lambda request, **kwargs: ("0xSender", make_inputs(**request)) - plugin._normalize_plugins_input = lambda request: request + plugin._normalize_plugins_input = lambda request, **kwargs: request plugin.deeploy_get_auth_result = lambda inputs: { DEEPLOY_KEYS.SENDER: "0xSender", DEEPLOY_KEYS.SENDER_ESCROW: "0xEscrow", @@ -97,6 +97,200 @@ def check_and_deploy_pipelines(**kwargs): plugin._queue_pipeline_persistence = lambda state: called.__setitem__("queued", called["queued"] + 1) return plugin, called + def _make_four_replica_cockroach_update_fixture(self, plugin): + nodes = ["0xai_node_a", "0xai_node_b", "0xai_node_c", "0xai_node_d"] + instance_id = "CONTAINER_APP_3ab323" + runtime_config = { + plugin.ct.CONFIG_INSTANCE.K_INSTANCE_ID: instance_id, + "IMAGE": "ghcr.io/ratio1/deeploy-cockroachdb-service:main", + "CONTAINER_RESOURCES": {"cpu": 1, "memory": "2g", "storage": "0g"}, + "FIXED_SIZE_VOLUMES": { + "cockroach_data": {"SIZE": "8G", "MOUNTING_POINT": "/cockroach/cockroach-data"}, + }, + "ENV": { + "CRDB_DATABASE": "appdb", + "CRDB_USER": "app_user", + "CRDB_PASSWORD": "sanitized-password", + "CRDB_NODE_COUNT": "4", + "CRDB_HOSTNAMES": "roach1,roach2,roach3,roach4", + }, + "PER_NODE_TARGET_NODES": nodes, + } + discovered_instances = [ + { + DEEPLOY_PLUGIN_DATA.INSTANCE_ID: instance_id, + DEEPLOY_PLUGIN_DATA.PLUGIN_SIGNATURE: "CONTAINER_APP_RUNNER", + DEEPLOY_PLUGIN_DATA.NODE: node, + DEEPLOY_PLUGIN_DATA.PLUGIN_INSTANCE: { + "instance_conf": copy.deepcopy(runtime_config), + }, + } + for node in nodes + ] + request_plugin = make_plugin_entry( + "CONTAINER_APP_RUNNER", + instance_id=instance_id, + IMAGE=runtime_config["IMAGE"], + CONTAINER_RESOURCES=copy.deepcopy(runtime_config["CONTAINER_RESOURCES"]), + FIXED_SIZE_VOLUMES=copy.deepcopy(runtime_config["FIXED_SIZE_VOLUMES"]), + ENV=copy.deepcopy(runtime_config["ENV"]), + PER_NODE_CONFIG={ + "byNode": { + node: {"ENV": {"CF_TUNNEL_TOKEN": f"sanitized-token-{index + 1}"}} + for index, node in enumerate(nodes) + }, + }, + ) + return nodes, discovered_instances, request_plugin + + def _make_legacy_service_update_request( + self, + nodes, + request_plugin, + top_level_instance_id=None, + nested_instance_id=None, + ): + app_params = { + key: copy.deepcopy(value) + for key, value in request_plugin.items() + if key not in (DEEPLOY_KEYS.PLUGIN_SIGNATURE, DEEPLOY_KEYS.PLUGIN_INSTANCE_ID) + } + if nested_instance_id is not None: + app_params[DEEPLOY_KEYS.PLUGIN_INSTANCE_ID] = nested_instance_id + request = { + DEEPLOY_KEYS.APP_ID: "cockroachdb_422ce92", + DEEPLOY_KEYS.APP_ALIAS: "cockroachdb", + DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.JOB_APP_TYPE: JOB_APP_TYPES.SERVICE, + DEEPLOY_KEYS.PIPELINE_INPUT_TYPE: "void", + DEEPLOY_KEYS.CHAINSTORE_RESPONSE: False, + DEEPLOY_KEYS.TARGET_NODES: nodes, + DEEPLOY_KEYS.TARGET_NODES_COUNT: len(nodes), + DEEPLOY_KEYS.PLUGIN_SIGNATURE: request_plugin[DEEPLOY_KEYS.PLUGIN_SIGNATURE], + DEEPLOY_KEYS.APP_PARAMS: app_params, + } + if top_level_instance_id is not None: + request[DEEPLOY_KEYS.PLUGIN_INSTANCE_ID] = top_level_instance_id + return request + + def test_normalize_legacy_update_copies_top_level_instance_id(self): + plugin = make_deeploy_plugin() + request = { + DEEPLOY_KEYS.PLUGIN_SIGNATURE: "CONTAINER_APP_RUNNER", + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID: "CONTAINER_APP_3ab323", + DEEPLOY_KEYS.APP_PARAMS: {"IMAGE": "repo/app:2.0"}, + } + + normalized = plugin._normalize_plugins_input( + plugin.deepcopy(request), + preserve_legacy_instance_id=True, + ) + + self.assertEqual( + normalized[DEEPLOY_KEYS.PLUGINS][0][DEEPLOY_KEYS.PLUGIN_INSTANCE_ID], + "CONTAINER_APP_3ab323", + ) + + def test_normalize_legacy_identity_duplicate_and_create_compatibility(self): + plugin = make_deeploy_plugin() + matching_request = { + DEEPLOY_KEYS.PLUGIN_SIGNATURE: "CONTAINER_APP_RUNNER", + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID: "CONTAINER_APP_3ab323", + DEEPLOY_KEYS.APP_PARAMS: { + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID: "CONTAINER_APP_3ab323", + "IMAGE": "repo/app:2.0", + }, + } + + normalized_update = plugin._normalize_plugins_input( + plugin.deepcopy(matching_request), + preserve_legacy_instance_id=True, + ) + normalized_create = plugin._normalize_plugins_input(plugin.deepcopy({ + DEEPLOY_KEYS.PLUGIN_SIGNATURE: "CONTAINER_APP_RUNNER", + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID: "CONTAINER_APP_3ab323", + DEEPLOY_KEYS.APP_PARAMS: {"IMAGE": "repo/app:2.0"}, + })) + nested_only = plugin._normalize_plugins_input( + plugin.deepcopy({ + DEEPLOY_KEYS.PLUGIN_SIGNATURE: "CONTAINER_APP_RUNNER", + DEEPLOY_KEYS.APP_PARAMS: { + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID: "CONTAINER_APP_3ab323", + "IMAGE": "repo/app:2.0", + }, + }), + preserve_legacy_instance_id=True, + ) + modern_plugins = plugin._normalize_plugins_input( + plugin.deepcopy({ + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID: "IGNORED_TOP_LEVEL_ID", + DEEPLOY_KEYS.PLUGINS: [ + make_plugin_entry( + "CONTAINER_APP_RUNNER", + instance_id="CONTAINER_APP_3ab323", + IMAGE="repo/app:2.0", + ), + ], + }), + preserve_legacy_instance_id=True, + ) + + self.assertEqual( + normalized_update[DEEPLOY_KEYS.PLUGINS][0][DEEPLOY_KEYS.PLUGIN_INSTANCE_ID], + "CONTAINER_APP_3ab323", + ) + self.assertNotIn( + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID, + normalized_create[DEEPLOY_KEYS.PLUGINS][0], + ) + self.assertEqual( + nested_only[DEEPLOY_KEYS.PLUGINS][0][DEEPLOY_KEYS.PLUGIN_INSTANCE_ID], + "CONTAINER_APP_3ab323", + ) + self.assertEqual( + modern_plugins[DEEPLOY_KEYS.PLUGINS][0][DEEPLOY_KEYS.PLUGIN_INSTANCE_ID], + "CONTAINER_APP_3ab323", + ) + + def test_normalize_legacy_update_rejects_conflicting_instance_ids(self): + plugin = make_deeploy_plugin() + request = { + DEEPLOY_KEYS.PLUGIN_SIGNATURE: "CONTAINER_APP_RUNNER", + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID: "CONTAINER_APP_3ab323", + DEEPLOY_KEYS.APP_PARAMS: { + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID: "CONTAINER_APP_other", + "IMAGE": "repo/app:2.0", + }, + } + + with self.assertRaisesRegex(ValueError, DEEPLOY_ERRORS.REQUEST3): + plugin._normalize_plugins_input( + request, + preserve_legacy_instance_id=True, + ) + + def test_normalize_legacy_update_preserves_blank_and_null_identity_for_validation(self): + plugin = make_deeploy_plugin() + for submitted_instance_id in ("", None): + with self.subTest(submitted_instance_id=submitted_instance_id): + normalized = plugin._normalize_plugins_input( + { + DEEPLOY_KEYS.PLUGIN_SIGNATURE: "CONTAINER_APP_RUNNER", + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID: submitted_instance_id, + DEEPLOY_KEYS.APP_PARAMS: {"IMAGE": "repo/app:2.0"}, + }, + preserve_legacy_instance_id=True, + ) + + self.assertIn( + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID, + normalized[DEEPLOY_KEYS.PLUGINS][0], + ) + self.assertEqual( + normalized[DEEPLOY_KEYS.PLUGINS][0][DEEPLOY_KEYS.PLUGIN_INSTANCE_ID], + submitted_instance_id, + ) + def test_prepare_single_plugin_instance_update_uses_plugin_config_and_strips_signature_fields(self): plugin = make_deeploy_plugin() @@ -119,6 +313,421 @@ def test_prepare_single_plugin_instance_update_uses_plugin_config_and_strips_sig self.assertNotIn(DEEPLOY_KEYS.PLUGIN_SIGNATURE, instance) self.assertNotIn("signature", instance) + def test_process_service_update_preserves_four_replica_identity_and_storage(self): + fixture_plugin = make_deeploy_plugin() + nodes, discovered_instances, request_plugin = self._make_four_replica_cockroach_update_fixture(fixture_plugin) + plugin, called = self._make_process_update_plugin( + discovered_instances=discovered_instances, + nodes=nodes, + deeploy_specs={ + DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.JOB_APP_TYPE: JOB_APP_TYPES.SERVICE, + DEEPLOY_KEYS.CURRENT_TARGET_NODES: nodes, + }, + ) + + response = plugin._process_pipeline_request( + { + DEEPLOY_KEYS.APP_ID: "cockroachdb_422ce92", + DEEPLOY_KEYS.APP_ALIAS: "cockroachdb", + DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.JOB_APP_TYPE: " SERVICE ", + DEEPLOY_KEYS.PIPELINE_INPUT_TYPE: "void", + DEEPLOY_KEYS.CHAINSTORE_RESPONSE: False, + DEEPLOY_KEYS.TARGET_NODES: nodes, + DEEPLOY_KEYS.TARGET_NODES_COUNT: len(nodes), + DEEPLOY_KEYS.PLUGINS: [request_plugin], + }, + is_create=False, + async_mode=True, + ) + + self.assertEqual(response[DEEPLOY_KEYS.STATUS], DEEPLOY_STATUS.COMMAND_DELIVERED) + self.assertEqual(called["delete"], 1) + self.assertEqual(called["deploy"], 1) + self.assertEqual(called["deploy_kwargs"]["job_app_type"], JOB_APP_TYPES.SERVICE) + redeploy_inputs = called["deploy_kwargs"]["inputs"] + self.assertEqual(len(redeploy_inputs[DEEPLOY_KEYS.PLUGINS]), 1) + self.assertEqual( + redeploy_inputs[DEEPLOY_KEYS.PLUGINS][0][DEEPLOY_KEYS.PLUGIN_INSTANCE_ID], + "CONTAINER_APP_3ab323", + ) + self.assertEqual( + plugin._aggregate_container_resources(redeploy_inputs)["storage"], + "8192m", + ) + + prepared_plan = called["deploy_kwargs"]["prepared_create_deploy_plan"] + self.assertEqual(set(prepared_plan["node_plugins_by_addr"]), set(nodes)) + for node_plugins in prepared_plan["node_plugins_by_addr"].values(): + self.assertEqual(len(node_plugins), 1) + instance = node_plugins[0][plugin.ct.CONFIG_PLUGIN.K_INSTANCES][0] + self.assertEqual(instance[plugin.ct.CONFIG_INSTANCE.K_INSTANCE_ID], "CONTAINER_APP_3ab323") + self.assertEqual(instance["PER_NODE_TARGET_NODES"], nodes) + self.assertEqual(instance["CONTAINER_RESOURCES"]["storage"], "0g") + self.assertEqual(instance["FIXED_SIZE_VOLUMES"]["cockroach_data"]["SIZE"], "8G") + + def test_process_legacy_service_update_preserves_four_replica_identity_and_storage(self): + fixture_plugin = make_deeploy_plugin() + nodes, discovered_instances, request_plugin = self._make_four_replica_cockroach_update_fixture(fixture_plugin) + plugin, called = self._make_process_update_plugin( + discovered_instances=discovered_instances, + nodes=nodes, + deeploy_specs={ + DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.JOB_APP_TYPE: JOB_APP_TYPES.SERVICE, + DEEPLOY_KEYS.CURRENT_TARGET_NODES: nodes, + }, + ) + plugin._normalize_plugins_input = types.MethodType( + DeeployManagerApiPlugin._normalize_plugins_input, + plugin, + ) + request = self._make_legacy_service_update_request( + nodes, + request_plugin, + top_level_instance_id="CONTAINER_APP_3ab323", + ) + + response = plugin._process_pipeline_request( + request, + is_create=False, + async_mode=True, + ) + + self.assertEqual(response[DEEPLOY_KEYS.STATUS], DEEPLOY_STATUS.COMMAND_DELIVERED) + self.assertEqual(called["delete"], 1) + self.assertEqual(called["deploy"], 1) + redeploy_inputs = called["deploy_kwargs"]["inputs"] + self.assertEqual(len(redeploy_inputs[DEEPLOY_KEYS.PLUGINS]), 1) + self.assertEqual( + redeploy_inputs[DEEPLOY_KEYS.PLUGINS][0][DEEPLOY_KEYS.PLUGIN_INSTANCE_ID], + "CONTAINER_APP_3ab323", + ) + self.assertEqual(plugin._aggregate_container_resources(redeploy_inputs)["storage"], "8192m") + prepared_plan = called["deploy_kwargs"]["prepared_create_deploy_plan"] + self.assertEqual(set(prepared_plan["node_plugins_by_addr"]), set(nodes)) + prepared_ids = { + instance[plugin.ct.CONFIG_INSTANCE.K_INSTANCE_ID] + for node_plugins in prepared_plan["node_plugins_by_addr"].values() + for node_plugin in node_plugins + for instance in node_plugin[plugin.ct.CONFIG_PLUGIN.K_INSTANCES] + } + self.assertEqual(prepared_ids, {"CONTAINER_APP_3ab323"}) + + def test_process_legacy_service_update_without_identity_fails_before_side_effects(self): + fixture_plugin = make_deeploy_plugin() + nodes, discovered_instances, request_plugin = self._make_four_replica_cockroach_update_fixture(fixture_plugin) + plugin, called = self._make_process_update_plugin( + discovered_instances=discovered_instances, + nodes=nodes, + deeploy_specs={ + DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.JOB_APP_TYPE: JOB_APP_TYPES.SERVICE, + DEEPLOY_KEYS.CURRENT_TARGET_NODES: nodes, + }, + ) + plugin._normalize_plugins_input = types.MethodType( + DeeployManagerApiPlugin._normalize_plugins_input, + plugin, + ) + phase_calls = defaultdict(int) + plugin._ensure_plugin_instance_ids = ( + lambda *args, **kwargs: phase_calls.__setitem__("backfill", phase_calls["backfill"] + 1) + ) + plugin.deeploy_check_payment_and_job_owner = ( + lambda *args, **kwargs: phase_calls.__setitem__("payment", phase_calls["payment"] + 1) or True + ) + plugin._prepare_create_pipeline_deploy_plan = ( + lambda **kwargs: phase_calls.__setitem__("preparation", phase_calls["preparation"] + 1) or {} + ) + plugin._reset_chainstore_response_keys = ( + lambda *args, **kwargs: phase_calls.__setitem__("reset", phase_calls["reset"] + 1) + ) + request = self._make_legacy_service_update_request(nodes, request_plugin) + + response = plugin._process_pipeline_request( + request, + is_create=False, + async_mode=True, + ) + + self.assertEqual(response[DEEPLOY_KEYS.STATUS], "failed") + self.assertIn(DEEPLOY_ERRORS.PLUGINS3, response[DEEPLOY_KEYS.ERROR]) + self.assertEqual(dict(phase_calls), {}) + self.assertEqual(called["delete"], 0) + self.assertEqual(called["deploy"], 0) + self.assertEqual(called["queued"], 0) + + def test_process_legacy_service_update_rejects_conflicting_identity_before_discovery(self): + fixture_plugin = make_deeploy_plugin() + nodes, discovered_instances, request_plugin = self._make_four_replica_cockroach_update_fixture(fixture_plugin) + plugin, called = self._make_process_update_plugin( + discovered_instances=discovered_instances, + nodes=nodes, + deeploy_specs={ + DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.JOB_APP_TYPE: JOB_APP_TYPES.SERVICE, + DEEPLOY_KEYS.CURRENT_TARGET_NODES: nodes, + }, + ) + plugin._normalize_plugins_input = types.MethodType( + DeeployManagerApiPlugin._normalize_plugins_input, + plugin, + ) + phase_calls = defaultdict(int) + plugin._gather_running_pipeline_context = ( + lambda **kwargs: phase_calls.__setitem__("discovery", phase_calls["discovery"] + 1) or {} + ) + plugin.deeploy_check_payment_and_job_owner = ( + lambda *args, **kwargs: phase_calls.__setitem__("payment", phase_calls["payment"] + 1) or True + ) + plugin._prepare_create_pipeline_deploy_plan = ( + lambda **kwargs: phase_calls.__setitem__("preparation", phase_calls["preparation"] + 1) or {} + ) + plugin._reset_chainstore_response_keys = ( + lambda *args, **kwargs: phase_calls.__setitem__("reset", phase_calls["reset"] + 1) + ) + request = self._make_legacy_service_update_request( + nodes, + request_plugin, + top_level_instance_id="CONTAINER_APP_3ab323", + nested_instance_id="CONTAINER_APP_other", + ) + + response = plugin._process_pipeline_request( + request, + is_create=False, + async_mode=True, + ) + + self.assertEqual(response[DEEPLOY_KEYS.STATUS], "failed") + self.assertIn(DEEPLOY_ERRORS.REQUEST3, response[DEEPLOY_KEYS.ERROR]) + self.assertIn("Conflicting legacy instance_id", response[DEEPLOY_KEYS.ERROR]) + self.assertEqual(dict(phase_calls), {}) + self.assertEqual(called["delete"], 0) + self.assertEqual(called["deploy"], 0) + self.assertEqual(called["queued"], 0) + + def test_process_update_without_job_app_type_fails_before_discovery_or_side_effects(self): + for persisted_job_app_type in (None, JOB_APP_TYPES.SERVICE): + with self.subTest(persisted_job_app_type=persisted_job_app_type): + fixture_plugin = make_deeploy_plugin() + nodes, discovered_instances, request_plugin = self._make_four_replica_cockroach_update_fixture( + fixture_plugin + ) + request_plugin.pop(DEEPLOY_KEYS.PLUGIN_INSTANCE_ID) + request_plugin["IMAGE"] = "ghcr.io/ratio1/deeploy-cockroachdb-service:review-repro" + deeploy_specs = { + DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.CURRENT_TARGET_NODES: nodes, + } + if persisted_job_app_type is not None: + deeploy_specs[DEEPLOY_KEYS.JOB_APP_TYPE] = persisted_job_app_type + plugin, called = self._make_process_update_plugin( + discovered_instances=discovered_instances, + nodes=nodes, + deeploy_specs=deeploy_specs, + ) + phase_calls = defaultdict(int) + + def gather_context(**kwargs): + phase_calls["discovery"] += 1 + return { + "discovered_instances": discovered_instances, + "nodes": nodes, + "deeploy_specs": deeploy_specs, + } + + plugin._gather_running_pipeline_context = gather_context + plugin.deeploy_check_payment_and_job_owner = ( + lambda *args, **kwargs: phase_calls.__setitem__("payment", phase_calls["payment"] + 1) or True + ) + plugin._check_nodes_availability = ( + lambda inputs: phase_calls.__setitem__("nodes", phase_calls["nodes"] + 1) or nodes + ) + plugin._prepare_create_pipeline_deploy_plan = ( + lambda **kwargs: phase_calls.__setitem__("preparation", phase_calls["preparation"] + 1) + or {"enable_chainstore_response": False, "response_keys": {}, "node_plugins_by_addr": {}} + ) + plugin._reset_chainstore_response_keys = ( + lambda *args, **kwargs: phase_calls.__setitem__("reset", phase_calls["reset"] + 1) + ) + + response = plugin._process_pipeline_request( + { + DEEPLOY_KEYS.APP_ID: "cockroachdb_422ce92", + DEEPLOY_KEYS.APP_ALIAS: "cockroachdb", + DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.PIPELINE_INPUT_TYPE: "void", + DEEPLOY_KEYS.CHAINSTORE_RESPONSE: True, + DEEPLOY_KEYS.TARGET_NODES: nodes, + DEEPLOY_KEYS.TARGET_NODES_COUNT: len(nodes), + DEEPLOY_KEYS.PLUGINS: [request_plugin], + }, + is_create=False, + async_mode=True, + ) + + self.assertEqual(response[DEEPLOY_KEYS.STATUS], "failed") + self.assertIn(DEEPLOY_ERRORS.REQUEST3, response[DEEPLOY_KEYS.ERROR]) + self.assertIn("job_app_type is required for update requests", response[DEEPLOY_KEYS.ERROR]) + self.assertEqual(dict(phase_calls), {}) + self.assertEqual(called["delete"], 0) + self.assertEqual(called["deploy"], 0) + self.assertEqual(called["queued"], 0) + + def test_process_update_rejects_blank_and_invalid_job_app_type_before_discovery(self): + for submitted_job_app_type in (" ", "unsupported"): + with self.subTest(submitted_job_app_type=submitted_job_app_type): + plugin, called = self._make_process_update_plugin(discovered_instances=[]) + discovery_calls = [] + plugin._gather_running_pipeline_context = lambda **kwargs: discovery_calls.append(kwargs) or {} + + response = plugin._process_pipeline_request( + { + DEEPLOY_KEYS.APP_ID: "app-123", + DEEPLOY_KEYS.APP_ALIAS: "app", + DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.JOB_APP_TYPE: submitted_job_app_type, + DEEPLOY_KEYS.PIPELINE_INPUT_TYPE: "void", + DEEPLOY_KEYS.CHAINSTORE_RESPONSE: False, + DEEPLOY_KEYS.TARGET_NODES: ["node-1"], + DEEPLOY_KEYS.TARGET_NODES_COUNT: 1, + DEEPLOY_KEYS.PLUGINS: [ + make_plugin_entry( + "CONTAINER_APP_RUNNER", + IMAGE="repo/app:2.0", + CONTAINER_RESOURCES={"cpu": 1, "memory": "256m", "storage": "1g"}, + ), + ], + }, + is_create=False, + async_mode=True, + ) + + self.assertEqual(response[DEEPLOY_KEYS.STATUS], "failed") + self.assertIn(DEEPLOY_ERRORS.REQUEST3, response[DEEPLOY_KEYS.ERROR]) + self.assertIn("job_app_type", response[DEEPLOY_KEYS.ERROR]) + self.assertEqual(discovery_calls, []) + self.assertEqual(called["delete"], 0) + self.assertEqual(called["deploy"], 0) + + def test_process_create_without_job_app_type_keeps_inference(self): + plugin, called = self._make_process_update_plugin( + discovered_instances=[], + nodes=["node-1"], + ) + + response = plugin._process_pipeline_request( + { + DEEPLOY_KEYS.APP_ALIAS: "app", + DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.PIPELINE_INPUT_TYPE: "void", + DEEPLOY_KEYS.CHAINSTORE_RESPONSE: False, + DEEPLOY_KEYS.TARGET_NODES: ["node-1"], + DEEPLOY_KEYS.TARGET_NODES_COUNT: 1, + DEEPLOY_KEYS.PLUGINS: [ + make_plugin_entry( + "CONTAINER_APP_RUNNER", + IMAGE="repo/app:1.0", + CONTAINER_RESOURCES={"cpu": 1, "memory": "256m", "storage": "1g"}, + ), + ], + }, + is_create=True, + async_mode=True, + ) + + self.assertEqual(response[DEEPLOY_KEYS.STATUS], DEEPLOY_STATUS.COMMAND_DELIVERED) + self.assertEqual(called["deploy"], 1) + self.assertEqual(called["deploy_kwargs"]["job_app_type"], JOB_APP_TYPES.GENERIC) + self.assertEqual( + called["deploy_kwargs"]["inputs"][DEEPLOY_KEYS.JOB_APP_TYPE], + JOB_APP_TYPES.GENERIC, + ) + + def test_process_legacy_create_ignores_top_level_instance_id(self): + plugin, called = self._make_process_update_plugin( + discovered_instances=[], + nodes=["node-1"], + ) + plugin._normalize_plugins_input = types.MethodType( + DeeployManagerApiPlugin._normalize_plugins_input, + plugin, + ) + + response = plugin._process_pipeline_request( + { + DEEPLOY_KEYS.APP_ALIAS: "app", + DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.PIPELINE_INPUT_TYPE: "void", + DEEPLOY_KEYS.CHAINSTORE_RESPONSE: False, + DEEPLOY_KEYS.TARGET_NODES: ["node-1"], + DEEPLOY_KEYS.TARGET_NODES_COUNT: 1, + DEEPLOY_KEYS.PLUGIN_SIGNATURE: "CONTAINER_APP_RUNNER", + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID: "CALLER_SUPPLIED_CREATE_ID", + DEEPLOY_KEYS.APP_PARAMS: { + "IMAGE": "repo/app:1.0", + "CONTAINER_RESOURCES": {"cpu": 1, "memory": "256m", "storage": "1g"}, + }, + }, + is_create=True, + async_mode=True, + ) + + self.assertEqual(response[DEEPLOY_KEYS.STATUS], DEEPLOY_STATUS.COMMAND_DELIVERED) + self.assertEqual(called["deploy"], 1) + self.assertNotIn( + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID, + called["deploy_kwargs"]["inputs"][DEEPLOY_KEYS.PLUGINS][0], + ) + + def test_process_service_update_without_resolved_id_fails_before_side_effects(self): + fixture_plugin = make_deeploy_plugin() + nodes, discovered_instances, request_plugin = self._make_four_replica_cockroach_update_fixture(fixture_plugin) + request_plugin.pop(DEEPLOY_KEYS.PLUGIN_INSTANCE_ID) + request_plugin["IMAGE"] = "repo/reconfigured-service:latest" + plugin, called = self._make_process_update_plugin( + discovered_instances=discovered_instances, + nodes=nodes, + deeploy_specs={ + DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.JOB_APP_TYPE: JOB_APP_TYPES.SERVICE, + DEEPLOY_KEYS.CURRENT_TARGET_NODES: nodes, + }, + ) + payment_calls = [] + reset_calls = [] + backfill_calls = [] + plugin.deeploy_check_payment_and_job_owner = lambda *args, **kwargs: payment_calls.append(args) or True + plugin._reset_chainstore_response_keys = lambda *args, **kwargs: reset_calls.append(args) + plugin._ensure_plugin_instance_ids = lambda *args, **kwargs: backfill_calls.append(args) + + response = plugin._process_pipeline_request( + { + DEEPLOY_KEYS.APP_ID: "cockroachdb_422ce92", + DEEPLOY_KEYS.APP_ALIAS: "cockroachdb", + DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.JOB_APP_TYPE: JOB_APP_TYPES.SERVICE, + DEEPLOY_KEYS.PIPELINE_INPUT_TYPE: "void", + DEEPLOY_KEYS.CHAINSTORE_RESPONSE: True, + DEEPLOY_KEYS.TARGET_NODES: nodes, + DEEPLOY_KEYS.TARGET_NODES_COUNT: len(nodes), + DEEPLOY_KEYS.PLUGINS: [request_plugin], + }, + is_create=False, + async_mode=True, + ) + + self.assertIn(DEEPLOY_ERRORS.PLUGINS3, response[DEEPLOY_KEYS.ERROR]) + self.assertIn("Service update plugins must include instance_id", response[DEEPLOY_KEYS.ERROR]) + self.assertEqual(backfill_calls, []) + self.assertEqual(payment_calls, []) + self.assertEqual(reset_calls, []) + self.assertEqual(called["delete"], 0) + self.assertEqual(called["deploy"], 0) + def test_prepare_single_plugin_instance_update_falls_back_to_instance_conf(self): plugin = make_deeploy_plugin() fallback_instance = { @@ -960,6 +1569,7 @@ def check_nodes_availability(inputs): DEEPLOY_KEYS.APP_ID: "app-123", DEEPLOY_KEYS.APP_ALIAS: "app", DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.JOB_APP_TYPE: "stack", DEEPLOY_KEYS.PIPELINE_INPUT_TYPE: "void", DEEPLOY_KEYS.CHAINSTORE_RESPONSE: False, DEEPLOY_KEYS.TARGET_NODES: ["node-1"], @@ -1202,7 +1812,7 @@ def test_process_update_ignores_invalid_omitted_live_plugin_config(self): ["api-instance"], ) - def test_process_update_detects_type_from_requested_replacement_only(self): + def test_process_update_uses_explicit_type_with_requested_replacement_only(self): plugin, called = self._make_process_update_plugin( discovered_instances=[ { @@ -1238,6 +1848,7 @@ def test_process_update_detects_type_from_requested_replacement_only(self): DEEPLOY_KEYS.APP_ID: "app-123", DEEPLOY_KEYS.APP_ALIAS: "app", DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.JOB_APP_TYPE: "generic", DEEPLOY_KEYS.PIPELINE_INPUT_TYPE: "void", DEEPLOY_KEYS.CHAINSTORE_RESPONSE: False, DEEPLOY_KEYS.TARGET_NODES: ["node-1"], diff --git a/ver.py b/ver.py index a529ab885..bf7ad7301 100644 --- a/ver.py +++ b/ver.py @@ -1 +1 @@ -__VER__ = '2.10.391' +__VER__ = '2.10.392' From 7c40cf6a2d6e73d0d02710beedb6232fe9cbfa0d Mon Sep 17 00:00:00 2001 From: Vitalii <87299468+toderian@users.noreply.github.com> Date: Fri, 24 Jul 2026 11:49:56 +0300 Subject: [PATCH 5/6] RM: bugfix & tasks finalization (#475) * fix: harden model-test backend contracts What changed: - Require and validate a server bearer token for model-test launch and preflight. - Reject unresolved credential references before persistence and emit canonical complete results. - Include sanitized model-test workers in local job listings with legacy read compatibility. Why: - Direct backend calls must fail closed and frontend/backend status and credential contracts must be truthful. Checks: - focused model-testing and endpoint-auth unittests in rm3: 97 passed - Python compile checks: passed - git diff --check: passed * fix: enforce launcher-owned lifecycle writes * fix: normalize model-test terminal events * fix: enforce launcher ownership for stop requests Reject all shared job writes from non-launcher nodes and fail foreign stop-monitoring requests before worker, timeline, or CStore mutation. Add direct service and write-boundary regression coverage. * fix: guard destructive job control ownership Reject foreign stop/delete, purge, engagement-redaction, and triage mutations before side effects. Route remaining production top-level job writes through the launcher-owned boundary and cover destructive foreign control paths. * fix: preserve foreign jobs during bulk purge Treat launcher-ownership conflicts as non-recoverable bulk-purge failures instead of force-purge candidates. Preserve the foreign job, artifacts, and secondary rows for handling by its launcher. * chore: increment version --- .../cybersec/red_mesh/model_testing/runner.py | 2 +- .../red_mesh/model_testing/security.py | 78 ++- .../cybersec/red_mesh/pentester_api_01.py | 495 ++++++++++++------ .../cybersec/red_mesh/services/control.py | 44 +- .../cybersec/red_mesh/services/query.py | 25 +- .../red_mesh/services/reconciliation.py | 11 +- .../cybersec/red_mesh/services/triage.py | 20 +- .../cybersec/red_mesh/tests/test_api.py | 334 ++++++++---- .../red_mesh/tests/test_integration.py | 36 +- .../red_mesh/tests/test_model_testing.py | 374 ++++++++++--- .../red_mesh/tests/test_reconciliation.py | 26 + ver.py | 2 +- 12 files changed, 1086 insertions(+), 361 deletions(-) diff --git a/extensions/business/cybersec/red_mesh/model_testing/runner.py b/extensions/business/cybersec/red_mesh/model_testing/runner.py index 79d25bbd3..3d1385ca7 100644 --- a/extensions/business/cybersec/red_mesh/model_testing/runner.py +++ b/extensions/business/cybersec/red_mesh/model_testing/runner.py @@ -795,7 +795,7 @@ def run(self): raw_evidence_cases.append(case_evidence) metrics["completed_cases"] += 1 results.append(case_result) - overall_status = "completed" if metrics["evaluated_cases"] == metrics["total_cases"] else "incomplete" + overall_status = "complete" if metrics["evaluated_cases"] == metrics["total_cases"] else "incomplete" if not results: overall_status = "failed" summary = { diff --git a/extensions/business/cybersec/red_mesh/model_testing/security.py b/extensions/business/cybersec/red_mesh/model_testing/security.py index 90994626f..b9553aa68 100644 --- a/extensions/business/cybersec/red_mesh/model_testing/security.py +++ b/extensions/business/cybersec/red_mesh/model_testing/security.py @@ -2,13 +2,16 @@ from __future__ import annotations +import hmac import ipaddress +import os import socket from urllib.parse import urlsplit MODEL_PROVIDER_CREDENTIAL_UNAVAILABLE = "credential_unavailable" -_MODEL_PROVIDER_REF_PREFIX = "model_provider/" +BACKEND_TOKEN_ENV = "REDMESH_BACKEND_TOKEN" +MIN_BACKEND_TOKEN_BYTES = 32 _PROVIDER_ALLOWED_KEYS = { "adapter", "provider_label", @@ -143,11 +146,51 @@ def validate_provider_url(base_url, *, resolver=None): def _credential_error(): return _validation_error( - MODEL_PROVIDER_CREDENTIAL_UNAVAILABLE, + "Provider requires an API key payload.", error_class=MODEL_PROVIDER_CREDENTIAL_UNAVAILABLE, ) +def _backend_auth_error(*, status_code, error, error_class, message): + return { + "status": "error", + "status_code": status_code, + "error": error, + "error_class": error_class, + "message": message, + } + + +def validate_backend_token(token): + """Validate the Navigator-to-edge bearer token without exposing token material.""" + expected = os.environ.get(BACKEND_TOKEN_ENV, "") + expected_bytes = expected.encode("utf-8") + if len(expected_bytes) < MIN_BACKEND_TOKEN_BYTES: + return _backend_auth_error( + status_code=401, + error="unauthorized", + error_class="backend_auth_unavailable", + message="Backend authentication is not configured.", + ) + + presented = token if isinstance(token, str) else "" + if not presented: + return _backend_auth_error( + status_code=401, + error="unauthorized", + error_class="backend_auth_required", + message="Backend authentication is required.", + ) + if not hmac.compare_digest(presented.encode("utf-8"), expected_bytes): + return _backend_auth_error( + status_code=403, + error="forbidden", + error_class="backend_auth_invalid", + message="Backend authentication failed.", + ) + return None + + def validate_provider_config_shape(provider, *, role): """Reject inline credential-bearing or unknown provider config fields.""" if not isinstance(provider, dict): @@ -182,32 +225,11 @@ def validate_model_provider_credentials( if isinstance(secret_payload, dict): api_key = str(secret_payload.get("api_key") or "") has_secret = bool(api_key) - if credential_ref and has_secret: - return None, _validation_error( - f"{role} may not supply both credential_ref and secret payload", - error_class="duplicate_credential_source", - ) + # Credential references are an accepted historical shape but there is no + # worker-local resolver yet. Reject them before launch/preflight persistence + # with one sanitized error that never includes the supplied identifier. + if credential_ref: + return None, _credential_error() if has_secret: return {"source": "secret_payload", "credential_ref_present": False}, None - if not credential_ref: - return None, _validation_error( - f"{role} requires credential_ref or secret payload", - error_class=MODEL_PROVIDER_CREDENTIAL_UNAVAILABLE, - ) - if not credential_ref.startswith(_MODEL_PROVIDER_REF_PREFIX): - return None, _credential_error() - ref_body = credential_ref[len(_MODEL_PROVIDER_REF_PREFIX):] - operator_prefix = f"operator/{created_by_id}/" - if ref_body.startswith(operator_prefix): - credential_id = ref_body[len(operator_prefix):] - if not credential_id or "/" in credential_id: - return None, _credential_error() - return {"source": "credential_ref", "credential_ref_present": True}, None - if ref_body.startswith("deploy/default_evaluator/"): - if role != "evaluator_model" or not use_default_evaluator_model: - return None, _credential_error() - credential_id = ref_body[len("deploy/default_evaluator/"):] - if not credential_id or "/" in credential_id: - return None, _credential_error() - return {"source": "credential_ref", "credential_ref_present": True}, None return None, _credential_error() diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index b2cc934a9..1fd6fe013 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -157,6 +157,7 @@ preflight_model_test_provider, sanitized_model_test_catalog, ) +from .model_testing.security import validate_backend_token from .model_testing.artifacts import ( sanitize_model_test_results, sanitize_model_test_summary, @@ -852,7 +853,7 @@ def __post_init(self): if isinstance(raw_report, dict): sample_value = next(iter(raw_report.values()), None) needs_aggregation = isinstance(sample_value, dict) and "local_worker_id" in sample_value - if needs_aggregation: + if needs_aggregation and normalized_spec.get("launcher") == self.ee_addr: # Merge stranded partial worker outputs left from prior deployments. self.P(f"Found incomplete report for {normalized_key}, aggregating...", color='r') agg_report = self._get_aggregated_report(raw_report) @@ -1022,10 +1023,13 @@ def _normalize_job_record(self, job_key, job_spec, migrate=False): normalized["job_revision"] = int(normalized.get("job_revision", 0) or 0) except (TypeError, ValueError): normalized["job_revision"] = 0 - if migrate and job_key != job_id: - PentesterApi01Plugin._write_job_record(self, job_id, normalized, context="normalize_migrate") - PentesterApi01Plugin._delete_job_record(self, job_key) - job_key = job_id + if migrate and job_key != job_id and launcher == self.ee_addr: + persisted = PentesterApi01Plugin._write_job_record( + self, job_id, normalized, context="normalize_migrate", + ) + if isinstance(persisted, dict): + PentesterApi01Plugin._delete_job_record(self, job_key) + job_key = job_id return job_key, normalized @@ -1472,7 +1476,6 @@ def _maybe_launch_model_test_jobs(self): worker_entry["model_test_worker_status"] = "running" worker_entry["worker_type"] = "model_test" worker_entry["assigned_at"] = worker_entry.get("assigned_at") or self.time() - self._write_job_record(job_id, job_specs, context="launch_model_test_worker") self._publish_model_test_progress(job_id, worker, job_specs, finished=False) def _publish_model_test_progress(self, job_id, worker, job_specs, *, finished=False, report_cid=None, worker_addr=None): @@ -1607,6 +1610,15 @@ def _maybe_close_model_test_jobs(self): result_payload, show_logs=False, ) + job_config = self._get_artifact_repository().get_job_config(job_specs) + if isinstance(job_config, dict): + write_raw_evidence_artifact( + self, + job_id, + job_config, + worker.state.get("raw_evidence_payload"), + get_model_testing_config(self), + ) worker_entry["finished"] = True worker_entry["canceled"] = bool(status.get("canceled")) worker_entry["model_test_worker_status"] = "canceled" if status.get("canceled") else "finished" @@ -1614,16 +1626,6 @@ def _maybe_close_model_test_jobs(self): worker_entry["result"] = None if report_cid else result_payload job_specs["model_test_summary"] = result_payload["model_test_summary"] job_specs["model_test_results"] = result_payload["model_test_results"] - finalized = PentesterApi01Plugin._finalize_model_test_job( - self, - job_id, - job_specs, - result_payload, - report_cid, - raw_evidence_payload=worker.state.get("raw_evidence_payload"), - ) - if not finalized: - self._write_job_record(job_id, job_specs, context="close_model_test_worker") self._publish_model_test_progress( job_id, worker, @@ -1633,11 +1635,107 @@ def _maybe_close_model_test_jobs(self): ) self.model_test_jobs.pop(job_id, None) + def _canonical_model_test_terminal_status(self, result_payload): + """Return the canonical result status for a new terminal model-test record.""" + payload = result_payload if isinstance(result_payload, dict) else {} + summary = payload.get("model_test_summary") or {} + results = payload.get("model_test_results") or {} + aliases = { + "complete": "complete", + "completed": "complete", + "passed": "complete", + "success": "complete", + "incomplete": "incomplete", + "failed": "failed", + "failure": "failed", + "error": "failed", + "canceled": "canceled", + "cancelled": "canceled", + } + for value in ( + summary.get("overall_status"), + results.get("overall_status"), + payload.get("status"), + ): + normalized = aliases.get(str(value or "").strip().lower()) + if normalized: + return normalized + return "failed" + + def _terminalize_model_test_job( + self, + job_specs, + result_status, + job_status, + *, + error_class=None, + ): + """Ensure exactly one safe terminal lifecycle event before archiving.""" + event_type = { + JOB_STATUS_FINALIZED: "finalized", + JOB_STATUS_FAILED: "failed", + JOB_STATUS_STOPPED: "canceled", + }[job_status] + event_label = { + "finalized": "Model test finalized", + "failed": "Model test failed", + "canceled": "Model test canceled", + }[event_type] + summary = job_specs.get("model_test_summary") or {} + meta = { + "overall_status": result_status, + "selected_execution_node": selected_model_test_worker_addr(job_specs), + "cases_completed": summary.get("cases_completed") or summary.get("completed_cases") or 0, + "cases_total": summary.get("cases_total") or summary.get("total_cases") or 0, + } + safe_error_class = sanitize_model_test_error_class(error_class) + if safe_error_class: + meta["error_class"] = safe_error_class + + terminal_event_types = { + "attestation_failed", + "canceled", + "completed", + "failed", + "finalized", + "stopped", + } + timeline = job_specs.setdefault("timeline", []) + matching = [ + entry for entry in timeline + if isinstance(entry, dict) + and entry.get("type") == event_type + and (entry.get("meta") or {}).get("overall_status") == result_status + ] + timeline[:] = [ + entry for entry in timeline + if not isinstance(entry, dict) or entry.get("type") not in terminal_event_types + ] + if matching: + timeline.append(matching[0]) + return matching[0] + PentesterApi01Plugin._emit_timeline_event( + self, + job_specs, + event_type, + event_label, + actor_type="system", + meta=meta, + ) + return job_specs["timeline"][-1] + def _finalize_model_test_job(self, job_id, job_specs, result_payload, worker_report_cid, raw_evidence_payload=None): """Write a model-test archive and prune CStore to a terminal stub.""" from .model_testing.artifacts import ModelTestArchive, ModelTestArtifactRepository from .services.resilience import run_bounded_retry + if job_specs.get("launcher") != self.ee_addr: + self.P(f"Skipping model-test finalization for non-launcher job {job_id}", color='y') + return False + current = PentesterApi01Plugin._get_job_state_repository(self).get_job(job_id) + if isinstance(current, dict) and current.get("job_cid"): + return True + artifacts = ModelTestArtifactRepository(self) job_config_cid = job_specs.get("job_config_cid", "") job_config = artifacts.get_json(job_config_cid) @@ -1668,38 +1766,23 @@ def _finalize_model_test_job(self, job_id, job_specs, result_payload, worker_rep except (TypeError, ValueError): duration = 0.0 + result_status = PentesterApi01Plugin._canonical_model_test_terminal_status(self, result_payload) model_test_results = sanitize_model_test_results((result_payload or {}).get("model_test_results") or {}) model_test_summary = sanitize_model_test_summary((result_payload or {}).get("model_test_summary") or {}) - result_status = (result_payload or {}).get("status") - job_status = JOB_STATUS_STOPPED if result_status in {"canceled", "failed"} else JOB_STATUS_FINALIZED - existing_event_types = { - entry.get("type") - for entry in job_specs.get("timeline", []) or [] - if isinstance(entry, dict) - } - if job_status == JOB_STATUS_FINALIZED and "completed" not in existing_event_types: - PentesterApi01Plugin._emit_timeline_event( - self, - job_specs, - "completed", - "Model test completed", - actor_type="system", - meta={ - "cases_completed": model_test_summary.get("cases_completed") - or model_test_summary.get("completed_cases"), - "cases_total": model_test_summary.get("cases_total") - or model_test_summary.get("total_cases"), - }, - ) - elif job_status == JOB_STATUS_STOPPED and "stopped" not in existing_event_types: - PentesterApi01Plugin._emit_timeline_event( - self, - job_specs, - "stopped", - "Model test stopped", - actor_type="system", - meta={"status": result_status or "stopped"}, - ) + model_test_results["overall_status"] = result_status + model_test_summary["overall_status"] = result_status + job_specs["model_test_results"] = model_test_results + job_specs["model_test_summary"] = model_test_summary + if result_status in {"complete", "incomplete"}: + job_status = JOB_STATUS_FINALIZED + elif result_status == "canceled": + job_status = JOB_STATUS_STOPPED + else: + job_status = JOB_STATUS_FAILED + terminal_error_class = ( + (result_payload or {}).get("error_class") + or model_test_summary.get("error_class") + ) required_end_attestation = bool( job_specs.get("blockchain_attestation_enabled") or job_config.get("blockchain_attestation_enabled") @@ -1724,17 +1807,8 @@ def _finalize_model_test_job(self, job_id, job_specs, result_payload, worker_rep message = "Required terminal blockchain attestation failed for model-test job." job_specs["failure_class"] = "attestation_failed" job_specs["failure_message"] = message - set_job_status(job_specs, JOB_STATUS_FAILED) - PentesterApi01Plugin._emit_timeline_event( - self, - job_specs, - "attestation_failed", - message, - actor_type="system", - meta={"failure_class": "attestation_failed"}, - ) - self._write_job_record(job_id, job_specs, context="model_test_attestation_failed") - return False + job_status = JOB_STATUS_FAILED + terminal_error_class = "finalization_failed" if redmesh_test_attestation: job_specs["redmesh_test_attestation"] = redmesh_test_attestation PentesterApi01Plugin._emit_timeline_event( @@ -1751,22 +1825,14 @@ def _finalize_model_test_job(self, job_id, job_specs, result_payload, worker_rep or {} ) + PentesterApi01Plugin._terminalize_model_test_job( + self, + job_specs, + result_status, + job_status, + error_class=terminal_error_class, + ) archive_timeline = list(job_specs.get("timeline") or []) - if job_status == JOB_STATUS_FINALIZED and not any( - isinstance(entry, dict) and entry.get("type") == "finalized" - for entry in archive_timeline - ): - archive_timeline.append({ - "type": "finalized", - "label": "Model test archive finalized", - "date": self.time(), - "actor": self.ee_id, - "actor_type": "system", - "meta": { - "archive_type": "model_test_archive_v1", - "raw_evidence_status": raw_evidence_metadata.get("status"), - }, - }) archive = ModelTestArchive( job_id=job_id, @@ -1833,17 +1899,20 @@ def _finalize_model_test_job(self, job_id, job_specs, result_payload, worker_rep blockchain_attestation_enabled=required_end_attestation, start_attestation_required=required_end_attestation, end_attestation_required=required_end_attestation, + failure_class=job_specs.get("failure_class", ""), + failure_message=job_specs.get("failure_message", ""), ) self._write_job_record(job_id, stub.to_dict(), context="model_test_archive_prune") self.P(f"Model-test job {job_id} archived. CID={job_cid}, CStore pruned to stub.") return True def _maybe_finalize_finished_model_test_jobs(self): - """Repair model-test jobs whose worker result is durable but job status is still RUNNING.""" + """Finalize worker-owned model-test result/live artifacts on the launcher.""" from .model_testing.artifacts import ModelTestArtifactRepository all_jobs = PentesterApi01Plugin._get_job_state_repository(self).list_jobs() or {} artifacts = ModelTestArtifactRepository(self) + live_payloads = PentesterApi01Plugin._get_job_state_repository(self).list_live_progress() or {} finalized = [] for job_key, raw_job_specs in list(all_jobs.items()): normalized_key, job_specs = self._normalize_job_record(job_key, raw_job_specs) @@ -1854,21 +1923,61 @@ def _maybe_finalize_finished_model_test_jobs(self): continue if job_specs.get("job_cid") or is_terminal_job_status(job_specs.get("job_status")): continue + if job_specs.get("launcher") != self.ee_addr: + continue workers = job_specs.get("workers") or {} - worker_entry = workers.get(self.ee_addr) - if not isinstance(worker_entry, dict) or not worker_entry.get("finished"): + worker_addr = selected_model_test_worker_addr(job_specs) + worker_entry = workers.get(worker_addr) + if not worker_addr or not isinstance(worker_entry, dict): + continue + + worker_entry = dict(worker_entry) + live_payload = live_payloads.get(f"{job_id}:{worker_addr}") + live = None + if isinstance(live_payload, dict): + try: + live = WorkerProgress.from_dict(live_payload) + except (KeyError, TypeError, ValueError): + live = None + if ( + live is not None + and live.assignment_revision_seen == PentesterApi01Plugin._get_worker_assignment_revision(worker_entry) + and live.finished + ): + worker_entry["finished"] = True + worker_entry["canceled"] = bool(live.canceled) + worker_entry["report_cid"] = live.report_cid + worker_entry["model_test_worker_status"] = ( + MODEL_TEST_PHASE_CANCELED if live.canceled else live.phase + ) + if not worker_entry.get("finished"): continue report_cid = worker_entry.get("report_cid") result_payload = artifacts.get_json(report_cid) if report_cid else None if not isinstance(result_payload, dict): result_payload = worker_entry.get("result") if isinstance(worker_entry.get("result"), dict) else None + if not isinstance(result_payload, dict) and live is not None and live.finished: + result_payload = { + "job_id": job_id, + "worker_addr": worker_addr, + "status": ( + MODEL_TEST_PHASE_CANCELED + if live.canceled + else (live.model_test_summary or {}).get("overall_status") + or (live.model_test_results or {}).get("overall_status") + or live.phase + ), + "model_test_results": sanitize_model_test_results(live.model_test_results or {}), + "model_test_summary": sanitize_model_test_summary(live.model_test_summary or {}), + "error_class": live.error_class, + } if not isinstance(result_payload, dict): continue result_payload.setdefault("job_id", job_id) - result_payload.setdefault("worker_addr", self.ee_addr) + result_payload.setdefault("worker_addr", worker_addr) result_payload.setdefault( "status", "canceled" if worker_entry.get("canceled") else worker_entry.get("model_test_worker_status", "finished"), @@ -2031,7 +2140,17 @@ def first_number(*values, default=0): report_cid, ) if not finalized: - set_job_status(job_specs, JOB_STATUS_STOPPED) + fallback_job_status = ( + JOB_STATUS_STOPPED if status == MODEL_TEST_PHASE_CANCELED else JOB_STATUS_FAILED + ) + PentesterApi01Plugin._terminalize_model_test_job( + self, + job_specs, + status, + fallback_job_status, + error_class=error_class, + ) + set_job_status(job_specs, fallback_job_status) self._write_job_record(job_id, job_specs, context=f"model_test_{status}") PentesterApi01Plugin._publish_model_test_progress( self, @@ -2090,13 +2209,6 @@ def _maybe_fail_stale_model_test_jobs(self): and worker_entry.get("cancel_requested") and live_progress is None ): - PentesterApi01Plugin._emit_timeline_event( - self, - job_specs, - "model_test_canceled_before_worker_start", - "Model test canceled before worker progress started", - meta={"worker_addr": worker_addr}, - ) PentesterApi01Plugin._write_terminal_model_test_worker_result( self, job_id, @@ -2203,56 +2315,29 @@ def _with_worker_assignment(job_config, worker_entry): def _mark_worker_terminal_error( self, job_specs, worker_addr, reason, error, context="worker_terminal_error", ): - """Mark one worker terminal in the shared job record and persist it. - - PR406 B8: instead of writing the launcher's stale snapshot back over - whatever the latest CStore record looks like, reload the current - record and patch only ``workers[worker_addr]``. Concurrent terminal - writes from two workers then merge by worker key instead of clobbering - each other. If the current record can't be loaded, fall back to the - incoming snapshot (with a warning). - """ + """Publish a worker-owned terminal launch failure in live progress.""" if not isinstance(job_specs, dict): return None - sanitize = getattr(getattr(self, "safety", None), "sanitize_error", None) - sanitized = sanitize(str(error)) if callable(sanitize) else str(error) - if not isinstance(sanitized, str): - sanitized = str(error) - - def _patch_worker(entry: dict): - entry["finished"] = True - entry["terminal_reason"] = reason - entry["error"] = sanitized - entry["result"] = None - return entry - job_id = job_specs.get("job_id", "") - current = None - if job_id: - current = PentesterApi01Plugin._get_job_state_repository(self).get_job(job_id) - - # Always reflect the patch in the caller's snapshot so any code that - # inspects job_specs after this call sees the worker as terminal. - workers_local = job_specs.setdefault("workers", {}) - _patch_worker(workers_local.setdefault(worker_addr, {})) - - if not isinstance(current, dict): - self.P( - f"[CSTORE] No current job record for {job_id}; writing stale snapshot for worker {worker_addr}", - color='y', - ) - return PentesterApi01Plugin._write_job_record( - self, job_id, job_specs, context=context, - ) - - # Merge: keep current top-level state, overlay the patched worker. - merged_workers = dict(current.get("workers") or {}) - merged_workers[worker_addr] = _patch_worker(dict(merged_workers.get(worker_addr) or {})) - merged = dict(current) - merged["workers"] = merged_workers - return PentesterApi01Plugin._write_job_record( - self, job_id, merged, context=context, + worker_entry = (job_specs.get("workers") or {}).get(worker_addr) or {} + now = self.time() + progress = WorkerProgress( + job_id=job_id, + worker_addr=worker_addr, + pass_nr=job_specs.get("job_pass", 1), + assignment_revision_seen=PentesterApi01Plugin._get_worker_assignment_revision(worker_entry), + progress=100.0, + phase="failed", + ports_scanned=0, + ports_total=0, + open_ports_found=[], + completed_tests=[], + updated_at=now, + last_seen_at=now, + finished=True, + error_class=reason, ) + return PentesterApi01Plugin._get_job_state_repository(self).put_live_progress_model(progress) def _log_audit_event(self, event_type, details): @@ -2711,18 +2796,22 @@ def _supports_guarded_job_writes(self): """ Return whether mutable RedMesh job writes have real guarded-write semantics. - The current chainstore API only exposes plain hget/hset primitives, so - RedMesh cannot claim compare-and-swap or optimistic concurrency guarantees. + RedMesh guards lifecycle ownership and terminal monotonicity before using + the underlying plain chainstore hset. This is not compare-and-swap. """ - return False + return True def _get_job_write_guarantees(self): """Describe the actual guarantees of mutable RedMesh job-state writes.""" return { - "mode": "detection_only", - "guarded_writes": False, + "mode": "launcher_single_writer", + "guarded_writes": True, "stale_write_detection": True, + "terminal_monotonicity": True, "job_revision": True, + "atomic_compare_and_swap": False, + "distributed_lease": False, + "duplicate_owner_exclusion": "operational", } def _write_job_record( @@ -2734,15 +2823,57 @@ def _write_job_record( reject_stale=False, ): """ - Persist mutable job state with revision bump and stale-write detection. + Persist launcher-owned lifecycle state with revision and terminal guards. - This does not provide compare-and-swap semantics. ``reject_stale`` closes - the detectable read-before-write window for sensitive completions, but a - distributed writer can still race the underlying plain hset. + This does not provide compare-and-swap semantics or a distributed lease; + duplicate launcher processes remain operationally excluded. """ - current = PentesterApi01Plugin._get_job_state_repository(self).get_job(job_id) + repo = PentesterApi01Plugin._get_job_state_repository(self) + current = repo.get_job(job_id) + incoming = job_specs if isinstance(job_specs, dict) else dict(job_specs) + + launcher = ( + current.get("launcher") if isinstance(current, dict) else None + ) or incoming.get("launcher") + if launcher and launcher != self.ee_addr: + self.P( + f"[CSTORE] Shared job write rejected for non-launcher {self.ee_addr}: " + f"job_id={job_id}, launcher={launcher}, context={context or 'unspecified'}", + color='y', + ) + self._log_audit_event("job_write_owner_rejected", { + "job_id": job_id, + "launcher": launcher, + "writer": self.ee_addr, + "context": context or "", + }) + return None + + if isinstance(current, dict) and is_terminal_job_status(current.get("job_status")): + same_status = incoming.get("job_status") == current.get("job_status") + current_cid = current.get("job_cid") + incoming_cid = incoming.get("job_cid") + same_archive = not current_cid or incoming_cid == current_cid + if not same_status or not same_archive: + self.P( + f"[CSTORE] Terminal lifecycle write rejected for job {job_id}: " + f"context={context or 'unspecified'}", + color='y', + ) + self._log_audit_event("terminal_write_rejected", { + "job_id": job_id, + "current_status": current.get("job_status"), + "incoming_status": incoming.get("job_status"), + "current_job_cid": current_cid, + "incoming_job_cid": incoming_cid, + "context": context or "", + }) + return current + if current_cid and context in {"archive_prune", "model_test_archive_prune"}: + return current + current_revision = PentesterApi01Plugin._get_job_revision(self, current) - incoming_revision = PentesterApi01Plugin._get_job_revision(self, job_specs) + incoming_revision = PentesterApi01Plugin._get_job_revision(self, incoming) if expected_revision is None: expected_revision = incoming_revision @@ -2777,9 +2908,9 @@ def _write_job_record( if reject_stale: return None - persisted = job_specs if isinstance(job_specs, dict) else dict(job_specs) + persisted = incoming persisted["job_revision"] = current_revision + 1 - normalized = PentesterApi01Plugin._get_job_state_repository(self).put_job(job_id, persisted) + normalized = repo.put_job(job_id, persisted) if isinstance(job_specs, dict) and isinstance(normalized, dict) and normalized is not job_specs: job_specs.clear() job_specs.update(normalized) @@ -2947,33 +3078,44 @@ def _close_job(self, job_id, canceled=False): ) self.P(f"Report saved to R1FS with CID: {report_cid}") else: - # Fallback: store report directly if R1FS fails - self.P("R1FS add_json returned None, storing report directly in CStore", color='y') + self.P("R1FS add_json returned None; publishing terminal storage failure", color='y') worker_entry["report_cid"] = None - worker_entry["result"] = report + worker_entry["result"] = None + PentesterApi01Plugin._mark_worker_terminal_error( + self, + job_specs, + self.ee_addr, + "report_storage_failed", + "Worker report artifact could not be stored", + ) except Exception as e: - # Fallback: store report directly if R1FS fails - self.P(f"Failed to save report to R1FS: {e}. Storing directly in CStore", color='r') + self.P(f"Failed to save report to R1FS: {e}", color='r') worker_entry["report_cid"] = None - worker_entry["result"] = report + worker_entry["result"] = None + PentesterApi01Plugin._mark_worker_terminal_error( + self, + job_specs, + self.ee_addr, + "report_storage_failed", + "Worker report artifact could not be stored", + ) else: self.P(f"No report data to save for job {job_id}", color='y') worker_entry["report_cid"] = None worker_entry["result"] = report - - # Re-read job_specs to avoid overwriting concurrent updates (e.g., pass_reports) - fresh_job_specs = PentesterApi01Plugin._get_job_state_repository(self).get_job(job_id) - if fresh_job_specs and isinstance(fresh_job_specs, dict): - fresh_job_specs["workers"][self.ee_addr] = worker_entry - job_specs = fresh_job_specs + PentesterApi01Plugin._mark_worker_terminal_error( + self, + job_specs, + self.ee_addr, + "report_unavailable", + "Worker report data was unavailable", + ) self.P("{} closing job_id {}:\n{}".format( closing, job_id, self.json_dumps(job_specs, indent=2) )) - PentesterApi01Plugin._write_job_record(self, job_id, job_specs, context="close_job") - # Audit: scan completed nr_findings = self._count_all_findings(report) self._log_audit_event("scan_completed", { @@ -3053,7 +3195,6 @@ def _maybe_stop_canceled_jobs(self): worker.stop() else: worker.stop() - self._write_job_record(job_id, job_specs, context="model_test_cancel_requested") self._publish_model_test_progress(job_id, worker, job_specs, finished=False, worker_addr=worker_addr) def _maybe_close_jobs(self): @@ -3126,6 +3267,12 @@ def _build_job_archive(self, job_key, job_specs): Full CStore job state. """ job_id = job_specs.get("job_id", job_key) + if job_specs.get("launcher") != self.ee_addr: + self.P(f"Skipping archive build for non-launcher job {job_id}", color='y') + return None + current = PentesterApi01Plugin._get_job_state_repository(self).get_job(job_key) + if isinstance(current, dict) and current.get("job_cid"): + return current.get("job_cid") # 1. Fetch job config and redact credentials for archive storage artifacts = PentesterApi01Plugin._get_artifact_repository(self) @@ -3860,9 +4007,10 @@ def launch_test( blockchain_attestation_enabled=blockchain_attestation_enabled, ) - @BasePlugin.endpoint(method="post") + @BasePlugin.endpoint(method="post", require_token=True) def launch_model_test( self, + token: str, task_name: str = "", task_description: str = "", selected_peers: list[str] = None, @@ -3882,6 +4030,9 @@ def launch_model_test( blockchain_attestation_enabled: bool = False, ): """Launch a Model Testing job through the model-test-specific gate.""" + auth_error = validate_backend_token(token) + if auth_error: + return auth_error return launch_model_test( self, task_name=task_name, @@ -3903,15 +4054,19 @@ def launch_model_test( blockchain_attestation_enabled=blockchain_attestation_enabled, ) - @BasePlugin.endpoint(method="post") + @BasePlugin.endpoint(method="post", require_token=True) def preflight_model_test_provider( self, + token: str, created_by_id: str = "", tested_model: dict = None, tested_model_secret_payload: dict = None, limits: dict = None, ): """Validate and transiently exercise a tested-model provider before launch.""" + auth_error = validate_backend_token(token) + if auth_error: + return auth_error return preflight_model_test_provider( self, created_by_id=created_by_id, @@ -4082,6 +4237,14 @@ def delete_job_engagement( if not isinstance(job_specs, dict): return {"error": "job_not_found", "message": f"job_specs for {job_id} not found"} + launcher = job_specs.get("launcher") + if launcher and launcher != self.ee_addr: + return { + "error": "job_launcher_mismatch", + "message": "Engagement deletion must be handled by the job launcher.", + "status_code": 409, + "job_id": job_id, + } if job_specs.get("job_cid"): return { "error": "unsupported_finalized_job", @@ -4120,7 +4283,11 @@ def delete_job_engagement( # Persist updated job_specs back to the running-state store so # the new job_config_cid + audit timeline survive. try: - state_repo.put_job(job_id, job_specs) + persisted = PentesterApi01Plugin._write_job_record( + self, job_id, job_specs, context="engagement_redaction", + ) + if persisted is None: + raise RuntimeError("launcher ownership changed before state persistence") except Exception as exc: self.P(f"engagement-deletion: sanitized config was written but " f"job_specs persistence failed: {exc}", color='r') @@ -4161,7 +4328,11 @@ def delete_job_engagement( ) break try: - state_repo.put_job(job_id, job_specs) + persisted = PentesterApi01Plugin._write_job_record( + self, job_id, job_specs, context="engagement_redaction_audit", + ) + if persisted is None: + raise RuntimeError("launcher ownership changed before audit persistence") except Exception as exc: self.P(f"engagement-deletion: document deletion completed but audit " f"persistence failed: {exc}", color='y') diff --git a/extensions/business/cybersec/red_mesh/services/control.py b/extensions/business/cybersec/red_mesh/services/control.py index 4864d4d6f..3ed003cca 100644 --- a/extensions/business/cybersec/red_mesh/services/control.py +++ b/extensions/business/cybersec/red_mesh/services/control.py @@ -48,11 +48,36 @@ def _delete_job_record(owner, job_id): _job_repo(owner).delete_job(job_id) +def _foreign_launcher_error(owner, job_id, job_specs, action): + launcher = job_specs.get("launcher") if isinstance(job_specs, dict) else None + if not launcher or launcher == getattr(owner, "ee_addr", None): + return None + owner.P(f"{action} rejected on non-launcher node for job {job_id}.", color='y') + owner._log_audit_event(f"job_{action}_owner_rejected", { + "job_id": job_id, + "writer": getattr(owner, "ee_addr", None), + }) + return { + "error": "job_launcher_mismatch", + "message": f"{action.replace('_', ' ').capitalize()} must be handled by the job launcher.", + "status_code": 409, + "job_id": job_id, + } + + def stop_and_delete_job(owner, job_id: str): """ Stop a running job, mark it stopped, then delegate to purge_job for full R1FS + CStore cleanup. """ + raw_job_specs = _job_repo(owner).get_job(job_id) + job_specs = None + if isinstance(raw_job_specs, dict): + _, job_specs = owner._normalize_job_record(job_id, raw_job_specs) + owner_error = _foreign_launcher_error(owner, job_id, job_specs, "stop_and_delete") + if owner_error: + return owner_error + local_workers = owner.scan_jobs.get(job_id) if local_workers: owner.P(f"Stopping and deleting job {job_id}.") @@ -62,9 +87,7 @@ def stop_and_delete_job(owner, job_id: str): owner.P(f"Job {job_id} stopped.") owner.scan_jobs.pop(job_id, None) - raw_job_specs = _job_repo(owner).get_job(job_id) - if isinstance(raw_job_specs, dict): - _, job_specs = owner._normalize_job_record(job_id, raw_job_specs) + if isinstance(job_specs, dict): workers_map = job_specs.setdefault("workers", {}) if is_model_test_job(job_specs): selected_worker = selected_model_test_worker_addr(job_specs, fallback=getattr(owner, "ee_addr", None)) @@ -122,6 +145,9 @@ def _purge_job_locked(owner, job_id: str): return {"status": "error", "message": f"Job {job_id} not found."} _, job_specs = owner._normalize_job_record(job_id, raw) + owner_error = _foreign_launcher_error(owner, job_id, job_specs, "purge") + if owner_error: + return owner_error job_status = job_specs.get("job_status", "") workers = job_specs.get("workers", {}) @@ -459,6 +485,15 @@ def purge_all_jobs(owner): terminal_statuses = (JOB_STATUS_FINALIZED, JOB_STATUS_STOPPED) for job_id, raw_payload in job_entries: + owner_error = _foreign_launcher_error(owner, job_id, raw_payload, "purge_all") + if owner_error: + jobs_failed += 1 + failed_job_ids.add(job_id) + errors.append({ + "job_id": job_id, + "message": owner_error["message"], + }) + continue raw_status = raw_payload.get("job_status") if isinstance(raw_payload, dict) else None use_direct_purge = raw_status in terminal_statuses try: @@ -745,6 +780,9 @@ def stop_monitoring(owner, job_id: str, stop_type: str = "SOFT"): return {"error": "Job not found", "job_id": job_id} _, job_specs = owner._normalize_job_record(job_id, raw_job_specs) + owner_error = _foreign_launcher_error(owner, job_id, job_specs, "stop_monitoring") + if owner_error: + return owner_error stop_type = str(stop_type).upper() is_continuous = job_specs.get("run_mode") == RUN_MODE_CONTINUOUS_MONITORING diff --git a/extensions/business/cybersec/red_mesh/services/query.py b/extensions/business/cybersec/red_mesh/services/query.py index d8f1087d8..b5d08c8de 100644 --- a/extensions/business/cybersec/red_mesh/services/query.py +++ b/extensions/business/cybersec/red_mesh/services/query.py @@ -503,7 +503,28 @@ def list_local_jobs(owner): """ Return jobs currently running on the local node. """ - return { + local_jobs = { job_id: owner._get_job_status(job_id) - for job_id, local_workers in owner.scan_jobs.items() + for job_id in getattr(owner, "scan_jobs", {}) } + for job_id, worker in getattr(owner, "model_test_jobs", {}).items(): + job_specs = _job_repo(owner).get_job(job_id) + if isinstance(job_specs, dict) and not _is_model_test_specs(job_specs): + continue + job_specs = job_specs if isinstance(job_specs, dict) else {} + worker_state = getattr(worker, "state", None) + live_summary = worker_state.get("model_test_summary") if isinstance(worker_state, dict) else None + job_status = job_specs.get("job_status") or "RUNNING" + local_jobs[job_id] = { + "job_id": job_id, + "status": job_status, + "job_status": job_status, + "job_type": MODEL_TEST_JOB_TYPE, + "task_kind": MODEL_TEST_JOB_TYPE, + "scan_type": MODEL_TEST_JOB_TYPE, + "model_test_summary": sanitize_model_test_summary( + live_summary or job_specs.get("model_test_summary") + ), + "model_test_node_selection": job_specs.get("model_test_node_selection"), + } + return local_jobs diff --git a/extensions/business/cybersec/red_mesh/services/reconciliation.py b/extensions/business/cybersec/red_mesh/services/reconciliation.py index 3c4c087f9..381929f34 100644 --- a/extensions/business/cybersec/red_mesh/services/reconciliation.py +++ b/extensions/business/cybersec/red_mesh/services/reconciliation.py @@ -1,3 +1,4 @@ +from ..constants import JOB_STATUS_STOPPED from ..models import WorkerProgress from .config import resolve_config_block @@ -228,7 +229,15 @@ def reconcile_workers_from_live(owner, job_id, *, live_payloads=None, now=None, if not live.finished: continue if not live.report_cid: - _stats_inc_once(stats, "ignored_no_report_cid", (job_id, worker_addr)) + if not live.error_class: + _stats_inc_once(stats, "ignored_no_report_cid", (job_id, worker_addr)) + continue + worker_entry["finished"] = True + worker_entry["terminal_reason"] = live.error_class + worker_entry["error_class"] = live.error_class + worker_entry["result"] = None + job_specs["job_status"] = JOB_STATUS_STOPPED + changed_workers.append(worker_addr) continue if not worker_entry.get("report_cid"): diff --git a/extensions/business/cybersec/red_mesh/services/triage.py b/extensions/business/cybersec/red_mesh/services/triage.py index addbff7d3..d8cd23595 100644 --- a/extensions/business/cybersec/red_mesh/services/triage.py +++ b/extensions/business/cybersec/red_mesh/services/triage.py @@ -22,6 +22,16 @@ def _artifact_repo(owner): return ArtifactRepository(owner) +def _write_job_record(owner, job_id, job_specs, context): + writer = getattr(type(owner), "_write_job_record", None) + if callable(writer): + return writer(owner, job_id, job_specs, context=context) + launcher = job_specs.get("launcher") if isinstance(job_specs, dict) else None + if launcher and launcher != getattr(owner, "ee_addr", None): + return None + return _job_repo(owner).put_job(job_id, job_specs) + + def _archive_contains_finding(archive: dict, finding_id: str) -> bool: return _find_archive_finding(archive, finding_id) is not None @@ -97,6 +107,14 @@ def _update_finding_triage_locked(owner, job_id: str, finding_id: str, status: s return {"error": "not_found", "message": f"Job {job_id} not found."} if not job_specs.get("job_cid"): return {"error": "not_available", "message": f"Job {job_id} is still running (triage requires archived findings)."} + launcher = job_specs.get("launcher") + if launcher and launcher != getattr(owner, "ee_addr", None): + return { + "error": "job_launcher_mismatch", + "message": "Finding triage must be handled by the job launcher.", + "status_code": 409, + "job_id": job_id, + } archive = _artifact_repo(owner).get_archive(job_specs) if not isinstance(archive, dict): @@ -140,7 +158,7 @@ def _update_finding_triage_locked(owner, job_id: str, finding_id: str, status: s event_action="triaged", ) if isinstance(job_specs.get("soc_event_status"), dict): - repo.put_job(job_id, job_specs) + _write_job_record(owner, job_id, job_specs, context="finding_triage_soc_event") return { "job_id": job_id, "finding_id": finding_id, diff --git a/extensions/business/cybersec/red_mesh/tests/test_api.py b/extensions/business/cybersec/red_mesh/tests/test_api.py index 0b6cbf642..532e2328b 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_api.py +++ b/extensions/business/cybersec/red_mesh/tests/test_api.py @@ -1,7 +1,10 @@ +import ast +import inspect import json import sys import struct import time +import textwrap import unittest from concurrent.futures import Future from copy import deepcopy @@ -3259,6 +3262,23 @@ def test_archive_written_to_r1fs(self): self.assertIn("ui_aggregate", archive_dict) self.assertIn("total_open_ports", archive_dict["ui_aggregate"]) + def test_duplicate_archive_finalization_reuses_existing_cid(self): + """Retrying finalization after prune is harmless and writes no artifact.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin() + plugin.chainstore_hget.return_value = { + "job_id": "test-job", + "job_status": "FINALIZED", + "launcher": "launcher-node", + "job_cid": "QmExistingArchive", + } + + result = Plugin._build_job_archive(plugin, "test-job", job_specs) + + self.assertEqual(result, "QmExistingArchive") + plugin.r1fs.add_json.assert_not_called() + plugin.chainstore_hset.assert_not_called() + def test_archive_ui_aggregate_includes_graybox_summary(self): """Archive UI aggregate preserves graybox scan metadata and scenario counts.""" Plugin = self._get_plugin_class() @@ -3904,17 +3924,21 @@ def test_write_job_record_bumps_revision(self): self.assertEqual(running.job_revision, 3) plugin._log_audit_event.assert_not_called() - def test_job_write_guarantees_report_detection_only_mode(self): - """RedMesh exposes detection-only semantics when chainstore lacks CAS.""" + def test_job_write_guarantees_report_launcher_guard_mode(self): + """Lifecycle guards are explicit without claiming chainstore CAS.""" Plugin = self._get_plugin_class() plugin = self._build_plugin({}) - self.assertFalse(Plugin._supports_guarded_job_writes(plugin)) + self.assertTrue(Plugin._supports_guarded_job_writes(plugin)) self.assertEqual(Plugin._get_job_write_guarantees(plugin), { - "mode": "detection_only", - "guarded_writes": False, + "mode": "launcher_single_writer", + "guarded_writes": True, "stale_write_detection": True, + "terminal_monotonicity": True, "job_revision": True, + "atomic_compare_and_swap": False, + "distributed_lease": False, + "duplicate_owner_exclusion": "operational", }) def test_write_job_record_logs_stale_write(self): @@ -3935,9 +3959,78 @@ def test_write_job_record_logs_stale_write(self): "expected_revision": 3, "current_revision": 5, "context": "close_job", - "write_mode": "detection_only", + "write_mode": "launcher_single_writer", }) + def test_shared_job_write_rejects_non_launcher(self): + Plugin = self._get_plugin_class() + current = { + "job_id": "job-1", + "job_status": "RUNNING", + "launcher": "launcher-node", + "job_revision": 2, + } + plugin = self._build_plugin({"job-1": current}) + plugin.ee_addr = "worker-node" + plugin.chainstore_hset = MagicMock() + plugin._log_audit_event = MagicMock() + plugin.P = MagicMock() + + result = Plugin._write_job_record( + plugin, "job-1", dict(current), context="stix_export", + ) + + self.assertIsNone(result) + plugin.chainstore_hset.assert_not_called() + plugin._log_audit_event.assert_called_once() + + def test_terminal_write_cannot_regress(self): + Plugin = self._get_plugin_class() + current = { + "job_id": "job-1", + "job_status": "FINALIZED", + "launcher": "launcher-node", + "job_cid": "cid-final", + "job_revision": 4, + } + plugin = self._build_plugin({"job-1": current}) + plugin.chainstore_hset = MagicMock() + plugin._log_audit_event = MagicMock() + plugin.P = MagicMock() + + result = Plugin._write_job_record( + plugin, + "job-1", + {**current, "job_status": "RUNNING", "job_cid": None}, + context="finalize_collecting", + ) + + self.assertEqual(result, current) + plugin.chainstore_hset.assert_not_called() + plugin._log_audit_event.assert_called_once() + + def test_duplicate_terminal_archive_write_is_idempotent(self): + Plugin = self._get_plugin_class() + current = { + "job_id": "job-1", + "job_status": "FINALIZED", + "launcher": "launcher-node", + "job_cid": "cid-final", + "job_revision": 4, + } + plugin = self._build_plugin({"job-1": current}) + plugin.chainstore_hset = MagicMock() + plugin._log_audit_event = MagicMock() + plugin.P = MagicMock() + + result = Plugin._write_job_record( + plugin, "job-1", dict(current), context="archive_prune", + ) + + self.assertEqual(result, current) + plugin.chainstore_hset.assert_not_called() + plugin._log_audit_event.assert_not_called() + def test_get_job_config_resolves_secret_ref_for_runtime(self): """Runtime config loading resolves secret_ref into inline credentials.""" Plugin = self._get_plugin_class() @@ -4008,7 +4101,7 @@ def test_get_job_config_fails_closed_for_malformed_secret_payload(self): ) self.assertEqual(len(plugin.r1fs.get_json.call_args_list), 2) - def test_mark_worker_terminal_error_sets_common_fields(self): + def test_mark_worker_terminal_error_publishes_live_terminal_state(self): Plugin = self._get_plugin_class() plugin = self._build_plugin({}) job_specs = { @@ -4016,92 +4109,27 @@ def test_mark_worker_terminal_error_sets_common_fields(self): "workers": {"worker-a": {"start_port": 443, "end_port": 443}}, } - with patch.object(Plugin, "_write_job_record", return_value=job_specs) as write: - Plugin._mark_worker_terminal_error( - plugin, - job_specs, - "worker-a", - "secret_resolution_failed", - "Failed to resolve graybox secret_ref", - context="test_terminal", - ) - - worker = job_specs["workers"]["worker-a"] - self.assertTrue(worker["finished"]) - self.assertEqual(worker["terminal_reason"], "secret_resolution_failed") - self.assertIn("secret_ref", worker["error"]) - write.assert_called_once() - - def test_mark_worker_terminal_error_merges_against_current_record(self): - """B8: concurrent terminal writes must merge by worker key, not overwrite.""" - Plugin = self._get_plugin_class() - # Current record in CStore has worker-A already terminal (written by - # worker A's concurrent failure). - current_record = { - "job_id": "job-concurrent", - "job_status": "RUNNING", - "job_pass": 1, - "run_mode": "SINGLEPASS", - "launcher": "launcher-node", - "target": "example.com", - "scan_type": "webapp", - "target_url": "https://example.com/app", - "start_port": 443, - "end_port": 443, - "date_created": 1000000.0, - "job_config_cid": "QmConfig", - "workers": { - "worker-A": { - "start_port": 443, "end_port": 443, - "finished": True, - "terminal_reason": "assignment_validation_failed", - "error": "A error", - }, - "worker-B": {"start_port": 443, "end_port": 443, "finished": False}, - }, - "timeline": [], - "pass_reports": [], - "job_revision": 7, - } - plugin = self._build_plugin({"job-concurrent": current_record}) - - # Worker-B's stale local snapshot doesn't know about A's terminal flag. - stale_snapshot = { - "job_id": "job-concurrent", - "workers": { - "worker-A": {"start_port": 443, "end_port": 443, "finished": False}, - "worker-B": {"start_port": 443, "end_port": 443, "finished": False}, - }, - } - - captured = {} - - def _capture(self_plugin, job_id, job_specs, expected_revision=None, context=""): - captured["job_id"] = job_id - captured["job_specs"] = dict(job_specs) - captured["context"] = context - return job_specs - - with patch.object(Plugin, "_write_job_record", side_effect=_capture): - Plugin._mark_worker_terminal_error( - plugin, - stale_snapshot, - "worker-B", - "launch_failed", - "B error", - context="b_terminal", - ) + plugin.time.return_value = 100.0 + Plugin._mark_worker_terminal_error( + plugin, + job_specs, + "worker-a", + "secret_resolution_failed", + "Failed to resolve graybox secret_ref", + context="test_terminal", + ) - persisted_workers = captured["job_specs"]["workers"] - # A's pre-existing terminal data survived the B write. - self.assertTrue(persisted_workers["worker-A"]["finished"]) - self.assertEqual(persisted_workers["worker-A"]["terminal_reason"], "assignment_validation_failed") - self.assertEqual(persisted_workers["worker-A"]["error"], "A error") - # B's terminal patch is applied. - self.assertTrue(persisted_workers["worker-B"]["finished"]) - self.assertEqual(persisted_workers["worker-B"]["terminal_reason"], "launch_failed") + live_call = next( + call for call in plugin.chainstore_hset.call_args_list + if call.kwargs.get("hkey") == "test-instance:live" + ) + live = live_call.kwargs["value"] + self.assertTrue(live["finished"]) + self.assertEqual(live["error_class"], "secret_resolution_failed") + self.assertNotIn("error", live) + self.assertNotIn("error_message", live) - def test_maybe_launch_jobs_secret_resolution_failure_marks_terminal(self): + def test_maybe_launch_jobs_secret_resolution_failure_publishes_terminal_live(self): Plugin = self._get_plugin_class() assignments, error = build_graybox_worker_assignments(["launcher-node"]) self.assertIsNone(error) @@ -4146,10 +4174,33 @@ def test_maybe_launch_jobs_secret_resolution_failure_marks_terminal(self): with patch.object(Plugin, "_write_job_record", return_value=job_specs) as write: Plugin._maybe_launch_jobs(plugin) - self.assertTrue(worker_entry["finished"]) - self.assertEqual(worker_entry["terminal_reason"], "secret_resolution_failed") - self.assertIn("secret_ref", worker_entry["error"]) - write.assert_called_once() + write.assert_not_called() + live_call = next( + call for call in plugin.chainstore_hset.call_args_list + if call.kwargs.get("hkey") == "test-instance:live" + ) + self.assertTrue(live_call.kwargs["value"]["finished"]) + self.assertEqual(live_call.kwargs["value"]["error_class"], "secret_resolution_failed") + + def test_worker_background_paths_have_no_shared_lifecycle_writer_calls(self): + """Structural guard: worker/background paths cannot call the shared writer.""" + Plugin = self._get_plugin_class() + functions = ( + Plugin._maybe_launch_model_test_jobs, + Plugin._maybe_close_model_test_jobs, + Plugin._maybe_stop_canceled_jobs, + Plugin._close_job, + Plugin._mark_worker_terminal_error, + ) + for function in functions: + tree = ast.parse(textwrap.dedent(inspect.getsource(function))) + writer_calls = [ + node for node in ast.walk(tree) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "_write_job_record" + ] + self.assertEqual(writer_calls, [], function.__name__) def test_get_job_data_running_last_5(self): """Running job with 8 passes returns last 5 refs only.""" @@ -4773,6 +4824,99 @@ def test_get_job_triage_not_found(self): self.assertEqual(result["audit"], []) +class TestModelTestingEndpointAuth(unittest.TestCase): + """Protected Model Testing endpoints validate the backend bearer token.""" + + @classmethod + def setUpClass(cls): + mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + cls.Plugin = PentesterApi01Plugin + + def test_missing_backend_token_configuration_fails_closed(self): + plugin = MagicMock() + with patch.dict("os.environ", {}, clear=True), patch( + "extensions.business.cybersec.red_mesh.pentester_api_01.launch_model_test" + ) as launch: + result = self.Plugin.launch_model_test(plugin, "presented-token") + + self.assertEqual(result["status_code"], 401) + self.assertEqual(result["error_class"], "backend_auth_unavailable") + self.assertNotIn("presented-token", str(result)) + launch.assert_not_called() + + def test_invalid_backend_token_is_forbidden_without_leak(self): + plugin = MagicMock() + expected = "expected-backend-token-material-32-bytes" + with patch.dict("os.environ", {"REDMESH_BACKEND_TOKEN": expected}), patch( + "extensions.business.cybersec.red_mesh.pentester_api_01.launch_model_test" + ) as launch: + result = self.Plugin.launch_model_test(plugin, "invalid-presented-token") + + self.assertEqual(result["status_code"], 403) + self.assertEqual(result["error_class"], "backend_auth_invalid") + self.assertNotIn(expected, str(result)) + self.assertNotIn("invalid-presented-token", str(result)) + launch.assert_not_called() + + def test_short_backend_token_configuration_fails_closed(self): + plugin = MagicMock() + with patch.dict("os.environ", {"REDMESH_BACKEND_TOKEN": "short-test-token"}), patch( + "extensions.business.cybersec.red_mesh.pentester_api_01.launch_model_test" + ) as launch: + result = self.Plugin.launch_model_test(plugin, "short-test-token") + + self.assertEqual(result["status_code"], 401) + self.assertEqual(result["error_class"], "backend_auth_unavailable") + self.assertNotIn("short-test-token", str(result)) + launch.assert_not_called() + + def test_empty_presented_token_is_unauthorized(self): + plugin = MagicMock() + expected = "expected-backend-token-material-32-bytes" + with patch.dict("os.environ", {"REDMESH_BACKEND_TOKEN": expected}), patch( + "extensions.business.cybersec.red_mesh.pentester_api_01.launch_model_test" + ) as launch: + result = self.Plugin.launch_model_test(plugin, "") + + self.assertEqual(result["status_code"], 401) + self.assertEqual(result["error_class"], "backend_auth_required") + self.assertNotIn(expected, str(result)) + launch.assert_not_called() + + def test_authenticated_launch_forwards_navigator_actor_assertion(self): + plugin = MagicMock() + token = "valid-backend-token-material-at-least-32-bytes" + with patch.dict("os.environ", {"REDMESH_BACKEND_TOKEN": token}), patch( + "extensions.business.cybersec.red_mesh.pentester_api_01.launch_model_test", + return_value={"status": "ok"}, + ) as launch: + result = self.Plugin.launch_model_test( + plugin, + token, + created_by_id="navigator-user-123", + ) + + self.assertEqual(result, {"status": "ok"}) + self.assertEqual(launch.call_args.kwargs["created_by_id"], "navigator-user-123") + + def test_authenticated_preflight_forwards_navigator_actor_assertion(self): + plugin = MagicMock() + token = "valid-backend-token-material-at-least-32-bytes" + with patch.dict("os.environ", {"REDMESH_BACKEND_TOKEN": token}), patch( + "extensions.business.cybersec.red_mesh.pentester_api_01.preflight_model_test_provider", + return_value={"status": "ok"}, + ) as preflight: + result = self.Plugin.preflight_model_test_provider( + plugin, + token, + created_by_id="navigator-user-123", + ) + + self.assertEqual(result, {"status": "ok"}) + self.assertEqual(preflight.call_args.kwargs["created_by_id"], "navigator-user-123") + + class TestPhase2AuditCounting(unittest.TestCase): """Phase 2: audit counts include graybox findings.""" diff --git a/extensions/business/cybersec/red_mesh/tests/test_integration.py b/extensions/business/cybersec/red_mesh/tests/test_integration.py index 6d26cc1c5..14d705628 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_integration.py +++ b/extensions/business/cybersec/red_mesh/tests/test_integration.py @@ -567,14 +567,17 @@ def test_publish_live_progress_zero_interval_uses_default(self): self.assertEqual(Plugin._get_progress_publish_interval(plugin), 30.0) plugin.chainstore_hset.assert_not_called() - def test_job_write_guarantees_are_detection_only(self): - """Mutable job writes explicitly advertise detection-only semantics.""" + def test_job_write_guarantees_are_launcher_guarded_without_cas(self): + """Mutable lifecycle writes advertise ownership guards, not atomic CAS.""" Plugin = self._get_plugin_class() plugin = MagicMock() - self.assertFalse(Plugin._supports_guarded_job_writes(plugin)) - self.assertEqual(Plugin._get_job_write_guarantees(plugin)["mode"], "detection_only") - self.assertFalse(Plugin._get_job_write_guarantees(plugin)["guarded_writes"]) + self.assertTrue(Plugin._supports_guarded_job_writes(plugin)) + guarantees = Plugin._get_job_write_guarantees(plugin) + self.assertEqual(guarantees["mode"], "launcher_single_writer") + self.assertTrue(guarantees["guarded_writes"]) + self.assertFalse(guarantees["atomic_compare_and_swap"]) + self.assertFalse(guarantees["distributed_lease"]) def test_live_hsync_due_uses_fixed_config_interval(self): """Launcher live-hsync schedule uses the normalized fixed interval.""" @@ -2113,6 +2116,29 @@ def test_status_error_falls_back_to_force_purge(self): deleted_cids = {c.args[0] for c in plugin.r1fs.delete_file.call_args_list} self.assertEqual(deleted_cids, {"cid-x"}) + def test_bulk_purge_preserves_foreign_launcher_jobs_without_force_fallback(self): + Plugin = self._get_plugin_class() + jobs = { + "job-foreign": { + "job_id": "job-foreign", + "job_status": "FINALIZED", + "job_cid": "cid-foreign", + "launcher": "node-B", + }, + } + plugin = self._make_plugin(jobs) + plugin.r1fs = MagicMock() + + result = Plugin.purge_all_redmesh_data(plugin, confirm=True) + + self.assertEqual(result["status"], "partial") + self.assertEqual(result["jobs_failed"], 1) + self.assertEqual(result["jobs_force_purged"], 0) + self.assertIn("job-foreign", plugin._hashes["test-instance"]) + plugin.purge_job.assert_not_called() + plugin.stop_and_delete_job.assert_not_called() + plugin.r1fs.delete_file.assert_not_called() + def test_partial_status_preserves_state_no_force_purge(self): """status='partial' is the retry contract — state preserved, no force-purge.""" Plugin = self._get_plugin_class() diff --git a/extensions/business/cybersec/red_mesh/tests/test_model_testing.py b/extensions/business/cybersec/red_mesh/tests/test_model_testing.py index 4f728c618..5b92efced 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_model_testing.py +++ b/extensions/business/cybersec/red_mesh/tests/test_model_testing.py @@ -934,17 +934,19 @@ def resolver(hostname, port, type=socket.SOCK_STREAM): self.assertIsNotNone(err) self.assertEqual(err["error_class"], "forbidden_destination") - def test_duplicate_credential_sources_fail_closed(self): + def test_credential_ref_with_secret_payload_is_rejected_without_identifier_leak(self): + credential_ref = "model_provider/operator/user-123/provider-a" _, err = validate_model_provider_credentials( { - "credential_ref": "model_provider/operator/user-123/provider-a", + "credential_ref": credential_ref, }, {"api_key": "secret"}, role="tested_model", created_by_id="user-123", ) - self.assertEqual(err["error_class"], "duplicate_credential_source") + self.assertEqual(err["error_class"], "credential_unavailable") + self.assertNotIn(credential_ref, str(err)) def test_inline_credential_fields_in_provider_config_fail_closed(self): owner = _owner(cfg_model_testing={"ENABLED": True}) @@ -1014,6 +1016,20 @@ def test_invalid_credential_refs_use_same_sanitized_error(self): self.assertEqual(err["error_class"], "credential_unavailable") self.assertNotIn(ref, str(err)) + def test_launch_rejects_validly_shaped_credential_ref_before_persistence(self): + owner = _owner(cfg_model_testing={"ENABLED": True}) + credential_ref = "model_provider/operator/user-123/provider-a" + kwargs = _valid_launch_kwargs() + kwargs["tested_model"] = _provider(credential_ref=credential_ref) + kwargs["tested_model_secret_payload"] = None + + result = launch_model_test(owner, **kwargs) + + self.assertEqual(result["error_class"], "credential_unavailable") + self.assertNotIn(credential_ref, str(result)) + owner.r1fs.add_json.assert_not_called() + owner.chainstore_hset.assert_not_called() + def test_preflight_model_test_provider_accepts_valid_remote_provider(self): owner = _owner(cfg_model_testing={"ENABLED": True}) secret = "sentinel-model-api-key" @@ -1064,17 +1080,19 @@ def test_preflight_model_test_provider_returns_sanitized_provider_auth_failure(s def test_preflight_model_test_provider_requires_api_key_payload(self): owner = _owner(cfg_model_testing={"ENABLED": True}) + credential_ref = "model_provider/operator/user-123/provider-a" result = preflight_model_test_provider( owner, created_by_id="user-123", - tested_model=_provider(credential_ref="model_provider/operator/user-123/provider-a"), + tested_model=_provider(credential_ref=credential_ref), tested_model_secret_payload=None, ) self.assertFalse(result["ok"]) self.assertEqual(result["error_class"], "credential_unavailable") self.assertIn("requires an API key", result["message"]) + self.assertNotIn(credential_ref, str(result)) owner.r1fs.add_json.assert_not_called() owner.chainstore_hset.assert_not_called() @@ -1379,6 +1397,11 @@ def test_model_test_finalization_requires_end_attestation_for_success(self): "job_type": "model_test", "blockchain_attestation_enabled": True, } + plugin.r1fs.add_json.return_value = "QmArchiveCID" + plugin.r1fs.get_json.side_effect = [ + {"job_type": "model_test", "blockchain_attestation_enabled": True}, + {"job_id": "job-123"}, + ] plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) job_specs = { "job_id": "job-123", @@ -1409,15 +1432,17 @@ def test_model_test_finalization_requires_end_attestation_for_success(self): "QmWorkerResult", ) - self.assertFalse(result) + self.assertTrue(result) plugin._submit_redmesh_test_attestation.assert_called_once() - plugin.r1fs.add_json.assert_not_called() - plugin._write_job_record.assert_called_with( - "job-123", - job_specs, - context="model_test_attestation_failed", - ) - self.assertEqual(job_specs["job_status"], "FAILED") + archive = plugin.r1fs.add_json.call_args.args[0] + terminal_events = [event for event in archive["timeline"] if event["type"] in {"finalized", "failed", "canceled"}] + self.assertEqual(len(terminal_events), 1) + self.assertEqual(terminal_events[0]["type"], "failed") + self.assertEqual(terminal_events[0]["meta"]["overall_status"], "complete") + self.assertEqual(terminal_events[0]["meta"]["error_class"], "finalization_failed") + stub = plugin._write_job_record.call_args.args[1] + self.assertEqual(stub["job_status"], "FAILED") + self.assertEqual(stub["failure_class"], "attestation_failed") self.assertEqual(job_specs["failure_class"], "attestation_failed") def test_model_test_finalization_end_attestation_exception_marks_failed(self): @@ -1434,6 +1459,11 @@ def test_model_test_finalization_end_attestation_exception_marks_failed(self): "job_type": "model_test", "blockchain_attestation_enabled": True, } + plugin.r1fs.add_json.return_value = "QmArchiveCID" + plugin.r1fs.get_json.side_effect = [ + {"job_type": "model_test", "blockchain_attestation_enabled": True}, + {"job_id": "job-123"}, + ] plugin._submit_redmesh_test_attestation = MagicMock(side_effect=RuntimeError("chain offline")) job_specs = { "job_id": "job-123", @@ -1464,9 +1494,11 @@ def test_model_test_finalization_end_attestation_exception_marks_failed(self): "QmWorkerResult", ) - self.assertFalse(result) - plugin.r1fs.add_json.assert_not_called() - self.assertEqual(job_specs["job_status"], "FAILED") + self.assertTrue(result) + archive = plugin.r1fs.add_json.call_args.args[0] + self.assertEqual(archive["timeline"][-1]["type"], "failed") + stub = plugin._write_job_record.call_args.args[1] + self.assertEqual(stub["job_status"], "FAILED") self.assertEqual(job_specs["failure_class"], "attestation_failed") def test_model_test_finalization_stores_successful_end_attestation(self): @@ -1526,6 +1558,43 @@ def test_model_test_finalization_stores_successful_end_attestation(self): self.assertEqual(stub["job_status"], "FINALIZED") self.assertTrue(stub["blockchain_attestation_enabled"]) + def test_model_test_terminal_event_is_idempotent(self): + mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + + plugin = MagicMock() + plugin.ee_addr = "launcher-node" + plugin.ee_id = "Launcher" + plugin.time.return_value = 200.0 + job_specs = { + "job_id": "job-idempotent", + "launcher": "launcher-node", + "model_test_summary": { + "overall_status": "incomplete", + "cases_completed": 4, + "cases_total": 12, + }, + "model_test_node_selection": {"selected_execution_node": "worker-node"}, + "workers": {"worker-node": {}}, + "timeline": [], + } + + PentesterApi01Plugin._terminalize_model_test_job( + plugin, job_specs, "incomplete", "FINALIZED", + ) + PentesterApi01Plugin._terminalize_model_test_job( + plugin, job_specs, "incomplete", "FINALIZED", + ) + + terminal_events = [event for event in job_specs["timeline"] if event["type"] == "finalized"] + self.assertEqual(len(terminal_events), 1) + self.assertEqual(terminal_events[0]["meta"], { + "overall_status": "incomplete", + "selected_execution_node": "worker-node", + "cases_completed": 4, + "cases_total": 12, + }) + def test_model_test_finalization_records_raw_evidence_capture_failed_when_requested_without_artifact(self): mock_plugin_modules() from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin @@ -1568,9 +1637,9 @@ def test_model_test_finalization_records_raw_evidence_capture_failed_when_reques "job-raw", job_specs, { - "status": "completed", - "model_test_results": {"overall_status": "completed", "cases": []}, - "model_test_summary": {"overall_status": "completed"}, + "status": "incomplete", + "model_test_results": {"overall_status": "incomplete", "cases": []}, + "model_test_summary": {"overall_status": "incomplete"}, }, "QmWorkerResult", ) @@ -1588,8 +1657,10 @@ def test_model_test_finalization_records_raw_evidence_capture_failed_when_reques self.assertEqual(stub["model_test_raw_evidence"]["status"], RAW_EVIDENCE_STATUS_CAPTURE_FAILED) self.assertEqual(stub["model_test_raw_evidence"]["error_class"], RAW_EVIDENCE_ERROR_CAPTURE_UNAVAILABLE) event_types = [event["type"] for event in archive_payload["timeline"]] - self.assertIn("completed", event_types) - self.assertIn("finalized", event_types) + self.assertEqual(event_types.count("finalized"), 1) + terminal_event = next(event for event in archive_payload["timeline"] if event["type"] == "finalized") + self.assertEqual(terminal_event["meta"]["overall_status"], "incomplete") + self.assertEqual(archive_payload["model_test_summary"]["overall_status"], "incomplete") def test_model_test_finalization_stores_requested_raw_evidence_in_restricted_lane(self): mock_plugin_modules() @@ -1677,8 +1748,9 @@ def add_json(payload, show_logs=False, secret=None): self.assertNotIn("raw prompt secret", str(archive_payload)) self.assertNotIn("raw answer secret", str(archive_payload)) event_types = [event["type"] for event in archive_payload["timeline"]] - self.assertIn("completed", event_types) - self.assertIn("finalized", event_types) + self.assertEqual(event_types.count("finalized"), 1) + terminal_event = next(event for event in archive_payload["timeline"] if event["type"] == "finalized") + self.assertEqual(terminal_event["meta"]["overall_status"], "complete") raw_metadata_write = next( call.kwargs["value"] @@ -2036,6 +2108,21 @@ def test_falsey_non_list_selected_peers_rejected(self): class TestModelTestingPersistenceContracts(unittest.TestCase): + def test_legacy_completed_status_remains_readable(self): + from extensions.business.cybersec.red_mesh.model_test_sanitization import ( + sanitize_model_test_results, + sanitize_model_test_summary, + ) + + self.assertEqual( + sanitize_model_test_summary({"overall_status": "completed"})["overall_status"], + "completed", + ) + self.assertEqual( + sanitize_model_test_results({"overall_status": "completed"})["overall_status"], + "completed", + ) + def test_model_test_artifact_serializers_strip_raw_payload_fields(self): from extensions.business.cybersec.red_mesh.model_testing.artifacts import ( ModelTestArchive, @@ -2475,6 +2562,39 @@ def test_model_test_listing_preserves_summary_and_node_selection(self): self.assertEqual(jobs["job-1"]["model_test_summary"]["overall_status"], "queued") self.assertEqual(jobs["job-1"]["model_test_node_selection"]["selected_execution_node"], "node-a") + def test_local_listing_includes_sanitized_model_test_job(self): + from extensions.business.cybersec.red_mesh.services.query import list_local_jobs + + job_specs = { + "job_id": "job-1", + "job_status": "RUNNING", + "job_type": "model_test", + "scan_type": "model_test", + "model_test_summary": {"overall_status": "queued"}, + "model_test_node_selection": {"selected_execution_node": "node-a"}, + } + worker = MagicMock() + worker.state = { + "model_test_summary": { + "overall_status": "running", + "error_message": "raw provider exception secret-token", + }, + "error_message": "raw worker exception secret-token", + } + owner = _owner() + owner.scan_jobs = {} + owner.model_test_jobs = {"job-1": worker} + owner.chainstore_hget = MagicMock() + owner.chainstore_hget.return_value = job_specs + + jobs = list_local_jobs(owner) + + self.assertEqual(jobs["job-1"]["job_type"], "model_test") + self.assertEqual(jobs["job-1"]["task_kind"], "model_test") + self.assertEqual(jobs["job-1"]["model_test_summary"]["overall_status"], "running") + self.assertNotIn("error_message", str(jobs["job-1"])) + self.assertNotIn("secret-token", str(jobs["job-1"])) + def test_finalized_cstore_model_preserves_model_test_fields(self): finalized = CStoreJobFinalized( job_id="job-1", @@ -2648,7 +2768,7 @@ def chat(self, messages, *, max_tokens, temperature): worker = selected.model_test_jobs["job-1"] self.assertIsInstance(worker, ModelTestWorker) worker.thread.join(timeout=1) - self.assertEqual(worker.state["model_test_summary"]["overall_status"], "completed") + self.assertEqual(worker.state["model_test_summary"]["overall_status"], "complete") self.assertEqual(worker.state["model_test_summary"]["evaluated_cases"], 12) self.assertEqual(len(worker.state["model_test_results"]["cases"]), 12) self.assertEqual(worker.state["model_test_results"]["cases"][0]["status"], "evaluated") @@ -2800,6 +2920,100 @@ def test_stop_monitoring_marks_selected_model_test_worker_cancel_requested(self) self.assertEqual(job_specs["model_test_summary"]["overall_status"], "cancel_requested") self.assertNotIn("launcher-node", job_specs["workers"]) + def test_stop_monitoring_rejects_foreign_launcher_without_mutation(self): + from extensions.business.cybersec.red_mesh.services.control import stop_monitoring + + stored = { + "job_id": "job-1", + "job_status": "RUNNING", + "job_type": "model_test", + "run_mode": "SINGLEPASS", + "launcher": "launcher-node", + "workers": {"node-a": {"worker_type": "model_test", "finished": False}}, + } + local_worker = MagicMock() + owner = _owner( + ee_addr="worker-node", + chainstore_hget=MagicMock(return_value=deepcopy(stored)), + ) + owner.scan_jobs = {} + owner.model_test_jobs = {"job-1": local_worker} + owner._normalize_job_record = MagicMock( + side_effect=lambda key, specs: (key, deepcopy(specs)), + ) + owner._emit_timeline_event = MagicMock() + owner._log_audit_event = MagicMock() + owner.P = MagicMock() + + result = stop_monitoring(owner, "job-1", stop_type="HARD") + + self.assertEqual(result["error"], "job_launcher_mismatch") + self.assertEqual(result["status_code"], 409) + self.assertEqual(stored["job_status"], "RUNNING") + owner.chainstore_hset.assert_not_called() + owner._emit_timeline_event.assert_not_called() + local_worker.stop.assert_not_called() + + def test_stop_and_delete_rejects_foreign_launcher_before_side_effects(self): + from extensions.business.cybersec.red_mesh.services.control import stop_and_delete_job + + stored = { + "job_id": "job-1", + "job_status": "RUNNING", + "job_type": "model_test", + "launcher": "launcher-node", + "workers": {"node-a": {"worker_type": "model_test", "finished": False}}, + } + local_worker = MagicMock() + owner = _owner( + ee_addr="worker-node", + chainstore_hget=MagicMock(return_value=deepcopy(stored)), + ) + owner.scan_jobs = {} + owner.model_test_jobs = {"job-1": local_worker} + owner._normalize_job_record = MagicMock( + side_effect=lambda key, specs: (key, deepcopy(specs)), + ) + owner._log_audit_event = MagicMock() + owner.P = MagicMock() + owner.purge_job = MagicMock() + + result = stop_and_delete_job(owner, "job-1") + + self.assertEqual(result["error"], "job_launcher_mismatch") + self.assertEqual(result["status_code"], 409) + owner.chainstore_hset.assert_not_called() + owner.purge_job.assert_not_called() + local_worker.stop.assert_not_called() + + def test_purge_rejects_foreign_launcher_before_artifact_deletion(self): + from extensions.business.cybersec.red_mesh.services.control import _purge_job_locked + + stored = { + "job_id": "job-1", + "job_status": "FINALIZED", + "launcher": "launcher-node", + "job_cid": "cid-archive", + "workers": {}, + } + owner = _owner( + ee_addr="worker-node", + chainstore_hget=MagicMock(return_value=deepcopy(stored)), + ) + owner._normalize_job_record = MagicMock( + side_effect=lambda key, specs: (key, deepcopy(specs)), + ) + owner._log_audit_event = MagicMock() + owner.P = MagicMock() + owner.r1fs.delete = MagicMock() + + result = _purge_job_locked(owner, "job-1") + + self.assertEqual(result["error"], "job_launcher_mismatch") + self.assertEqual(result["status_code"], 409) + owner.r1fs.delete.assert_not_called() + owner.chainstore_hset.assert_not_called() + def test_maybe_stop_canceled_jobs_stops_active_model_test_worker(self): mock_plugin_modules() from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin @@ -2882,7 +3096,7 @@ def test_close_model_test_worker_writes_result_and_removes_tracking(self): "done": True, "canceled": False, "model_test_results": { - "overall_status": "completed", + "overall_status": "complete", "cases": [ { "case_id": "cbrn-chemical-001", @@ -2900,7 +3114,7 @@ def test_close_model_test_worker_writes_result_and_removes_tracking(self): ], }, "model_test_summary": { - "overall_status": "completed", + "overall_status": "complete", "cases_total": 12, "cases_completed": 12, "evaluated_cases": 12, @@ -3003,43 +3217,19 @@ def get_json(cid): PentesterApi01Plugin._maybe_close_model_test_jobs(plugin) self.assertEqual(plugin.model_test_jobs, {}) - self.assertEqual(plugin.r1fs.add_json.call_count, 2) + self.assertEqual(plugin.r1fs.add_json.call_count, 1) stored_result = stored_artifacts[0] self.assertEqual(stored_result["schema_version"], "model_test_worker_result_v1") self.assertEqual(stored_result["job_id"], "job-1") self.assertEqual(stored_result["worker_addr"], "node-a") - self.assertEqual(stored_result["status"], "completed") - self.assertEqual(stored_result["model_test_summary"]["overall_status"], "completed") + self.assertEqual(stored_result["status"], "complete") + self.assertEqual(stored_result["model_test_summary"]["overall_status"], "complete") self.assertEqual(stored_result["model_test_results"]["cases"][0]["status"], "evaluated") - stored_archive = stored_artifacts[1] - self.assertEqual(stored_archive["schema_version"], "model_test_archive_v1") - self.assertEqual(stored_archive["job_id"], "job-1") - self.assertEqual(stored_archive["job_type"], "model_test") - self.assertEqual(stored_archive["job_config"]["job_id"], "job-1") - self.assertNotIn("model_provider_secret_ref", stored_archive["job_config"]) - self.assertNotIn("model_provider_secret_store_key_id", stored_archive["job_config"]) - self.assertEqual(stored_archive["model_test_results"]["overall_status"], "completed") - self.assertEqual(stored_archive["model_test_results"]["cases"][0]["case_id"], "cbrn-chemical-001") - self.assertEqual(stored_archive["model_test_summary"]["overall_status"], "completed") - self.assertEqual(stored_archive["model_test_node_selection"]["selected_execution_node"], "node-a") - self.assertEqual(stored_archive["ui_aggregate"]["scan_type"], "model_test") - self.assertEqual(stored_archive["ui_aggregate"]["finding_count"], 0) - self.assertEqual(stored_archive["duration"], 10.0) - self.assertEqual(len(written_records), 1) - persisted_specs = written_records[0][1] - self.assertEqual(written_records[0][2], "model_test_archive_prune") - self.assertEqual(persisted_specs["job_status"], "FINALIZED") - self.assertEqual(persisted_specs["job_type"], "model_test") - self.assertEqual(persisted_specs["scan_type"], "model_test") - self.assertEqual(persisted_specs["job_cid"], "cid-archive") - self.assertEqual(persisted_specs["job_config_cid"], "cid-config") - self.assertEqual(persisted_specs["model_test_summary"]["overall_status"], "completed") - self.assertEqual(persisted_specs["model_test_node_selection"]["selected_execution_node"], "node-a") - self.assertNotIn("workers", persisted_specs) + self.assertEqual(written_records, []) plugin._publish_model_test_progress.assert_called_once() _, _, progress_specs = plugin._publish_model_test_progress.call_args.args[:3] self.assertEqual(progress_specs["workers"]["node-a"]["report_cid"], "cid-result") - self.assertEqual(progress_specs["model_test_summary"]["overall_status"], "completed") + self.assertEqual(progress_specs["model_test_summary"]["overall_status"], "complete") self.assertEqual(plugin.scan_jobs, {}) def test_finished_model_test_job_recovery_finalizes_stale_running_record(self): @@ -3076,10 +3266,9 @@ def test_finished_model_test_job_recovery_finalizes_stale_running_record(self): "worker_type": "model_test", "start_port": 0, "end_port": 0, - "finished": True, + "finished": False, "canceled": False, - "model_test_worker_status": "finished", - "report_cid": "cid-result", + "model_test_worker_status": "assigned", "assignment_revision": 1, "assigned_at": 123.0, }, @@ -3132,10 +3321,31 @@ def test_finished_model_test_job_recovery_finalizes_stale_running_record(self): }, } plugin = MagicMock() - plugin.ee_addr = "node-a" + plugin.ee_addr = "launcher-node" plugin.cfg_instance_id = "instance" plugin.time.return_value = 130.0 - plugin.chainstore_hgetall.return_value = {"job-stale": job_specs} + terminal_live = { + "job-stale:node-a": { + "job_id": "job-stale", + "worker_addr": "node-a", + "pass_nr": 1, + "assignment_revision_seen": 1, + "progress": 100.0, + "phase": "done", + "ports_scanned": 0, + "ports_total": 0, + "open_ports_found": [], + "completed_tests": [], + "updated_at": 130.0, + "finished": True, + "report_cid": "cid-result", + "scan_type": "model_test", + "job_type": "model_test", + }, + } + plugin.chainstore_hgetall.side_effect = lambda hkey: ( + terminal_live if hkey == "instance:live" else {"job-stale": job_specs} + ) plugin._normalize_job_record.side_effect = lambda key, specs, migrate=False: (key, specs) written_records = [] stored_artifacts = [] @@ -3169,16 +3379,40 @@ def get_json(cid): stored_archive = stored_artifacts[0] self.assertEqual(stored_archive["schema_version"], "model_test_archive_v1") self.assertEqual(stored_archive["job_id"], "job-stale") - self.assertEqual(stored_archive["model_test_summary"]["overall_status"], "completed") + self.assertEqual(stored_archive["model_test_summary"]["overall_status"], "complete") self.assertEqual(stored_archive["model_test_results"]["cases"][0]["case_id"], "cbrn-chemical-001") self.assertEqual(len(written_records), 1) persisted_specs = written_records[0][1] self.assertEqual(written_records[0][2], "model_test_archive_prune") self.assertEqual(persisted_specs["job_status"], "FINALIZED") self.assertEqual(persisted_specs["job_cid"], "cid-archive") - self.assertEqual(persisted_specs["model_test_summary"]["overall_status"], "completed") + self.assertEqual(persisted_specs["model_test_summary"]["overall_status"], "complete") self.assertNotIn("workers", persisted_specs) + def test_non_launcher_cannot_finalize_model_test_job(self): + mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + + plugin = MagicMock() + plugin.ee_addr = "worker-node" + plugin.r1fs = MagicMock() + job_specs = { + "job_id": "job-owned", + "launcher": "launcher-node", + "job_config_cid": "cid-config", + } + + finalized = PentesterApi01Plugin._finalize_model_test_job( + plugin, + "job-owned", + job_specs, + {"status": "complete", "model_test_results": {}, "model_test_summary": {}}, + "cid-result", + ) + + self.assertFalse(finalized) + plugin.r1fs.add_json.assert_not_called() + def test_finished_model_test_recovery_skips_failed_attestation_record(self): mock_plugin_modules() from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin @@ -3403,8 +3637,16 @@ def get_json(cid): self.assertEqual(stored_artifacts[1]["schema_version"], "model_test_archive_v1") self.assertEqual(stored_artifacts[1]["model_test_summary"]["cases_completed"], 4) self.assertEqual(stored_artifacts[1]["model_test_results"]["cases"][0]["case_id"], "case-1") + terminal_events = [ + event for event in stored_artifacts[1]["timeline"] + if event["type"] in {"finalized", "failed", "canceled"} + ] + self.assertEqual(len(terminal_events), 1) + self.assertEqual(terminal_events[0]["type"], "failed") + self.assertEqual(terminal_events[0]["meta"]["overall_status"], "failed") + self.assertEqual(terminal_events[0]["meta"]["error_class"], MODEL_TEST_ERROR_WORKER_LOST) persisted_specs = written_records[-1][1] - self.assertEqual(persisted_specs["job_status"], "STOPPED") + self.assertEqual(persisted_specs["job_status"], "FAILED") self.assertEqual(persisted_specs["model_test_summary"]["error_class"], MODEL_TEST_ERROR_WORKER_LOST) self.assertEqual(persisted_specs["model_test_summary"]["cases_completed"], 4) self.assertEqual(persisted_specs["job_cid"], "cid-archive") @@ -3514,6 +3756,14 @@ def get_json(cid): self.assertEqual(failed, []) self.assertEqual(stored_artifacts[0]["status"], "canceled") self.assertEqual(stored_artifacts[0]["error_class"], MODEL_TEST_ERROR_CANCELED_BY_USER) + terminal_events = [ + event for event in stored_artifacts[1]["timeline"] + if event["type"] in {"finalized", "failed", "canceled"} + ] + self.assertEqual(len(terminal_events), 1) + self.assertEqual(terminal_events[0]["type"], "canceled") + self.assertEqual(terminal_events[0]["meta"]["overall_status"], "canceled") + self.assertNotIn("stopped", [event["type"] for event in stored_artifacts[1]["timeline"]]) persisted_specs = written_records[-1][1] self.assertEqual(persisted_specs["job_status"], "STOPPED") self.assertEqual(persisted_specs["model_test_summary"]["overall_status"], "canceled") diff --git a/extensions/business/cybersec/red_mesh/tests/test_reconciliation.py b/extensions/business/cybersec/red_mesh/tests/test_reconciliation.py index bb916ebff..12fda53b0 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_reconciliation.py +++ b/extensions/business/cybersec/red_mesh/tests/test_reconciliation.py @@ -521,6 +521,32 @@ def test_reconcile_workers_from_live_requires_report_cid(self): self.assertNotIn("report_cid", job_specs["workers"]["worker-A"]) owner._write_job_record.assert_not_called() + def test_reconcile_workers_from_terminal_error_stops_job(self): + job_specs = { + "job_id": "job-1", + "job_status": "RUNNING", + "job_pass": 2, + "launcher": "launcher-A", + "workers": { + "worker-A": {"start_port": 1, "end_port": 10, "assignment_revision": 3}, + }, + } + live_payloads = self._terminal_live_payload(cid=None) + live_payloads["job-1:worker-A"]["phase"] = "failed" + live_payloads["job-1:worker-A"]["error_class"] = "launch_failed" + owner, _repo = self._make_live_reconcile_owner(job_specs, live_payloads) + + changed = reconcile_workers_from_live(owner, "job-1") + + self.assertTrue(changed) + self.assertEqual(job_specs["job_status"], "STOPPED") + worker = job_specs["workers"]["worker-A"] + self.assertTrue(worker["finished"]) + self.assertEqual(worker["terminal_reason"], "launch_failed") + owner._write_job_record.assert_called_once_with( + "job-1", job_specs, context="reconcile_from_live", + ) + def test_reconcile_workers_from_live_skips_canceled_and_unreachable(self): job_specs = { "job_id": "job-1", diff --git a/ver.py b/ver.py index bf7ad7301..76cafa307 100644 --- a/ver.py +++ b/ver.py @@ -1 +1 @@ -__VER__ = '2.10.392' +__VER__ = '2.10.393' From 23241b0ea68b277cc5693feb74d79e2619d03989 Mon Sep 17 00:00:00 2001 From: Cristi Bleotiu <164478159+cristibleotiu@users.noreply.github.com> Date: Fri, 24 Jul 2026 19:36:45 +0300 Subject: [PATCH 6/6] chore: inc ver --- ver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ver.py b/ver.py index 76cafa307..c239b235f 100644 --- a/ver.py +++ b/ver.py @@ -1 +1 @@ -__VER__ = '2.10.393' +__VER__ = '2.10.400'