diff --git a/extensions/business/cybersec/red_mesh/mixins/report.py b/extensions/business/cybersec/red_mesh/mixins/report.py index daed3d58d..67b7ca3f1 100644 --- a/extensions/business/cybersec/red_mesh/mixins/report.py +++ b/extensions/business/cybersec/red_mesh/mixins/report.py @@ -355,10 +355,13 @@ def _stamp_worker_source(self, local_job_status, worker_id, node_addr): self._stamp_finding_list(local_job_status.get("findings"), worker_id, node_addr) - def _get_aggregated_report(self, local_jobs, worker_cls=None): + def _get_aggregated_report(self, local_jobs, worker_cls=None, log_details=True): """ Aggregate results from multiple local workers. + ``log_details=False`` is used at request trust boundaries where aggregate + content must not be copied into runtime logs on failure. + Parameters ---------- local_jobs : dict @@ -429,11 +432,14 @@ def _get_aggregated_report(self, local_jobs, worker_cls=None): self.P(f"Report aggregation done.") # endif we have local jobs except Exception as exc: - self.P("Error during report aggregation: {}:\n{}\n{}\ntype_or_func={}, field={}".format( - exc, self.trace_info(), - self.json_dumps(dct_aggregated_report, indent=2), - type_or_func, field - )) + if log_details: + self.P("Error during report aggregation: {}:\n{}\n{}\ntype_or_func={}, field={}".format( + exc, self.trace_info(), + self.json_dumps(dct_aggregated_report, indent=2), + type_or_func, field + )) + else: + self.P("Manual report aggregation failed", color='y') # Phase 0 dedup pass: collapse findings duplicated across workers # because each worker stamps its own _source_worker_id / # _source_node_addr before merge. The JSON-key fallback in diff --git a/extensions/business/cybersec/red_mesh/model_testing/runner.py b/extensions/business/cybersec/red_mesh/model_testing/runner.py index 79d25bbd3..3d1385ca7 100644 --- a/extensions/business/cybersec/red_mesh/model_testing/runner.py +++ b/extensions/business/cybersec/red_mesh/model_testing/runner.py @@ -795,7 +795,7 @@ def run(self): raw_evidence_cases.append(case_evidence) metrics["completed_cases"] += 1 results.append(case_result) - overall_status = "completed" if metrics["evaluated_cases"] == metrics["total_cases"] else "incomplete" + overall_status = "complete" if metrics["evaluated_cases"] == metrics["total_cases"] else "incomplete" if not results: overall_status = "failed" summary = { diff --git a/extensions/business/cybersec/red_mesh/model_testing/security.py b/extensions/business/cybersec/red_mesh/model_testing/security.py index 90994626f..b9553aa68 100644 --- a/extensions/business/cybersec/red_mesh/model_testing/security.py +++ b/extensions/business/cybersec/red_mesh/model_testing/security.py @@ -2,13 +2,16 @@ from __future__ import annotations +import hmac import ipaddress +import os import socket from urllib.parse import urlsplit MODEL_PROVIDER_CREDENTIAL_UNAVAILABLE = "credential_unavailable" -_MODEL_PROVIDER_REF_PREFIX = "model_provider/" +BACKEND_TOKEN_ENV = "REDMESH_BACKEND_TOKEN" +MIN_BACKEND_TOKEN_BYTES = 32 _PROVIDER_ALLOWED_KEYS = { "adapter", "provider_label", @@ -143,11 +146,51 @@ def validate_provider_url(base_url, *, resolver=None): def _credential_error(): return _validation_error( - MODEL_PROVIDER_CREDENTIAL_UNAVAILABLE, + "Provider requires an API key payload.", error_class=MODEL_PROVIDER_CREDENTIAL_UNAVAILABLE, ) +def _backend_auth_error(*, status_code, error, error_class, message): + return { + "status": "error", + "status_code": status_code, + "error": error, + "error_class": error_class, + "message": message, + } + + +def validate_backend_token(token): + """Validate the Navigator-to-edge bearer token without exposing token material.""" + expected = os.environ.get(BACKEND_TOKEN_ENV, "") + expected_bytes = expected.encode("utf-8") + if len(expected_bytes) < MIN_BACKEND_TOKEN_BYTES: + return _backend_auth_error( + status_code=401, + error="unauthorized", + error_class="backend_auth_unavailable", + message="Backend authentication is not configured.", + ) + + presented = token if isinstance(token, str) else "" + if not presented: + return _backend_auth_error( + status_code=401, + error="unauthorized", + error_class="backend_auth_required", + message="Backend authentication is required.", + ) + if not hmac.compare_digest(presented.encode("utf-8"), expected_bytes): + return _backend_auth_error( + status_code=403, + error="forbidden", + error_class="backend_auth_invalid", + message="Backend authentication failed.", + ) + return None + + def validate_provider_config_shape(provider, *, role): """Reject inline credential-bearing or unknown provider config fields.""" if not isinstance(provider, dict): @@ -182,32 +225,11 @@ def validate_model_provider_credentials( if isinstance(secret_payload, dict): api_key = str(secret_payload.get("api_key") or "") has_secret = bool(api_key) - if credential_ref and has_secret: - return None, _validation_error( - f"{role} may not supply both credential_ref and secret payload", - error_class="duplicate_credential_source", - ) + # Credential references are an accepted historical shape but there is no + # worker-local resolver yet. Reject them before launch/preflight persistence + # with one sanitized error that never includes the supplied identifier. + if credential_ref: + return None, _credential_error() if has_secret: return {"source": "secret_payload", "credential_ref_present": False}, None - if not credential_ref: - return None, _validation_error( - f"{role} requires credential_ref or secret payload", - error_class=MODEL_PROVIDER_CREDENTIAL_UNAVAILABLE, - ) - if not credential_ref.startswith(_MODEL_PROVIDER_REF_PREFIX): - return None, _credential_error() - ref_body = credential_ref[len(_MODEL_PROVIDER_REF_PREFIX):] - operator_prefix = f"operator/{created_by_id}/" - if ref_body.startswith(operator_prefix): - credential_id = ref_body[len(operator_prefix):] - if not credential_id or "/" in credential_id: - return None, _credential_error() - return {"source": "credential_ref", "credential_ref_present": True}, None - if ref_body.startswith("deploy/default_evaluator/"): - if role != "evaluator_model" or not use_default_evaluator_model: - return None, _credential_error() - credential_id = ref_body[len("deploy/default_evaluator/"):] - if not credential_id or "/" in credential_id: - return None, _credential_error() - return {"source": "credential_ref", "credential_ref_present": True}, None return None, _credential_error() diff --git a/extensions/business/cybersec/red_mesh/models/__init__.py b/extensions/business/cybersec/red_mesh/models/__init__.py index 75dd64ccf..b542ca7e0 100644 --- a/extensions/business/cybersec/red_mesh/models/__init__.py +++ b/extensions/business/cybersec/red_mesh/models/__init__.py @@ -52,6 +52,20 @@ FindingTriageState, VALID_TRIAGE_STATUSES, ) +from extensions.business.cybersec.red_mesh.models.rulebook import ( + RULEBOOK_ASSESSMENT_SCHEMA, + RULEBOOK_ASSESSMENT_SCHEMA_VERSION, + RULEBOOK_SUBMISSION_CONTRACT_VERSION, + RulebookPendingSubmission, + RulebookReviewAuditEntry, + RulebookReviewState, + RulebookSubmissionReference, + RulebookSubmissionRegistry, + VALID_RULEBOOK_ANSWER_VALUES, + VALID_RULEBOOK_CHECK_STATUSES, + VALID_RULEBOOK_REVIEW_STATES, + VALID_RULEBOOK_SUBMISSION_STATES, +) from extensions.business.cybersec.red_mesh.models.engagement import ( ASSET_EXPOSURES, AuthorizationRef, @@ -109,6 +123,18 @@ "FindingTriageState", "FindingTriageAuditEntry", "VALID_TRIAGE_STATUSES", + "RULEBOOK_ASSESSMENT_SCHEMA", + "RULEBOOK_ASSESSMENT_SCHEMA_VERSION", + "RULEBOOK_SUBMISSION_CONTRACT_VERSION", + "RulebookReviewState", + "RulebookReviewAuditEntry", + "RulebookSubmissionReference", + "RulebookPendingSubmission", + "RulebookSubmissionRegistry", + "VALID_RULEBOOK_ANSWER_VALUES", + "VALID_RULEBOOK_CHECK_STATUSES", + "VALID_RULEBOOK_REVIEW_STATES", + "VALID_RULEBOOK_SUBMISSION_STATES", # engagement "Contact", "EngagementContext", diff --git a/extensions/business/cybersec/red_mesh/models/cstore.py b/extensions/business/cybersec/red_mesh/models/cstore.py index f88b53c58..f3962b7a4 100644 --- a/extensions/business/cybersec/red_mesh/models/cstore.py +++ b/extensions/business/cybersec/red_mesh/models/cstore.py @@ -146,6 +146,7 @@ class CStoreJobRunning: stix_export: dict = None opencti_export: dict = None taxii_export: dict = None + rulebook_assessments: dict = None graybox_assignment_summary: dict = None blockchain_attestation_enabled: bool = False start_attestation_required: bool = False @@ -195,6 +196,7 @@ def from_dict(cls, d: dict) -> CStoreJobRunning: stix_export=d.get("stix_export"), opencti_export=d.get("opencti_export"), taxii_export=d.get("taxii_export"), + rulebook_assessments=d.get("rulebook_assessments"), graybox_assignment_summary=d.get("graybox_assignment_summary"), blockchain_attestation_enabled=d.get("blockchain_attestation_enabled", False), start_attestation_required=d.get("start_attestation_required", False), @@ -243,6 +245,7 @@ class CStoreJobFinalized: stix_export: dict = None opencti_export: dict = None taxii_export: dict = None + rulebook_assessments: dict = None graybox_assignment_summary: dict = None blockchain_attestation_enabled: bool = False start_attestation_required: bool = False @@ -290,6 +293,7 @@ def from_dict(cls, d: dict) -> CStoreJobFinalized: stix_export=d.get("stix_export"), opencti_export=d.get("opencti_export"), taxii_export=d.get("taxii_export"), + rulebook_assessments=d.get("rulebook_assessments"), graybox_assignment_summary=d.get("graybox_assignment_summary"), blockchain_attestation_enabled=d.get("blockchain_attestation_enabled", False), start_attestation_required=d.get("start_attestation_required", False), diff --git a/extensions/business/cybersec/red_mesh/models/rulebook.py b/extensions/business/cybersec/red_mesh/models/rulebook.py new file mode 100644 index 000000000..9bbd86fbd --- /dev/null +++ b/extensions/business/cybersec/red_mesh/models/rulebook.py @@ -0,0 +1,283 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass + +from extensions.business.cybersec.red_mesh.models.shared import _strip_none + + +RULEBOOK_ASSESSMENT_SCHEMA = "redmesh.rulebook_assessment.v1" +RULEBOOK_ASSESSMENT_SCHEMA_VERSION = "1.1.0" +RULEBOOK_SUBMISSION_CONTRACT_VERSION = "1.0.0" + +VALID_RULEBOOK_CHECK_STATUSES = frozenset({ + "supported", + "gap", + "needs_review", + "not_observable", + "not_applicable", +}) + +VALID_RULEBOOK_REVIEW_STATES = frozenset({ + "draft", + "submitted", + "reviewed", +}) + +VALID_RULEBOOK_SUBMISSION_STATES = frozenset({ + "prepared", + "artifact_written", + "reference_recorded", +}) + +VALID_RULEBOOK_ANSWER_VALUES = frozenset({ + "yes", + "no", + "unknown", + "not_applicable", +}) + + +def _coerce_answers(value): + if not isinstance(value, dict): + return {} + answers = {} + for question_id, raw_answer in value.items(): + if not isinstance(question_id, str) or not question_id.strip(): + continue + payload = raw_answer if isinstance(raw_answer, dict) else {"value": raw_answer} + answer_value = str(payload.get("value") or "unknown").strip().lower() + if answer_value not in VALID_RULEBOOK_ANSWER_VALUES: + answer_value = "unknown" + answers[question_id.strip()] = _strip_none({ + "value": answer_value, + "note": str(payload.get("note") or "")[:1000], + "reviewer": str(payload.get("reviewer") or "")[:200], + "updated_at": float(payload.get("updated_at", 0.0) or 0.0), + }) + return answers + + +@dataclass(frozen=True) +class RulebookReviewState: + job_id: str + profile_id: str + profile_version: str = "" + review_state: str = "draft" + reviewer: str = "" + note: str = "" + answers: dict = None + updated_at: float = 0.0 + review_revision: int = 0 + last_reopen_idempotency_key: str = "" + last_reopen_from_revision: int = 0 + last_reopen_actor: str = "" + + def to_dict(self) -> dict: + return _strip_none({ + **asdict(self), + "answers": _coerce_answers(self.answers), + }) + + @classmethod + def from_dict(cls, payload: dict) -> "RulebookReviewState": + review_state = str(payload.get("review_state") or "draft").strip().lower() + if review_state not in VALID_RULEBOOK_REVIEW_STATES: + raise ValueError(f"Unsupported rulebook review state: {review_state}") + return cls( + job_id=str(payload["job_id"]), + profile_id=str(payload["profile_id"]), + profile_version=str(payload.get("profile_version") or ""), + review_state=review_state, + reviewer=str(payload.get("reviewer") or "")[:200], + note=str(payload.get("note") or "")[:1000], + answers=_coerce_answers(payload.get("answers")), + updated_at=float(payload.get("updated_at", 0.0) or 0.0), + review_revision=max(0, int(payload.get("review_revision", 0) or 0)), + last_reopen_idempotency_key=str(payload.get("last_reopen_idempotency_key") or "")[:200], + last_reopen_from_revision=max(0, int(payload.get("last_reopen_from_revision", 0) or 0)), + last_reopen_actor=str(payload.get("last_reopen_actor") or "")[:200], + ) + + +@dataclass(frozen=True) +class RulebookReviewAuditEntry: + job_id: str + profile_id: str + profile_version: str + review_state: str + reviewer: str = "" + note: str = "" + changed_question_ids: list = None + previous_answers: dict = None + current_answers: dict = None + timestamp: float = 0.0 + review_revision: int = 0 + + def to_dict(self) -> dict: + return _strip_none({ + **asdict(self), + "changed_question_ids": list(self.changed_question_ids or []), + "previous_answers": _coerce_answers(self.previous_answers), + "current_answers": _coerce_answers(self.current_answers), + }) + + @classmethod + def from_dict(cls, payload: dict) -> "RulebookReviewAuditEntry": + review_state = str(payload.get("review_state") or "draft").strip().lower() + if review_state not in VALID_RULEBOOK_REVIEW_STATES: + raise ValueError(f"Unsupported rulebook review state: {review_state}") + return cls( + job_id=str(payload["job_id"]), + profile_id=str(payload["profile_id"]), + profile_version=str(payload.get("profile_version") or ""), + review_state=review_state, + reviewer=str(payload.get("reviewer") or "")[:200], + note=str(payload.get("note") or "")[:1000], + changed_question_ids=[ + str(item) + for item in (payload.get("changed_question_ids") or []) + if isinstance(item, str) and item + ], + previous_answers=_coerce_answers(payload.get("previous_answers")), + current_answers=_coerce_answers(payload.get("current_answers")), + timestamp=float(payload.get("timestamp", 0.0) or 0.0), + review_revision=max(0, int(payload.get("review_revision", 0) or 0)), + ) + + +@dataclass(frozen=True) +class RulebookSubmissionReference: + revision: int + cid: str + submitted_at: float + actor: str + pass_nr: int + profile_id: str + profile_version: str + schema_version: str + review_revision: int + idempotency_key: str = "" + fingerprint: str = "" + legacy: bool = False + + def to_dict(self) -> dict: + return _strip_none(asdict(self)) + + @classmethod + def from_dict(cls, payload: dict) -> "RulebookSubmissionReference": + revision = int(payload.get("revision", 0) or 0) + if revision < 0: + raise ValueError("Submission revision cannot be negative") + cid = str(payload.get("cid") or payload.get("artifact_cid") or "").strip() + if not cid: + raise ValueError("Submission reference requires a CID") + return cls( + revision=revision, + cid=cid, + submitted_at=float(payload.get("submitted_at", 0.0) or 0.0), + actor=str(payload.get("actor") or "")[:200], + pass_nr=max(0, int(payload.get("pass_nr", 0) or 0)), + profile_id=str(payload.get("profile_id") or ""), + profile_version=str(payload.get("profile_version") or ""), + schema_version=str(payload.get("schema_version") or ""), + review_revision=max(0, int(payload.get("review_revision", 0) or 0)), + idempotency_key=str(payload.get("idempotency_key") or "")[:200], + fingerprint=str(payload.get("fingerprint") or "")[:128], + legacy=bool(payload.get("legacy", False)), + ) + + +@dataclass(frozen=True) +class RulebookPendingSubmission: + target_revision: int + expected_review_revision: int + expected_pass_nr: int + expected_profile_version: str + actor: str + idempotency_key: str + fingerprint: str + state: str = "prepared" + created_at: float = 0.0 + updated_at: float = 0.0 + attempt_count: int = 1 + cid: str = "" + last_error: dict = None + + def to_dict(self) -> dict: + return _strip_none(asdict(self)) + + @classmethod + def from_dict(cls, payload: dict) -> "RulebookPendingSubmission": + state = str(payload.get("state") or "prepared") + if state not in VALID_RULEBOOK_SUBMISSION_STATES: + raise ValueError(f"Unsupported pending submission state: {state}") + target_revision = int(payload.get("target_revision", 0) or 0) + if target_revision < 1: + raise ValueError("Pending submission requires a positive target revision") + idempotency_key = str(payload.get("idempotency_key") or "").strip() + fingerprint = str(payload.get("fingerprint") or "").strip() + if not idempotency_key or not fingerprint: + raise ValueError("Pending submission requires idempotency key and fingerprint") + last_error = payload.get("last_error") + return cls( + target_revision=target_revision, + expected_review_revision=max(0, int(payload.get("expected_review_revision", 0) or 0)), + expected_pass_nr=max(0, int(payload.get("expected_pass_nr", 0) or 0)), + expected_profile_version=str(payload.get("expected_profile_version") or ""), + actor=str(payload.get("actor") or "")[:200], + idempotency_key=idempotency_key[:200], + fingerprint=fingerprint[:128], + state=state, + created_at=float(payload.get("created_at", 0.0) or 0.0), + updated_at=float(payload.get("updated_at", 0.0) or 0.0), + attempt_count=max(1, int(payload.get("attempt_count", 1) or 1)), + cid=str(payload.get("cid") or ""), + last_error=dict(last_error) if isinstance(last_error, dict) else None, + ) + + +@dataclass(frozen=True) +class RulebookSubmissionRegistry: + contract_version: str = RULEBOOK_SUBMISSION_CONTRACT_VERSION + latest_revision: int = 0 + submissions: list = None + pending: dict = None + + def to_dict(self) -> dict: + submissions = [ + item.to_dict() if isinstance(item, RulebookSubmissionReference) else RulebookSubmissionReference.from_dict(item).to_dict() + for item in (self.submissions or []) + ] + pending = self.pending + if isinstance(pending, RulebookPendingSubmission): + pending = pending.to_dict() + elif isinstance(pending, dict): + pending = RulebookPendingSubmission.from_dict(pending).to_dict() + return _strip_none({ + "contract_version": self.contract_version, + "latest_revision": max([self.latest_revision] + [item["revision"] for item in submissions]), + "submissions": submissions, + "pending": pending, + }) + + @classmethod + def from_dict(cls, payload: dict) -> "RulebookSubmissionRegistry": + contract_version = str(payload.get("contract_version") or RULEBOOK_SUBMISSION_CONTRACT_VERSION) + if contract_version != RULEBOOK_SUBMISSION_CONTRACT_VERSION: + raise ValueError(f"Unsupported submission contract version: {contract_version}") + submissions = [ + RulebookSubmissionReference.from_dict(item).to_dict() + for item in (payload.get("submissions") or []) + if isinstance(item, dict) + ] + pending_payload = payload.get("pending") + pending = RulebookPendingSubmission.from_dict(pending_payload).to_dict() if isinstance(pending_payload, dict) else None + latest_revision = max( + [max(0, int(payload.get("latest_revision", 0) or 0))] + [item["revision"] for item in submissions] + ) + return cls( + contract_version=contract_version, + latest_revision=latest_revision, + submissions=submissions, + pending=pending, + ) diff --git a/extensions/business/cybersec/red_mesh/pentester_api_01.py b/extensions/business/cybersec/red_mesh/pentester_api_01.py index e8bb699f1..1fd6fe013 100644 --- a/extensions/business/cybersec/red_mesh/pentester_api_01.py +++ b/extensions/business/cybersec/red_mesh/pentester_api_01.py @@ -30,11 +30,18 @@ """ +import asyncio +import json import random +import time from collections import deque +from concurrent.futures import CancelledError, ThreadPoolExecutor from copy import deepcopy +from dataclasses import dataclass +from uuid import uuid4 from naeural_core.business.default.web_app.fast_api_web_app import FastApiWebAppPlugin as BasePlugin +from .llm_input_builder import LlmInput, build_llm_input from .mixins import ( _RedMeshLlmAgentMixin, _AttestationMixin, _RiskScoringMixin, _ReportMixin, _LiveProgressMixin, _MispExportMixin, @@ -77,22 +84,23 @@ AuthorizationUploadError, build_network_workers, build_webapp_workers, - cancel_api_operation, collect_engagement_document_cids, - create_analyze_job_operation, delete_engagement_data, DeleteEngagementError, correlate_suricata_eve, dry_run_opencti_export, dry_run_taxii_export, - DEFAULT_API_OPERATIONS_CONFIG, export_stix_bundle, - get_api_operation_result, - get_api_operation_status, get_detection_correlation, get_opencti_export_status, + get_rulebook_assessment_status, + get_rulebook_review, + reopen_rulebook_review, get_stix_export_status, get_taxii_export_status, + generate_rulebook_assessment, + save_rulebook_review_draft, + submit_rulebook_review, push_to_opencti, publish_to_taxii, get_job_analysis, @@ -113,7 +121,6 @@ list_local_jobs, list_network_jobs, maybe_finalize_pass, - maybe_start_api_operation_worker, normalize_common_launch_options, parse_exceptions, purge_all_jobs, @@ -140,6 +147,7 @@ stop_monitoring, test_event_export, update_finding_triage, + update_rulebook_review, validation_error, ) from .model_testing import ( @@ -149,6 +157,7 @@ preflight_model_test_provider, sanitized_model_test_catalog, ) +from .model_testing.security import validate_backend_token from .model_testing.artifacts import ( sanitize_model_test_results, sanitize_model_test_summary, @@ -176,6 +185,15 @@ from .model_testing.secrets import sanitize_model_test_job_config_for_archive from .model_testing.worker import ModelTestWorker from .repositories import ArtifactRepository, JobStateRepository +from .services.llm_structured import ( + PROMPT_PROFILE_AUTO, + PROVIDER_PATH_LOCAL, + PROVIDER_PATH_REMOTE, + build_response_format_for_prompt_profile, + generate_exec_summary, + infer_provider_path, + resolve_prompt_profile, +) from .graybox.scenario_runtime import ( GrayboxWorkerAssignment, rehash_worker_assignment_dict, @@ -272,6 +290,262 @@ def _sanitize_model_test_status_job(job_specs: dict) -> dict: "weak_auth": "Testing credentials", } +_ANALYZE_TOTAL_TIMEOUT_SECONDS = 90.0 +_ANALYZE_PROVIDER_TIMEOUT_SECONDS = 30.0 +_ANALYZE_POLL_INTERVAL_SECONDS = 0.1 +_ANALYZE_MAX_WORKERS = 64 +_ANALYZE_MAX_REPORT_BYTES = 8 * 1024 * 1024 +_ANALYZE_MAX_PROVIDER_RESPONSE_BYTES = 2 * 1024 * 1024 +_ANALYZE_MAX_FOCUS_AREAS = 8 +_ANALYZE_MAX_TEXT_CHARS = 64 + + +async def _bounded_provider_post_async( + url: str, + payload: dict, + timeout_seconds: float, + max_response_bytes: int, +) -> bytes: + """POST JSON with a hard total deadline and bounded streamed response.""" + import aiohttp + + timeout = aiohttp.ClientTimeout( + total=timeout_seconds, + connect=min(10.0, timeout_seconds), + ) + try: + async with aiohttp.ClientSession(timeout=timeout) as session: + async with session.post( + url, + json=payload, + headers={"Content-Type": "application/json"}, + ) as response: + if response.status != 200: + raise RuntimeError("manual analysis provider rejected the request") + if ( + response.content_length is not None + and response.content_length > max_response_bytes + ): + raise RuntimeError("manual analysis provider response is too large") + + raw_response = bytearray() + async for chunk in response.content.iter_chunked(64 * 1024): + raw_response.extend(chunk) + if len(raw_response) > max_response_bytes: + raise RuntimeError("manual analysis provider response is too large") + return bytes(raw_response) + except asyncio.TimeoutError as exc: + raise TimeoutError("manual analysis provider timed out") from exc + except RuntimeError: + raise + except Exception as exc: + raise RuntimeError("manual analysis provider unavailable") from exc + + +def _bounded_provider_post( + url: str, + payload: dict, + timeout_seconds: float, + max_response_bytes: int, +) -> bytes: + """Run the bounded async transport from the isolated worker thread.""" + return asyncio.run(_bounded_provider_post_async( + url=url, + payload=payload, + timeout_seconds=timeout_seconds, + max_response_bytes=max_response_bytes, + )) + + +@dataclass(frozen=True) +class _ManualAnalysisWork: + """Immutable input owned exclusively by the manual-analysis worker.""" + + llm_input: LlmInput + llm_config: dict + api_host: str + api_port: int + deadline_monotonic: float + + +@dataclass(frozen=True) +class _ManualAnalysisOutcome: + """Sanitized worker result returned to the serialized plugin loop.""" + + sections: dict | None + failed: bool + deadline_exceeded: bool = False + + +class _ManualAnalysisWorker: + """Run structured generation without a reference to the RedMesh plugin.""" + + def __init__(self, work: _ManualAnalysisWork): + self._work = work + + @staticmethod + def _safe_validation(validation) -> dict: + issues = tuple(getattr(validation, "issues", ()) or ()) + + def _items(severity): + return [ + { + "code": str(getattr(issue, "code", "") or "")[:80], + "field": str(getattr(issue, "field", "") or "")[:120], + } + for issue in issues + if getattr(issue, "severity", "") == severity + ] + + errors = _items("error") + return { + "ok": not errors, + "errors": errors, + "warnings": _items("warning"), + } + + @staticmethod + def _safe_attempt_logs(result) -> list: + public_logs = [] + for item in tuple(getattr(result, "attempt_logs", ()) or ()): + if not isinstance(item, dict): + continue + public_logs.append({ + "attempt": item.get("attempt"), + "chunk": item.get("chunk") or None, + "elapsed_seconds": item.get("elapsed_seconds"), + "raw_len": item.get("raw_len"), + "validation_codes": [ + str(code)[:80] for code in list(item.get("validation_codes") or [])[:16] + ], + }) + return public_logs + + def _remaining_seconds(self) -> float: + return self._work.deadline_monotonic - time.monotonic() + + def _chat(self, messages: list, max_tokens: int, temperature: float, response_format=None) -> str: + remaining = self._remaining_seconds() + if remaining <= 0: + raise TimeoutError("manual analysis deadline exceeded") + + request_timeout = min(_ANALYZE_PROVIDER_TIMEOUT_SECONDS, remaining) + payload = { + "messages": messages, + "max_tokens": max_tokens, + "temperature": temperature, + } + if response_format is not None: + payload["response_format"] = response_format + + try: + raw_response = _bounded_provider_post( + f"http://{self._work.api_host}:{self._work.api_port}/chat", + payload, + request_timeout, + _ANALYZE_MAX_PROVIDER_RESPONSE_BYTES, + ) + except TimeoutError: + raise + except Exception as exc: + raise RuntimeError("manual analysis provider unavailable") from exc + + try: + response_data = json.loads(raw_response) + except Exception as exc: + raise RuntimeError("manual analysis provider returned invalid JSON") from exc + if isinstance(response_data, dict) and "result" in response_data: + response_data = response_data["result"] + if not isinstance(response_data, dict) or "error" in response_data: + raise RuntimeError("manual analysis provider returned an invalid response") + + choices = response_data.get("choices") + if not isinstance(choices, list) or not choices: + raise RuntimeError("manual analysis provider returned no choices") + choice = choices[0] if isinstance(choices[0], dict) else {} + message = choice.get("message") if isinstance(choice, dict) else {} + content = message.get("content") if isinstance(message, dict) else None + if not isinstance(content, str) or not content.strip(): + raise RuntimeError("manual analysis provider returned empty content") + return content + + def run(self) -> _ManualAnalysisOutcome: + config = self._work.llm_config + model_name = str(config.get("MODEL") or "CyberSecQwen-4B.Q4_K_M.gguf") + provider_path = str(config.get("PROVIDER") or "local") + requested_profile = str(config.get("PROMPT_PROFILE") or PROMPT_PROFILE_AUTO) + if requested_profile == PROMPT_PROFILE_AUTO: + effective_path = infer_provider_path( + provider_path=provider_path, + model_name=model_name, + ) + requested_profile = ( + config.get("REMOTE_PROMPT_PROFILE") + if effective_path == PROVIDER_PATH_REMOTE + else config.get("LOCAL_PROMPT_PROFILE") + ) + profile = resolve_prompt_profile( + requested_profile, + provider_path=provider_path, + model_name=model_name, + ) + response_format = None + if profile.provider_path != PROVIDER_PATH_LOCAL: + response_format = build_response_format_for_prompt_profile(profile) + + def _chat(messages, max_tokens, temperature): + return self._chat( + messages, + max_tokens, + temperature, + response_format=response_format, + ) + + try: + result = generate_exec_summary( + llm_call=_chat, + prepared_input=self._work.llm_input, + model_name=model_name, + provider_path=provider_path, + prompt_profile=profile.id, + max_findings=config.get("STRUCTURED_MAX_FINDINGS", 1), + max_tokens=config.get("STRUCTURED_MAX_TOKENS", 1024), + temperature=config.get("STRUCTURED_TEMPERATURE"), + ) + except Exception: + return _ManualAnalysisOutcome( + sections=None, + failed=True, + deadline_exceeded=self._remaining_seconds() <= 0, + ) + + if self._remaining_seconds() <= 0: + return _ManualAnalysisOutcome( + sections=None, + failed=True, + deadline_exceeded=True, + ) + + sections = result.sections.to_dict() + sections["prompt_profile"] = result.prompt_profile + sections["provider_path"] = result.provider_path + if result.error: + sections["validation"] = self._safe_validation(result.validation) + sections["error"] = True + sections["attempts"] = result.attempts + sections["diagnostics"] = { + "attempt_logs": self._safe_attempt_logs(result), + } + return _ManualAnalysisOutcome( + sections=sections, + failed=bool(result.error), + ) + + +def _run_manual_analysis_worker(work: _ManualAnalysisWork) -> _ManualAnalysisOutcome: + return _ManualAnalysisWorker(work).run() + + __VER__ = '0.9.0' @@ -343,8 +617,6 @@ def _sanitize_model_test_status_job(job_specs: dict) -> dict: }, "LLM_AGENT_API_HOST": "127.0.0.1", # Host where LLM Agent API is running "LLM_AGENT_API_PORT": None, # Port for LLM Agent API (required if enabled) - "API_OPERATIONS": dict(DEFAULT_API_OPERATIONS_CONFIG), - # Security hardening controls "REDACT_CREDENTIALS": True, # Strip passwords from persisted reports "ICS_SAFE_MODE": True, # Halt probing when ICS/SCADA indicators detected @@ -423,6 +695,12 @@ def on_init(self): None """ super(PentesterApi01Plugin, self).on_init() + self._manual_analysis_executor = ThreadPoolExecutor( + max_workers=1, + thread_name_prefix="redmesh-manual-analysis", + ) + self._manual_analysis_state = None + self._automatic_analysis_state = None self.__features = self._get_all_features() self._validate_feature_catalog() # Track active and completed jobs by target @@ -487,6 +765,20 @@ def _setup_semaphore_env(self): def on_close(self): + for state_name in ("_manual_analysis_state", "_automatic_analysis_state"): + state = self.__dict__.get(state_name) + if isinstance(state, dict): + future = state.get("future") + if future is not None: + future.cancel() + setattr(self, state_name, None) + executor = self.__dict__.get("_manual_analysis_executor") + if executor is not None: + try: + executor.shutdown(wait=False, cancel_futures=True) + except TypeError: + executor.shutdown(wait=False) + self._manual_analysis_executor = None super(PentesterApi01Plugin, self).on_close() return @@ -561,7 +853,7 @@ def __post_init(self): if isinstance(raw_report, dict): sample_value = next(iter(raw_report.values()), None) needs_aggregation = isinstance(sample_value, dict) and "local_worker_id" in sample_value - if needs_aggregation: + if needs_aggregation and normalized_spec.get("launcher") == self.ee_addr: # Merge stranded partial worker outputs left from prior deployments. self.P(f"Found incomplete report for {normalized_key}, aggregating...", color='r') agg_report = self._get_aggregated_report(raw_report) @@ -731,10 +1023,13 @@ def _normalize_job_record(self, job_key, job_spec, migrate=False): normalized["job_revision"] = int(normalized.get("job_revision", 0) or 0) except (TypeError, ValueError): normalized["job_revision"] = 0 - if migrate and job_key != job_id: - PentesterApi01Plugin._write_job_record(self, job_id, normalized, context="normalize_migrate") - PentesterApi01Plugin._delete_job_record(self, job_key) - job_key = job_id + if migrate and job_key != job_id and launcher == self.ee_addr: + persisted = PentesterApi01Plugin._write_job_record( + self, job_id, normalized, context="normalize_migrate", + ) + if isinstance(persisted, dict): + PentesterApi01Plugin._delete_job_record(self, job_key) + job_key = job_id return job_key, normalized @@ -1181,7 +1476,6 @@ def _maybe_launch_model_test_jobs(self): worker_entry["model_test_worker_status"] = "running" worker_entry["worker_type"] = "model_test" worker_entry["assigned_at"] = worker_entry.get("assigned_at") or self.time() - self._write_job_record(job_id, job_specs, context="launch_model_test_worker") self._publish_model_test_progress(job_id, worker, job_specs, finished=False) def _publish_model_test_progress(self, job_id, worker, job_specs, *, finished=False, report_cid=None, worker_addr=None): @@ -1316,6 +1610,15 @@ def _maybe_close_model_test_jobs(self): result_payload, show_logs=False, ) + job_config = self._get_artifact_repository().get_job_config(job_specs) + if isinstance(job_config, dict): + write_raw_evidence_artifact( + self, + job_id, + job_config, + worker.state.get("raw_evidence_payload"), + get_model_testing_config(self), + ) worker_entry["finished"] = True worker_entry["canceled"] = bool(status.get("canceled")) worker_entry["model_test_worker_status"] = "canceled" if status.get("canceled") else "finished" @@ -1323,16 +1626,6 @@ def _maybe_close_model_test_jobs(self): worker_entry["result"] = None if report_cid else result_payload job_specs["model_test_summary"] = result_payload["model_test_summary"] job_specs["model_test_results"] = result_payload["model_test_results"] - finalized = PentesterApi01Plugin._finalize_model_test_job( - self, - job_id, - job_specs, - result_payload, - report_cid, - raw_evidence_payload=worker.state.get("raw_evidence_payload"), - ) - if not finalized: - self._write_job_record(job_id, job_specs, context="close_model_test_worker") self._publish_model_test_progress( job_id, worker, @@ -1342,11 +1635,107 @@ def _maybe_close_model_test_jobs(self): ) self.model_test_jobs.pop(job_id, None) + def _canonical_model_test_terminal_status(self, result_payload): + """Return the canonical result status for a new terminal model-test record.""" + payload = result_payload if isinstance(result_payload, dict) else {} + summary = payload.get("model_test_summary") or {} + results = payload.get("model_test_results") or {} + aliases = { + "complete": "complete", + "completed": "complete", + "passed": "complete", + "success": "complete", + "incomplete": "incomplete", + "failed": "failed", + "failure": "failed", + "error": "failed", + "canceled": "canceled", + "cancelled": "canceled", + } + for value in ( + summary.get("overall_status"), + results.get("overall_status"), + payload.get("status"), + ): + normalized = aliases.get(str(value or "").strip().lower()) + if normalized: + return normalized + return "failed" + + def _terminalize_model_test_job( + self, + job_specs, + result_status, + job_status, + *, + error_class=None, + ): + """Ensure exactly one safe terminal lifecycle event before archiving.""" + event_type = { + JOB_STATUS_FINALIZED: "finalized", + JOB_STATUS_FAILED: "failed", + JOB_STATUS_STOPPED: "canceled", + }[job_status] + event_label = { + "finalized": "Model test finalized", + "failed": "Model test failed", + "canceled": "Model test canceled", + }[event_type] + summary = job_specs.get("model_test_summary") or {} + meta = { + "overall_status": result_status, + "selected_execution_node": selected_model_test_worker_addr(job_specs), + "cases_completed": summary.get("cases_completed") or summary.get("completed_cases") or 0, + "cases_total": summary.get("cases_total") or summary.get("total_cases") or 0, + } + safe_error_class = sanitize_model_test_error_class(error_class) + if safe_error_class: + meta["error_class"] = safe_error_class + + terminal_event_types = { + "attestation_failed", + "canceled", + "completed", + "failed", + "finalized", + "stopped", + } + timeline = job_specs.setdefault("timeline", []) + matching = [ + entry for entry in timeline + if isinstance(entry, dict) + and entry.get("type") == event_type + and (entry.get("meta") or {}).get("overall_status") == result_status + ] + timeline[:] = [ + entry for entry in timeline + if not isinstance(entry, dict) or entry.get("type") not in terminal_event_types + ] + if matching: + timeline.append(matching[0]) + return matching[0] + PentesterApi01Plugin._emit_timeline_event( + self, + job_specs, + event_type, + event_label, + actor_type="system", + meta=meta, + ) + return job_specs["timeline"][-1] + def _finalize_model_test_job(self, job_id, job_specs, result_payload, worker_report_cid, raw_evidence_payload=None): """Write a model-test archive and prune CStore to a terminal stub.""" from .model_testing.artifacts import ModelTestArchive, ModelTestArtifactRepository from .services.resilience import run_bounded_retry + if job_specs.get("launcher") != self.ee_addr: + self.P(f"Skipping model-test finalization for non-launcher job {job_id}", color='y') + return False + current = PentesterApi01Plugin._get_job_state_repository(self).get_job(job_id) + if isinstance(current, dict) and current.get("job_cid"): + return True + artifacts = ModelTestArtifactRepository(self) job_config_cid = job_specs.get("job_config_cid", "") job_config = artifacts.get_json(job_config_cid) @@ -1377,38 +1766,23 @@ def _finalize_model_test_job(self, job_id, job_specs, result_payload, worker_rep except (TypeError, ValueError): duration = 0.0 + result_status = PentesterApi01Plugin._canonical_model_test_terminal_status(self, result_payload) model_test_results = sanitize_model_test_results((result_payload or {}).get("model_test_results") or {}) model_test_summary = sanitize_model_test_summary((result_payload or {}).get("model_test_summary") or {}) - result_status = (result_payload or {}).get("status") - job_status = JOB_STATUS_STOPPED if result_status in {"canceled", "failed"} else JOB_STATUS_FINALIZED - existing_event_types = { - entry.get("type") - for entry in job_specs.get("timeline", []) or [] - if isinstance(entry, dict) - } - if job_status == JOB_STATUS_FINALIZED and "completed" not in existing_event_types: - PentesterApi01Plugin._emit_timeline_event( - self, - job_specs, - "completed", - "Model test completed", - actor_type="system", - meta={ - "cases_completed": model_test_summary.get("cases_completed") - or model_test_summary.get("completed_cases"), - "cases_total": model_test_summary.get("cases_total") - or model_test_summary.get("total_cases"), - }, - ) - elif job_status == JOB_STATUS_STOPPED and "stopped" not in existing_event_types: - PentesterApi01Plugin._emit_timeline_event( - self, - job_specs, - "stopped", - "Model test stopped", - actor_type="system", - meta={"status": result_status or "stopped"}, - ) + model_test_results["overall_status"] = result_status + model_test_summary["overall_status"] = result_status + job_specs["model_test_results"] = model_test_results + job_specs["model_test_summary"] = model_test_summary + if result_status in {"complete", "incomplete"}: + job_status = JOB_STATUS_FINALIZED + elif result_status == "canceled": + job_status = JOB_STATUS_STOPPED + else: + job_status = JOB_STATUS_FAILED + terminal_error_class = ( + (result_payload or {}).get("error_class") + or model_test_summary.get("error_class") + ) required_end_attestation = bool( job_specs.get("blockchain_attestation_enabled") or job_config.get("blockchain_attestation_enabled") @@ -1433,17 +1807,8 @@ def _finalize_model_test_job(self, job_id, job_specs, result_payload, worker_rep message = "Required terminal blockchain attestation failed for model-test job." job_specs["failure_class"] = "attestation_failed" job_specs["failure_message"] = message - set_job_status(job_specs, JOB_STATUS_FAILED) - PentesterApi01Plugin._emit_timeline_event( - self, - job_specs, - "attestation_failed", - message, - actor_type="system", - meta={"failure_class": "attestation_failed"}, - ) - self._write_job_record(job_id, job_specs, context="model_test_attestation_failed") - return False + job_status = JOB_STATUS_FAILED + terminal_error_class = "finalization_failed" if redmesh_test_attestation: job_specs["redmesh_test_attestation"] = redmesh_test_attestation PentesterApi01Plugin._emit_timeline_event( @@ -1460,22 +1825,14 @@ def _finalize_model_test_job(self, job_id, job_specs, result_payload, worker_rep or {} ) + PentesterApi01Plugin._terminalize_model_test_job( + self, + job_specs, + result_status, + job_status, + error_class=terminal_error_class, + ) archive_timeline = list(job_specs.get("timeline") or []) - if job_status == JOB_STATUS_FINALIZED and not any( - isinstance(entry, dict) and entry.get("type") == "finalized" - for entry in archive_timeline - ): - archive_timeline.append({ - "type": "finalized", - "label": "Model test archive finalized", - "date": self.time(), - "actor": self.ee_id, - "actor_type": "system", - "meta": { - "archive_type": "model_test_archive_v1", - "raw_evidence_status": raw_evidence_metadata.get("status"), - }, - }) archive = ModelTestArchive( job_id=job_id, @@ -1542,17 +1899,20 @@ def _finalize_model_test_job(self, job_id, job_specs, result_payload, worker_rep blockchain_attestation_enabled=required_end_attestation, start_attestation_required=required_end_attestation, end_attestation_required=required_end_attestation, + failure_class=job_specs.get("failure_class", ""), + failure_message=job_specs.get("failure_message", ""), ) self._write_job_record(job_id, stub.to_dict(), context="model_test_archive_prune") self.P(f"Model-test job {job_id} archived. CID={job_cid}, CStore pruned to stub.") return True def _maybe_finalize_finished_model_test_jobs(self): - """Repair model-test jobs whose worker result is durable but job status is still RUNNING.""" + """Finalize worker-owned model-test result/live artifacts on the launcher.""" from .model_testing.artifacts import ModelTestArtifactRepository all_jobs = PentesterApi01Plugin._get_job_state_repository(self).list_jobs() or {} artifacts = ModelTestArtifactRepository(self) + live_payloads = PentesterApi01Plugin._get_job_state_repository(self).list_live_progress() or {} finalized = [] for job_key, raw_job_specs in list(all_jobs.items()): normalized_key, job_specs = self._normalize_job_record(job_key, raw_job_specs) @@ -1563,21 +1923,61 @@ def _maybe_finalize_finished_model_test_jobs(self): continue if job_specs.get("job_cid") or is_terminal_job_status(job_specs.get("job_status")): continue + if job_specs.get("launcher") != self.ee_addr: + continue workers = job_specs.get("workers") or {} - worker_entry = workers.get(self.ee_addr) - if not isinstance(worker_entry, dict) or not worker_entry.get("finished"): + worker_addr = selected_model_test_worker_addr(job_specs) + worker_entry = workers.get(worker_addr) + if not worker_addr or not isinstance(worker_entry, dict): + continue + + worker_entry = dict(worker_entry) + live_payload = live_payloads.get(f"{job_id}:{worker_addr}") + live = None + if isinstance(live_payload, dict): + try: + live = WorkerProgress.from_dict(live_payload) + except (KeyError, TypeError, ValueError): + live = None + if ( + live is not None + and live.assignment_revision_seen == PentesterApi01Plugin._get_worker_assignment_revision(worker_entry) + and live.finished + ): + worker_entry["finished"] = True + worker_entry["canceled"] = bool(live.canceled) + worker_entry["report_cid"] = live.report_cid + worker_entry["model_test_worker_status"] = ( + MODEL_TEST_PHASE_CANCELED if live.canceled else live.phase + ) + if not worker_entry.get("finished"): continue report_cid = worker_entry.get("report_cid") result_payload = artifacts.get_json(report_cid) if report_cid else None if not isinstance(result_payload, dict): result_payload = worker_entry.get("result") if isinstance(worker_entry.get("result"), dict) else None + if not isinstance(result_payload, dict) and live is not None and live.finished: + result_payload = { + "job_id": job_id, + "worker_addr": worker_addr, + "status": ( + MODEL_TEST_PHASE_CANCELED + if live.canceled + else (live.model_test_summary or {}).get("overall_status") + or (live.model_test_results or {}).get("overall_status") + or live.phase + ), + "model_test_results": sanitize_model_test_results(live.model_test_results or {}), + "model_test_summary": sanitize_model_test_summary(live.model_test_summary or {}), + "error_class": live.error_class, + } if not isinstance(result_payload, dict): continue result_payload.setdefault("job_id", job_id) - result_payload.setdefault("worker_addr", self.ee_addr) + result_payload.setdefault("worker_addr", worker_addr) result_payload.setdefault( "status", "canceled" if worker_entry.get("canceled") else worker_entry.get("model_test_worker_status", "finished"), @@ -1740,7 +2140,17 @@ def first_number(*values, default=0): report_cid, ) if not finalized: - set_job_status(job_specs, JOB_STATUS_STOPPED) + fallback_job_status = ( + JOB_STATUS_STOPPED if status == MODEL_TEST_PHASE_CANCELED else JOB_STATUS_FAILED + ) + PentesterApi01Plugin._terminalize_model_test_job( + self, + job_specs, + status, + fallback_job_status, + error_class=error_class, + ) + set_job_status(job_specs, fallback_job_status) self._write_job_record(job_id, job_specs, context=f"model_test_{status}") PentesterApi01Plugin._publish_model_test_progress( self, @@ -1799,13 +2209,6 @@ def _maybe_fail_stale_model_test_jobs(self): and worker_entry.get("cancel_requested") and live_progress is None ): - PentesterApi01Plugin._emit_timeline_event( - self, - job_specs, - "model_test_canceled_before_worker_start", - "Model test canceled before worker progress started", - meta={"worker_addr": worker_addr}, - ) PentesterApi01Plugin._write_terminal_model_test_worker_result( self, job_id, @@ -1912,56 +2315,29 @@ def _with_worker_assignment(job_config, worker_entry): def _mark_worker_terminal_error( self, job_specs, worker_addr, reason, error, context="worker_terminal_error", ): - """Mark one worker terminal in the shared job record and persist it. - - PR406 B8: instead of writing the launcher's stale snapshot back over - whatever the latest CStore record looks like, reload the current - record and patch only ``workers[worker_addr]``. Concurrent terminal - writes from two workers then merge by worker key instead of clobbering - each other. If the current record can't be loaded, fall back to the - incoming snapshot (with a warning). - """ + """Publish a worker-owned terminal launch failure in live progress.""" if not isinstance(job_specs, dict): return None - sanitize = getattr(getattr(self, "safety", None), "sanitize_error", None) - sanitized = sanitize(str(error)) if callable(sanitize) else str(error) - if not isinstance(sanitized, str): - sanitized = str(error) - - def _patch_worker(entry: dict): - entry["finished"] = True - entry["terminal_reason"] = reason - entry["error"] = sanitized - entry["result"] = None - return entry - job_id = job_specs.get("job_id", "") - current = None - if job_id: - current = PentesterApi01Plugin._get_job_state_repository(self).get_job(job_id) - - # Always reflect the patch in the caller's snapshot so any code that - # inspects job_specs after this call sees the worker as terminal. - workers_local = job_specs.setdefault("workers", {}) - _patch_worker(workers_local.setdefault(worker_addr, {})) - - if not isinstance(current, dict): - self.P( - f"[CSTORE] No current job record for {job_id}; writing stale snapshot for worker {worker_addr}", - color='y', - ) - return PentesterApi01Plugin._write_job_record( - self, job_id, job_specs, context=context, - ) - - # Merge: keep current top-level state, overlay the patched worker. - merged_workers = dict(current.get("workers") or {}) - merged_workers[worker_addr] = _patch_worker(dict(merged_workers.get(worker_addr) or {})) - merged = dict(current) - merged["workers"] = merged_workers - return PentesterApi01Plugin._write_job_record( - self, job_id, merged, context=context, + worker_entry = (job_specs.get("workers") or {}).get(worker_addr) or {} + now = self.time() + progress = WorkerProgress( + job_id=job_id, + worker_addr=worker_addr, + pass_nr=job_specs.get("job_pass", 1), + assignment_revision_seen=PentesterApi01Plugin._get_worker_assignment_revision(worker_entry), + progress=100.0, + phase="failed", + ports_scanned=0, + ports_total=0, + open_ports_found=[], + completed_tests=[], + updated_at=now, + last_seen_at=now, + finished=True, + error_class=reason, ) + return PentesterApi01Plugin._get_job_state_repository(self).put_live_progress_model(progress) def _log_audit_event(self, event_type, details): @@ -2420,32 +2796,102 @@ def _supports_guarded_job_writes(self): """ Return whether mutable RedMesh job writes have real guarded-write semantics. - The current chainstore API only exposes plain hget/hset primitives, so - RedMesh cannot claim compare-and-swap or optimistic concurrency guarantees. + RedMesh guards lifecycle ownership and terminal monotonicity before using + the underlying plain chainstore hset. This is not compare-and-swap. """ - return False + return True def _get_job_write_guarantees(self): """Describe the actual guarantees of mutable RedMesh job-state writes.""" return { - "mode": "detection_only", - "guarded_writes": False, + "mode": "launcher_single_writer", + "guarded_writes": True, "stale_write_detection": True, + "terminal_monotonicity": True, "job_revision": True, + "atomic_compare_and_swap": False, + "distributed_lease": False, + "duplicate_owner_exclusion": "operational", } - def _write_job_record(self, job_id, job_specs, expected_revision=None, context=""): + def _write_job_record( + self, + job_id, + job_specs, + expected_revision=None, + context="", + reject_stale=False, + ): """ - Persist mutable job state with revision bump and stale-write detection. + Persist launcher-owned lifecycle state with revision and terminal guards. - This is observability only; it does not provide compare-and-swap semantics. + This does not provide compare-and-swap semantics or a distributed lease; + duplicate launcher processes remain operationally excluded. """ - current = PentesterApi01Plugin._get_job_state_repository(self).get_job(job_id) + repo = PentesterApi01Plugin._get_job_state_repository(self) + current = repo.get_job(job_id) + incoming = job_specs if isinstance(job_specs, dict) else dict(job_specs) + + launcher = ( + current.get("launcher") if isinstance(current, dict) else None + ) or incoming.get("launcher") + if launcher and launcher != self.ee_addr: + self.P( + f"[CSTORE] Shared job write rejected for non-launcher {self.ee_addr}: " + f"job_id={job_id}, launcher={launcher}, context={context or 'unspecified'}", + color='y', + ) + self._log_audit_event("job_write_owner_rejected", { + "job_id": job_id, + "launcher": launcher, + "writer": self.ee_addr, + "context": context or "", + }) + return None + + if isinstance(current, dict) and is_terminal_job_status(current.get("job_status")): + same_status = incoming.get("job_status") == current.get("job_status") + current_cid = current.get("job_cid") + incoming_cid = incoming.get("job_cid") + same_archive = not current_cid or incoming_cid == current_cid + if not same_status or not same_archive: + self.P( + f"[CSTORE] Terminal lifecycle write rejected for job {job_id}: " + f"context={context or 'unspecified'}", + color='y', + ) + self._log_audit_event("terminal_write_rejected", { + "job_id": job_id, + "current_status": current.get("job_status"), + "incoming_status": incoming.get("job_status"), + "current_job_cid": current_cid, + "incoming_job_cid": incoming_cid, + "context": context or "", + }) + return current + if current_cid and context in {"archive_prune", "model_test_archive_prune"}: + return current + current_revision = PentesterApi01Plugin._get_job_revision(self, current) - incoming_revision = PentesterApi01Plugin._get_job_revision(self, job_specs) + incoming_revision = PentesterApi01Plugin._get_job_revision(self, incoming) if expected_revision is None: expected_revision = incoming_revision + if reject_stale and not isinstance(current, dict): + self.P( + f"[CSTORE] Guarded write rejected for missing job {job_id}: " + f"context={context or 'unspecified'}", + color='y', + ) + self._log_audit_event("stale_write_detected", { + "job_id": job_id, + "expected_revision": expected_revision, + "current_revision": None, + "context": context or "", + "write_mode": PentesterApi01Plugin._get_job_write_guarantees(self)["mode"], + }) + return None + if isinstance(current, dict) and current_revision != expected_revision: self.P( f"[CSTORE] Stale write detected for job {job_id}: " @@ -2459,10 +2905,12 @@ def _write_job_record(self, job_id, job_specs, expected_revision=None, context=" "context": context or "", "write_mode": PentesterApi01Plugin._get_job_write_guarantees(self)["mode"], }) + if reject_stale: + return None - persisted = job_specs if isinstance(job_specs, dict) else dict(job_specs) + persisted = incoming persisted["job_revision"] = current_revision + 1 - normalized = PentesterApi01Plugin._get_job_state_repository(self).put_job(job_id, persisted) + normalized = repo.put_job(job_id, persisted) if isinstance(job_specs, dict) and isinstance(normalized, dict) and normalized is not job_specs: job_specs.clear() job_specs.update(normalized) @@ -2630,33 +3078,44 @@ def _close_job(self, job_id, canceled=False): ) self.P(f"Report saved to R1FS with CID: {report_cid}") else: - # Fallback: store report directly if R1FS fails - self.P("R1FS add_json returned None, storing report directly in CStore", color='y') + self.P("R1FS add_json returned None; publishing terminal storage failure", color='y') worker_entry["report_cid"] = None - worker_entry["result"] = report + worker_entry["result"] = None + PentesterApi01Plugin._mark_worker_terminal_error( + self, + job_specs, + self.ee_addr, + "report_storage_failed", + "Worker report artifact could not be stored", + ) except Exception as e: - # Fallback: store report directly if R1FS fails - self.P(f"Failed to save report to R1FS: {e}. Storing directly in CStore", color='r') + self.P(f"Failed to save report to R1FS: {e}", color='r') worker_entry["report_cid"] = None - worker_entry["result"] = report + worker_entry["result"] = None + PentesterApi01Plugin._mark_worker_terminal_error( + self, + job_specs, + self.ee_addr, + "report_storage_failed", + "Worker report artifact could not be stored", + ) else: self.P(f"No report data to save for job {job_id}", color='y') worker_entry["report_cid"] = None worker_entry["result"] = report - - # Re-read job_specs to avoid overwriting concurrent updates (e.g., pass_reports) - fresh_job_specs = PentesterApi01Plugin._get_job_state_repository(self).get_job(job_id) - if fresh_job_specs and isinstance(fresh_job_specs, dict): - fresh_job_specs["workers"][self.ee_addr] = worker_entry - job_specs = fresh_job_specs + PentesterApi01Plugin._mark_worker_terminal_error( + self, + job_specs, + self.ee_addr, + "report_unavailable", + "Worker report data was unavailable", + ) self.P("{} closing job_id {}:\n{}".format( closing, job_id, self.json_dumps(job_specs, indent=2) )) - PentesterApi01Plugin._write_job_record(self, job_id, job_specs, context="close_job") - # Audit: scan completed nr_findings = self._count_all_findings(report) self._log_audit_event("scan_completed", { @@ -2736,7 +3195,6 @@ def _maybe_stop_canceled_jobs(self): worker.stop() else: worker.stop() - self._write_job_record(job_id, job_specs, context="model_test_cancel_requested") self._publish_model_test_progress(job_id, worker, job_specs, finished=False, worker_addr=worker_addr) def _maybe_close_jobs(self): @@ -2809,6 +3267,12 @@ def _build_job_archive(self, job_key, job_specs): Full CStore job state. """ job_id = job_specs.get("job_id", job_key) + if job_specs.get("launcher") != self.ee_addr: + self.P(f"Skipping archive build for non-launcher job {job_id}", color='y') + return None + current = PentesterApi01Plugin._get_job_state_repository(self).get_job(job_key) + if isinstance(current, dict) and current.get("job_cid"): + return current.get("job_cid") # 1. Fetch job config and redact credentials for archive storage artifacts = PentesterApi01Plugin._get_artifact_repository(self) @@ -2920,6 +3384,7 @@ def _build_job_archive(self, job_key, job_specs): stix_export=job_specs.get("stix_export"), opencti_export=job_specs.get("opencti_export"), taxii_export=job_specs.get("taxii_export"), + rulebook_assessments=job_specs.get("rulebook_assessments"), blockchain_attestation_enabled=bool(job_specs.get("blockchain_attestation_enabled", False)), start_attestation_required=bool(job_specs.get("start_attestation_required", False)), end_attestation_required=bool(job_specs.get("end_attestation_required", False)), @@ -3542,9 +4007,10 @@ def launch_test( blockchain_attestation_enabled=blockchain_attestation_enabled, ) - @BasePlugin.endpoint(method="post") + @BasePlugin.endpoint(method="post", require_token=True) def launch_model_test( self, + token: str, task_name: str = "", task_description: str = "", selected_peers: list[str] = None, @@ -3564,6 +4030,9 @@ def launch_model_test( blockchain_attestation_enabled: bool = False, ): """Launch a Model Testing job through the model-test-specific gate.""" + auth_error = validate_backend_token(token) + if auth_error: + return auth_error return launch_model_test( self, task_name=task_name, @@ -3585,15 +4054,19 @@ def launch_model_test( blockchain_attestation_enabled=blockchain_attestation_enabled, ) - @BasePlugin.endpoint(method="post") + @BasePlugin.endpoint(method="post", require_token=True) def preflight_model_test_provider( self, + token: str, created_by_id: str = "", tested_model: dict = None, tested_model_secret_payload: dict = None, limits: dict = None, ): """Validate and transiently exercise a tested-model provider before launch.""" + auth_error = validate_backend_token(token) + if auth_error: + return auth_error return preflight_model_test_provider( self, created_by_id=created_by_id, @@ -3764,6 +4237,14 @@ def delete_job_engagement( if not isinstance(job_specs, dict): return {"error": "job_not_found", "message": f"job_specs for {job_id} not found"} + launcher = job_specs.get("launcher") + if launcher and launcher != self.ee_addr: + return { + "error": "job_launcher_mismatch", + "message": "Engagement deletion must be handled by the job launcher.", + "status_code": 409, + "job_id": job_id, + } if job_specs.get("job_cid"): return { "error": "unsupported_finalized_job", @@ -3802,7 +4283,11 @@ def delete_job_engagement( # Persist updated job_specs back to the running-state store so # the new job_config_cid + audit timeline survive. try: - state_repo.put_job(job_id, job_specs) + persisted = PentesterApi01Plugin._write_job_record( + self, job_id, job_specs, context="engagement_redaction", + ) + if persisted is None: + raise RuntimeError("launcher ownership changed before state persistence") except Exception as exc: self.P(f"engagement-deletion: sanitized config was written but " f"job_specs persistence failed: {exc}", color='r') @@ -3843,7 +4328,11 @@ def delete_job_engagement( ) break try: - state_repo.put_job(job_id, job_specs) + persisted = PentesterApi01Plugin._write_job_record( + self, job_id, job_specs, context="engagement_redaction_audit", + ) + if persisted is None: + raise RuntimeError("launcher ownership changed before audit persistence") except Exception as exc: self.P(f"engagement-deletion: document deletion completed but audit " f"persistence failed: {exc}", color='y') @@ -3962,6 +4451,119 @@ def get_stix_export_status(self, job_id: str): """Return STIX 2.1 manual export status for a job.""" return get_stix_export_status(self, job_id) + @BasePlugin.endpoint(method="post") + def generate_rulebook_assessment( + self, + job_id: str, + profile_id: str = None, + pass_nr: int = None, + persist: bool = True, + force: bool = True, + ): + """Build and optionally persist a RedMesh rulebook evidence assessment.""" + return generate_rulebook_assessment( + self, + job_id, + profile_id=profile_id, + pass_nr=pass_nr, + persist=persist, + force=force, + ) + + @BasePlugin.endpoint + def get_rulebook_assessment_status(self, job_id: str, profile_id: str = None): + """Return RedMesh rulebook assessment generation status for a job.""" + return get_rulebook_assessment_status(self, job_id, profile_id=profile_id) + + @BasePlugin.endpoint + def get_rulebook_review(self, job_id: str, profile_id: str = None): + """Return mutable reviewer state for a RedMesh rulebook profile.""" + return get_rulebook_review(self, job_id, profile_id=profile_id) + + @BasePlugin.endpoint(method="post") + def save_rulebook_review_draft( + self, + job_id: str, + profile_id: str = None, + answers: dict = None, + actor: str = "", + note: str = "", + expected_review_revision: int = None, + ): + """Save mutable rulebook review draft state without persisting an R1FS artifact.""" + return save_rulebook_review_draft( + self, + job_id, + profile_id=profile_id, + answers=answers, + actor=actor, + note=note, + expected_review_revision=expected_review_revision, + ) + + @BasePlugin.endpoint(method="post") + def submit_rulebook_review( + self, + job_id: str, + profile_id: str = None, + expected_review_revision: int = None, + expected_pass_nr: int = None, + expected_profile_version: str = None, + idempotency_key: str = "", + actor: str = "", + ): + """Persist and register one immutable rulebook review submission revision.""" + return submit_rulebook_review( + self, + job_id, + profile_id=profile_id, + expected_review_revision=expected_review_revision, + expected_pass_nr=expected_pass_nr, + expected_profile_version=expected_profile_version, + idempotency_key=idempotency_key, + actor=actor, + ) + + @BasePlugin.endpoint(method="post") + def reopen_rulebook_review( + self, + job_id: str, + profile_id: str = None, + expected_review_revision: int = None, + idempotency_key: str = "", + actor: str = "", + ): + """Create an editable draft while preserving every submitted revision.""" + return reopen_rulebook_review( + self, + job_id, + profile_id=profile_id, + expected_review_revision=expected_review_revision, + idempotency_key=idempotency_key, + actor=actor, + ) + + @BasePlugin.endpoint(method="post") + def update_rulebook_review( + self, + job_id: str, + profile_id: str = None, + answers: dict = None, + reviewer: str = "", + note: str = "", + review_state: str = "draft", + ): + """Save mutable reviewer answers for a RedMesh rulebook profile.""" + return update_rulebook_review( + self, + job_id, + profile_id=profile_id, + answers=answers, + reviewer=reviewer, + note=note, + review_state=review_state, + ) + @BasePlugin.endpoint(method="post") def dry_run_opencti_export(self, job_id: str, pass_nr: int = None): """Build and persist a redacted OpenCTI STIX bundle without pushing it.""" @@ -4042,7 +4644,7 @@ def get_report(self, cid: str): if not cid: return {"error": "No CID provided"} try: - report = self.r1fs.get_json(cid) + report = self.r1fs.get_json(cid, pin=False) if report is None: return {"error": "Report not found", "cid": cid} if is_restricted_raw_evidence_artifact(report): @@ -4092,159 +4694,486 @@ def stop_monitoring(self, job_id: str, stop_type: str = "SOFT"): return stop_monitoring(self, job_id, stop_type=stop_type) - @BasePlugin.endpoint(method="post") - def analyze_job( - self, - job_id: str, - analysis_type: str = "", - focus_areas: list[str] = None - ): - """ - Manually trigger LLM analysis for a completed job. + @staticmethod + def _manual_analysis_error(code, message, status_code, *, retryable=False, job_id=""): + result = { + "error": code, + "message": message, + "status_code": status_code, + "retryable": bool(retryable), + } + if job_id: + result["job_id"] = job_id + return result - Aggregates reports from all workers and runs analysis on the combined data. + def _discard_drained_manual_analysis(self): + state = self.__dict__.get("_manual_analysis_state") + if not isinstance(state, dict) or not state.get("discard_result"): + return + future = state.get("future") + if future is not None and future.done(): + try: + future.result() + except Exception: + pass + self._manual_analysis_state = None + return - Parameters - ---------- - job_id : str - Identifier of the job to analyze. - analysis_type : str, optional - Type of analysis: "security_assessment", "vulnerability_summary", "remediation_plan". - focus_areas : list[str], optional - Areas to focus on: ["web", "network", "databases", "authentication"]. + def _get_manual_analysis_executor(self): + """Return the single model worker shared by manual and automatic analysis.""" + executor = self.__dict__.get("_manual_analysis_executor") + if executor is None: + executor = ThreadPoolExecutor( + max_workers=1, + thread_name_prefix="redmesh-manual-analysis", + ) + self._manual_analysis_executor = executor + return executor - Returns - ------- - dict - LLM analysis result or error message. - """ + @staticmethod + def _validate_manual_analysis_request(job_id, analysis_type, focus_areas): + if not isinstance(job_id, str) or not job_id.strip() or len(job_id) > 128: + return PentesterApi01Plugin._manual_analysis_error( + "invalid_job_id", + "A valid job_id is required", + 400, + ) + if not isinstance(analysis_type, str) or len(analysis_type) > _ANALYZE_MAX_TEXT_CHARS: + return PentesterApi01Plugin._manual_analysis_error( + "invalid_analysis_type", + "analysis_type is invalid", + 400, + job_id=job_id, + ) + if focus_areas is None: + return None + if ( + not isinstance(focus_areas, list) + or len(focus_areas) > _ANALYZE_MAX_FOCUS_AREAS + or any( + not isinstance(item, str) or not item.strip() or len(item) > _ANALYZE_MAX_TEXT_CHARS + for item in focus_areas + ) + ): + return PentesterApi01Plugin._manual_analysis_error( + "invalid_focus_areas", + "focus_areas is invalid", + 400, + job_id=job_id, + ) + return None + + def _collect_bounded_manual_analysis_reports(self, workers): + reports = {} + total_bytes = 0 + for worker_addr, worker_entry in workers.items(): + if not isinstance(worker_entry, dict): + continue + report = None + report_cid = worker_entry.get("report_cid") + if report_cid: + try: + report = self.r1fs.get_json(report_cid) + except Exception: + self.P("Manual analysis could not read one worker report", color='y') + if not report: + report = worker_entry.get("result") + if not isinstance(report, dict): + continue + report = deepcopy(report) + try: + total_bytes += len( + json.dumps(report, separators=(",", ":"), default=str).encode("utf-8") + ) + except Exception: + return None, PentesterApi01Plugin._manual_analysis_error( + "analysis_input_invalid", + "Manual analysis report data is invalid", + 422, + ) + if total_bytes > _ANALYZE_MAX_REPORT_BYTES: + return None, PentesterApi01Plugin._manual_analysis_error( + "analysis_input_too_large", + "Manual analysis report data exceeds the supported limit", + 413, + ) + reports[str(worker_addr)] = report + return reports, None + + def _prepare_manual_analysis(self, job_id): llm_cfg = get_llm_agent_config(self) if not llm_cfg["ENABLED"]: - return {"error": "LLM Agent API is not enabled", "job_id": job_id} - + return None, {"error": "LLM Agent API is not enabled", "job_id": job_id} if not self.cfg_llm_agent_api_port: - return {"error": "LLM Agent API port not configured", "job_id": job_id} + return None, {"error": "LLM Agent API port not configured", "job_id": job_id} + native_timeout = getattr(self, "cfg_request_timeout", 120) + if not isinstance(native_timeout, (int, float)) or native_timeout < 120: + return None, PentesterApi01Plugin._manual_analysis_error( + "analysis_timeout_unavailable", + "Manual analysis requires a native request timeout of at least 120 seconds", + 503, + job_id=job_id, + ) - # Get job from CStore job_specs = self._get_job_from_cstore(job_id) - if not job_specs: - return {"error": "Job not found", "job_id": job_id} - - workers = job_specs.get("workers", {}) - if not workers: - return {"error": "No workers found for this job", "job_id": job_id} - - # Check if all workers have finished - all_finished = all(w.get("finished") for w in workers.values()) - if not all_finished: - return {"error": "Job not yet complete, some workers still running", "job_id": job_id} - - # Collect and aggregate reports from all workers - node_reports = self._collect_node_reports(workers) - aggregated_report = self._get_aggregated_report(node_reports) if node_reports else {} + if not isinstance(job_specs, dict): + return None, {"error": "Job not found", "job_id": job_id} + + workers = job_specs.get("workers") + if not isinstance(workers, dict) or not workers: + return None, {"error": "No workers found for this job", "job_id": job_id} + if len(workers) > _ANALYZE_MAX_WORKERS: + return None, PentesterApi01Plugin._manual_analysis_error( + "analysis_input_too_large", + "Manual analysis worker count exceeds the supported limit", + 413, + job_id=job_id, + ) + if not all(isinstance(worker, dict) and worker.get("finished") for worker in workers.values()): + return None, { + "error": "Job not yet complete, some workers still running", + "job_id": job_id, + } + pass_reports = job_specs.get("pass_reports") + if not isinstance(pass_reports, list) or not pass_reports or not isinstance(pass_reports[-1], dict): + return None, {"error": "No report data available for this job", "job_id": job_id} + latest_ref = pass_reports[-1] + expected_report_cid = str(latest_ref.get("report_cid") or "") + if not expected_report_cid: + return None, {"error": "No report data available for this job", "job_id": job_id} + + node_reports, report_error = self._collect_bounded_manual_analysis_reports(workers) + if report_error is not None: + report_error["job_id"] = job_id + return None, report_error + aggregated_report = ( + self._get_aggregated_report(node_reports, log_details=False) + if node_reports else {} + ) if not aggregated_report: - return {"error": "No report data available for this job", "job_id": job_id} + return None, {"error": "No report data available for this job", "job_id": job_id} - target = job_specs.get("target", "unknown") job_config = self._get_job_config(job_specs) _risk_result, flat_findings = self._compute_risk_and_findings(aggregated_report) - - llm_report_sections = self._run_structured_report_sections( - job_id=job_id, + prepared_input = build_llm_input( findings=flat_findings, aggregated_report=aggregated_report, engagement=job_config.get("engagement") if isinstance(job_config, dict) else None, + max_findings=llm_cfg.get("STRUCTURED_MAX_FINDINGS", 6), + ) + work = _ManualAnalysisWork( + llm_input=deepcopy(prepared_input), + llm_config=deepcopy(llm_cfg), + api_host=str(self.cfg_llm_agent_api_host or "127.0.0.1"), + api_port=int(self.cfg_llm_agent_api_port), + deadline_monotonic=time.monotonic() + _ANALYZE_TOTAL_TIMEOUT_SECONDS, + ) + current_pass = job_specs.get("job_pass", 1) + state = { + "pending_id": uuid4().hex, + "job_id": job_id, + "job_revision": PentesterApi01Plugin._get_job_revision(self, job_specs), + "pass_nr": latest_ref.get("pass_nr", current_pass), + "report_cid": expected_report_cid, + "target": job_specs.get("target", "unknown"), + "num_workers": len(workers), + "deadline_monotonic": time.monotonic() + _ANALYZE_TOTAL_TIMEOUT_SECONDS, + "next_check_monotonic": 0.0, + "discard_result": False, + "work": work, + } + return state, None + + def _postpone_manual_analysis(self, pending_id): + return self.create_postponed_request( + solver_method=self.solve_postponed_analyze_job, + method_kwargs={"pending_id": pending_id}, ) - structured_failed = bool(getattr(self, "_last_structured_llm_failed", False)) - if llm_report_sections is None: + + def _manual_analysis_state_matches(self, job_specs, state): + if not isinstance(job_specs, dict): + return False + if PentesterApi01Plugin._get_job_revision(self, job_specs) != state.get("job_revision"): + return False + pass_reports = job_specs.get("pass_reports") + if not isinstance(pass_reports, list) or not pass_reports: + return False + latest_ref = pass_reports[-1] + if not isinstance(latest_ref, dict): + return False + return ( + latest_ref.get("pass_nr", job_specs.get("job_pass", 1)) == state.get("pass_nr") + and str(latest_ref.get("report_cid") or "") == state.get("report_cid") + ) + + def _finalize_manual_analysis(self, state, outcome): + job_id = state["job_id"] + if not isinstance(outcome, _ManualAnalysisOutcome) or outcome.sections is None: return { "error": "Structured LLM report generation failed", "status": "structured_llm_failed", "job_id": job_id, } - # Update the latest pass report with manual structured sections. - pass_reports = job_specs.get("pass_reports", []) - current_pass = job_specs.get("job_pass", 1) + current_job = self._get_job_from_cstore(job_id) + if not PentesterApi01Plugin._manual_analysis_state_matches(self, current_job, state): + return PentesterApi01Plugin._manual_analysis_error( + "analysis_state_changed", + "The job changed while manual analysis was running", + 409, + retryable=True, + job_id=job_id, + ) - if pass_reports: - latest_ref = pass_reports[-1] - try: - pass_data = self.r1fs.get_json(latest_ref["report_cid"]) - if pass_data: - pass_data["llm_report_sections"] = llm_report_sections - if not structured_failed: - llm_text, summary_text = render_legacy_llm_fields(llm_report_sections) - if llm_text: - pass_data["llm_analysis"] = llm_text - if summary_text: - pass_data["quick_summary"] = summary_text - if structured_failed: - pass_data["llm_failed"] = True - else: - pass_data.pop("llm_failed", None) - updated_cid = self.r1fs.add_json(pass_data, show_logs=False) - if updated_cid: - latest_ref["report_cid"] = updated_cid - self._emit_timeline_event( - job_specs, "llm_analysis", - "Manual structured LLM report sections completed", - actor_type="user", - meta={"report_cid": updated_cid, "pass_nr": latest_ref.get("pass_nr", current_pass)} - ) - PentesterApi01Plugin._write_job_record(self, job_id, job_specs, context="manual_llm_update") - self.P(f"Manual structured LLM sections saved for job {job_id}, updated pass report CID: {updated_cid}") - except Exception as e: - self.P(f"Failed to update pass report with structured LLM sections: {e}", color='y') + job_specs = deepcopy(current_job) + latest_ref = job_specs["pass_reports"][-1] + try: + pass_data = self.r1fs.get_json(state["report_cid"]) + if not isinstance(pass_data, dict): + raise ValueError("pass report unavailable") + pass_data = deepcopy(pass_data) + pass_data["llm_report_sections"] = deepcopy(outcome.sections) + if not outcome.failed: + llm_text, summary_text = render_legacy_llm_fields(outcome.sections) + if llm_text: + pass_data["llm_analysis"] = llm_text + if summary_text: + pass_data["quick_summary"] = summary_text + pass_data.pop("llm_failed", None) + else: + pass_data["llm_failed"] = True + updated_cid = self.r1fs.add_json(pass_data, show_logs=False) + if not updated_cid: + raise RuntimeError("pass report write failed") + + latest_job = self._get_job_from_cstore(job_id) + if not PentesterApi01Plugin._manual_analysis_state_matches(self, latest_job, state): + return PentesterApi01Plugin._manual_analysis_error( + "analysis_state_changed", + "The job changed while manual analysis was running", + 409, + retryable=True, + job_id=job_id, + ) + job_specs = deepcopy(latest_job) + latest_ref = job_specs["pass_reports"][-1] + latest_ref["report_cid"] = updated_cid + self._emit_timeline_event( + job_specs, + "llm_analysis", + "Manual structured LLM report sections completed", + actor_type="user", + meta={"report_cid": updated_cid, "pass_nr": state["pass_nr"]}, + ) + persisted = PentesterApi01Plugin._write_job_record( + self, + job_id, + job_specs, + expected_revision=state["job_revision"], + context="manual_llm_update", + reject_stale=True, + ) + if not isinstance(persisted, dict): + return PentesterApi01Plugin._manual_analysis_error( + "analysis_state_changed", + "The job changed while manual analysis was running", + 409, + retryable=True, + job_id=job_id, + ) + persisted_job = self._get_job_from_cstore(job_id) + persisted_refs = ( + persisted_job.get("pass_reports") + if isinstance(persisted_job, dict) else None + ) + if ( + not isinstance(persisted_refs, list) + or not persisted_refs + or not isinstance(persisted_refs[-1], dict) + or str(persisted_refs[-1].get("report_cid") or "") != str(updated_cid) + ): + raise RuntimeError("manual analysis job update was not confirmed") + except Exception: + self.P("Manual structured LLM persistence failed", color='y') + return PentesterApi01Plugin._manual_analysis_error( + "analysis_persistence_failed", + "Manual analysis could not be persisted", + 503, + retryable=True, + job_id=job_id, + ) + self.P(f"Manual structured LLM sections saved for job {job_id}") return { "job_id": job_id, - "target": target, - "num_workers": len(workers), - "pass_nr": pass_reports[-1].get("pass_nr", current_pass) if pass_reports else current_pass, + "target": state["target"], + "num_workers": state["num_workers"], + "pass_nr": state["pass_nr"], "analysis_type": "structured_report_sections", - "llm_failed": structured_failed, - "llm_report_sections": llm_report_sections, + "llm_failed": outcome.failed, + "llm_report_sections": outcome.sections, } - @BasePlugin.endpoint(method="post", require_token=True) - def create_analyze_job_operation( + def solve_postponed_analyze_job(self, pending_id: str): + try: + state = self.__dict__.get("_manual_analysis_state") + if not isinstance(state, dict) or state.get("pending_id") != pending_id: + return PentesterApi01Plugin._manual_analysis_error( + "analysis_request_unavailable", + "The manual analysis request is no longer available", + 410, + retryable=True, + ) + future = state.get("future") + if future is None: + self._manual_analysis_state = None + return PentesterApi01Plugin._manual_analysis_error( + "analysis_executor_failed", + "Manual analysis could not be completed", + 503, + retryable=True, + job_id=state.get("job_id", ""), + ) + + now = time.monotonic() + if now < state.get("next_check_monotonic", 0): + return PentesterApi01Plugin._postpone_manual_analysis(self, pending_id) + state["next_check_monotonic"] = now + _ANALYZE_POLL_INTERVAL_SECONDS + + if future.done(): + try: + outcome = future.result() + except CancelledError: + result = PentesterApi01Plugin._manual_analysis_error( + "analysis_canceled", + "Manual analysis was canceled", + 503, + retryable=True, + job_id=state["job_id"], + ) + except Exception: + result = PentesterApi01Plugin._manual_analysis_error( + "analysis_executor_failed", + "Manual analysis could not be completed", + 503, + retryable=True, + job_id=state["job_id"], + ) + else: + if ( + isinstance(outcome, _ManualAnalysisOutcome) + and outcome.deadline_exceeded + ): + result = PentesterApi01Plugin._manual_analysis_error( + "analysis_timeout", + "Manual analysis exceeded its deadline", + 504, + retryable=True, + job_id=state["job_id"], + ) + else: + result = PentesterApi01Plugin._finalize_manual_analysis( + self, + state, + outcome, + ) + self._manual_analysis_state = None + return result + + if now >= state["deadline_monotonic"]: + state["discard_result"] = True + future.cancel() + return PentesterApi01Plugin._manual_analysis_error( + "analysis_timeout", + "Manual analysis exceeded its deadline", + 504, + retryable=True, + job_id=state["job_id"], + ) + return PentesterApi01Plugin._postpone_manual_analysis(self, pending_id) + except Exception: + state = self.__dict__.get("_manual_analysis_state") + if isinstance(state, dict): + state["discard_result"] = True + return PentesterApi01Plugin._manual_analysis_error( + "analysis_executor_failed", + "Manual analysis could not be completed", + 503, + retryable=True, + ) + + @BasePlugin.endpoint(method="post") + def analyze_job( self, - token: str, job_id: str, analysis_type: str = "", focus_areas: list[str] = None, - idempotency_key: str = "", ): - """Create an async operation for manual structured LLM job analysis.""" - return create_analyze_job_operation( - self, - token=token, - job_id=job_id, - analysis_type=analysis_type, - focus_areas=focus_areas, - idempotency_key=idempotency_key, - ) - - - @BasePlugin.endpoint(require_token=True) - def get_api_operation_status(self, token: str, operation_id: str): - """Return a sanitized RedMesh API operation status.""" - return get_api_operation_status(self, token=token, operation_id=operation_id) - - - @BasePlugin.endpoint(method="post", require_token=True) - def cancel_api_operation(self, token: str, operation_id: str, reason: str = ""): - """Request cancellation of a RedMesh API operation.""" - return cancel_api_operation(self, token=token, operation_id=operation_id, reason=reason) - - - @BasePlugin.endpoint(require_token=True) - def get_api_operation_result(self, token: str, result_handle: str): - """Resolve an opaque RedMesh API operation result handle.""" - return get_api_operation_result(self, token=token, result_handle=result_handle) + """Run one manual analysis through a native postponed request.""" + try: + request_error = PentesterApi01Plugin._validate_manual_analysis_request( + job_id, + analysis_type, + focus_areas, + ) + if request_error is not None: + return request_error + + PentesterApi01Plugin._discard_drained_manual_analysis(self) + if isinstance(self.__dict__.get("_manual_analysis_state"), dict): + return PentesterApi01Plugin._manual_analysis_error( + "analysis_busy", + "Another manual analysis is already running", + 409, + retryable=True, + job_id=job_id, + ) + state, error = PentesterApi01Plugin._prepare_manual_analysis(self, job_id) + if error is not None: + return error + self._manual_analysis_state = state + try: + executor = PentesterApi01Plugin._get_manual_analysis_executor(self) + state["future"] = executor.submit(_run_manual_analysis_worker, state.pop("work")) + return PentesterApi01Plugin._postpone_manual_analysis(self, state["pending_id"]) + except Exception: + future = state.get("future") + canceled = False + if future is not None: + canceled = future.cancel() + if future is None or canceled or future.done(): + self._manual_analysis_state = None + else: + state["discard_result"] = True + return PentesterApi01Plugin._manual_analysis_error( + "analysis_executor_failed", + "Manual analysis could not be started", + 503, + retryable=True, + job_id=job_id, + ) + except Exception: + # Keep admission failures on the typed public error path. + state = self.__dict__.get("_manual_analysis_state") + if isinstance(state, dict): + future = state.get("future") + try: + canceled = future is None or future.cancel() + except Exception: + canceled = False + if canceled or (future is not None and future.done()): + self._manual_analysis_state = None + else: + state["discard_result"] = True + return PentesterApi01Plugin._manual_analysis_error( + "analysis_executor_failed", + "Manual analysis could not be started", + 503, + retryable=True, + job_id=job_id, + ) @BasePlugin.endpoint def get_analysis(self, job_id: str = "", cid: str = "", pass_nr: int = None): @@ -4292,6 +5221,7 @@ def process(self): ------- None """ + PentesterApi01Plugin._discard_drained_manual_analysis(self) super(PentesterApi01Plugin, self).process() # Wait for semaphore dependencies before signaling own readiness @@ -4326,6 +5256,4 @@ def process(self): self._maybe_reannounce_worker_assignments() # Finalize completed passes and handle continuous monitoring (launcher only) self._maybe_finalize_pass() - # Start at most one background API operation worker without blocking request handling. - maybe_start_api_operation_worker(self) return diff --git a/extensions/business/cybersec/red_mesh/repositories/artifacts.py b/extensions/business/cybersec/red_mesh/repositories/artifacts.py index 323f328d9..15ab21534 100644 --- a/extensions/business/cybersec/red_mesh/repositories/artifacts.py +++ b/extensions/business/cybersec/red_mesh/repositories/artifacts.py @@ -27,9 +27,18 @@ def put_json(self, payload, *, show_logs=False, secret=None): return self.owner.r1fs.add_json(payload, show_logs=show_logs, secret=secret) return self.owner.r1fs.add_json(payload, show_logs=show_logs) - def delete(self, cid, *, show_logs=False, raise_on_error=False): + def delete(self, cid, *, show_logs=False, raise_on_error=False, purge=False): if not cid: return False + if purge: + return self.owner.r1fs.delete_file( + cid, + unpin_remote=True, + run_gc=True, + cleanup_local_files=True, + show_logs=show_logs, + raise_on_error=raise_on_error, + ) return self.owner.r1fs.delete_file(cid, show_logs=show_logs, raise_on_error=raise_on_error) def get_job_config(self, job_specs): diff --git a/extensions/business/cybersec/red_mesh/repositories/cstore.py b/extensions/business/cybersec/red_mesh/repositories/cstore.py index a919ef093..15b01b0bd 100644 --- a/extensions/business/cybersec/red_mesh/repositories/cstore.py +++ b/extensions/business/cybersec/red_mesh/repositories/cstore.py @@ -3,6 +3,9 @@ CStoreJobRunning, FindingTriageAuditEntry, FindingTriageState, + RulebookReviewAuditEntry, + RulebookReviewState, + RulebookSubmissionRegistry, WorkerProgress, ) @@ -46,6 +49,18 @@ def _triage_audit_hkey(self): def _model_test_raw_evidence_hkey(self): return f"{self.owner.cfg_instance_id}:model_test_raw_evidence" + @property + def _rulebook_review_hkey(self): + return f"{self.owner.cfg_instance_id}:rulebook_review" + + @property + def _rulebook_review_audit_hkey(self): + return f"{self.owner.cfg_instance_id}:rulebook_review:audit" + + @property + def _rulebook_review_submissions_hkey(self): + return f"{self.owner.cfg_instance_id}:rulebook_review:submissions" + def get_job(self, job_id): return self.owner.chainstore_hget(hkey=self._jobs_hkey, key=job_id) @@ -304,3 +319,97 @@ def delete_job_triage(self, job_id): value=None, ) return + + @staticmethod + def rulebook_key(job_id, profile_id): + return f"{job_id}:{profile_id}" + + def get_rulebook_review(self, job_id, profile_id): + return self.owner.chainstore_hget( + hkey=self._rulebook_review_hkey, + key=self.rulebook_key(job_id, profile_id), + ) + + def get_rulebook_review_model(self, job_id, profile_id): + payload = self.get_rulebook_review(job_id, profile_id) + if not isinstance(payload, dict): + return None + return RulebookReviewState.from_dict(payload) + + def put_rulebook_review(self, review): + if isinstance(review, RulebookReviewState): + payload = review.to_dict() + else: + payload = RulebookReviewState.from_dict(review).to_dict() + self.owner.chainstore_hset( + hkey=self._rulebook_review_hkey, + key=self.rulebook_key(payload["job_id"], payload["profile_id"]), + value=payload, + ) + return payload + + def get_rulebook_review_audit(self, job_id, profile_id): + payload = self.owner.chainstore_hget( + hkey=self._rulebook_review_audit_hkey, + key=self.rulebook_key(job_id, profile_id), + ) + return payload if isinstance(payload, list) else [] + + def append_rulebook_review_audit(self, entry): + if isinstance(entry, RulebookReviewAuditEntry): + payload = entry.to_dict() + else: + payload = RulebookReviewAuditEntry.from_dict(entry).to_dict() + key = self.rulebook_key(payload["job_id"], payload["profile_id"]) + audit_log = list(self.get_rulebook_review_audit(payload["job_id"], payload["profile_id"])) + audit_log.append(payload) + self.owner.chainstore_hset(hkey=self._rulebook_review_audit_hkey, key=key, value=audit_log) + return audit_log + + def get_rulebook_submission_registry(self, job_id, profile_id): + return self.owner.chainstore_hget( + hkey=self._rulebook_review_submissions_hkey, + key=self.rulebook_key(job_id, profile_id), + ) + + def get_rulebook_submission_registry_model(self, job_id, profile_id): + payload = self.get_rulebook_submission_registry(job_id, profile_id) + if not isinstance(payload, dict): + return None + return RulebookSubmissionRegistry.from_dict(payload) + + def put_rulebook_submission_registry(self, job_id, profile_id, registry): + if isinstance(registry, RulebookSubmissionRegistry): + payload = registry.to_dict() + else: + payload = RulebookSubmissionRegistry.from_dict(registry).to_dict() + self.owner.chainstore_hset( + hkey=self._rulebook_review_submissions_hkey, + key=self.rulebook_key(job_id, profile_id), + value=payload, + ) + return payload + + def list_job_rulebook_submission_registries(self, job_id): + payload = self.owner.chainstore_hgetall(hkey=self._rulebook_review_submissions_hkey) or {} + prefix = f"{job_id}:" + return { + key[len(prefix):]: value + for key, value in payload.items() + if isinstance(key, str) and key.startswith(prefix) and isinstance(value, dict) + } + + def delete_job_rulebook_reviews(self, job_id): + prefix = f"{job_id}:" + for hkey in ( + self._rulebook_review_hkey, + self._rulebook_review_audit_hkey, + self._rulebook_review_submissions_hkey, + ): + payload = self.owner.chainstore_hgetall(hkey=hkey) or {} + if not isinstance(payload, dict): + continue + for key in list(payload): + if isinstance(key, str) and key.startswith(prefix): + self.owner.chainstore_hset(hkey=hkey, key=key, value=None) + return diff --git a/extensions/business/cybersec/red_mesh/services/__init__.py b/extensions/business/cybersec/red_mesh/services/__init__.py index 7abf84570..bdd77b61e 100644 --- a/extensions/business/cybersec/red_mesh/services/__init__.py +++ b/extensions/business/cybersec/red_mesh/services/__init__.py @@ -1,5 +1,4 @@ from .config import ( - DEFAULT_API_OPERATIONS_CONFIG, DEFAULT_EVENT_EXPORT_CONFIG, DEFAULT_MODEL_TESTING_CONFIG, DEFAULT_OPENCTI_EXPORT_CONFIG, @@ -8,7 +7,6 @@ DEFAULT_TAXII_EXPORT_CONFIG, DEFAULT_WAZUH_EXPORT_CONFIG, get_attestation_config, - get_api_operations_config, get_event_export_config, get_graybox_budgets_config, get_llm_agent_config, @@ -20,17 +18,6 @@ get_wazuh_export_config, resolve_config_block, ) -from .api_operations import ( - ApiOperationRepository, - cancel_api_operation, - create_analyze_job_operation, - derive_operation_auth_context, - execute_api_operation_worker, - get_api_operation_result, - get_api_operation_status, - maybe_start_api_operation_worker, - public_operation_view, -) from .misp_config import get_misp_export_config from .misp_export import ( build_misp_event, @@ -43,6 +30,18 @@ export_stix_bundle, get_stix_export_status, ) +from .rulebook_assessment import ( + DEFAULT_RULEBOOK_PROFILE_ID, + build_rulebook_assessment, + generate_rulebook_assessment, + get_rulebook_assessment_status, + get_rulebook_review, + list_rulebook_profiles, + reopen_rulebook_review, + save_rulebook_review_draft, + submit_rulebook_review, + update_rulebook_review, +) from .opencti_export import ( dry_run_opencti_export, get_opencti_export_status, @@ -179,25 +178,17 @@ "INTERMEDIATE_JOB_STATUSES", "ScanStrategy", "TERMINAL_JOB_STATUSES", - "DEFAULT_API_OPERATIONS_CONFIG", "DEFAULT_EVENT_EXPORT_CONFIG", "DEFAULT_MODEL_TESTING_CONFIG", "DEFAULT_OPENCTI_EXPORT_CONFIG", + "DEFAULT_RULEBOOK_PROFILE_ID", "DEFAULT_STIX_EXPORT_CONFIG", "DEFAULT_SURICATA_CORRELATION_CONFIG", "DEFAULT_TAXII_EXPORT_CONFIG", "DEFAULT_WAZUH_EXPORT_CONFIG", - "ApiOperationRepository", "can_transition_job_status", - "cancel_api_operation", "coerce_scan_type", - "create_analyze_job_operation", - "derive_operation_auth_context", - "execute_api_operation_worker", "get_attestation_config", - "get_api_operation_result", - "get_api_operation_status", - "get_api_operations_config", "get_event_export_config", "get_graybox_budgets_config", "get_llm_agent_config", @@ -213,6 +204,7 @@ "build_finding_event", "build_lifecycle_event", "build_misp_event", + "build_rulebook_assessment", "build_stix_bundle", "build_redmesh_event", "build_service_observed_event", @@ -223,15 +215,16 @@ "dry_run_taxii_export", "export_misp_json", "export_stix_bundle", + "generate_rulebook_assessment", "get_misp_export_status", "get_opencti_export_status", + "get_rulebook_assessment_status", + "get_rulebook_review", "get_stix_export_status", "get_taxii_export_status", "push_to_misp", "push_to_opencti", "publish_to_taxii", - "maybe_start_api_operation_worker", - "public_operation_view", "redact_event_payload", "resolve_config_block", "stable_hmac_pseudonym", @@ -268,6 +261,10 @@ "launch_webapp_scan", "list_local_jobs", "list_network_jobs", + "list_rulebook_profiles", + "reopen_rulebook_review", + "save_rulebook_review_draft", + "submit_rulebook_review", "maybe_finalize_pass", "normalize_common_launch_options", "parse_exceptions", @@ -292,6 +289,7 @@ "get_job_archive_with_triage", "get_job_triage", "update_finding_triage", + "update_rulebook_review", "validation_error", # engagement context (Phase 3) "AuthorizationUploadError", diff --git a/extensions/business/cybersec/red_mesh/services/api_operations.py b/extensions/business/cybersec/red_mesh/services/api_operations.py deleted file mode 100644 index 699aa64fd..000000000 --- a/extensions/business/cybersec/red_mesh/services/api_operations.py +++ /dev/null @@ -1,1363 +0,0 @@ -from __future__ import annotations - -import copy -import hashlib -import hmac -import json -import os -import re -import threading -import uuid -from datetime import datetime, timedelta, timezone - -from ..models import render_legacy_llm_fields -from .config import get_api_operations_config, get_llm_agent_config -from .scan_strategy import coerce_scan_type, get_scan_strategy - - -OPERATION_SCHEMA_VERSION = "redmesh_api_operation_v1" -OPERATION_TYPE_ANALYZE_JOB = "analyze_job" - -STATE_QUEUED = "queued" -STATE_RUNNING = "running" -STATE_SUCCEEDED = "succeeded" -STATE_FAILED = "failed" -STATE_CANCEL_REQUESTED = "cancel_requested" -STATE_CANCELED = "canceled" -STATE_EXPIRED = "expired" - -ACTIVE_OPERATION_STATES = { - STATE_QUEUED, - STATE_RUNNING, - STATE_CANCEL_REQUESTED, -} -TERMINAL_OPERATION_STATES = { - STATE_SUCCEEDED, - STATE_FAILED, - STATE_CANCELED, - STATE_EXPIRED, -} - -PUBLIC_FAILURE_KEYS = { - "failure_class", - "retryable", - "phase", - "short_message", - "attempt_count", -} -RESULT_HANDLE_PREFIX = "opres_" - -ALLOWED_ANALYSIS_TYPES = { - "structured_report_sections", -} -ALLOWED_FOCUS_AREAS = { - "api", - "authentication", - "authorization", - "databases", - "network", - "services", - "tls", - "web", -} -_CID_RE = re.compile(r"(?i)\b(?:Qm[1-9A-HJ-NP-Za-km-z]{20,}|bafy[a-z2-7]{20,}|r1fs:[^\s,;]+)\b") -_URL_RE = re.compile(r"(?i)\bhttps?://[^\s,;]+") -_SENSITIVE_RE = re.compile(r"(?i)(api[_-]?key|authorization|bearer|password|secret|token|credential|prompt)") -_TOKEN_VALUE_RE = re.compile( - r"(?i)\b(?:sk|pk|rk|xox[baprs]|gh[pousr]|eyJ)[A-Za-z0-9._=-]{8,}\b" -) - - -class ApiOperationError(Exception): - def __init__(self, code: str, message: str = "", *, retryable: bool = False): - super().__init__(message or code) - self.code = code - self.message = message or code - self.retryable = bool(retryable) - - -def _utc_timestamp(): - return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _utc_timestamp_after(seconds: int): - delta = timedelta(seconds=max(_safe_int(seconds, 0), 0)) - return (datetime.now(timezone.utc) + delta).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _safe_int(value, default=0): - try: - return int(value) - except (TypeError, ValueError): - return default - - -def _sha256(value: str) -> str: - return hashlib.sha256(str(value or "").encode("utf-8")).hexdigest() - - -def _hmac_sha256(secret: str, value: str) -> str: - return hmac.new( - str(secret or "").encode("utf-8"), - str(value or "").encode("utf-8"), - hashlib.sha256, - ).hexdigest() - - -def _constant_time_in(value: str, candidates: list[str]) -> bool: - found = False - for candidate in candidates: - found = hmac.compare_digest(value, str(candidate or "")) or found - return found - - -def _error(code: str, message: str = "", *, retryable: bool = False, **extra): - payload = { - "error": code, - "message": message or code, - "retryable": bool(retryable), - } - payload.update(extra) - return payload - - -def _not_found(): - return _error("operation_not_found", "Operation not found") - - -def _operation_store_lock(owner): - lock = getattr(owner, "_api_operation_store_lock", None) - if lock is None: - lock = threading.RLock() - setattr(owner, "_api_operation_store_lock", lock) - return lock - - -def _job_operation_lock(owner, job_id: str): - locks = getattr(owner, "_api_operation_job_locks", None) - if not isinstance(locks, dict): - locks = {} - setattr(owner, "_api_operation_job_locks", locks) - key = str(job_id or "") - lock = locks.get(key) - if lock is None: - lock = threading.RLock() - locks[key] = lock - return lock - - -def _safe_log(owner, message: str, color=None): - logger = getattr(owner, "P", None) - if callable(logger): - if color is None: - logger(message) - else: - logger(message, color=color) - - -def _resolve_operation_hmac_secret(config: dict) -> str: - inline = str(config.get("HMAC_SECRET") or "").strip() - if inline: - return inline - env_name = str(config.get("HMAC_SECRET_ENV") or "").strip() - if env_name: - return str(os.environ.get(env_name, "") or "").strip() - return "" - - -def _valid_token_hashes(config: dict) -> list[str]: - hashes = [str(item or "").strip().lower() for item in config.get("TOKEN_HASHES") or [] if item] - env_name = str(config.get("TOKEN_ENV") or "").strip() - if env_name: - env_token = str(os.environ.get(env_name, "") or "").strip() - if env_token: - hashes.append(_sha256(env_token)) - return hashes - - -def derive_operation_auth_context(owner, token: str) -> dict: - config = get_api_operations_config(owner) - if not config.get("ENABLED"): - raise ApiOperationError("operation_auth_disabled", "API operations are disabled") - - token = str(token or "").strip() - if not token: - raise ApiOperationError("operation_auth_required", "Bearer token is required") - - token_hash = _sha256(token) - valid_hashes = _valid_token_hashes(config) - if not valid_hashes: - raise ApiOperationError("operation_auth_not_configured", "API operation auth is not configured") - if not _constant_time_in(token_hash, valid_hashes): - raise ApiOperationError("operation_auth_denied", "Bearer token is not authorized") - - secret = _resolve_operation_hmac_secret(config) - if not secret: - raise ApiOperationError("operation_auth_not_configured", "API operation HMAC secret is not configured") - - scope_id = str(getattr(owner, "cfg_instance_id", "") or "redmesh").strip() or "redmesh" - actor_digest = _hmac_sha256(secret, f"token:{token}")[:32] - actor_id = f"token:{actor_digest}" - return { - "tenant_id": scope_id, - "scope_id": scope_id, - "scope_hash": _hmac_sha256(secret, f"scope:{scope_id}"), - "actor_id": actor_id, - "actor_hash": _hmac_sha256(secret, f"actor:{actor_id}"), - "auth_source": "redmesh_api_operations", - "hmac_secret": secret, - "config": config, - } - - -class ApiOperationRepository: - """Repository for RedMesh API operation rows stored in CStore.""" - - def __init__(self, owner): - self.owner = owner - - @property - def operations_hkey(self): - return f"{getattr(self.owner, 'cfg_instance_id', 'redmesh')}:api_operations" - - @property - def idempotency_hkey(self): - return f"{getattr(self.owner, 'cfg_instance_id', 'redmesh')}:api_operations:idempotency" - - def get_operation(self, operation_id: str): - return self.owner.chainstore_hget(hkey=self.operations_hkey, key=operation_id) - - def list_operations(self): - payload = self.owner.chainstore_hgetall(hkey=self.operations_hkey) - return payload if isinstance(payload, dict) else {} - - def put_operation(self, operation: dict, *, expected_revision=None, context: str = "", allow_stale=False) -> dict: - operation_id = str((operation or {}).get("operation_id") or "") - current = self.get_operation(operation_id) - current_revision = _safe_int((current or {}).get("revision"), 0) if isinstance(current, dict) else 0 - incoming_revision = _safe_int((operation or {}).get("revision"), 0) - if expected_revision is None: - expected_revision = incoming_revision - - if isinstance(current, dict) and current_revision != expected_revision: - audit = getattr(self.owner, "_log_audit_event", None) - if callable(audit): - audit("api_operation_stale_write_detected", { - "operation_id": operation_id, - "expected_revision": expected_revision, - "current_revision": current_revision, - "context": context or "", - "write_mode": "detection_only", - }) - if not allow_stale: - return current - - payload = dict(operation or {}) - payload["revision"] = current_revision + 1 - payload["updated_at"] = _utc_timestamp() - self.owner.chainstore_hset(hkey=self.operations_hkey, key=operation_id, value=payload) - return payload - - def get_idempotency(self, key: str): - return self.owner.chainstore_hget(hkey=self.idempotency_hkey, key=key) - - def put_idempotency(self, key: str, value: dict): - self.owner.chainstore_hset(hkey=self.idempotency_hkey, key=key, value=dict(value or {})) - return value - - -def _normalize_focus_areas(value, config: dict) -> list[str]: - if value is None: - raw_values = [] - elif isinstance(value, (list, tuple, set)): - raw_values = list(value) - else: - raw_values = [value] - - max_items = _safe_int(config.get("MAX_FOCUS_AREAS"), 8) - max_len = _safe_int(config.get("MAX_FOCUS_AREA_LENGTH"), 80) - normalized = [] - seen = set() - for item in raw_values: - text = str(item or "").strip() - if not text: - continue - text = text[:max_len] - key = text.lower().replace("-", "_").replace(" ", "_") - if key not in ALLOWED_FOCUS_AREAS: - raise ApiOperationError("invalid_focus_area", "focus_areas contains an unsupported value") - if key in seen: - continue - seen.add(key) - normalized.append(key) - if len(normalized) >= max_items: - break - return sorted(normalized, key=lambda item: item.lower()) - - -def _normalize_analysis_type(value: str) -> str: - normalized = str(value or "structured_report_sections").strip().lower() - if not normalized: - normalized = "structured_report_sections" - if normalized not in ALLOWED_ANALYSIS_TYPES: - raise ApiOperationError("invalid_analysis_type", "analysis_type is not supported for async analyze operations") - return normalized - - -def _redact_public_text(value: str, limit=160) -> str: - text = str(value or "") - text = _CID_RE.sub("[redacted-cid]", text) - text = _URL_RE.sub("[redacted-url]", text) - text = _TOKEN_VALUE_RE.sub("[redacted-token]", text) - if _SENSITIVE_RE.search(text): - text = _SENSITIVE_RE.sub("[redacted]", text) - if len(text) > limit: - text = text[:limit].rstrip() + "..." - return text - - -def _sanitize_public_value(value, *, depth=0): - if depth > 3: - return None - if isinstance(value, str): - return _redact_public_text(value) - if isinstance(value, bool) or value is None: - return value - if isinstance(value, (int, float)): - return value - if isinstance(value, list): - return [ - item for item in ( - _sanitize_public_value(item, depth=depth + 1) - for item in value[:16] - ) - if item is not None - ] - if isinstance(value, dict): - public = {} - for key, item in value.items(): - key_text = str(key or "") - if _SENSITIVE_RE.search(key_text) or key_text.endswith("_cid") or key_text in {"cid", "url", "details"}: - continue - sanitized = _sanitize_public_value(item, depth=depth + 1) - if sanitized is not None: - public[key_text[:64]] = sanitized - return public - return _redact_public_text(value) - - -def _canonical_fingerprint(secret: str, payload: dict) -> str: - canonical = json.dumps(payload, sort_keys=True, separators=(",", ":")) - return _hmac_sha256(secret, canonical) - - -def _idempotency_hash(context: dict, idempotency_key: str) -> str: - return _hmac_sha256(context["hmac_secret"], f"idempotency:{idempotency_key}") - - -def _idempotency_index_key(context: dict, operation_type: str, idempotency_key_hash: str) -> str: - return ":".join([ - context["scope_hash"], - context["actor_hash"], - operation_type, - idempotency_key_hash, - ]) - - -def _operation_visible_to_context(operation: dict, context: dict) -> bool: - return ( - isinstance(operation, dict) - and operation.get("actor_hash") == context.get("actor_hash") - and operation.get("scope_hash") == context.get("scope_hash") - ) - - -def _parse_utc_timestamp(value: str): - try: - return datetime.fromisoformat(str(value or "").replace("Z", "+00:00")) - except ValueError: - return None - - -def _operation_expired(operation: dict) -> bool: - expires_at = _parse_utc_timestamp((operation or {}).get("expires_at")) - if expires_at is None: - return False - return datetime.now(timezone.utc) >= expires_at - - -def _lease_expired(operation: dict) -> bool: - lease = (operation or {}).get("lease") - if not isinstance(lease, dict): - return False - expires_at = _parse_utc_timestamp(lease.get("expires_at")) - if expires_at is None: - return False - return datetime.now(timezone.utc) >= expires_at - - -def _expire_operation_if_needed(repo: ApiOperationRepository, operation: dict) -> dict: - if not isinstance(operation, dict) or not _operation_expired(operation): - return operation - if operation.get("state") in TERMINAL_OPERATION_STATES: - return operation - updated = dict(operation) - updated["state"] = STATE_EXPIRED - updated["phase"] = "expired" - updated["finished_at"] = updated.get("finished_at") or _utc_timestamp() - return repo.put_operation( - updated, - expected_revision=operation.get("revision"), - context="expire_api_operation", - ) - - -def _recover_stale_operation_if_needed(repo: ApiOperationRepository, operation: dict) -> dict: - operation = _expire_operation_if_needed(repo, operation) - if not isinstance(operation, dict): - return operation - if operation.get("operation_type") != OPERATION_TYPE_ANALYZE_JOB: - return operation - if not _operation_owned_by_node(repo.owner, operation): - return operation - if not _operation_job_visible(repo.owner, operation): - return operation - if operation.get("state") in TERMINAL_OPERATION_STATES: - return operation - if operation.get("state") not in {STATE_RUNNING, STATE_CANCEL_REQUESTED}: - return operation - if not _lease_expired(operation): - return operation - - updated = dict(operation) - now = _utc_timestamp() - if operation.get("state") == STATE_CANCEL_REQUESTED: - cancel = dict(updated.get("cancel") or {}) - cancel["requested"] = True - cancel["observed_at"] = now - cancel["side_effects"] = "unknown_after_restart" - updated.update({ - "state": STATE_CANCELED, - "phase": "canceled", - "finished_at": now, - "lease": {}, - "cancel": cancel, - }) - elif _safe_int(operation.get("attempt"), 0) < _safe_int(operation.get("max_attempts"), 1): - updated.update({ - "state": STATE_QUEUED, - "phase": "recovered", - "lease": {}, - "retryable": True, - "recovered_at": now, - }) - else: - updated.update({ - "state": STATE_FAILED, - "phase": "lease_expired", - "finished_at": now, - "lease": {}, - "retryable": False, - "failure": _failure_payload( - "operation_failed", - "lease_expired", - "Operation worker lease expired", - retryable=False, - attempt_count=operation.get("attempt"), - ), - }) - - return repo.put_operation( - updated, - expected_revision=operation.get("revision"), - context="recover_stale_api_operation", - ) - - -def _operation_job_visible(owner, operation: dict) -> bool: - job_id = str((operation or {}).get("related_job_id") or "").strip() - if not job_id: - return False - return isinstance(_get_job(owner, job_id), dict) - - -def _operation_owned_by_node(owner, operation: dict) -> bool: - owner_node = str((operation or {}).get("owner_node") or "").strip() - local_node = str(getattr(owner, "ee_addr", "") or "").strip() - return not owner_node or not local_node or owner_node == local_node - - -def _operation_cancel_requested(operation: dict) -> bool: - cancel = (operation or {}).get("cancel") - return ( - (operation or {}).get("state") == STATE_CANCEL_REQUESTED - or isinstance(cancel, dict) and bool(cancel.get("requested")) - ) - - -def _failure_payload(failure_class: str, phase: str, message: str, *, retryable=False, attempt_count=0): - public_class = failure_class if failure_class in { - "job_changed", - "job_has_no_report_data", - "job_not_found", - "llm_disabled", - "llm_config_error", - "result_persist_failed", - "structured_llm_failed", - "unsupported_operation_type", - "worker_start_failed", - } else "operation_failed" - public_message = message if isinstance(message, str) and len(message) <= 240 else public_class - if public_class == "operation_failed": - public_message = "Operation failed" - return { - "failure_class": str(public_class or "operation_failed"), - "phase": str(phase or "failed"), - "short_message": _redact_public_text(public_message or public_class or "Operation failed"), - "retryable": bool(retryable), - "attempt_count": _safe_int(attempt_count, 0), - } - - -def _worker_thread_alive(owner) -> bool: - worker = getattr(owner, "_api_operation_worker_thread", None) - return bool(worker and worker.is_alive()) - - -def _operation_lease_valid(operation: dict, lease_token: str) -> bool: - lease = (operation or {}).get("lease") - return isinstance(lease, dict) and hmac.compare_digest( - str(lease.get("token") or ""), - str(lease_token or ""), - ) - - -def _result_handle(owner, operation_id: str, job_id: str, pass_nr) -> str: - secret = _resolve_operation_hmac_secret(get_api_operations_config(owner)) - if not secret: - return f"{RESULT_HANDLE_PREFIX}{uuid.uuid4().hex}" - digest = _hmac_sha256(secret, f"{operation_id}:{job_id}:{pass_nr}")[:32] - return f"{RESULT_HANDLE_PREFIX}{digest}" - - -def _write_job_record(owner, job_id: str, job_specs: dict, *, expected_revision=None, context=""): - writer = getattr(type(owner), "_write_job_record", None) - if callable(writer): - return writer(owner, job_id, job_specs, expected_revision=expected_revision, context=context) - writer = getattr(owner, "_write_job_record", None) - if callable(writer): - return writer(job_id, job_specs, expected_revision=expected_revision, context=context) - owner.chainstore_hset(hkey=getattr(owner, "cfg_instance_id", "redmesh"), key=job_id, value=job_specs) - return job_specs - - -def _job_revision(job_specs: dict) -> int: - return _safe_int((job_specs or {}).get("job_revision"), 0) - - -def _latest_pass_matches(job_specs: dict, *, expected_revision: int, report_cid: str, pass_nr) -> bool: - if not isinstance(job_specs, dict): - return False - if _job_revision(job_specs) != _safe_int(expected_revision, 0): - return False - pass_reports = job_specs.get("pass_reports") or [] - if not pass_reports: - return False - latest_ref = pass_reports[-1] - if latest_ref.get("report_cid") != report_cid: - return False - if pass_nr is not None and latest_ref.get("pass_nr") != pass_nr: - return False - return True - - -def _operation_target_matches_job(operation: dict, job_specs: dict) -> bool: - pass_reports = (job_specs or {}).get("pass_reports") or [] - latest_ref = pass_reports[-1] if pass_reports else {} - target_revision = _safe_int((operation or {}).get("target_job_revision_at_create"), 0) - return _latest_pass_matches( - job_specs, - expected_revision=target_revision, - report_cid=(operation or {}).get("target_report_cid_at_create"), - pass_nr=(operation or {}).get("target_pass_nr_at_create"), - ) and bool(latest_ref) - - -def _fail_job_changed(owner, operation_id: str, lease_token: str, *, artifact_written=False): - failure = _failure_payload( - "job_changed", - "job_changed", - "Job pass report changed before the operation could update it", - retryable=False, - ) - if artifact_written: - failure["side_effects"] = "result_artifact_written" - return _finish_worker_operation( - owner, - operation_id, - lease_token, - state=STATE_FAILED, - phase="job_changed", - failure=failure, - retryable=False, - ) - - -def _emit_operation_timeline(owner, job_specs: dict, event_type: str, message: str, *, pass_nr=None): - emitter = getattr(owner, "_emit_timeline_event", None) - if not callable(emitter): - return - meta = {} - if pass_nr is not None: - meta["pass_nr"] = pass_nr - emitter(job_specs, event_type, message, actor_type="user", meta=meta) - - -def _load_worker_operation(repo: ApiOperationRepository, operation_id: str, lease_token: str): - operation = repo.get_operation(operation_id) - operation = _expire_operation_if_needed(repo, operation) - if not isinstance(operation, dict): - return None - if operation.get("state") in TERMINAL_OPERATION_STATES: - return None - if not _operation_lease_valid(operation, lease_token): - return None - return operation - - -def _update_worker_operation(owner, operation_id: str, lease_token: str, updates: dict, *, context: str): - with _operation_store_lock(owner): - repo = ApiOperationRepository(owner) - operation = _load_worker_operation(repo, operation_id, lease_token) - if not isinstance(operation, dict) or operation.get("state") in TERMINAL_OPERATION_STATES: - return operation - - updated = dict(operation) - updated.update(dict(updates or {})) - lease = dict(updated.get("lease") or {}) - lease["heartbeat_at"] = _utc_timestamp() - lease["expires_at"] = _utc_timestamp_after(context_config(owner).get("LEASE_SECONDS")) - updated["lease"] = lease - return repo.put_operation( - updated, - expected_revision=operation.get("revision"), - context=context, - ) - - -def context_config(owner) -> dict: - return get_api_operations_config(owner) - - -def _finish_worker_operation( - owner, - operation_id: str, - lease_token: str, - *, - state: str, - phase: str, - failure=None, - result_public=None, - retryable=False, - cancel_side_effects=None, -): - with _operation_store_lock(owner): - repo = ApiOperationRepository(owner) - operation = _load_worker_operation(repo, operation_id, lease_token) - if not isinstance(operation, dict): - return None - if operation.get("state") in TERMINAL_OPERATION_STATES: - return operation - - updated = dict(operation) - updated["state"] = state - updated["phase"] = phase - updated["finished_at"] = _utc_timestamp() - updated["retryable"] = bool(retryable) - if failure: - updated["failure"] = failure - if result_public: - updated["result_public"] = result_public - if cancel_side_effects is not None: - cancel = dict(updated.get("cancel") or {}) - cancel["requested"] = True - cancel["observed_at"] = updated["finished_at"] - cancel["side_effects"] = cancel_side_effects - updated["cancel"] = cancel - return repo.put_operation( - updated, - expected_revision=operation.get("revision"), - context=f"finish_{phase}", - ) - - -def _fail_worker_operation(owner, operation_id: str, lease_token: str, phase: str, exc, *, retryable=False): - failure = _failure_payload( - exc.__class__.__name__ if not isinstance(exc, ApiOperationError) else exc.code, - phase, - getattr(exc, "message", str(exc)), - retryable=retryable or bool(getattr(exc, "retryable", False)), - attempt_count=0, - ) - return _finish_worker_operation( - owner, - operation_id, - lease_token, - state=STATE_FAILED, - phase=phase, - failure=failure, - retryable=failure["retryable"], - ) - - -def _cancel_worker_operation(owner, operation_id: str, lease_token: str, *, side_effects: str): - return _finish_worker_operation( - owner, - operation_id, - lease_token, - state=STATE_CANCELED, - phase="canceled", - cancel_side_effects=side_effects, - ) - - -def _public_failure(failure): - if not isinstance(failure, dict): - return None - return { - key: _sanitize_public_value(failure[key]) - for key in PUBLIC_FAILURE_KEYS - if key in failure - } - - -def _public_result(result): - if not isinstance(result, dict): - return None - public = {} - if result.get("kind"): - public["kind"] = result.get("kind") - if result.get("handle"): - public["handle"] = result.get("handle") - if result.get("pass_nr") is not None: - public["pass_nr"] = result.get("pass_nr") - summary = result.get("summary") - if isinstance(summary, dict): - public["summary"] = _sanitize_public_value(summary) - return public or None - - -def public_operation_view(operation: dict) -> dict: - if not isinstance(operation, dict): - return {} - view = { - "operation_id": operation.get("operation_id"), - "operation_type": operation.get("operation_type"), - "state": operation.get("state"), - "phase": operation.get("phase"), - "related_job_id": operation.get("related_job_id"), - "created_at": operation.get("created_at"), - "updated_at": operation.get("updated_at"), - "started_at": operation.get("started_at"), - "finished_at": operation.get("finished_at"), - "expires_at": operation.get("expires_at"), - "retryable": bool(operation.get("retryable", False)), - "attempt": operation.get("attempt"), - "max_attempts": operation.get("max_attempts"), - } - poll_after_ms = operation.get("poll_after_ms") - if poll_after_ms is not None: - view["poll_after_ms"] = poll_after_ms - failure = _public_failure(operation.get("failure")) - if failure: - view["failure"] = failure - result = _public_result(operation.get("result_public")) - if result: - view["result"] = result - cancel = operation.get("cancel") - if isinstance(cancel, dict): - view["cancel"] = { - key: cancel.get(key) - for key in ("requested", "requested_at", "observed_at", "side_effects") - if key in cancel - } - return {key: value for key, value in view.items() if value is not None} - - -def _active_operations(repo: ApiOperationRepository): - return [ - operation for operation in (repo.list_operations() or {}).values() - if ( - isinstance(operation, dict) - and operation.get("state") in ACTIVE_OPERATION_STATES - and not _operation_expired(operation) - and not _lease_expired(operation) - ) - ] - - -def _check_backpressure(repo: ApiOperationRepository, context: dict, job_id: str): - config = context["config"] - active = _active_operations(repo) - if len(active) >= _safe_int(config.get("MAX_QUEUE_GLOBAL"), 32): - raise ApiOperationError("operation_backpressure", "API operation queue is full", retryable=True) - - actor_count = sum(1 for operation in active if operation.get("actor_hash") == context["actor_hash"]) - if actor_count >= _safe_int(config.get("MAX_QUEUE_PER_ACTOR"), 8): - raise ApiOperationError("operation_backpressure", "API operation actor quota is full", retryable=True) - - job_count = sum( - 1 for operation in active - if operation.get("operation_type") == OPERATION_TYPE_ANALYZE_JOB - and operation.get("related_job_id") == job_id - ) - if job_count >= _safe_int(config.get("MAX_QUEUE_PER_JOB"), 1): - raise ApiOperationError("operation_backpressure", "API operation job quota is full", retryable=True) - - -def _get_job(owner, job_id: str): - getter = getattr(owner, "_get_job_from_cstore", None) - if callable(getter): - return getter(job_id) - return owner.chainstore_hget(hkey=getattr(owner, "cfg_instance_id", "redmesh"), key=job_id) - - -def _has_report_data(job_specs: dict) -> bool: - pass_reports = job_specs.get("pass_reports") - if isinstance(pass_reports, list) and pass_reports: - return True - workers = job_specs.get("workers") or {} - if not isinstance(workers, dict): - return False - for worker in workers.values(): - if not isinstance(worker, dict): - continue - if worker.get("report_cid") or worker.get("result"): - return True - return False - - -def _validate_analyze_job_admission(owner, job_id: str) -> dict: - llm_cfg = get_llm_agent_config(owner) - if not llm_cfg.get("ENABLED"): - raise ApiOperationError("llm_disabled", "LLM Agent API is not enabled") - if not getattr(owner, "cfg_llm_agent_api_port", None): - raise ApiOperationError("llm_config_error", "LLM Agent API port not configured") - - job_specs = _get_job(owner, job_id) - if not isinstance(job_specs, dict): - raise ApiOperationError("job_not_found", "Job not found") - - workers = job_specs.get("workers") or {} - if not isinstance(workers, dict) or not workers: - raise ApiOperationError("job_has_no_workers", "No workers found for this job") - if not all(isinstance(worker, dict) and worker.get("finished") for worker in workers.values()): - raise ApiOperationError("job_not_complete", "Job not yet complete") - if not _has_report_data(job_specs): - raise ApiOperationError("job_has_no_report_data", "No report data available for this job") - return job_specs - - -def _aggregate_node_reports(owner, job_specs: dict, node_reports: dict) -> dict: - worker_cls = None - try: - strategy = get_scan_strategy(coerce_scan_type((job_specs or {}).get("scan_type"))) - worker_cls = getattr(strategy, "worker_cls", None) - except Exception: - worker_cls = None - - if worker_cls is not None: - try: - return owner._get_aggregated_report(node_reports, worker_cls=worker_cls) - except TypeError: - pass - return owner._get_aggregated_report(node_reports) - - -def create_analyze_job_operation( - owner, - token: str, - job_id: str, - analysis_type: str = "", - focus_areas=None, - idempotency_key: str = "", -): - try: - context = derive_operation_auth_context(owner, token) - config = context["config"] - job_id = str(job_id or "").strip() - if not job_id: - raise ApiOperationError("invalid_job_id", "job_id is required") - - idempotency_key = str(idempotency_key or "").strip() - max_key_len = _safe_int(config.get("MAX_IDEMPOTENCY_KEY_LENGTH"), 128) - if idempotency_key and len(idempotency_key) > max_key_len: - raise ApiOperationError("invalid_idempotency_key", "Idempotency key is too long") - - normalized_focus = _normalize_focus_areas(focus_areas, config) - normalized_analysis = _normalize_analysis_type(analysis_type) - job_specs = _validate_analyze_job_admission(owner, job_id) - request_public = { - "analysis_type": normalized_analysis, - "focus_areas": normalized_focus, - } - request_fingerprint = _canonical_fingerprint(context["hmac_secret"], { - "operation_type": OPERATION_TYPE_ANALYZE_JOB, - "job_id": job_id, - "analysis_type": normalized_analysis, - "focus_areas": normalized_focus, - "scope_hash": context["scope_hash"], - "actor_hash": context["actor_hash"], - }) - - with _operation_store_lock(owner): - repo = ApiOperationRepository(owner) - idempotency_key_hash = "" - index_key = "" - if idempotency_key: - idempotency_key_hash = _idempotency_hash(context, idempotency_key) - index_key = _idempotency_index_key(context, OPERATION_TYPE_ANALYZE_JOB, idempotency_key_hash) - existing = repo.get_idempotency(index_key) - if isinstance(existing, dict): - if existing.get("request_fingerprint") != request_fingerprint: - return _error("idempotency_conflict", "Idempotency key was used with a different request") - operation = repo.get_operation(existing.get("operation_id", "")) - operation = _expire_operation_if_needed(repo, operation) - if _operation_visible_to_context(operation, context): - return { - "status": "accepted", - "idempotent_replay": True, - "operation": public_operation_view(operation), - } - - _check_backpressure(repo, context, job_id) - - pass_reports = job_specs.get("pass_reports") or [] - latest_ref = pass_reports[-1] if pass_reports else {} - now = _utc_timestamp() - operation_id = f"op_{uuid.uuid4().hex}" - operation = { - "schema_version": OPERATION_SCHEMA_VERSION, - "operation_id": operation_id, - "operation_type": OPERATION_TYPE_ANALYZE_JOB, - "owner_node": str(getattr(owner, "ee_addr", "") or ""), - "state": STATE_QUEUED, - "phase": "queued", - "actor": { - "actor_id": context["actor_id"], - "auth_source": context["auth_source"], - }, - "scope": { - "tenant_id": context["tenant_id"], - "scope_id": context["scope_id"], - }, - "actor_hash": context["actor_hash"], - "scope_hash": context["scope_hash"], - "related_job_id": job_id, - "target_job_revision_at_create": _safe_int(job_specs.get("job_revision"), 0), - "target_pass_nr_at_create": latest_ref.get("pass_nr"), - "target_report_cid_at_create": latest_ref.get("report_cid"), - "request_public": request_public, - "request_fingerprint": request_fingerprint, - "idempotency_key_hash": idempotency_key_hash, - "attempt": 0, - "max_attempts": max(_safe_int(getattr(owner, "cfg_llm_api_retries", 1), 1), 1), - "revision": 0, - "lease": {}, - "retryable": False, - "result_public": None, - "cancel": {}, - "created_at": now, - "updated_at": now, - "expires_at": _utc_timestamp_after(config.get("OPERATION_TTL_SECONDS")), - "poll_after_ms": _safe_int(config.get("POLL_AFTER_MS"), 1000), - } - operation = repo.put_operation(operation, expected_revision=0, context="create_analyze_job_operation") - if index_key: - repo.put_idempotency(index_key, { - "operation_id": operation_id, - "request_fingerprint": request_fingerprint, - "created_at": now, - "expires_at": operation.get("expires_at"), - }) - - return { - "status": "accepted", - "operation": public_operation_view(operation), - } - except ApiOperationError as exc: - extra = {} - if exc.code == "operation_backpressure": - extra["retry_after_ms"] = _safe_int(get_api_operations_config(owner).get("POLL_AFTER_MS"), 1000) - return _error(exc.code, exc.message, retryable=exc.retryable, **extra) - - -def _claim_next_api_operation_locked(owner): - repo = ApiOperationRepository(owner) - lease_seconds = context_config(owner).get("LEASE_SECONDS") - now = _utc_timestamp() - - operations = sorted( - (repo.list_operations() or {}).values(), - key=lambda item: str((item or {}).get("created_at") or ""), - ) - for operation in operations: - operation = _recover_stale_operation_if_needed(repo, operation) - if not isinstance(operation, dict): - continue - if operation.get("state") != STATE_QUEUED: - continue - if operation.get("operation_type") != OPERATION_TYPE_ANALYZE_JOB: - continue - if not _operation_owned_by_node(owner, operation): - continue - if not _operation_job_visible(owner, operation): - continue - - lease_token = uuid.uuid4().hex - claimed = dict(operation) - attempt = _safe_int(claimed.get("attempt"), 0) + 1 - claimed.update({ - "state": STATE_RUNNING, - "phase": "claimed", - "started_at": claimed.get("started_at") or now, - "attempt": attempt, - "lease": { - "owner_node": str(getattr(owner, "ee_addr", "") or ""), - "token": lease_token, - "acquired_at": now, - "heartbeat_at": now, - "expires_at": _utc_timestamp_after(lease_seconds), - }, - }) - updated = repo.put_operation( - claimed, - expected_revision=operation.get("revision"), - context="claim_api_operation", - ) - if _operation_lease_valid(updated, lease_token) and updated.get("state") == STATE_RUNNING: - return updated["operation_id"], lease_token - return None - - -def maybe_start_api_operation_worker(owner) -> bool: - """Start one bounded background API operation worker if queued work exists.""" - config = get_api_operations_config(owner) - if not config.get("ENABLED"): - return False - - with _operation_store_lock(owner): - if _worker_thread_alive(owner): - return False - claimed = _claim_next_api_operation_locked(owner) - if not claimed: - return False - operation_id, lease_token = claimed - thread = threading.Thread( - target=execute_api_operation_worker, - args=(owner, operation_id, lease_token), - name=f"redmesh-api-operation-{operation_id[:12]}", - daemon=True, - ) - setattr(owner, "_api_operation_worker_thread", thread) - - try: - thread.start() - except Exception as exc: - _safe_log(owner, f"Failed to start API operation worker for {operation_id}: {exc}", color="y") - _fail_worker_operation(owner, operation_id, lease_token, "worker_start_failed", exc, retryable=True) - return False - return True - - -def _execute_analyze_job_operation(owner, operation: dict, lease_token: str): - operation_id = operation["operation_id"] - job_id = str(operation.get("related_job_id") or "") - - _update_worker_operation( - owner, - operation_id, - lease_token, - {"phase": "collecting_reports"}, - context="api_operation_collecting_reports", - ) - - job_specs = _validate_analyze_job_admission(owner, job_id) - if not _operation_target_matches_job(operation, job_specs): - _fail_job_changed(owner, operation_id, lease_token) - return - - workers = job_specs.get("workers") or {} - node_reports = owner._collect_node_reports(workers) - aggregated_report = _aggregate_node_reports(owner, job_specs, node_reports) if node_reports else {} - if not aggregated_report: - raise ApiOperationError("job_has_no_report_data", "No report data available for this job") - - job_config = owner._get_job_config(job_specs) - _risk_result, flat_findings = owner._compute_risk_and_findings(aggregated_report) - - _update_worker_operation( - owner, - operation_id, - lease_token, - {"phase": "llm_pending"}, - context="api_operation_llm_pending", - ) - - llm_report_sections = owner._run_structured_report_sections( - job_id=job_id, - findings=flat_findings, - aggregated_report=aggregated_report, - engagement=job_config.get("engagement") if isinstance(job_config, dict) else None, - ) - structured_failed = bool(getattr(owner, "_last_structured_llm_failed", False)) - if llm_report_sections is None: - raise ApiOperationError( - "structured_llm_failed", - "Structured LLM report generation failed", - retryable=True, - ) - - with _operation_store_lock(owner): - repo = ApiOperationRepository(owner) - current = _load_worker_operation(repo, operation_id, lease_token) - if not isinstance(current, dict): - return - if _operation_cancel_requested(current): - _cancel_worker_operation(owner, operation_id, lease_token, side_effects="none") - return - - _update_worker_operation( - owner, - operation_id, - lease_token, - {"phase": "persisting_result"}, - context="api_operation_persisting_result", - ) - - target_report_cid = operation.get("target_report_cid_at_create") - target_pass_nr = operation.get("target_pass_nr_at_create") - with _job_operation_lock(owner, job_id): - latest_job_specs = _get_job(owner, job_id) - if not isinstance(latest_job_specs, dict): - raise ApiOperationError("job_not_found", "Job not found") - latest_job_specs = copy.deepcopy(latest_job_specs) - expected_job_revision = _safe_int(operation.get("target_job_revision_at_create"), 0) - - pass_reports = latest_job_specs.get("pass_reports") or [] - if not pass_reports: - raise ApiOperationError("job_has_no_report_data", "No pass report data available for this job") - latest_ref = pass_reports[-1] - current_report_cid = latest_ref.get("report_cid") - current_pass_nr = latest_ref.get("pass_nr", latest_job_specs.get("job_pass", 1)) - if target_report_cid and current_report_cid != target_report_cid: - _fail_job_changed(owner, operation_id, lease_token) - return - if target_pass_nr is not None and current_pass_nr != target_pass_nr: - _fail_job_changed(owner, operation_id, lease_token) - return - - pass_data = owner.r1fs.get_json(current_report_cid) - if not isinstance(pass_data, dict): - raise ApiOperationError("job_has_no_report_data", "Pass report data is not available") - - fresh_job_specs = _get_job(owner, job_id) - if not _latest_pass_matches( - fresh_job_specs, - expected_revision=expected_job_revision, - report_cid=current_report_cid, - pass_nr=current_pass_nr, - ): - _fail_job_changed(owner, operation_id, lease_token) - return - - pass_data = dict(pass_data) - pass_data["llm_report_sections"] = llm_report_sections - pass_data["llm_operation_id"] = operation_id - if structured_failed: - pass_data["llm_failed"] = True - else: - pass_data.pop("llm_failed", None) - llm_text, summary_text = render_legacy_llm_fields(llm_report_sections) - if llm_text: - pass_data["llm_analysis"] = llm_text - if summary_text: - pass_data["quick_summary"] = summary_text - - with _operation_store_lock(owner): - repo = ApiOperationRepository(owner) - current = _load_worker_operation(repo, operation_id, lease_token) - if not isinstance(current, dict): - return - if _operation_cancel_requested(current): - _cancel_worker_operation(owner, operation_id, lease_token, side_effects="none") - return - - updated_cid = owner.r1fs.add_json(pass_data, show_logs=False) - if not updated_cid: - raise ApiOperationError("result_persist_failed", "Failed to persist updated pass report", retryable=True) - - fresh_job_specs = _get_job(owner, job_id) - if not _latest_pass_matches( - fresh_job_specs, - expected_revision=expected_job_revision, - report_cid=current_report_cid, - pass_nr=current_pass_nr, - ): - _fail_job_changed(owner, operation_id, lease_token, artifact_written=True) - return - - with _operation_store_lock(owner): - repo = ApiOperationRepository(owner) - current = _load_worker_operation(repo, operation_id, lease_token) - if not isinstance(current, dict): - return - if _operation_cancel_requested(current): - _cancel_worker_operation(owner, operation_id, lease_token, side_effects="result_artifact_written") - return - - latest_job_specs = copy.deepcopy(fresh_job_specs) - pass_reports = latest_job_specs.get("pass_reports") or [] - latest_ref = pass_reports[-1] - latest_ref["report_cid"] = updated_cid - _emit_operation_timeline( - owner, - latest_job_specs, - "llm_analysis", - "Manual structured LLM report sections completed", - pass_nr=current_pass_nr, - ) - _write_job_record( - owner, - job_id, - latest_job_specs, - expected_revision=_safe_int(latest_job_specs.get("job_revision"), 0), - context="manual_llm_operation_update", - ) - - handle = _result_handle(owner, operation_id, job_id, current_pass_nr) - result_public = { - "kind": "redmesh_analyze_job_operation_result", - "handle": handle, - "pass_nr": current_pass_nr, - "summary": { - "llm_report_sections_available": True, - "llm_failed": structured_failed, - "job_updated": True, - }, - } - _finish_worker_operation( - owner, - operation_id, - lease_token, - state=STATE_SUCCEEDED, - phase="succeeded", - result_public=result_public, - ) - - -def execute_api_operation_worker(owner, operation_id: str, lease_token: str): - try: - repo = ApiOperationRepository(owner) - with _operation_store_lock(owner): - operation = _load_worker_operation(repo, operation_id, lease_token) - if not isinstance(operation, dict) or operation.get("state") in TERMINAL_OPERATION_STATES: - return - if operation.get("operation_type") != OPERATION_TYPE_ANALYZE_JOB: - raise ApiOperationError("unsupported_operation_type", "Operation type is not supported") - _execute_analyze_job_operation(owner, operation, lease_token) - except Exception as exc: - _safe_log(owner, f"API operation {operation_id} failed: {exc}", color="y") - _fail_worker_operation(owner, operation_id, lease_token, "failed", exc, retryable=getattr(exc, "retryable", False)) - finally: - with _operation_store_lock(owner): - worker = getattr(owner, "_api_operation_worker_thread", None) - if worker is threading.current_thread(): - setattr(owner, "_api_operation_worker_thread", None) - - -def get_api_operation_status(owner, token: str, operation_id: str): - try: - context = derive_operation_auth_context(owner, token) - except ApiOperationError as exc: - return _error(exc.code, exc.message, retryable=exc.retryable) - - with _operation_store_lock(owner): - repo = ApiOperationRepository(owner) - operation = repo.get_operation(str(operation_id or "").strip()) - operation = _expire_operation_if_needed(repo, operation) - if not _operation_visible_to_context(operation, context): - return _not_found() - if not _operation_job_visible(owner, operation): - return _not_found() - return {"operation": public_operation_view(operation)} - - -def cancel_api_operation(owner, token: str, operation_id: str, reason: str = ""): - try: - context = derive_operation_auth_context(owner, token) - except ApiOperationError as exc: - return _error(exc.code, exc.message, retryable=exc.retryable) - - with _operation_store_lock(owner): - repo = ApiOperationRepository(owner) - operation = repo.get_operation(str(operation_id or "").strip()) - operation = _expire_operation_if_needed(repo, operation) - if not _operation_visible_to_context(operation, context): - return _not_found() - if not _operation_job_visible(owner, operation): - return _not_found() - - state = operation.get("state") - if state in TERMINAL_OPERATION_STATES: - return {"operation": public_operation_view(operation)} - - cancel = dict(operation.get("cancel") or {}) - cancel.update({ - "requested": True, - "requested_at": cancel.get("requested_at") or _utc_timestamp(), - "reason": _redact_public_text(str(reason or ""), limit=160), - "side_effects": cancel.get("side_effects") or "none", - }) - operation["cancel"] = cancel - if state == STATE_QUEUED: - operation["state"] = STATE_CANCELED - operation["phase"] = "canceled" - operation["finished_at"] = _utc_timestamp() - cancel["observed_at"] = operation["finished_at"] - else: - operation["state"] = STATE_CANCEL_REQUESTED - operation["phase"] = "cancel_requested" - updated = repo.put_operation(operation, expected_revision=operation.get("revision"), context="cancel_api_operation") - return {"operation": public_operation_view(updated)} - - -def get_api_operation_result(owner, token: str, result_handle: str): - try: - context = derive_operation_auth_context(owner, token) - except ApiOperationError as exc: - return _error(exc.code, exc.message, retryable=exc.retryable) - - handle = str(result_handle or "").strip() - if _CID_RE.search(handle): - return _error("invalid_result_handle", "Operation result handles are opaque; raw CIDs are not accepted") - if not handle.startswith(RESULT_HANDLE_PREFIX): - return _not_found() - - with _operation_store_lock(owner): - repo = ApiOperationRepository(owner) - for operation in (repo.list_operations() or {}).values(): - operation = _expire_operation_if_needed(repo, operation) - if not _operation_visible_to_context(operation, context): - continue - if not _operation_job_visible(owner, operation): - continue - if operation.get("state") != STATE_SUCCEEDED: - continue - result = _public_result(operation.get("result_public")) - if isinstance(result, dict) and result.get("handle") == handle: - return { - "operation_id": operation.get("operation_id"), - "result": result, - } - return _not_found() diff --git a/extensions/business/cybersec/red_mesh/services/config.py b/extensions/business/cybersec/red_mesh/services/config.py index adb820249..f2f7c94e7 100644 --- a/extensions/business/cybersec/red_mesh/services/config.py +++ b/extensions/business/cybersec/red_mesh/services/config.py @@ -160,23 +160,6 @@ def resolve_config_block(owner, block_name, defaults, normalizer=None): }, } -DEFAULT_API_OPERATIONS_CONFIG = { - "ENABLED": False, - "TOKEN_HASHES": [], - "TOKEN_ENV": "REDMESH_API_OPERATION_TOKEN", - "HMAC_SECRET": "", - "HMAC_SECRET_ENV": "REDMESH_API_OPERATION_HMAC_SECRET", - "MAX_IDEMPOTENCY_KEY_LENGTH": 128, - "MAX_FOCUS_AREAS": 8, - "MAX_FOCUS_AREA_LENGTH": 80, - "MAX_QUEUE_GLOBAL": 32, - "MAX_QUEUE_PER_ACTOR": 8, - "MAX_QUEUE_PER_JOB": 1, - "OPERATION_TTL_SECONDS": 86400, - "LEASE_SECONDS": 300, - "POLL_AFTER_MS": 1000, -} - _WAZUH_AUTH_MODES = {"static", "wazuh_jwt"} _TAXII_AUTH_MODES = {"static", "basic"} _OPENCTI_AUTH_MODES = {"static"} @@ -555,91 +538,6 @@ def _normalize(merged, defaults): ) -def get_api_operations_config(owner): - """Return normalized RedMesh async API operation config.""" - def _normalize_hashes(value): - if isinstance(value, str): - values = [item.strip() for item in value.split(",")] - elif isinstance(value, (list, tuple, set)): - values = [str(item or "").strip() for item in value] - else: - values = [] - return [item.lower() for item in values if item] - - def _normalize(merged, defaults): - return { - "ENABLED": bool(merged.get("ENABLED", defaults["ENABLED"])), - "TOKEN_HASHES": _normalize_hashes(merged.get("TOKEN_HASHES", defaults["TOKEN_HASHES"])), - "TOKEN_ENV": _safe_secret_env(merged.get("TOKEN_ENV"), defaults["TOKEN_ENV"]), - "HMAC_SECRET": str(merged.get("HMAC_SECRET") or defaults["HMAC_SECRET"]), - "HMAC_SECRET_ENV": _safe_secret_env( - merged.get("HMAC_SECRET_ENV"), - defaults["HMAC_SECRET_ENV"], - ), - "MAX_IDEMPOTENCY_KEY_LENGTH": _bounded_int( - merged.get("MAX_IDEMPOTENCY_KEY_LENGTH"), - defaults["MAX_IDEMPOTENCY_KEY_LENGTH"], - minimum=16, - maximum=512, - ), - "MAX_FOCUS_AREAS": _bounded_int( - merged.get("MAX_FOCUS_AREAS"), - defaults["MAX_FOCUS_AREAS"], - minimum=0, - maximum=32, - ), - "MAX_FOCUS_AREA_LENGTH": _bounded_int( - merged.get("MAX_FOCUS_AREA_LENGTH"), - defaults["MAX_FOCUS_AREA_LENGTH"], - minimum=8, - maximum=256, - ), - "MAX_QUEUE_GLOBAL": _bounded_int( - merged.get("MAX_QUEUE_GLOBAL"), - defaults["MAX_QUEUE_GLOBAL"], - minimum=1, - maximum=1024, - ), - "MAX_QUEUE_PER_ACTOR": _bounded_int( - merged.get("MAX_QUEUE_PER_ACTOR"), - defaults["MAX_QUEUE_PER_ACTOR"], - minimum=1, - maximum=256, - ), - "MAX_QUEUE_PER_JOB": _bounded_int( - merged.get("MAX_QUEUE_PER_JOB"), - defaults["MAX_QUEUE_PER_JOB"], - minimum=1, - maximum=16, - ), - "OPERATION_TTL_SECONDS": _bounded_int( - merged.get("OPERATION_TTL_SECONDS"), - defaults["OPERATION_TTL_SECONDS"], - minimum=60, - maximum=30 * 86400, - ), - "LEASE_SECONDS": _bounded_int( - merged.get("LEASE_SECONDS"), - defaults["LEASE_SECONDS"], - minimum=10, - maximum=3600, - ), - "POLL_AFTER_MS": _bounded_int( - merged.get("POLL_AFTER_MS"), - defaults["POLL_AFTER_MS"], - minimum=250, - maximum=60000, - ), - } - - return resolve_config_block( - owner, - "API_OPERATIONS", - DEFAULT_API_OPERATIONS_CONFIG, - normalizer=_normalize, - ) - - def get_event_export_config(owner): """Return normalized canonical RedMesh event export config.""" def _normalize(merged, defaults): diff --git a/extensions/business/cybersec/red_mesh/services/control.py b/extensions/business/cybersec/red_mesh/services/control.py index 5412a70fe..3ed003cca 100644 --- a/extensions/business/cybersec/red_mesh/services/control.py +++ b/extensions/business/cybersec/red_mesh/services/control.py @@ -1,3 +1,5 @@ +from contextlib import ExitStack + from ..constants import ( JOB_STATUS_FINALIZED, JOB_STATUS_RUNNING, @@ -46,11 +48,36 @@ def _delete_job_record(owner, job_id): _job_repo(owner).delete_job(job_id) +def _foreign_launcher_error(owner, job_id, job_specs, action): + launcher = job_specs.get("launcher") if isinstance(job_specs, dict) else None + if not launcher or launcher == getattr(owner, "ee_addr", None): + return None + owner.P(f"{action} rejected on non-launcher node for job {job_id}.", color='y') + owner._log_audit_event(f"job_{action}_owner_rejected", { + "job_id": job_id, + "writer": getattr(owner, "ee_addr", None), + }) + return { + "error": "job_launcher_mismatch", + "message": f"{action.replace('_', ' ').capitalize()} must be handled by the job launcher.", + "status_code": 409, + "job_id": job_id, + } + + def stop_and_delete_job(owner, job_id: str): """ Stop a running job, mark it stopped, then delegate to purge_job for full R1FS + CStore cleanup. """ + raw_job_specs = _job_repo(owner).get_job(job_id) + job_specs = None + if isinstance(raw_job_specs, dict): + _, job_specs = owner._normalize_job_record(job_id, raw_job_specs) + owner_error = _foreign_launcher_error(owner, job_id, job_specs, "stop_and_delete") + if owner_error: + return owner_error + local_workers = owner.scan_jobs.get(job_id) if local_workers: owner.P(f"Stopping and deleting job {job_id}.") @@ -60,9 +87,7 @@ def stop_and_delete_job(owner, job_id: str): owner.P(f"Job {job_id} stopped.") owner.scan_jobs.pop(job_id, None) - raw_job_specs = _job_repo(owner).get_job(job_id) - if isinstance(raw_job_specs, dict): - _, job_specs = owner._normalize_job_record(job_id, raw_job_specs) + if isinstance(job_specs, dict): workers_map = job_specs.setdefault("workers", {}) if is_model_test_job(job_specs): selected_worker = selected_model_test_worker_addr(job_specs, fallback=getattr(owner, "ee_addr", None)) @@ -101,6 +126,16 @@ def stop_and_delete_job(owner, job_id: str): def purge_job(owner, job_id: str): + """Serialize purge with every supported rulebook review mutation for the job.""" + from .rulebook_assessment import _submission_lock, list_rulebook_profiles + + with ExitStack() as stack: + for profile in sorted(list_rulebook_profiles(), key=lambda item: item["profile_id"]): + stack.enter_context(_submission_lock(owner, job_id, profile["profile_id"])) + return _purge_job_locked(owner, job_id) + + +def _purge_job_locked(owner, job_id: str): """ Purge a job: delete all R1FS artifacts, clean up live progress keys, then tombstone the CStore entry. @@ -110,6 +145,9 @@ def purge_job(owner, job_id: str): return {"status": "error", "message": f"Job {job_id} not found."} _, job_specs = owner._normalize_job_record(job_id, raw) + owner_error = _foreign_launcher_error(owner, job_id, job_specs, "purge") + if owner_error: + return owner_error job_status = job_specs.get("job_status", "") workers = job_specs.get("workers", {}) @@ -151,6 +189,59 @@ def _track(cid, source): for addr, w in workers.items(): _track(w.get("report_cid"), f"workers[{addr}].report_cid") + rulebook_assessments = job_specs.get("rulebook_assessments") + if isinstance(rulebook_assessments, dict): + for profile_id, meta in rulebook_assessments.items(): + if isinstance(meta, dict): + _track(meta.get("artifact_cid"), f"rulebook_assessments[{profile_id}].artifact_cid") + for hi, historical in enumerate(meta.get("history") or []): + if isinstance(historical, dict): + _track(historical.get("artifact_cid"), f"rulebook_assessments[{profile_id}].history[{hi}].artifact_cid") + + submission_hkey = f"{owner.cfg_instance_id}:rulebook_review:submissions" + submission_key_prefix = f"{job_id}:" + all_submission_rows = owner.chainstore_hgetall(hkey=submission_hkey) or {} + formal_submission_cids = set() + if isinstance(all_submission_rows, dict): + for key, registry in all_submission_rows.items(): + if not isinstance(key, str) or not key.startswith(submission_key_prefix) or not isinstance(registry, dict): + continue + for reference in registry.get("submissions") or []: + if isinstance(reference, dict) and isinstance(reference.get("cid"), str) and reference.get("cid"): + formal_submission_cids.add(reference["cid"]) + _track(reference["cid"], f"rulebook_review_submissions[{key}].submissions") + pending = registry.get("pending") + if isinstance(pending, dict) and isinstance(pending.get("cid"), str) and pending.get("cid"): + formal_submission_cids.add(pending["cid"]) + _track(pending["cid"], f"rulebook_review_submissions[{key}].pending") + + other_job_cids = set() + for key, registry in all_submission_rows.items(): + if not isinstance(key, str) or key.startswith(submission_key_prefix) or not isinstance(registry, dict): + continue + for reference in registry.get("submissions") or []: + if isinstance(reference, dict) and isinstance(reference.get("cid"), str) and reference.get("cid"): + other_job_cids.add(reference["cid"]) + pending = registry.get("pending") + if isinstance(pending, dict) and isinstance(pending.get("cid"), str) and pending.get("cid"): + other_job_cids.add(pending["cid"]) + all_jobs = _job_repo(owner).list_jobs() or {} + if isinstance(all_jobs, dict): + for other_job_id, other_payload in all_jobs.items(): + if other_job_id != job_id and isinstance(other_payload, dict): + other_job_cids.update(_collect_cids_from_raw(other_payload)) + shared_cids = formal_submission_cids & other_job_cids + if shared_cids: + owner.P(f"[PURGE] Shared submission CIDs retained: {sorted(shared_cids)}", color='r') + return { + "status": "partial", + "job_id": job_id, + "cids_deleted": 0, + "cids_failed": len(shared_cids), + "cids_total": len(cids), + "message": "Submission artifacts are referenced by another job; CStore was kept for retry.", + } + for ri, ref in enumerate(job_specs.get("pass_reports", [])): report_cid = ref.get("report_cid") if report_cid: @@ -170,9 +261,11 @@ def _track(cid, source): owner.P(f"[PURGE] Total CIDs collected: {len(cids)}: {sorted(cids)}") deleted, failed = 0, 0 + # R1FS deletion acknowledges local/relay unpin and garbage-collection requests. + # The relay sweep can take up to 24 hours, and reading here can rehydrate the CID. for cid in cids: try: - success = artifacts.delete(cid, show_logs=True, raise_on_error=False) + success = artifacts.delete(cid, show_logs=True, raise_on_error=False, purge=True) if success: deleted += 1 owner.P(f"[PURGE] Deleted CID {cid}") @@ -202,6 +295,7 @@ def _track(cid, source): _job_repo(owner).delete_live_progress(key) _job_repo(owner).delete_job_triage(job_id) + _job_repo(owner).delete_job_rulebook_reviews(job_id) _delete_job_record(owner, job_id) owner.P(f"Purged job {job_id}: {deleted}/{len(cids)} CIDs deleted.") @@ -223,36 +317,122 @@ def _collect_cids_from_raw(payload): yield from _collect_cids_from_raw(item) +def _collect_rulebook_submission_cids(payload): + if not isinstance(payload, dict): + return set() + cids = { + reference.get("cid") + for reference in payload.get("submissions") or [] + if isinstance(reference, dict) and isinstance(reference.get("cid"), str) and reference.get("cid") + } + pending = payload.get("pending") + if isinstance(pending, dict) and isinstance(pending.get("cid"), str) and pending.get("cid"): + cids.add(pending["cid"]) + return cids + + def _force_purge_job(owner, job_id, raw_payload, errors): + from .rulebook_assessment import _submission_lock, list_rulebook_profiles + + with ExitStack() as stack: + for profile in sorted(list_rulebook_profiles(), key=lambda item: item["profile_id"]): + stack.enter_context(_submission_lock(owner, job_id, profile["profile_id"])) + return _force_purge_job_locked(owner, job_id, raw_payload, errors) + + +def _force_purge_job_locked(owner, job_id, raw_payload, errors): """ Best-effort wipe of a job whose record could not be parsed/purged by ``stop_and_delete_job``. Returns (cids_deleted, cids_failed). - Scans the raw payload for ``*_cid`` fields, attempts R1FS deletion, then - tombstones the CStore record and matching live/triage rows regardless. + Scans the raw payload for ``*_cid`` fields and attempts R1FS deletion. + Formal submission pointers remain retryable unless R1FS acknowledges deletion; + other legacy artifacts retain the existing best-effort force-wipe behavior. """ - cids = sorted({c for c in _collect_cids_from_raw(raw_payload) if isinstance(c, str)}) + cids = {c for c in _collect_cids_from_raw(raw_payload) if isinstance(c, str)} + submission_hkey = f"{owner.cfg_instance_id}:rulebook_review:submissions" + try: + submission_rows = owner.chainstore_hgetall(hkey=submission_hkey) or {} + except Exception as exc: + errors.append({"job_id": job_id, "scope": submission_hkey, "message": f"{type(exc).__name__}: {exc}"}) + owner.P(f"[PURGE_ALL_FORCE] Could not inspect submission registry for {job_id}; retaining rows.", color='r') + return 0, 1 + if not isinstance(submission_rows, dict): + errors.append({"job_id": job_id, "scope": submission_hkey, "message": "unexpected non-dict submission registry"}) + owner.P(f"[PURGE_ALL_FORCE] Invalid submission registry for {job_id}; retaining rows.", color='r') + return 0, 1 + shared_submission_cids = set() + job_submission_cids = set() + prefix = f"{job_id}:" + other_submission_cids = set() + for key, registry in submission_rows.items(): + if isinstance(key, str) and key.startswith(prefix): + job_submission_cids.update(_collect_rulebook_submission_cids(registry)) + elif isinstance(key, str): + other_submission_cids.update(_collect_rulebook_submission_cids(registry)) + shared_submission_cids = job_submission_cids & other_submission_cids + if job_submission_cids: + try: + all_jobs = _job_repo(owner).list_jobs() or {} + if not isinstance(all_jobs, dict): + raise TypeError("unexpected non-dict job registry") + for other_job_id, other_payload in all_jobs.items(): + if other_job_id != job_id and isinstance(other_payload, dict): + other_submission_cids.update(_collect_cids_from_raw(other_payload)) + shared_submission_cids = job_submission_cids & other_submission_cids + except Exception as exc: + errors.append({"job_id": job_id, "scope": owner.cfg_instance_id, "message": f"{type(exc).__name__}: {exc}"}) + owner.P(f"[PURGE_ALL_FORCE] Could not inspect shared CID references for {job_id}; retaining rows.", color='r') + return 0, len(job_submission_cids) + cids.update(job_submission_cids - shared_submission_cids) + cids.difference_update(shared_submission_cids) + cids = sorted(cids) cids_deleted = 0 - cids_failed = 0 + cids_failed = len(shared_submission_cids) + if shared_submission_cids: + errors.append({ + "job_id": job_id, + "scope": submission_hkey, + "message": f"retained shared submission CIDs: {sorted(shared_submission_cids)}", + }) artifacts = _artifact_repo(owner) + failed_submission_cids = set(shared_submission_cids) for cid in cids: try: - success = artifacts.delete(cid, show_logs=True, raise_on_error=False) + success = artifacts.delete(cid, show_logs=True, raise_on_error=False, purge=True) if success: cids_deleted += 1 owner.P(f"[PURGE_ALL_FORCE] Deleted CID {cid} for {job_id}") else: cids_failed += 1 + if cid in job_submission_cids: + failed_submission_cids.add(cid) owner.P(f"[PURGE_ALL_FORCE] delete returned False for CID {cid} ({job_id})", color='y') except Exception as exc: cids_failed += 1 + if cid in job_submission_cids: + failed_submission_cids.add(cid) owner.P(f"[PURGE_ALL_FORCE] Failed to delete CID {cid} ({job_id}): {exc}", color='r') errors.append({"job_id": job_id, "scope": "r1fs", "message": f"{type(exc).__name__}: {exc}"}) + if failed_submission_cids: + owner.P( + f"[PURGE_ALL_FORCE] Retaining CStore rows for {job_id}; formal submission CIDs require retry.", + color='r', + ) + return cids_deleted, cids_failed + cfg_instance_id = owner.cfg_instance_id prefix = f"{job_id}:" - for hkey in (f"{cfg_instance_id}:live", f"{cfg_instance_id}:triage", f"{cfg_instance_id}:triage:audit"): + for hkey in ( + f"{cfg_instance_id}:live", + f"{cfg_instance_id}:triage", + f"{cfg_instance_id}:triage:audit", + f"{cfg_instance_id}:rulebook_review", + f"{cfg_instance_id}:rulebook_review:audit", + f"{cfg_instance_id}:rulebook_review:submissions", + ): try: rows = owner.chainstore_hgetall(hkey=hkey) except Exception as exc: @@ -305,6 +485,15 @@ def purge_all_jobs(owner): terminal_statuses = (JOB_STATUS_FINALIZED, JOB_STATUS_STOPPED) for job_id, raw_payload in job_entries: + owner_error = _foreign_launcher_error(owner, job_id, raw_payload, "purge_all") + if owner_error: + jobs_failed += 1 + failed_job_ids.add(job_id) + errors.append({ + "job_id": job_id, + "message": owner_error["message"], + }) + continue raw_status = raw_payload.get("job_status") if isinstance(raw_payload, dict) else None use_direct_purge = raw_status in terminal_statuses try: @@ -320,6 +509,8 @@ def purge_all_jobs(owner): cids_failed += fc_failed jobs_failed += 1 jobs_force_purged += 1 + if fc_failed: + failed_job_ids.add(job_id) continue if not isinstance(result, dict): @@ -329,6 +520,8 @@ def purge_all_jobs(owner): cids_failed += fc_failed jobs_failed += 1 jobs_force_purged += 1 + if fc_failed: + failed_job_ids.add(job_id) continue status = result.get("status") @@ -354,11 +547,16 @@ def purge_all_jobs(owner): cids_failed += fc_failed jobs_failed += 1 jobs_force_purged += 1 + if fc_failed: + failed_job_ids.add(job_id) cfg_instance_id = owner.cfg_instance_id live_hkey = f"{cfg_instance_id}:live" triage_hkey = f"{cfg_instance_id}:triage" triage_audit_hkey = f"{cfg_instance_id}:triage:audit" + rulebook_review_hkey = f"{cfg_instance_id}:rulebook_review" + rulebook_review_audit_hkey = f"{cfg_instance_id}:rulebook_review:audit" + rulebook_review_submissions_hkey = f"{cfg_instance_id}:rulebook_review:submissions" integrations_hkey = f"{cfg_instance_id}:integrations" def _job_id_from_compound_key(key): @@ -387,9 +585,145 @@ def _sweep_hash(hkey, expected_value_types): errors.append({"job_id": job_id_prefix or "", "scope": hkey, "message": f"{type(exc).__name__}: {exc}"}) return rows_deleted + def _sweep_submission_hash(): + nonlocal cids_deleted, cids_failed + + from .rulebook_assessment import _submission_lock, list_rulebook_profiles + + try: + initial_rows = owner.chainstore_hgetall(hkey=rulebook_review_submissions_hkey) + except Exception as exc: + cids_failed += 1 + failed_job_ids.update(raw_jobs) + errors.append({ + "job_id": "", + "scope": rulebook_review_submissions_hkey, + "message": f"{type(exc).__name__}: {exc}", + }) + return 0 + if not isinstance(initial_rows, dict): + cids_failed += 1 + failed_job_ids.update(raw_jobs) + errors.append({ + "job_id": "", + "scope": rulebook_review_submissions_hkey, + "message": "unexpected non-dict submission registry", + }) + return 0 + + initial_keys = list(initial_rows) + lock_targets = set() + for key in initial_keys: + job_id_prefix = _job_id_from_compound_key(key) + if job_id_prefix and isinstance(key, str) and ":" in key: + lock_targets.add((job_id_prefix, key.split(":", 1)[1])) + profile_ids = { + profile["profile_id"] + for profile in list_rulebook_profiles() + if isinstance(profile, dict) and isinstance(profile.get("profile_id"), str) + } + for failed_job_id in failed_job_ids: + for profile_id in profile_ids: + lock_targets.add((failed_job_id, profile_id)) + + rows_deleted = 0 + deletion_results = {} + artifacts = _artifact_repo(owner) + with ExitStack() as stack: + for lock_job_id, lock_profile_id in sorted(lock_targets): + stack.enter_context(_submission_lock(owner, lock_job_id, lock_profile_id)) + try: + rows = owner.chainstore_hgetall(hkey=rulebook_review_submissions_hkey) + except Exception as exc: + cids_failed += 1 + failed_job_ids.update(raw_jobs) + errors.append({ + "job_id": "", + "scope": rulebook_review_submissions_hkey, + "message": f"{type(exc).__name__}: {exc}", + }) + return 0 + if not isinstance(rows, dict): + cids_failed += 1 + failed_job_ids.update(raw_jobs) + errors.append({ + "job_id": "", + "scope": rulebook_review_submissions_hkey, + "message": "unexpected non-dict submission registry", + }) + return 0 + + protected_cids = set() + for failed_job_id in failed_job_ids: + raw_failed_job = raw_jobs.get(failed_job_id) + if isinstance(raw_failed_job, dict): + protected_cids.update(_collect_cids_from_raw(raw_failed_job)) + for key, value in rows.items(): + if _job_id_from_compound_key(key) in failed_job_ids: + protected_cids.update(_collect_rulebook_submission_cids(value)) + + for key in initial_keys: + value = rows.get(key) + job_id_prefix = _job_id_from_compound_key(key) + if not job_id_prefix or job_id_prefix in failed_job_ids or not isinstance(value, dict): + continue + row_cids = _collect_rulebook_submission_cids(value) + shared_cids = row_cids & protected_cids + if shared_cids: + failed_job_ids.add(job_id_prefix) + cids_failed += len(shared_cids) + errors.append({ + "job_id": job_id_prefix, + "scope": rulebook_review_submissions_hkey, + "message": f"retained orphan registry with shared CIDs: {sorted(shared_cids)}", + }) + continue + + row_failed = False + for cid in sorted(row_cids): + success = deletion_results.get(cid) + if success is None: + try: + success = artifacts.delete(cid, show_logs=True, raise_on_error=False, purge=True) + except Exception as exc: + success = False + errors.append({ + "job_id": job_id_prefix, + "scope": "r1fs", + "message": f"{type(exc).__name__}: {exc}", + }) + deletion_results[cid] = success + if success: + cids_deleted += 1 + else: + cids_failed += 1 + if not success: + row_failed = True + if row_failed: + failed_job_ids.add(job_id_prefix) + errors.append({ + "job_id": job_id_prefix, + "scope": rulebook_review_submissions_hkey, + "message": "orphan submission CID deletion was not acknowledged", + }) + continue + try: + owner.chainstore_hset(hkey=rulebook_review_submissions_hkey, key=key, value=None) + rows_deleted += 1 + except Exception as exc: + errors.append({ + "job_id": job_id_prefix, + "scope": rulebook_review_submissions_hkey, + "message": f"{type(exc).__name__}: {exc}", + }) + return rows_deleted + _sweep_hash(live_hkey, dict) _sweep_hash(triage_hkey, dict) _sweep_hash(triage_audit_hkey, list) + _sweep_hash(rulebook_review_hkey, dict) + _sweep_hash(rulebook_review_audit_hkey, list) + _sweep_submission_hash() integration_status_rows_deleted = 0 if jobs_failed == 0 and cids_failed == 0: integration_status_rows_deleted = _sweep_hash(integrations_hkey, dict) @@ -446,6 +780,9 @@ def stop_monitoring(owner, job_id: str, stop_type: str = "SOFT"): return {"error": "Job not found", "job_id": job_id} _, job_specs = owner._normalize_job_record(job_id, raw_job_specs) + owner_error = _foreign_launcher_error(owner, job_id, job_specs, "stop_monitoring") + if owner_error: + return owner_error stop_type = str(stop_type).upper() is_continuous = job_specs.get("run_mode") == RUN_MODE_CONTINUOUS_MONITORING diff --git a/extensions/business/cybersec/red_mesh/services/finalization.py b/extensions/business/cybersec/red_mesh/services/finalization.py index c61d6e570..ec67d5b6f 100644 --- a/extensions/business/cybersec/red_mesh/services/finalization.py +++ b/extensions/business/cybersec/red_mesh/services/finalization.py @@ -28,6 +28,7 @@ emit_finding_event, emit_lifecycle_event, ) +from .rulebook_assessment import ensure_rulebook_assessment from .scan_strategy import coerce_scan_type, get_scan_strategy from .state_machine import is_intermediate_job_status, is_terminal_job_status, set_job_status @@ -97,6 +98,82 @@ def _record_stale_intermediate_recovery(owner, job_specs, previous_status, worke }) +def _automatic_analysis_report_identity(workers): + return tuple(sorted( + (str(address), str(worker.get("report_cid") or "")) + for address, worker in (workers or {}).items() + if isinstance(worker, dict) + )) + + +def _automatic_analysis_state_matches(job_specs, state): + if not isinstance(job_specs, dict) or not isinstance(state, dict): + return False + return ( + job_specs.get("job_id") == state.get("job_id") + and job_specs.get("job_pass", 1) == state.get("pass_nr") + and _automatic_analysis_report_identity(job_specs.get("workers")) + == state.get("report_identity") + ) + + +def _poll_automatic_analysis(owner, all_jobs): + """Yield while automatic model work runs, then return its single result.""" + state = owner.__dict__.get("_automatic_analysis_state") + if not isinstance(state, dict): + return None, False + + current_job = None + for job_key, job_specs in all_jobs.items(): + normalized_key, normalized = owner._normalize_job_record(job_key, job_specs) + if normalized_key is None: + continue + if isinstance(normalized, dict) and normalized.get("job_id") == state.get("job_id"): + current_job = normalized + break + + future = state.get("future") + invalidated = ( + not _automatic_analysis_state_matches(current_job, state) + or is_terminal_job_status((current_job or {}).get("job_status")) + ) + if invalidated: + state["discard_result"] = True + if future is not None: + future.cancel() + + if future is None: + owner._automatic_analysis_state = None + return None, False + if not future.done(): + return None, True + if state.get("discard_result"): + try: + future.result() + except Exception: + pass + owner._automatic_analysis_state = None + return None, False + + try: + sections = future.result() + except Exception as exc: + owner.P( + f"Structured LLM call raised for job {state.get('job_id')}: {exc}", + color='y', + ) + sections = None + failed = True + else: + failed = bool(getattr(owner, "_last_structured_llm_failed", None)) + owner._automatic_analysis_state = None + return { + "state": state, + "sections": sections, + "failed": failed, + }, False + + def _attestation_required(job_specs, job_config) -> bool: if isinstance(job_specs, dict) and "blockchain_attestation_enabled" in job_specs: return bool(job_specs.get("blockchain_attestation_enabled")) @@ -129,12 +206,38 @@ def _mark_attestation_failed(owner, job_key, job_specs, *, job_id, pass_nr, mess owner._clear_live_progress(job_id, list((job_specs.get("workers") or {}).keys())) +def _ensure_rulebook_assessment_after_pass(owner, job_specs, *, job_id, pass_nr): + try: + result = ensure_rulebook_assessment(owner, job_id, pass_nr=pass_nr) + except Exception as exc: + owner.P(f"[NIS2] Rulebook assessment ensure failed for job {job_id} pass {pass_nr}: {exc}", color='y') + return job_specs + + if not isinstance(result, dict) or result.get("status") != "ok": + error = result.get("error") if isinstance(result, dict) else "unknown_error" + owner.P(f"[NIS2] Rulebook assessment not generated for job {job_id} pass {pass_nr}: {error}", color='y') + return job_specs + + owner.P( + f"[NIS2] Rulebook assessment ready for job {job_id} pass {pass_nr}: " + f"{result.get('artifact_cid') or 'cached'}" + ) + refreshed = owner._get_job_from_cstore(job_id) + return refreshed if isinstance(refreshed, dict) else job_specs + + def maybe_finalize_pass(owner): """ Launcher finalizes completed passes and orchestrates continuous monitoring. """ all_jobs = _job_repo(owner).list_jobs() artifacts = _artifact_repo(owner) + automatic_completion, should_yield = _poll_automatic_analysis( + owner, + all_jobs, + ) + if should_yield: + return for job_key, job_specs in all_jobs.items(): normalized_key, job_specs = owner._normalize_job_record(job_key, job_specs) @@ -157,12 +260,24 @@ def maybe_finalize_pass(owner): next_pass_at = job_specs.get("next_pass_at") job_pass = job_specs.get("job_pass", 1) job_id = job_specs.get("job_id") + resumed_automatic_analysis = ( + isinstance(automatic_completion, dict) + and _automatic_analysis_state_matches( + job_specs, + automatic_completion.get("state"), + ) + ) + if resumed_automatic_analysis: + if job_status != JOB_STATUS_SCHEDULED_FOR_STOP: + job_status = automatic_completion["state"]["job_status"] + elif isinstance(automatic_completion, dict): + continue if is_terminal_job_status(job_status): if not job_specs.get("job_cid") and job_specs.get("pass_reports"): owner.P(f"[STUCK RECOVERY] {job_id} is {job_status} but has no job_cid — retrying archive build", color='y') owner._build_job_archive(job_id, job_specs) continue - if is_intermediate_job_status(job_status): + if is_intermediate_job_status(job_status) and not resumed_automatic_analysis: if not _is_stale_intermediate_recovery_candidate(job_specs, workers, job_status, next_pass_at): continue _record_stale_intermediate_recovery(owner, job_specs, job_status, len(workers)) @@ -171,7 +286,10 @@ def maybe_finalize_pass(owner): if all_finished and next_pass_at is None: pass_date_started = owner._get_timeline_date(job_specs, "pass_started") or owner._get_timeline_date(job_specs, "created") - pass_date_completed = owner.time() + pass_date_completed = ( + automatic_completion["state"]["pass_date_completed"] + if resumed_automatic_analysis else owner.time() + ) now_ts = pass_date_completed set_job_status(job_specs, JOB_STATUS_COLLECTING) @@ -217,28 +335,47 @@ def maybe_finalize_pass(owner): llm_report_sections = None structured_llm_failed = None if llm_cfg["ENABLED"] and aggregated: - set_job_status(job_specs, JOB_STATUS_ANALYZING) - job_specs = _write_job_record(owner, job_key, job_specs, context="finalize_analyzing") - # PTES report narrative uses only the structured LLM path. - # Legacy aggregate/quick-summary calls accepted raw scan-shaped - # payloads and are intentionally bypassed for report finalization. - try: - llm_report_sections = owner._run_structured_report_sections( - job_id=job_id, - findings=flat_findings, - aggregated_report=aggregated, - engagement=job_config.get("engagement") if isinstance(job_config, dict) else None, - ) - structured_llm_failed = getattr(owner, "_last_structured_llm_failed", None) - if llm_report_sections and not structured_llm_failed: - llm_text, summary_text = render_legacy_llm_fields(llm_report_sections) - except Exception as exc: - owner.P( - f"Structured LLM call raised for job {job_id}: {exc}", - color='y', - ) - llm_report_sections = None - structured_llm_failed = True + if resumed_automatic_analysis: + llm_report_sections = automatic_completion["sections"] + structured_llm_failed = automatic_completion["failed"] + else: + set_job_status(job_specs, JOB_STATUS_ANALYZING) + job_specs = _write_job_record(owner, job_key, job_specs, context="finalize_analyzing") + try: + executor = owner._get_manual_analysis_executor() + # HTTP work yields with PostponedRequest. Automatic work has no + # request to postpone, so process() yields by checking this future + # on later turns. + future = executor.submit( + owner._run_structured_report_sections, + job_id=job_id, + findings=flat_findings, + aggregated_report=aggregated, + engagement=( + job_config.get("engagement") + if isinstance(job_config, dict) else None + ), + ) + owner._automatic_analysis_state = { + "job_id": job_id, + "pass_nr": job_pass, + "report_identity": _automatic_analysis_report_identity(workers), + "job_status": job_status, + "pass_date_completed": pass_date_completed, + "future": future, + "discard_result": False, + } + return + except Exception as exc: + owner._automatic_analysis_state = None + owner.P( + f"Structured LLM call raised for job {job_id}: {exc}", + color='y', + ) + structured_llm_failed = True + + if llm_report_sections and not structured_llm_failed: + llm_text, summary_text = render_legacy_llm_fields(llm_report_sections) llm_failed = True if (llm_cfg["ENABLED"] and structured_llm_failed) else None if llm_failed: @@ -421,6 +558,7 @@ def maybe_finalize_pass(owner): set_job_status(job_specs, JOB_STATUS_FINALIZING) job_specs = _write_job_record(owner, job_key, job_specs, context="finalize_finalizing") + job_specs = _ensure_rulebook_assessment_after_pass(owner, job_specs, job_id=job_id, pass_nr=job_pass) if required_attestation_failed: _mark_attestation_failed( diff --git a/extensions/business/cybersec/red_mesh/services/llm_structured.py b/extensions/business/cybersec/red_mesh/services/llm_structured.py index 4cb7c6132..c169adaed 100644 --- a/extensions/business/cybersec/red_mesh/services/llm_structured.py +++ b/extensions/business/cybersec/red_mesh/services/llm_structured.py @@ -502,9 +502,10 @@ def build_response_format_for_prompt_profile( def generate_exec_summary( *, llm_call: LlmCall, - findings: list[dict] | None, + findings: list[dict] | None = None, aggregated_report: dict | None = None, engagement: dict | None = None, + prepared_input: LlmInput | None = None, model_name: str = "", provider_path: str | None = None, prompt_profile: str | None = None, @@ -530,6 +531,9 @@ def generate_exec_summary( fields ONLY; raw blobs are dropped by build_llm_input. engagement : dict | None EngagementContext.to_dict() output. + prepared_input : LlmInput | None + Sanitized trust-boundary output prepared by the caller. When supplied, + raw findings/report/engagement inputs are ignored. model_name : str Stamped onto the resulting LlmReportSections.model. max_findings : int | None @@ -555,12 +559,17 @@ def generate_exec_summary( temperature = profile.default_temperature # --- Trust boundary: scrub inputs through build_llm_input. --- - llm_input = build_llm_input( - findings=findings, - aggregated_report=aggregated_report, - engagement=engagement, - max_findings=max_findings, - ) + if prepared_input is None: + llm_input = build_llm_input( + findings=findings, + aggregated_report=aggregated_report, + engagement=engagement, + max_findings=max_findings, + ) + elif isinstance(prepared_input, LlmInput): + llm_input = prepared_input + else: + raise TypeError("prepared_input must be an LlmInput") compact_findings = _compact_findings_for_structured_prompt(llm_input.findings) messages = _build_messages_for_profile( diff --git a/extensions/business/cybersec/red_mesh/services/query.py b/extensions/business/cybersec/red_mesh/services/query.py index d8f1087d8..b5d08c8de 100644 --- a/extensions/business/cybersec/red_mesh/services/query.py +++ b/extensions/business/cybersec/red_mesh/services/query.py @@ -503,7 +503,28 @@ def list_local_jobs(owner): """ Return jobs currently running on the local node. """ - return { + local_jobs = { job_id: owner._get_job_status(job_id) - for job_id, local_workers in owner.scan_jobs.items() + for job_id in getattr(owner, "scan_jobs", {}) } + for job_id, worker in getattr(owner, "model_test_jobs", {}).items(): + job_specs = _job_repo(owner).get_job(job_id) + if isinstance(job_specs, dict) and not _is_model_test_specs(job_specs): + continue + job_specs = job_specs if isinstance(job_specs, dict) else {} + worker_state = getattr(worker, "state", None) + live_summary = worker_state.get("model_test_summary") if isinstance(worker_state, dict) else None + job_status = job_specs.get("job_status") or "RUNNING" + local_jobs[job_id] = { + "job_id": job_id, + "status": job_status, + "job_status": job_status, + "job_type": MODEL_TEST_JOB_TYPE, + "task_kind": MODEL_TEST_JOB_TYPE, + "scan_type": MODEL_TEST_JOB_TYPE, + "model_test_summary": sanitize_model_test_summary( + live_summary or job_specs.get("model_test_summary") + ), + "model_test_node_selection": job_specs.get("model_test_node_selection"), + } + return local_jobs diff --git a/extensions/business/cybersec/red_mesh/services/reconciliation.py b/extensions/business/cybersec/red_mesh/services/reconciliation.py index 3c4c087f9..381929f34 100644 --- a/extensions/business/cybersec/red_mesh/services/reconciliation.py +++ b/extensions/business/cybersec/red_mesh/services/reconciliation.py @@ -1,3 +1,4 @@ +from ..constants import JOB_STATUS_STOPPED from ..models import WorkerProgress from .config import resolve_config_block @@ -228,7 +229,15 @@ def reconcile_workers_from_live(owner, job_id, *, live_payloads=None, now=None, if not live.finished: continue if not live.report_cid: - _stats_inc_once(stats, "ignored_no_report_cid", (job_id, worker_addr)) + if not live.error_class: + _stats_inc_once(stats, "ignored_no_report_cid", (job_id, worker_addr)) + continue + worker_entry["finished"] = True + worker_entry["terminal_reason"] = live.error_class + worker_entry["error_class"] = live.error_class + worker_entry["result"] = None + job_specs["job_status"] = JOB_STATUS_STOPPED + changed_workers.append(worker_addr) continue if not worker_entry.get("report_cid"): diff --git a/extensions/business/cybersec/red_mesh/services/rulebook_assessment.py b/extensions/business/cybersec/red_mesh/services/rulebook_assessment.py new file mode 100644 index 000000000..f62893269 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/services/rulebook_assessment.py @@ -0,0 +1,2094 @@ +from __future__ import annotations + +import copy +import hashlib +import json +import re +import threading +import time as _time +from datetime import datetime, timezone +from urllib.parse import urlsplit + +from ..constants import JOB_STATUS_FINALIZED +from ..models import ( + RULEBOOK_ASSESSMENT_SCHEMA, + RULEBOOK_ASSESSMENT_SCHEMA_VERSION, + RULEBOOK_SUBMISSION_CONTRACT_VERSION, + RulebookPendingSubmission, + RulebookReviewAuditEntry, + RulebookReviewState, + RulebookSubmissionReference, + RulebookSubmissionRegistry, + VALID_RULEBOOK_ANSWER_VALUES, + VALID_RULEBOOK_CHECK_STATUSES, + VALID_RULEBOOK_REVIEW_STATES, +) +from ..repositories import ArtifactRepository, JobStateRepository +from .event_redaction import stable_hmac_pseudonym, strip_sensitive_fields +from .scan_guards import reject_model_test_for_scan_operation + + +DEFAULT_RULEBOOK_PROFILE_ID = "nis2.eu_baseline.v1" +RULEBOOK_PROFILE_VERSION = "1.0.0" + +_SUBMISSION_LOCKS = {} +_SUBMISSION_LOCKS_GUARD = threading.Lock() + +_IPV4_RE = re.compile( + r"(?" + + def _replace_ip(match): + return stable_hmac_pseudonym(match.group(0), hmac_secret, prefix="ip") + + text = _PEM_PRIVATE_KEY_RE.sub("", text) + text = _BEARER_TOKEN_RE.sub("Authorization: Bearer ", text) + text = _JWT_RE.sub("", text) + text = _SECRET_ASSIGNMENT_RE.sub(_replace_secret, text) + text = _PROVIDER_TOKEN_RE.sub("", text) + public_references = [] + + def _preserve_public_reference(match): + marker = f"publicrefmarker{len(public_references)}" + public_references.append((marker, match.group(0))) + return marker + + text = _PUBLIC_REFERENCE_RE.sub(_preserve_public_reference, text) + text = _UNLABELLED_HEX_RE.sub("", text) + text = _UNLABELLED_TOKEN_RE.sub("", text) + for marker, public_reference in public_references: + text = text.replace(marker, public_reference) + text = _IPV4_RE.sub(_replace_ip, text) + text = " ".join(text.split()) + return text[:max_len] + + +def _error(code, job_id, **extra): + return { + "status": "error", + "error": code, + "job_id": job_id, + **extra, + } + + +def _error_message(payload): + return str(payload.get("message") or payload.get("error") or "Rulebook assessment failed.") + + +def _sanitize_error(owner, payload): + hmac_secret = str(getattr(owner, "cfg_instance_id", "") or "redmesh-rulebook") + return { + "error": _safe_text(payload.get("error") or "rulebook_assessment_failed", hmac_secret=hmac_secret, max_len=120), + "message": _safe_text(_error_message(payload), hmac_secret=hmac_secret, max_len=280), + "retryable": bool(payload.get("retryable", True)), + "at": _utc_timestamp(), + } + + +def _profile(profile_id): + return _PROFILES.get(profile_id or DEFAULT_RULEBOOK_PROFILE_ID) + + +def list_rulebook_profiles(): + return [ + { + "profile_id": profile["profile_id"], + "profile_version": profile["profile_version"], + "title": profile["title"], + "jurisdiction_scope": profile["jurisdiction_scope"], + } + for profile in _PROFILES.values() + ] + + +def _resolve_scan_context(owner, job_id, pass_nr=None): + job_specs = owner._get_job_from_cstore(job_id) + if not isinstance(job_specs, dict): + return None, _error("job_not_found", job_id) + + unsupported = reject_model_test_for_scan_operation(job_specs, job_id, "rulebook_assessment") + if unsupported: + return None, { + **unsupported, + "error": "model_test_not_supported", + "error_class": unsupported.get("error_class") or unsupported.get("error"), + } + + artifacts = _artifact_repo(owner) + job_cid = job_specs.get("job_cid") + archive = {} + if job_cid: + archive = artifacts.get_archive(job_specs) + if not isinstance(archive, dict): + return None, _error("artifact_not_found", job_id, artifact="archive", artifact_cid=job_cid) + job_config = archive.get("job_config") or artifacts.get_job_config(job_specs) or {} + passes = archive.get("passes") or [] + if not passes: + return None, _error("no_completed_passes", job_id) + if pass_nr is None: + pass_data = passes[-1] + else: + pass_data = next((item for item in passes if item.get("pass_nr") == pass_nr), None) + if not isinstance(pass_data, dict): + return None, _error( + "pass_not_found", + job_id, + available_passes=[item.get("pass_nr") for item in passes if isinstance(item, dict)], + ) + agg_cid = pass_data.get("aggregated_report_cid") + aggregated = artifacts.get_json(agg_cid) if agg_cid else {} + else: + pass_reports = job_specs.get("pass_reports") or [] + if not pass_reports: + return None, _error("no_completed_passes", job_id) + if pass_nr is None: + pass_ref = pass_reports[-1] + else: + pass_ref = next((item for item in pass_reports if item.get("pass_nr") == pass_nr), None) + if not isinstance(pass_ref, dict): + return None, _error( + "pass_not_found", + job_id, + available_passes=[item.get("pass_nr") for item in pass_reports if isinstance(item, dict)], + ) + pass_data = artifacts.get_pass_report(pass_ref.get("report_cid")) + if not isinstance(pass_data, dict): + return None, _error("artifact_not_found", job_id, artifact="pass_report", artifact_cid=pass_ref.get("report_cid")) + agg_cid = pass_data.get("aggregated_report_cid") + aggregated = artifacts.get_json(agg_cid) if agg_cid else {} + job_config = artifacts.get_job_config(job_specs) or {} + + return { + "job_specs": job_specs, + "job_config": job_config if isinstance(job_config, dict) else {}, + "archive": archive if isinstance(archive, dict) else {}, + "pass_data": pass_data, + "aggregated": aggregated if isinstance(aggregated, dict) else {}, + }, None + + +def _finding_id(finding): + return str(finding.get("finding_id") or finding.get("id") or "").strip() + + +def _severity(finding): + return str(finding.get("severity") or "").strip().upper() + + +def _triage_status(finding, triage_map): + triage = triage_map.get(_finding_id(finding)) + if isinstance(triage, dict): + return str(triage.get("status") or "").strip().lower() + return str((finding.get("triage") or {}).get("status") or finding.get("triage_state") or "").strip().lower() + + +def _is_actionable_finding(finding, triage_map): + if not isinstance(finding, dict): + return False + finding_status = str(finding.get("status") or "").strip().lower() + if finding_status == "not_vulnerable": + return False + if _triage_status(finding, triage_map) in _CLOSED_TRIAGE_STATUSES: + return False + return bool(_finding_id(finding) or finding.get("title") or finding.get("description")) + + +def _search_text(finding): + parts = [ + finding.get("title"), + finding.get("description"), + finding.get("category"), + finding.get("probe"), + finding.get("cwe_id"), + finding.get("owasp_id"), + finding.get("cve"), + finding.get("cve_id"), + ] + return " ".join(str(part).lower() for part in parts if part) + + +def _matches_keywords(finding, keywords): + text = _search_text(finding) + return any(keyword in text for keyword in keywords) + + +def _safe_finding_ref(finding, *, hmac_secret, redaction_values, triage_map): + payload = { + "type": "finding", + "finding_id": _safe_text(_finding_id(finding), hmac_secret=hmac_secret, redaction_values=redaction_values, max_len=160), + "severity": _severity(finding), + "title": _safe_text(finding.get("title") or "Untitled finding", hmac_secret=hmac_secret, redaction_values=redaction_values, max_len=240), + "category": _safe_text(finding.get("category") or "", hmac_secret=hmac_secret, redaction_values=redaction_values, max_len=120), + "probe": _safe_text(finding.get("probe") or "", hmac_secret=hmac_secret, redaction_values=redaction_values, max_len=160), + "cwe_id": _safe_text(finding.get("cwe_id") or "", hmac_secret=hmac_secret, redaction_values=redaction_values, max_len=80), + "owasp_id": _safe_text(finding.get("owasp_id") or "", hmac_secret=hmac_secret, redaction_values=redaction_values, max_len=80), + "triage_status": _triage_status(finding, triage_map) or None, + } + return {key: value for key, value in payload.items() if value not in (None, "", [])} + + +def _source_refs(ctx, actual_pass_nr): + pass_data = ctx["pass_data"] + refs = [{"type": "job", "job_id": ctx["job_specs"].get("job_id")}] + refs.append({"type": "pass", "pass_nr": actual_pass_nr}) + if pass_data.get("aggregated_report_cid"): + refs.append({"type": "artifact", "artifact_kind": "aggregated_report", "artifact_cid": pass_data.get("aggregated_report_cid")}) + return refs + + +def _auto_from_findings(check, findings, *, hmac_secret, redaction_values, triage_map): + gap_findings = [finding for finding in findings if _severity(finding) in _GAP_SEVERITIES] + review_findings = [ + finding + for finding in findings + if _severity(finding) not in _GAP_SEVERITIES or _severity(finding) in _REVIEW_SEVERITIES + ] + refs = [ + _safe_finding_ref(finding, hmac_secret=hmac_secret, redaction_values=redaction_values, triage_map=triage_map) + for finding in gap_findings[:10] + ] + if gap_findings: + return { + "status": "gap", + "summary": f"{len(gap_findings)} unresolved medium-or-higher finding(s) match this check.", + "gap_reason": "Unresolved scan findings need treatment evidence.", + "evidence_refs": refs, + } + if review_findings: + return { + "status": "needs_review", + "summary": f"{len(review_findings)} lower-severity or inconclusive finding(s) need reviewer interpretation.", + "gap_reason": "", + "evidence_refs": [ + _safe_finding_ref(finding, hmac_secret=hmac_secret, redaction_values=redaction_values, triage_map=triage_map) + for finding in review_findings[:10] + ], + } + return { + "status": "supported", + "summary": "No unresolved matching findings were observed in the selected RedMesh pass.", + "gap_reason": "", + "evidence_refs": [], + } + + +def _automated_check_result(check, ctx, *, hmac_secret, redaction_values, triage_map): + pass_data = ctx["pass_data"] + findings = [ + finding + for finding in (pass_data.get("findings") or []) + if _is_actionable_finding(finding, triage_map) + ] + signal_kind = check.get("signal_kind") + + if signal_kind == "vulnerability_management": + result = _auto_from_findings(check, findings, hmac_secret=hmac_secret, redaction_values=redaction_values, triage_map=triage_map) + elif signal_kind == "auth_access_findings": + matched = [finding for finding in findings if _matches_keywords(finding, _ACCESS_KEYWORDS)] + result = _auto_from_findings(check, matched, hmac_secret=hmac_secret, redaction_values=redaction_values, triage_map=triage_map) + elif signal_kind == "crypto_transport_findings": + matched = [finding for finding in findings if _matches_keywords(finding, _CRYPTO_KEYWORDS)] + result = _auto_from_findings(check, matched, hmac_secret=hmac_secret, redaction_values=redaction_values, triage_map=triage_map) + elif signal_kind == "assessment_performed": + result = { + "status": "supported", + "summary": "A completed RedMesh scan pass is available as retained technical assessment evidence.", + "gap_reason": "", + "evidence_refs": _source_refs(ctx, pass_data.get("pass_nr") or 1), + } + elif signal_kind == "soc_incident_evidence": + soc = ctx["job_specs"].get("soc_event_status") or ctx["archive"].get("soc_event_status") + detection = ctx["job_specs"].get("detection_correlation") or ctx["archive"].get("detection_correlation") + status_text = " ".join( + str(item).lower() + for item in [ + (soc or {}).get("last_status") if isinstance(soc, dict) else "", + (soc or {}).get("status") if isinstance(soc, dict) else "", + (soc or {}).get("outcome") if isinstance(soc, dict) else "", + (detection or {}).get("status") if isinstance(detection, dict) else "", + ] + if item + ) + if any(token in status_text for token in ("sent", "completed", "success", "ok")): + result = { + "status": "supported", + "summary": "RedMesh has SOC export or detection-correlation metadata for this job.", + "gap_reason": "", + "evidence_refs": [{"type": "job_metadata", "fields": ["soc_event_status", "detection_correlation"]}], + } + elif isinstance(soc, dict) or isinstance(detection, dict): + result = { + "status": "needs_review", + "summary": "SOC or detection metadata exists but does not show a completed export/correlation outcome.", + "gap_reason": "", + "evidence_refs": [{"type": "job_metadata", "fields": ["soc_event_status", "detection_correlation"]}], + } + else: + result = { + "status": "needs_review", + "summary": "No SOC export or detection-correlation metadata was observed for this job.", + "gap_reason": "", + "evidence_refs": [], + } + else: + result = { + "status": "not_observable", + "summary": "This check needs reviewer evidence outside the RedMesh scan archive.", + "gap_reason": "", + "evidence_refs": [], + } + + return { + "check_id": check["check_id"], + "title": check["title"], + "theme": check["theme"], + "article_refs": list(check.get("article_refs") or []), + "status": result["status"], + "automated_status": result["status"], + "source": "automated" if result["status"] != "not_observable" else "not_observable", + "summary": result["summary"], + "gap_reason": result.get("gap_reason") or "", + "evidence_refs": result.get("evidence_refs") or [], + "review_question_id": check.get("review_question_id"), + "review_answer": None, + "limitations": [check.get("limitations")] if check.get("limitations") else [], + } + + +def _apply_review(check_result, review): + question_id = check_result.get("review_question_id") + answers = (review.to_dict().get("answers") if review else {}) or {} + answer = answers.get(question_id) + if not answer: + return check_result + + result = dict(check_result) + result["review_answer"] = answer + auto_status = result.get("automated_status") + value = answer.get("value") + if auto_status == "gap": + result["status"] = "gap" + result["source"] = "mixed" + return result + + if value == "yes": + result["status"] = "supported" + result["gap_reason"] = "" + elif value == "no": + result["status"] = "gap" + result["gap_reason"] = "Reviewer indicated evidence is missing or insufficient." + elif value == "not_applicable": + result["status"] = "not_applicable" + result["gap_reason"] = "" + else: + result["status"] = "needs_review" + result["source"] = "reviewer" + return result + + +def _status_counts(checks): + counts = {status: 0 for status in sorted(VALID_RULEBOOK_CHECK_STATUSES)} + for check in checks: + status = check.get("status") + if status in counts: + counts[status] += 1 + return counts + + +def _review_view(review, *, hmac_secret): + if review is None: + return {"review_state": "draft", "answers": {}} + payload = review.to_dict() + payload.pop("last_reopen_idempotency_key", None) + payload.pop("last_reopen_from_revision", None) + payload.pop("last_reopen_actor", None) + payload["note"] = _safe_text(payload.get("note") or "", hmac_secret=hmac_secret, max_len=1000) + for answer in (payload.get("answers") or {}).values(): + if isinstance(answer, dict): + answer["note"] = _safe_text(answer.get("note") or "", hmac_secret=hmac_secret, max_len=1000) + return payload + + +def _profile_meta(job_specs, profile_id): + assessments = job_specs.get("rulebook_assessments") or {} + if not isinstance(assessments, dict): + return {} + meta = assessments.get(profile_id) + return meta if isinstance(meta, dict) else {} + + +def _meta_pass_nr(meta): + value = meta.get("latest_pass_nr", meta.get("pass_nr")) + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _existing_same_pass_artifact(owner, job_id, profile_id, pass_nr): + job_specs = owner._get_job_from_cstore(job_id) + if not isinstance(job_specs, dict): + return None + meta = _profile_meta(job_specs, profile_id) + artifact_cid = meta.get("artifact_cid") + if not artifact_cid or _meta_pass_nr(meta) != int(pass_nr): + return None + if meta.get("run_state") not in (None, "succeeded"): + return None + assessment = _artifact_repo(owner).get_json(artifact_cid) + if not isinstance(assessment, dict): + return None + if ( + assessment.get("schema_version") != RULEBOOK_ASSESSMENT_SCHEMA_VERSION + or assessment.get("artifact_kind") != "generated_assessment" + ): + return None + return meta, assessment + + +def _history_with_previous(existing_meta, new_meta): + history = [] + for item in existing_meta.get("history") or []: + if isinstance(item, dict): + history.append(dict(item)) + previous_cid = existing_meta.get("artifact_cid") + if previous_cid and previous_cid != new_meta.get("artifact_cid"): + previous = { + "artifact_cid": previous_cid, + "pass_nr": existing_meta.get("latest_pass_nr", existing_meta.get("pass_nr")), + "profile_version": existing_meta.get("profile_version"), + "schema_version": existing_meta.get("schema_version"), + "artifact_kind": existing_meta.get("artifact_kind"), + "last_generated_at": existing_meta.get("last_generated_at"), + "status_counts": existing_meta.get("status_counts"), + "review_state": existing_meta.get("review_state"), + } + if not any(item.get("artifact_cid") == previous_cid for item in history): + history.append({key: value for key, value in previous.items() if value not in (None, "", [])}) + return history[-20:] + + +def _write_assessment_meta(owner, job_id, profile_id, meta, *, context): + job_specs = owner._get_job_from_cstore(job_id) + if not isinstance(job_specs, dict): + return None + assessments = dict(job_specs.get("rulebook_assessments") or {}) + existing_meta = assessments.get(profile_id) if isinstance(assessments.get(profile_id), dict) else {} + if meta.get("run_state") == "succeeded": + meta = dict(meta) + meta["history"] = _history_with_previous(existing_meta, meta) + assessments[profile_id] = meta + job_specs["rulebook_assessments"] = assessments + return _write_job_record(owner, job_id, job_specs, context=context) + + +def _success_meta(result, artifact_cid): + generated_at = result["assessment"]["generated_at"] + return { + "schema": RULEBOOK_ASSESSMENT_SCHEMA, + "schema_version": RULEBOOK_ASSESSMENT_SCHEMA_VERSION, + "artifact_kind": "generated_assessment", + "profile_id": result["profile_id"], + "profile_version": result["profile_version"], + "artifact_cid": artifact_cid, + "last_generated_at": generated_at, + "pass_nr": result["pass_nr"], + "latest_pass_nr": result["pass_nr"], + "status_counts": result["status_counts"], + "review_state": result["assessment"]["review_state"].get("review_state", "draft"), + "auto_enabled": True, + "run_state": "succeeded", + } + + +def _failed_meta(owner, job_id, profile_id, payload): + profile = _profile(profile_id) + existing = _profile_meta(owner._get_job_from_cstore(job_id) or {}, profile_id) + meta = dict(existing) + meta.update({ + "schema": RULEBOOK_ASSESSMENT_SCHEMA, + "schema_version": RULEBOOK_ASSESSMENT_SCHEMA_VERSION, + "profile_id": profile_id, + "profile_version": profile["profile_version"] if profile else existing.get("profile_version"), + "auto_enabled": True, + "run_state": "failed", + "last_error": _sanitize_error(owner, payload), + }) + return meta + + +def build_rulebook_assessment( + owner, + job_id, + profile_id=DEFAULT_RULEBOOK_PROFILE_ID, + pass_nr=None, + *, + include_review=True, + artifact_kind="generated_assessment", + submission=None, +): + profile = _profile(profile_id) + if not profile: + return _error("invalid_profile", job_id, profile_id=profile_id) + ctx, err = _resolve_scan_context(owner, job_id, pass_nr=pass_nr) + if err: + return err + + job_specs = ctx["job_specs"] + actual_pass_nr = ctx["pass_data"].get("pass_nr") or pass_nr or 1 + hmac_secret = str(getattr(owner, "cfg_instance_id", "") or "redmesh-rulebook") + redaction_values = _target_values(ctx["job_config"], job_specs) + target_value = (redaction_values or [job_id or "unknown"])[0] + target_pseudonym = stable_hmac_pseudonym(target_value, hmac_secret, prefix="target") + triage_map = _job_repo(owner).list_job_triage(job_id) + review = _job_repo(owner).get_rulebook_review_model(job_id, profile["profile_id"]) if include_review else None + + checks = [] + for check in profile["checks"]: + check_result = _automated_check_result( + check, + ctx, + hmac_secret=hmac_secret, + redaction_values=redaction_values, + triage_map=triage_map, + ) + checks.append(_apply_review(check_result, review)) + + counts = _status_counts(checks) + assessment = { + "schema": RULEBOOK_ASSESSMENT_SCHEMA, + "schema_version": RULEBOOK_ASSESSMENT_SCHEMA_VERSION, + "artifact_kind": artifact_kind, + "job_id": job_id, + "generated_at": _utc_timestamp(), + "profile": { + "profile_id": profile["profile_id"], + "profile_version": profile["profile_version"], + "title": profile["title"], + "jurisdiction_scope": profile["jurisdiction_scope"], + "source_refs": list(profile.get("source_refs") or []), + "legal_notice": profile["legal_notice"], + }, + "scan_context": { + "target_ref": target_pseudonym, + "scan_type": job_specs.get("scan_type"), + "job_status": job_specs.get("job_status"), + "pass_nr": actual_pass_nr, + "job_created_at": _utc_timestamp(job_specs.get("date_created")) if job_specs.get("date_created") else None, + "job_completed_at": _utc_timestamp(job_specs.get("date_completed")) if job_specs.get("date_completed") else None, + "job_archive_cid": job_specs.get("job_cid"), + "aggregated_report_cid": ctx["pass_data"].get("aggregated_report_cid"), + "finding_count": len(ctx["pass_data"].get("findings") or []), + "worker_count": job_specs.get("worker_count") or len(job_specs.get("workers") or {}), + }, + "status_counts": counts, + "checks": checks, + "review_state": _review_view(review, hmac_secret=hmac_secret), + "limitations": [ + "Evidence states are based on RedMesh scan data plus reviewer input.", + "National transposition, governance approval, contracts, and reporting operations need separate review.", + "A clean technical scan does not prove organization-wide control operation.", + ], + } + if isinstance(submission, dict): + assessment["submission"] = dict(submission) + return { + "status": "ok", + "job_id": job_id, + "profile_id": profile["profile_id"], + "profile_version": profile["profile_version"], + "pass_nr": actual_pass_nr, + "status_counts": counts, + "assessment": assessment, + } + + +def generate_rulebook_assessment(owner, job_id, profile_id=DEFAULT_RULEBOOK_PROFILE_ID, pass_nr=None, persist=True, force=True): + result = build_rulebook_assessment( + owner, + job_id, + profile_id=profile_id, + pass_nr=pass_nr, + include_review=not persist, + ) + if result.get("status") != "ok": + profile = _profile(profile_id) + if profile and persist: + _write_assessment_meta( + owner, + job_id, + profile["profile_id"], + _failed_meta(owner, job_id, profile["profile_id"], result), + context="rulebook_assessment_failed", + ) + return result + + artifact_cid = None + if persist: + existing = None if force else _existing_same_pass_artifact(owner, job_id, result["profile_id"], result["pass_nr"]) + if existing: + meta, assessment = existing + return { + **result, + "assessment": assessment, + "artifact_cid": meta.get("artifact_cid"), + "generated": True, + "cached": True, + } + + artifact_cid = _artifact_repo(owner).put_json(result["assessment"], show_logs=False) + if not artifact_cid: + failed = _error("artifact_write_failed", job_id, profile_id=profile_id) + _write_assessment_meta( + owner, + job_id, + result["profile_id"], + _failed_meta(owner, job_id, result["profile_id"], failed), + context="rulebook_assessment_failed", + ) + return failed + + _write_assessment_meta( + owner, + job_id, + result["profile_id"], + _success_meta(result, artifact_cid), + context="rulebook_assessment", + ) + + return { + **result, + "artifact_cid": artifact_cid, + "generated": bool(artifact_cid), + } + + +def ensure_rulebook_assessment(owner, job_id, profile_id=DEFAULT_RULEBOOK_PROFILE_ID, pass_nr=None): + return generate_rulebook_assessment( + owner, + job_id, + profile_id=profile_id, + pass_nr=pass_nr, + persist=True, + force=False, + ) + + +def get_rulebook_assessment_status(owner, job_id, profile_id=DEFAULT_RULEBOOK_PROFILE_ID): + profile = _profile(profile_id) + if not profile: + return _error("invalid_profile", job_id, profile_id=profile_id) + job_specs = owner._get_job_from_cstore(job_id) + if not isinstance(job_specs, dict): + return {"job_id": job_id, "found": False, "generated": False, "profile_id": profile_id} + unsupported = reject_model_test_for_scan_operation(job_specs, job_id, "rulebook_assessment_status") + if unsupported: + return { + **unsupported, + "error": "model_test_not_supported", + "error_class": unsupported.get("error_class") or unsupported.get("error"), + "found": True, + "generated": False, + } + meta = (job_specs.get("rulebook_assessments") or {}).get(profile["profile_id"]) + if not isinstance(meta, dict) or not meta: + result = { + "job_id": job_id, + "found": True, + "generated": False, + "profile_id": profile["profile_id"], + "profile_version": profile["profile_version"], + "schema": RULEBOOK_ASSESSMENT_SCHEMA, + "schema_version": RULEBOOK_ASSESSMENT_SCHEMA_VERSION, + } + else: + result = { + "job_id": job_id, + "found": True, + "generated": bool(meta.get("artifact_cid")) and meta.get("run_state") != "failed", + **meta, + } + result["submission_contract_version"] = RULEBOOK_SUBMISSION_CONTRACT_VERSION + try: + review = _job_repo(owner).get_rulebook_review_model(job_id, profile["profile_id"]) + result.update(_submission_public_state(owner, job_id, job_specs, profile, review)) + except ValueError: + result.update({ + "submission_contract_version": None, + "submission_contract_unsupported": True, + }) + return result + + +def _submission_lock(owner, job_id, profile_id): + key = f"{getattr(owner, 'cfg_instance_id', '')}:{job_id}:{profile_id}" + with _SUBMISSION_LOCKS_GUARD: + lock = _SUBMISSION_LOCKS.get(key) + if lock is None: + lock = threading.RLock() + _SUBMISSION_LOCKS[key] = lock + return lock + + +def _empty_submission_registry(): + return RulebookSubmissionRegistry() + + +def _submission_registry(repo, job_id, profile_id): + return repo.get_rulebook_submission_registry_model(job_id, profile_id) or _empty_submission_registry() + + +def _owner_time(owner): + return float(getattr(owner, "time", _time.time)()) + + +def _latest_pass_nr(owner, job_id): + ctx, err = _resolve_scan_context(owner, job_id) + if err: + return None, err + raw = ctx["pass_data"].get("pass_nr") or 1 + try: + return int(raw), None + except (TypeError, ValueError): + return None, _error("pass_not_found", job_id) + + +def _legacy_submission_reference(job_specs, profile, review): + if review is None or review.review_state != "reviewed": + return None + meta = _profile_meta(job_specs, profile["profile_id"]) + cid = str(meta.get("artifact_cid") or "").strip() + if not cid: + return None + try: + pass_nr = int(meta.get("latest_pass_nr", meta.get("pass_nr", 0)) or 0) + except (TypeError, ValueError): + pass_nr = 0 + return RulebookSubmissionReference( + revision=0, + cid=cid, + submitted_at=review.updated_at, + actor=review.reviewer, + pass_nr=pass_nr, + profile_id=profile["profile_id"], + profile_version=str(meta.get("profile_version") or review.profile_version or profile["profile_version"]), + schema_version=str(meta.get("schema_version") or "1.0.0"), + review_revision=review.review_revision, + legacy=True, + ).to_dict() + + +def _submission_reference_view(reference, *, latest_pass_nr, profile_version): + payload = reference.to_dict() if isinstance(reference, RulebookSubmissionReference) else dict(reference) + stale_reasons = [] + if latest_pass_nr is not None and int(payload.get("pass_nr", 0) or 0) != int(latest_pass_nr): + stale_reasons.append("newer_scan_pass") + if payload.get("profile_version") != profile_version: + stale_reasons.append("newer_profile_version") + return { + **payload, + "artifact_cid": payload.get("cid"), + "stale": bool(stale_reasons), + "stale_reasons": stale_reasons, + } + + +def _submission_public_state(owner, job_id, job_specs, profile, review): + repo = _job_repo(owner) + registry = _submission_registry(repo, job_id, profile["profile_id"]) + registry_payload = registry.to_dict() + latest_pass_nr, _ = _latest_pass_nr(owner, job_id) + references = list(registry_payload.get("submissions") or []) + legacy_reference = _legacy_submission_reference(job_specs, profile, review) + if legacy_reference and not any(item.get("cid") == legacy_reference["cid"] for item in references): + references.append(legacy_reference) + views = [ + _submission_reference_view( + reference, + latest_pass_nr=latest_pass_nr, + profile_version=profile["profile_version"], + ) + for reference in sorted(references, key=lambda item: int(item.get("revision", 0) or 0), reverse=True) + ] + pending = registry_payload.get("pending") + latest = views[0] if views else None + migration_required = bool(review and review.review_state == "reviewed" and legacy_reference is None) + effective_state = "draft" + if not pending and latest and review and review.review_state in {"submitted", "reviewed"}: + effective_state = "submitted" + operation_state = None + if pending: + operation_state = "failed" if pending.get("last_error") else "submitting" + return { + "submission_contract_version": RULEBOOK_SUBMISSION_CONTRACT_VERSION, + "effective_review_state": effective_state, + "review_revision": review.review_revision if review else 0, + "latest_submission": latest, + "submissions": views, + "submission_operation_state": operation_state, + "submission_error": (pending or {}).get("last_error") if isinstance(pending, dict) else None, + "migration_submission_required": migration_required, + } + + +def _submission_error(code, job_id, profile_id, message, *, retryable=False, **extra): + return _error( + code, + job_id, + profile_id=profile_id, + submission_contract_version=RULEBOOK_SUBMISSION_CONTRACT_VERSION, + message=message, + retryable=retryable, + **extra, + ) + + +def _unsupported_submission_registry_error(job_id, profile_id): + return _submission_error( + "submission_contract_unsupported", + job_id, + profile_id, + "Submission registry contract version is not supported by this backend.", + ) + + +def get_rulebook_review(owner, job_id, profile_id=DEFAULT_RULEBOOK_PROFILE_ID): + profile = _profile(profile_id) + if not profile: + return _error("invalid_profile", job_id, profile_id=profile_id) + job_specs = owner._get_job_from_cstore(job_id) + if not isinstance(job_specs, dict): + return _error("job_not_found", job_id, profile_id=profile_id) + unsupported = reject_model_test_for_scan_operation(job_specs, job_id, "rulebook_review") + if unsupported: + return { + **unsupported, + "error": "model_test_not_supported", + "error_class": unsupported.get("error_class") or unsupported.get("error"), + } + if job_specs.get("job_status") != JOB_STATUS_FINALIZED: + return _error( + "job_not_finalized", + job_id, + profile_id=profile["profile_id"], + job_status=job_specs.get("job_status"), + ) + hmac_secret = str(getattr(owner, "cfg_instance_id", "") or "redmesh-rulebook") + repo = _job_repo(owner) + review = repo.get_rulebook_review_model(job_id, profile["profile_id"]) + try: + submission_state = _submission_public_state(owner, job_id, job_specs, profile, review) + except ValueError: + return _submission_error( + "submission_contract_unsupported", + job_id, + profile["profile_id"], + "Submission registry contract version is not supported by this backend.", + ) + return { + "status": "ok", + "job_id": job_id, + "profile": { + "profile_id": profile["profile_id"], + "profile_version": profile["profile_version"], + "title": profile["title"], + "checks": [ + { + "check_id": check["check_id"], + "title": check["title"], + "theme": check["theme"], + "review_question_id": check.get("review_question_id"), + "control_intent": check.get("control_intent"), + "limitations": check.get("limitations"), + } + for check in profile["checks"] + ], + }, + "found": review is not None, + "review": _review_view(review, hmac_secret=hmac_secret), + "audit": repo.get_rulebook_review_audit(job_id, profile["profile_id"]), + **submission_state, + } + + +def _validate_review_answers(profile, answers): + if answers is None: + return {} + if not isinstance(answers, dict): + raise ValueError("answers must be an object") + known_questions = { + check.get("review_question_id") + for check in profile["checks"] + if check.get("review_question_id") + } + validated = {} + for question_id, raw_answer in answers.items(): + if question_id not in known_questions: + raise ValueError(f"Unknown review question: {question_id}") + payload = raw_answer if isinstance(raw_answer, dict) else {"value": raw_answer} + answer_value = str(payload.get("value") or "unknown").strip().lower() + if answer_value not in VALID_RULEBOOK_ANSWER_VALUES: + raise ValueError(f"Unsupported answer value for {question_id}: {answer_value}") + validated[question_id] = { + "value": answer_value, + "note": str(payload.get("note") or "")[:1000], + } + return validated + + +def _validated_expected_revision(value, job_id, profile_id): + if value is None: + return None, _submission_error( + "review_revision_conflict", + job_id, + profile_id, + "expected_review_revision is required.", + ) + try: + revision = int(value) + except (TypeError, ValueError): + return None, _submission_error( + "review_revision_conflict", + job_id, + profile_id, + "expected_review_revision must be a non-negative integer.", + ) + if revision < 0: + return None, _submission_error( + "review_revision_conflict", + job_id, + profile_id, + "expected_review_revision must be a non-negative integer.", + ) + return revision, None + + +def _sanitize_review_patch(owner, answers, actor, note): + hmac_secret = str(getattr(owner, "cfg_instance_id", "") or "redmesh-rulebook") + actor = _safe_text(actor or "", hmac_secret=hmac_secret, max_len=200) + note = _safe_text(note or "", hmac_secret=hmac_secret, max_len=1000) + now = _owner_time(owner) + sanitized_answers = {} + for question_id, answer in answers.items(): + sanitized_answers[question_id] = { + "value": answer["value"], + "note": _safe_text(answer.get("note") or "", hmac_secret=hmac_secret, max_len=1000), + "reviewer": actor, + "updated_at": now, + } + return sanitized_answers, actor, note, now + + +def _put_review_with_audit( + owner, + repo, + *, + job_id, + profile, + previous, + state, + changed_question_ids, + event_type, +): + previous_answers = (previous.to_dict().get("answers") if previous else {}) or {} + current_answers = (state.to_dict().get("answers") if state else {}) or {} + review_payload = repo.put_rulebook_review(state) + audit_payload = repo.append_rulebook_review_audit(RulebookReviewAuditEntry( + job_id=job_id, + profile_id=profile["profile_id"], + profile_version=profile["profile_version"], + review_state=state.review_state, + reviewer=state.reviewer, + note=state.note, + changed_question_ids=list(changed_question_ids or []), + previous_answers={question_id: previous_answers.get(question_id) for question_id in changed_question_ids or []}, + current_answers={question_id: current_answers.get(question_id) for question_id in changed_question_ids or []}, + timestamp=state.updated_at, + review_revision=state.review_revision, + )) + if hasattr(owner, "_log_audit_event"): + owner._log_audit_event(event_type, { + "job_id": job_id, + "profile_id": profile["profile_id"], + "review_state": state.review_state, + "review_revision": state.review_revision, + "changed_question_ids": list(changed_question_ids or []), + }) + return review_payload, audit_payload + + +def save_rulebook_review_draft( + owner, + job_id, + profile_id=DEFAULT_RULEBOOK_PROFILE_ID, + answers=None, + actor="", + note="", + expected_review_revision=None, +): + profile = _profile(profile_id) + if not profile: + return _error("invalid_profile", job_id, profile_id=profile_id) + expected_revision, err = _validated_expected_revision( + expected_review_revision, + job_id, + profile["profile_id"], + ) + if err: + return err + try: + validated_answers = _validate_review_answers(profile, answers) + except ValueError as exc: + return _error("invalid_review_answer", job_id, profile_id=profile["profile_id"], message=str(exc)) + + with _submission_lock(owner, job_id, profile["profile_id"]): + job_specs = owner._get_job_from_cstore(job_id) + if not isinstance(job_specs, dict): + return _error("job_not_found", job_id, profile_id=profile["profile_id"]) + if job_specs.get("job_status") != JOB_STATUS_FINALIZED: + return _error("job_not_finalized", job_id, profile_id=profile["profile_id"]) + unsupported = reject_model_test_for_scan_operation(job_specs, job_id, "rulebook_review") + if unsupported: + return {**unsupported, "error": "model_test_not_supported"} + repo = _job_repo(owner) + previous = repo.get_rulebook_review_model(job_id, profile["profile_id"]) + try: + registry = _submission_registry(repo, job_id, profile["profile_id"]).to_dict() + except ValueError: + return _unsupported_submission_registry_error(job_id, profile["profile_id"]) + if registry.get("pending"): + return _submission_error( + "submission_in_progress", + job_id, + profile["profile_id"], + "A submission retry must finish before the draft can change.", + retryable=True, + ) + current_revision = previous.review_revision if previous else 0 + if expected_revision != current_revision: + return _submission_error( + "review_revision_conflict", + job_id, + profile["profile_id"], + "The review changed. Reload before saving this draft.", + expected_review_revision=expected_revision, + current_review_revision=current_revision, + ) + legacy_reference = _legacy_submission_reference(job_specs, profile, previous) + if (previous and previous.review_state == "submitted") or legacy_reference: + return _submission_error( + "review_already_submitted", + job_id, + profile["profile_id"], + "Reopen the submitted review before editing it.", + ) + + sanitized_answers, actor, note, now = _sanitize_review_patch(owner, validated_answers, actor, note) + previous_answers = (previous.to_dict().get("answers") if previous else {}) or {} + current_answers = dict(previous_answers) + current_answers.update(sanitized_answers) + changed = sorted( + question_id + for question_id in set(previous_answers) | set(current_answers) + if previous_answers.get(question_id) != current_answers.get(question_id) + ) + state = RulebookReviewState( + job_id=job_id, + profile_id=profile["profile_id"], + profile_version=profile["profile_version"], + review_state="draft", + reviewer=actor, + note=note, + answers=current_answers, + updated_at=now, + review_revision=current_revision + 1, + ) + try: + _put_review_with_audit( + owner, + repo, + job_id=job_id, + profile=profile, + previous=previous, + state=state, + changed_question_ids=changed, + event_type="rulebook_review_draft_saved", + ) + except Exception: + return _submission_error( + "review_draft_save_failed", + job_id, + profile["profile_id"], + "Unable to save the review draft.", + retryable=True, + ) + return get_rulebook_review(owner, job_id, profile["profile_id"]) + + +def _validate_submission_answers(profile, review): + answers = (review.to_dict().get("answers") if review else {}) or {} + missing = [] + comments_required = [] + for check in profile["checks"]: + question_id = check.get("review_question_id") + if not question_id: + continue + answer = answers.get(question_id) + if not isinstance(answer, dict) or answer.get("value") not in VALID_RULEBOOK_ANSWER_VALUES: + missing.append(question_id) + continue + if answer.get("value") in {"no", "unknown", "not_applicable"} and not str(answer.get("note") or "").strip(): + comments_required.append(question_id) + return missing, comments_required + + +def _submission_fingerprint(assessment): + canonical = json.dumps(assessment, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def _build_submission_snapshot( + owner, + job_id, + profile, + *, + pass_nr, + revision, + submitted_at, + actor, + review_revision, +): + submission_meta = { + "contract_version": RULEBOOK_SUBMISSION_CONTRACT_VERSION, + "revision": revision, + "submitted_at": submitted_at, + "actor": actor, + "pass_nr": pass_nr, + "profile_id": profile["profile_id"], + "profile_version": profile["profile_version"], + "review_revision": review_revision, + } + built = build_rulebook_assessment( + owner, + job_id, + profile_id=profile["profile_id"], + pass_nr=pass_nr, + include_review=True, + artifact_kind="review_submission", + submission=submission_meta, + ) + if built.get("status") != "ok": + return built + assessment = built["assessment"] + assessment["generated_at"] = _utc_timestamp(submitted_at) + assessment["review_state"].update({ + "review_state": "submitted", + "reviewer": actor, + "updated_at": submitted_at, + "review_revision": review_revision, + }) + return { + "status": "ok", + "assessment": assessment, + "fingerprint": _submission_fingerprint(assessment), + } + + +def _registry_with(registry, *, submissions=None, pending=None, keep_pending=False): + payload = registry.to_dict() if isinstance(registry, RulebookSubmissionRegistry) else dict(registry) + if submissions is not None: + payload["submissions"] = submissions + if keep_pending or pending is not None: + payload["pending"] = pending + else: + payload.pop("pending", None) + return RulebookSubmissionRegistry.from_dict(payload) + + +def _persist_pending_error(repo, job_id, profile_id, registry, pending, payload): + try: + next_pending = dict(pending) + next_pending["last_error"] = payload + next_pending["updated_at"] = payload.get("at_epoch", next_pending.get("updated_at", 0.0)) + repo.put_rulebook_submission_registry( + job_id, + profile_id, + _registry_with(registry, pending=next_pending, keep_pending=True), + ) + except Exception: + pass + + +def submit_rulebook_review( + owner, + job_id, + profile_id=DEFAULT_RULEBOOK_PROFILE_ID, + expected_review_revision=None, + expected_pass_nr=None, + expected_profile_version=None, + idempotency_key="", + actor="", +): + profile = _profile(profile_id) + if not profile: + return _error("invalid_profile", job_id, profile_id=profile_id) + expected_revision, err = _validated_expected_revision( + expected_review_revision, + job_id, + profile["profile_id"], + ) + if err: + return err + try: + expected_pass = int(expected_pass_nr) + except (TypeError, ValueError): + return _submission_error( + "submission_pass_stale", + job_id, + profile["profile_id"], + "expected_pass_nr must identify the latest completed pass.", + ) + idempotency_key = str(idempotency_key or "").strip() + if not idempotency_key or len(idempotency_key) > 200: + return _submission_error( + "submission_idempotency_conflict", + job_id, + profile["profile_id"], + "A bounded idempotency key is required.", + ) + if str(expected_profile_version or "") != profile["profile_version"]: + return _submission_error( + "submission_profile_stale", + job_id, + profile["profile_id"], + "The rulebook profile changed. Reload before submitting.", + current_profile_version=profile["profile_version"], + ) + hmac_secret = str(getattr(owner, "cfg_instance_id", "") or "redmesh-rulebook") + actor = _safe_text(actor or "", hmac_secret=hmac_secret, max_len=200) + if not actor: + return _submission_error( + "invalid_review_actor", + job_id, + profile["profile_id"], + "A server-derived review actor is required.", + ) + + with _submission_lock(owner, job_id, profile["profile_id"]): + job_specs = owner._get_job_from_cstore(job_id) + if not isinstance(job_specs, dict): + return _error("job_not_found", job_id, profile_id=profile["profile_id"]) + if job_specs.get("job_status") != JOB_STATUS_FINALIZED: + return _error("job_not_finalized", job_id, profile_id=profile["profile_id"]) + unsupported = reject_model_test_for_scan_operation(job_specs, job_id, "rulebook_review") + if unsupported: + return {**unsupported, "error": "model_test_not_supported"} + repo = _job_repo(owner) + review = repo.get_rulebook_review_model(job_id, profile["profile_id"]) + try: + registry = _submission_registry(repo, job_id, profile["profile_id"]) + except ValueError: + return _unsupported_submission_registry_error(job_id, profile["profile_id"]) + registry_payload = registry.to_dict() + existing_pending = registry_payload.get("pending") + + for reference in registry_payload.get("submissions") or []: + if reference.get("idempotency_key") != idempotency_key: + continue + if existing_pending: + break + if ( + int(reference.get("review_revision", -1)) == expected_revision + and int(reference.get("pass_nr", -1)) == expected_pass + and reference.get("profile_version") == profile["profile_version"] + and reference.get("actor") == actor + ): + replay_snapshot = _build_submission_snapshot( + owner, + job_id, + profile, + pass_nr=int(reference["pass_nr"]), + revision=int(reference["revision"]), + submitted_at=float(reference["submitted_at"]), + actor=actor, + review_revision=expected_revision, + ) + if replay_snapshot.get("status") != "ok": + return replay_snapshot + if replay_snapshot["fingerprint"] != reference.get("fingerprint"): + return _submission_error( + "submission_idempotency_conflict", + job_id, + profile["profile_id"], + "The idempotency key was already used for different submission inputs.", + ) + result = get_rulebook_review(owner, job_id, profile["profile_id"]) + result.update({"submission": _submission_reference_view( + reference, + latest_pass_nr=expected_pass, + profile_version=profile["profile_version"], + ), "idempotent_replay": True}) + return result + return _submission_error( + "submission_idempotency_conflict", + job_id, + profile["profile_id"], + "The idempotency key was already used for different submission inputs.", + ) + + current_revision = review.review_revision if review else 0 + if expected_revision != current_revision: + return _submission_error( + "review_revision_conflict", + job_id, + profile["profile_id"], + "The review changed. Reload before submitting.", + expected_review_revision=expected_revision, + current_review_revision=current_revision, + ) + latest_pass, pass_error = _latest_pass_nr(owner, job_id) + if pass_error: + return pass_error + if expected_pass != latest_pass: + return _submission_error( + "submission_pass_stale", + job_id, + profile["profile_id"], + "Newer scan evidence exists. Reload before submitting.", + expected_pass_nr=expected_pass, + current_pass_nr=latest_pass, + ) + + pending = existing_pending + if pending and pending.get("idempotency_key") != idempotency_key: + return _submission_error( + "submission_in_progress", + job_id, + profile["profile_id"], + "Another submission is pending for this review.", + retryable=True, + ) + if review and review.review_state == "submitted" and not pending: + return _submission_error( + "review_already_submitted", + job_id, + profile["profile_id"], + "Reopen the review before creating another submission revision.", + ) + legacy_reference = _legacy_submission_reference(job_specs, profile, review) + if legacy_reference and not pending: + return _submission_error( + "review_already_submitted", + job_id, + profile["profile_id"], + "Reopen the legacy submitted review before creating a native revision.", + ) + + missing, comments_required = _validate_submission_answers(profile, review) + if missing or comments_required: + return _submission_error( + "submission_comments_required", + job_id, + profile["profile_id"], + "Every review question must be answered; No, Unknown, and Not applicable require comments.", + missing_question_ids=missing, + comment_required_question_ids=comments_required, + ) + + now = _owner_time(owner) + target_revision = int((pending or {}).get("target_revision") or (registry.latest_revision + 1)) + submitted_at = float((pending or {}).get("created_at") or now) + built = _build_submission_snapshot( + owner, + job_id, + profile, + pass_nr=latest_pass, + revision=target_revision, + submitted_at=submitted_at, + actor=actor, + review_revision=expected_revision, + ) + if built.get("status") != "ok": + return built + assessment = built["assessment"] + fingerprint = built["fingerprint"] + + if pending: + if pending.get("fingerprint") != fingerprint: + return _submission_error( + "submission_idempotency_conflict", + job_id, + profile["profile_id"], + "The pending submission no longer matches the canonical review snapshot.", + ) + pending = { + **pending, + "attempt_count": int(pending.get("attempt_count", 1) or 1) + 1, + "updated_at": now, + "last_error": None, + } + else: + pending = RulebookPendingSubmission( + target_revision=target_revision, + expected_review_revision=expected_revision, + expected_pass_nr=latest_pass, + expected_profile_version=profile["profile_version"], + actor=actor, + idempotency_key=idempotency_key, + fingerprint=fingerprint, + state="prepared", + created_at=submitted_at, + updated_at=now, + ).to_dict() + try: + registry = _registry_with(registry, pending=pending, keep_pending=True) + repo.put_rulebook_submission_registry(job_id, profile["profile_id"], registry) + except Exception: + return _submission_error( + "submission_record_failed", + job_id, + profile["profile_id"], + "Unable to record pending submission state.", + retryable=True, + ) + + state_order = {"prepared": 0, "artifact_written": 1, "reference_recorded": 2} + if state_order[pending["state"]] < state_order["artifact_written"]: + try: + cid = _artifact_repo(owner).put_json(assessment, show_logs=False) + except Exception: + cid = None + if not cid: + failure = { + "error": "submission_persist_failed", + "message": "Unable to persist the review submission artifact.", + "retryable": True, + "at": _utc_timestamp(), + "at_epoch": now, + } + _persist_pending_error(repo, job_id, profile["profile_id"], registry, pending, failure) + return _submission_error( + "submission_persist_failed", + job_id, + profile["profile_id"], + failure["message"], + retryable=True, + ) + pending = {**pending, "state": "artifact_written", "cid": cid, "updated_at": now} + try: + registry = _registry_with(registry, pending=pending, keep_pending=True) + repo.put_rulebook_submission_registry(job_id, profile["profile_id"], registry) + except Exception: + return _submission_error( + "submission_record_failed", + job_id, + profile["profile_id"], + "The artifact exists but its CID could not be recorded. Retry with the same key.", + retryable=True, + ) + + submissions = list(registry.to_dict().get("submissions") or []) + reference = next((item for item in submissions if int(item.get("revision", -1)) == target_revision), None) + if reference is None: + reference = RulebookSubmissionReference( + revision=target_revision, + cid=pending["cid"], + submitted_at=submitted_at, + actor=actor, + pass_nr=latest_pass, + profile_id=profile["profile_id"], + profile_version=profile["profile_version"], + schema_version=RULEBOOK_ASSESSMENT_SCHEMA_VERSION, + review_revision=expected_revision, + idempotency_key=idempotency_key, + fingerprint=fingerprint, + ).to_dict() + submissions.append(reference) + if state_order[pending["state"]] < state_order["reference_recorded"]: + pending = {**pending, "state": "reference_recorded", "updated_at": now} + try: + registry = _registry_with(registry, submissions=submissions, pending=pending, keep_pending=True) + repo.put_rulebook_submission_registry(job_id, profile["profile_id"], registry) + except Exception: + failure = { + "error": "submission_record_failed", + "message": "The submission reference could not be recorded.", + "retryable": True, + "at": _utc_timestamp(), + "at_epoch": now, + } + _persist_pending_error(repo, job_id, profile["profile_id"], registry, pending, failure) + return _submission_error( + "submission_record_failed", + job_id, + profile["profile_id"], + "The submission reference could not be recorded. Retry with the same key.", + retryable=True, + ) + + submitted_state = RulebookReviewState( + job_id=job_id, + profile_id=profile["profile_id"], + profile_version=profile["profile_version"], + review_state="submitted", + reviewer=actor, + note=review.note if review else "", + answers=(review.to_dict().get("answers") if review else {}) or {}, + updated_at=submitted_at, + review_revision=expected_revision, + ) + try: + if not review or review.review_state != "submitted": + _put_review_with_audit( + owner, + repo, + job_id=job_id, + profile=profile, + previous=review, + state=submitted_state, + changed_question_ids=[], + event_type="rulebook_review_submitted", + ) + committed_registry = RulebookSubmissionRegistry( + contract_version=RULEBOOK_SUBMISSION_CONTRACT_VERSION, + latest_revision=max(registry.latest_revision, target_revision), + submissions=submissions, + pending=None, + ) + repo.put_rulebook_submission_registry(job_id, profile["profile_id"], committed_registry) + except Exception: + failure = { + "error": "submission_record_failed", + "message": "The final submission state was not committed.", + "retryable": True, + "at": _utc_timestamp(), + "at_epoch": now, + } + _persist_pending_error(repo, job_id, profile["profile_id"], registry, pending, failure) + return _submission_error( + "submission_record_failed", + job_id, + profile["profile_id"], + "The submission is recoverable but its final state was not committed. Retry with the same key.", + retryable=True, + ) + + result = get_rulebook_review(owner, job_id, profile["profile_id"]) + result.update({ + "submission": _submission_reference_view( + reference, + latest_pass_nr=latest_pass, + profile_version=profile["profile_version"], + ), + "idempotent_replay": False, + }) + return result + + +def reopen_rulebook_review( + owner, + job_id, + profile_id=DEFAULT_RULEBOOK_PROFILE_ID, + expected_review_revision=None, + idempotency_key="", + actor="", +): + profile = _profile(profile_id) + if not profile: + return _error("invalid_profile", job_id, profile_id=profile_id) + expected_revision, err = _validated_expected_revision( + expected_review_revision, + job_id, + profile["profile_id"], + ) + if err: + return err + idempotency_key = str(idempotency_key or "").strip() + if not idempotency_key or len(idempotency_key) > 200: + return _submission_error( + "reopen_idempotency_conflict", + job_id, + profile["profile_id"], + "A bounded reopen idempotency key is required.", + ) + hmac_secret = str(getattr(owner, "cfg_instance_id", "") or "redmesh-rulebook") + actor = _safe_text(actor or "", hmac_secret=hmac_secret, max_len=200) + if not actor: + return _submission_error("invalid_review_actor", job_id, profile["profile_id"], "A server-derived actor is required.") + + with _submission_lock(owner, job_id, profile["profile_id"]): + job_specs = owner._get_job_from_cstore(job_id) + if not isinstance(job_specs, dict): + return _error("job_not_found", job_id, profile_id=profile["profile_id"]) + if job_specs.get("job_status") != JOB_STATUS_FINALIZED: + return _error("job_not_finalized", job_id, profile_id=profile["profile_id"]) + unsupported = reject_model_test_for_scan_operation(job_specs, job_id, "rulebook_review") + if unsupported: + return {**unsupported, "error": "model_test_not_supported"} + repo = _job_repo(owner) + review = repo.get_rulebook_review_model(job_id, profile["profile_id"]) + try: + registry = _submission_registry(repo, job_id, profile["profile_id"]) + except ValueError: + return _unsupported_submission_registry_error(job_id, profile["profile_id"]) + registry_payload = registry.to_dict() + if registry_payload.get("pending"): + return _submission_error( + "submission_in_progress", + job_id, + profile["profile_id"], + "Finish the pending submission before reopening.", + retryable=True, + ) + current_revision = review.review_revision if review else 0 + if review and review.last_reopen_idempotency_key == idempotency_key: + if ( + review.review_state == "draft" + and review.last_reopen_from_revision == expected_revision + and review.last_reopen_actor == actor + ): + result = get_rulebook_review(owner, job_id, profile["profile_id"]) + result["idempotent_replay"] = True + return result + return _submission_error( + "reopen_idempotency_conflict", + job_id, + profile["profile_id"], + "The reopen idempotency key was already used for different inputs.", + ) + if expected_revision != current_revision: + return _submission_error( + "review_revision_conflict", + job_id, + profile["profile_id"], + "The review changed. Reload before reopening.", + current_review_revision=current_revision, + ) + legacy_reference = _legacy_submission_reference(job_specs, profile, review) + if not registry_payload.get("submissions") and not legacy_reference: + return _submission_error( + "review_not_submitted", + job_id, + profile["profile_id"], + "Only a submitted review can be reopened.", + ) + if not review or review.review_state not in {"submitted", "reviewed"}: + return _submission_error( + "review_not_submitted", + job_id, + profile["profile_id"], + "Only a submitted review can be reopened.", + ) + now = _owner_time(owner) + reopened = RulebookReviewState( + job_id=job_id, + profile_id=profile["profile_id"], + profile_version=profile["profile_version"], + review_state="draft", + reviewer=actor, + note=review.note, + answers=review.to_dict().get("answers") or {}, + updated_at=now, + review_revision=current_revision + 1, + last_reopen_idempotency_key=idempotency_key, + last_reopen_from_revision=current_revision, + last_reopen_actor=actor, + ) + try: + _put_review_with_audit( + owner, + repo, + job_id=job_id, + profile=profile, + previous=review, + state=reopened, + changed_question_ids=[], + event_type="rulebook_review_reopened", + ) + except Exception: + return _submission_error( + "review_reopen_failed", + job_id, + profile["profile_id"], + "Unable to reopen the submitted review.", + retryable=True, + ) + result = get_rulebook_review(owner, job_id, profile["profile_id"]) + result["idempotent_replay"] = False + return result + + +def _update_rulebook_review_locked( + owner, + job_id, + profile, + job_specs, + validated_answers, + reviewer, + note, + review_state, +): + repo = _job_repo(owner) + previous = repo.get_rulebook_review_model(job_id, profile["profile_id"]) + try: + registry = _submission_registry(repo, job_id, profile["profile_id"]).to_dict() + except ValueError: + return _unsupported_submission_registry_error(job_id, profile["profile_id"]) + formal_history_blocks_legacy_write = bool( + registry.get("submissions") + and ( + previous is None + or previous.review_state != "draft" + or review_state != "draft" + ) + ) + if registry.get("pending") or formal_history_blocks_legacy_write or _legacy_submission_reference(job_specs, profile, previous): + return _submission_error( + "review_already_submitted", + job_id, + profile["profile_id"], + "Use the explicit reopen operation before changing a submitted review.", + ) + now = float(getattr(owner, "time", _time.time)()) + hmac_secret = str(getattr(owner, "cfg_instance_id", "") or "redmesh-rulebook") + reviewer = _safe_text(reviewer or "", hmac_secret=hmac_secret, max_len=200) + note = _safe_text(note or "", hmac_secret=hmac_secret, max_len=1000) + sanitized_answers = {} + for question_id, answer in validated_answers.items(): + sanitized_answers[question_id] = { + "value": answer["value"], + "note": _safe_text(answer.get("note") or "", hmac_secret=hmac_secret, max_len=1000), + "reviewer": reviewer, + "updated_at": now, + } + + previous_answers = (previous.to_dict().get("answers") if previous else {}) or {} + current_answers = dict(previous_answers) + current_answers.update(sanitized_answers) + changed = sorted( + question_id + for question_id in set(previous_answers) | set(current_answers) + if previous_answers.get(question_id) != current_answers.get(question_id) + ) + state = RulebookReviewState( + job_id=job_id, + profile_id=profile["profile_id"], + profile_version=profile["profile_version"], + review_state=review_state, + reviewer=reviewer, + note=note, + answers=current_answers, + updated_at=now, + review_revision=(previous.review_revision if previous else 0) + 1, + ) + review_payload = repo.put_rulebook_review(state) + audit_payload = repo.append_rulebook_review_audit(RulebookReviewAuditEntry( + job_id=job_id, + profile_id=profile["profile_id"], + profile_version=profile["profile_version"], + review_state=review_state, + reviewer=reviewer, + note=note, + changed_question_ids=changed, + previous_answers={question_id: previous_answers.get(question_id) for question_id in changed}, + current_answers={question_id: current_answers.get(question_id) for question_id in changed}, + timestamp=now, + review_revision=state.review_revision, + )) + if hasattr(owner, "_log_audit_event"): + owner._log_audit_event("rulebook_review_updated", { + "job_id": job_id, + "profile_id": profile["profile_id"], + "review_state": review_state, + "review_revision": state.review_revision, + "changed_question_ids": changed, + }) + return { + "status": "ok", + "job_id": job_id, + "profile_id": profile["profile_id"], + "review": review_payload, + "audit": audit_payload, + } + + +def update_rulebook_review( + owner, + job_id, + profile_id=DEFAULT_RULEBOOK_PROFILE_ID, + answers=None, + reviewer="", + note="", + review_state="draft", +): + profile = _profile(profile_id) + if not profile: + return _error("invalid_profile", job_id, profile_id=profile_id) + if review_state not in VALID_RULEBOOK_REVIEW_STATES: + return _error("invalid_review_answer", job_id, profile_id=profile_id, message="Unsupported review_state.") + try: + validated_answers = _validate_review_answers(profile, answers) + except ValueError as exc: + return _error("invalid_review_answer", job_id, profile_id=profile_id, message=str(exc)) + with _submission_lock(owner, job_id, profile["profile_id"]): + job_specs = owner._get_job_from_cstore(job_id) + if not isinstance(job_specs, dict): + return _error("job_not_found", job_id, profile_id=profile_id) + unsupported = reject_model_test_for_scan_operation(job_specs, job_id, "rulebook_review") + if unsupported: + return { + **unsupported, + "error": "model_test_not_supported", + "error_class": unsupported.get("error_class") or unsupported.get("error"), + } + if job_specs.get("job_status") != JOB_STATUS_FINALIZED: + return _error( + "job_not_finalized", + job_id, + profile_id=profile["profile_id"], + job_status=job_specs.get("job_status"), + ) + return _update_rulebook_review_locked( + owner, + job_id, + profile, + job_specs, + validated_answers, + reviewer, + note, + review_state, + ) diff --git a/extensions/business/cybersec/red_mesh/services/state_machine.py b/extensions/business/cybersec/red_mesh/services/state_machine.py index 00f4e28d6..bbf06b1b8 100644 --- a/extensions/business/cybersec/red_mesh/services/state_machine.py +++ b/extensions/business/cybersec/red_mesh/services/state_machine.py @@ -31,6 +31,7 @@ JOB_STATUS_ANALYZING: { JOB_STATUS_COLLECTING, JOB_STATUS_FINALIZING, + JOB_STATUS_SCHEDULED_FOR_STOP, JOB_STATUS_STOPPED, JOB_STATUS_FAILED, }, diff --git a/extensions/business/cybersec/red_mesh/services/triage.py b/extensions/business/cybersec/red_mesh/services/triage.py index d4b8a1b77..d8cd23595 100644 --- a/extensions/business/cybersec/red_mesh/services/triage.py +++ b/extensions/business/cybersec/red_mesh/services/triage.py @@ -1,4 +1,5 @@ from copy import deepcopy +from contextlib import ExitStack from ..model_testing.artifacts import ModelTestArchive from ..model_testing.constants import is_model_test_job @@ -21,6 +22,16 @@ def _artifact_repo(owner): return ArtifactRepository(owner) +def _write_job_record(owner, job_id, job_specs, context): + writer = getattr(type(owner), "_write_job_record", None) + if callable(writer): + return writer(owner, job_id, job_specs, context=context) + launcher = job_specs.get("launcher") if isinstance(job_specs, dict) else None + if launcher and launcher != getattr(owner, "ee_addr", None): + return None + return _job_repo(owner).put_job(job_id, job_specs) + + def _archive_contains_finding(archive: dict, finding_id: str) -> bool: return _find_archive_finding(archive, finding_id) is not None @@ -65,6 +76,26 @@ def get_job_triage(owner, job_id: str, finding_id: str = ""): def update_finding_triage(owner, job_id: str, finding_id: str, status: str, note: str = "", actor: str = "", review_at: float = 0): + from .rulebook_assessment import _submission_lock, list_rulebook_profiles + + with ExitStack() as stack: + profiles = sorted(list_rulebook_profiles(), key=lambda item: item["profile_id"]) + for profile in profiles: + stack.enter_context(_submission_lock(owner, job_id, profile["profile_id"])) + repo = _job_repo(owner) + for profile in profiles: + registry = repo.get_rulebook_submission_registry(job_id, profile["profile_id"]) + if isinstance(registry, dict) and registry.get("pending"): + return { + "error": "submission_in_progress", + "message": "Finding triage cannot change while a formal review submission is pending.", + "job_id": job_id, + "finding_id": finding_id, + } + return _update_finding_triage_locked(owner, job_id, finding_id, status, note, actor, review_at) + + +def _update_finding_triage_locked(owner, job_id: str, finding_id: str, status: str, note: str = "", actor: str = "", review_at: float = 0): if status not in VALID_TRIAGE_STATUSES: return { "error": "validation_error", @@ -76,6 +107,14 @@ def update_finding_triage(owner, job_id: str, finding_id: str, status: str, note return {"error": "not_found", "message": f"Job {job_id} not found."} if not job_specs.get("job_cid"): return {"error": "not_available", "message": f"Job {job_id} is still running (triage requires archived findings)."} + launcher = job_specs.get("launcher") + if launcher and launcher != getattr(owner, "ee_addr", None): + return { + "error": "job_launcher_mismatch", + "message": "Finding triage must be handled by the job launcher.", + "status_code": 409, + "job_id": job_id, + } archive = _artifact_repo(owner).get_archive(job_specs) if not isinstance(archive, dict): @@ -119,7 +158,7 @@ def update_finding_triage(owner, job_id: str, finding_id: str, status: str, note event_action="triaged", ) if isinstance(job_specs.get("soc_event_status"), dict): - repo.put_job(job_id, job_specs) + _write_job_record(owner, job_id, job_specs, context="finding_triage_soc_event") return { "job_id": job_id, "finding_id": finding_id, diff --git a/extensions/business/cybersec/red_mesh/tests/conftest.py b/extensions/business/cybersec/red_mesh/tests/conftest.py index cfa44f92f..4e0a90b8c 100644 --- a/extensions/business/cybersec/red_mesh/tests/conftest.py +++ b/extensions/business/cybersec/red_mesh/tests/conftest.py @@ -155,6 +155,19 @@ class FakeBasePlugin: CONFIG = {'VALIDATION_RULES': {}} endpoint = staticmethod(endpoint_decorator) + def create_postponed_request(self, solver_method, method_kwargs=None): + return { + "solver_method": solver_method, + "method_kwargs": dict(method_kwargs or {}), + } + + def process(self): + return None + + def on_close(self): + self._base_closed = True + return None + mock_module = MagicMock() mock_module.FastApiWebAppPlugin = FakeBasePlugin diff --git a/extensions/business/cybersec/red_mesh/tests/test_api.py b/extensions/business/cybersec/red_mesh/tests/test_api.py index 15d9cfced..532e2328b 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_api.py +++ b/extensions/business/cybersec/red_mesh/tests/test_api.py @@ -1,7 +1,13 @@ +import ast +import inspect import json import sys import struct +import time +import textwrap import unittest +from concurrent.futures import Future +from copy import deepcopy from unittest.mock import MagicMock, patch from extensions.business.cybersec.red_mesh.constants import JOB_ARCHIVE_VERSION, MAX_CONTINUOUS_PASSES @@ -1735,6 +1741,20 @@ def _build_finalize_plugin(self, job_id="test-job", job_pass=1, run_mode="SINGLE plugin.cfg_attestation = {"ENABLED": True, "PRIVATE_KEY": "", "MIN_SECONDS_BETWEEN_SUBMITS": 300, "RETRIES": 2} plugin.time.return_value = 1000100.0 plugin.json_dumps.return_value = "{}" + plugin._automatic_analysis_state = None + if llm_enabled: + executor = MagicMock() + + def submit_immediately(fn, *args, **kwargs): + future = Future() + try: + future.set_result(fn(*args, **kwargs)) + except Exception as exc: + future.set_exception(exc) + return future + + executor.submit.side_effect = submit_immediately + plugin._get_manual_analysis_executor.return_value = executor # R1FS mock plugin.r1fs = MagicMock() @@ -2247,6 +2267,7 @@ def _structured_failure(*_args, **_kwargs): plugin._run_aggregated_llm_analysis = MagicMock(side_effect=AssertionError("legacy raw LLM path must not run")) plugin._run_quick_summary_analysis = MagicMock(side_effect=AssertionError("legacy quick summary path must not run")) + PentesterApi01Plugin._maybe_finalize_pass(plugin) PentesterApi01Plugin._maybe_finalize_pass(plugin) # Check PassReport has llm_failed=True @@ -2298,6 +2319,7 @@ def _structured_success(*_args, **_kwargs): plugin._run_aggregated_llm_analysis = MagicMock(side_effect=AssertionError("legacy raw LLM path must not run")) plugin._run_quick_summary_analysis = MagicMock(side_effect=AssertionError("legacy quick summary path must not run")) + PentesterApi01Plugin._maybe_finalize_pass(plugin) PentesterApi01Plugin._maybe_finalize_pass(plugin) plugin._run_quick_summary_analysis.assert_not_called() @@ -2308,6 +2330,155 @@ def _structured_success(*_args, **_kwargs): self.assertIn("## Overall Posture", pass_report_dict["llm_analysis"]) self.assertIn("Structured posture", pass_report_dict["llm_analysis"]) + def test_automatic_analysis_yields_and_resumes_without_stale_recovery(self): + """Pending model work returns promptly and is consumed once on a later turn.""" + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin(llm_enabled=True) + self._configure_successful_pass_finalization(plugin, job_specs) + future = Future() + executor = MagicMock() + executor.submit.return_value = future + plugin._get_manual_analysis_executor.return_value = executor + sections = { + "executive_headline": "Structured headline", + "overall_posture": "Structured posture", + "recommendation_summary": ["Patch exposed services"], + "conclusion": "Structured conclusion", + } + + started = time.monotonic() + PentesterApi01Plugin._maybe_finalize_pass(plugin) + self.assertLess(time.monotonic() - started, 0.1) + + self.assertEqual(job_specs["job_status"], "ANALYZING") + self.assertEqual(job_specs["pass_reports"], []) + self.assertIsNotNone(plugin._automatic_analysis_state) + executor.submit.assert_called_once() + + started = time.monotonic() + PentesterApi01Plugin._maybe_finalize_pass(plugin) + self.assertLess(time.monotonic() - started, 0.1) + + executor.submit.assert_called_once() + plugin._collect_node_reports.assert_called_once() + plugin._log_audit_event.assert_not_called() + + plugin._last_structured_llm_failed = False + future.set_result(sections) + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + self.assertEqual(job_specs["job_status"], "FINALIZED") + self.assertEqual(len(job_specs["pass_reports"]), 1) + self.assertIsNone(plugin._automatic_analysis_state) + executor.submit.assert_called_once() + plugin._log_audit_event.assert_not_called() + + def test_soft_stop_during_automatic_analysis_is_preserved_on_resume(self): + """A responsive soft stop requested while analysis runs must end the pass.""" + PentesterApi01Plugin = self._get_plugin_class() + from extensions.business.cybersec.red_mesh.services.control import stop_monitoring + + plugin, job_specs = self._build_finalize_plugin( + run_mode="CONTINUOUS_MONITORING", + llm_enabled=True, + ) + self._configure_successful_pass_finalization(plugin, job_specs) + future = Future() + executor = MagicMock() + executor.submit.return_value = future + plugin._get_manual_analysis_executor.return_value = executor + plugin.chainstore_hget.return_value = job_specs + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + self.assertEqual(job_specs["job_status"], "ANALYZING") + + stop_result = stop_monitoring(plugin, job_specs["job_id"], stop_type="SOFT") + self.assertEqual(stop_result["job_status"], "SCHEDULED_FOR_STOP") + + plugin._last_structured_llm_failed = False + future.set_result({"executive_headline": "Analysis complete"}) + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + self.assertEqual(job_specs["job_status"], "STOPPED") + self.assertEqual(len(job_specs["pass_reports"]), 1) + self.assertIsNone(plugin._automatic_analysis_state) + executor.submit.assert_called_once() + + def test_automatic_analysis_future_failure_keeps_existing_llm_failure_path(self): + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin(llm_enabled=True) + self._configure_successful_pass_finalization(plugin, job_specs) + future = Future() + executor = MagicMock() + executor.submit.return_value = future + plugin._get_manual_analysis_executor.return_value = executor + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + future.set_exception(RuntimeError("provider failed")) + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + pass_report_dict = plugin.r1fs.add_json.call_args_list[1][0][0] + self.assertTrue(pass_report_dict["llm_failed"]) + self.assertIsNone(pass_report_dict.get("llm_report_sections")) + self.assertEqual(job_specs["job_status"], "FINALIZED") + self.assertIsNone(plugin._automatic_analysis_state) + + def test_automatic_analysis_submit_failure_keeps_existing_llm_failure_path(self): + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin(llm_enabled=True) + self._configure_successful_pass_finalization(plugin, job_specs) + executor = MagicMock() + executor.submit.side_effect = RuntimeError("executor unavailable") + plugin._get_manual_analysis_executor.return_value = executor + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + pass_report_dict = plugin.r1fs.add_json.call_args_list[1][0][0] + self.assertTrue(pass_report_dict["llm_failed"]) + self.assertEqual(job_specs["job_status"], "FINALIZED") + self.assertIsNone(plugin._automatic_analysis_state) + + def test_terminal_job_cancels_pending_automatic_analysis(self): + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin(llm_enabled=True) + self._configure_successful_pass_finalization(plugin, job_specs) + future = Future() + executor = MagicMock() + executor.submit.return_value = future + plugin._get_manual_analysis_executor.return_value = executor + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + job_specs["job_status"] = "FINALIZED" + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + self.assertTrue(future.cancelled()) + self.assertIsNone(plugin._automatic_analysis_state) + executor.submit.assert_called_once() + self.assertEqual(job_specs["pass_reports"], []) + + def test_changed_report_identity_cancels_pending_automatic_analysis(self): + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin(llm_enabled=True) + self._configure_successful_pass_finalization(plugin, job_specs) + future = Future() + replacement_future = Future() + executor = MagicMock() + executor.submit.side_effect = [future, replacement_future] + plugin._get_manual_analysis_executor.return_value = executor + + PentesterApi01Plugin._maybe_finalize_pass(plugin) + job_specs["workers"]["worker-A"]["report_cid"] = "QmChanged" + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + self.assertTrue(future.cancelled()) + self.assertIsNotNone(plugin._automatic_analysis_state) + self.assertIn( + ("worker-A", "QmChanged"), + plugin._automatic_analysis_state["report_identity"], + ) + self.assertEqual(executor.submit.call_count, 2) + self.assertEqual(job_specs["pass_reports"], []) + def test_pass_reports_survive_typed_job_record_rewrites(self): """Pass reports must stay attached after typed repository rewrites the job dict.""" PentesterApi01Plugin = self._get_plugin_class() @@ -2338,6 +2509,55 @@ def test_pass_reports_survive_typed_job_record_rewrites(self): archived_job_specs = plugin._build_job_archive.call_args[0][1] self.assertEqual(len(archived_job_specs["pass_reports"]), 1) + def test_finalization_runs_nis2_ensure_after_pass_report(self): + """Completed pass finalization triggers default-on NIS2 assessment ensure.""" + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin() + self._configure_successful_pass_finalization(plugin, job_specs) + + with patch( + "extensions.business.cybersec.red_mesh.services.finalization.ensure_rulebook_assessment", + return_value={"status": "ok", "artifact_cid": "QmRulebook", "pass_nr": 1}, + ) as ensure_mock: + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + ensure_mock.assert_called_once_with(plugin, job_specs["job_id"], pass_nr=1) + self.assertEqual(job_specs["job_status"], "FINALIZED") + plugin._build_job_archive.assert_called_once_with(job_specs["job_id"], job_specs) + + def test_nis2_ensure_failure_does_not_block_finalization(self): + """NIS2 generation is best-effort and must not fail the scan.""" + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin() + self._configure_successful_pass_finalization(plugin, job_specs) + + with patch( + "extensions.business.cybersec.red_mesh.services.finalization.ensure_rulebook_assessment", + return_value={"status": "error", "error": "artifact_write_failed"}, + ) as ensure_mock: + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + ensure_mock.assert_called_once_with(plugin, job_specs["job_id"], pass_nr=1) + self.assertEqual(job_specs["job_status"], "FINALIZED") + plugin._build_job_archive.assert_called_once_with(job_specs["job_id"], job_specs) + + def test_continuous_pass_runs_nis2_ensure_before_next_pass_schedule(self): + """Continuous jobs refresh NIS2 readiness against each completed pass.""" + PentesterApi01Plugin = self._get_plugin_class() + plugin, job_specs = self._build_finalize_plugin(run_mode="CONTINUOUS_MONITORING") + self._configure_successful_pass_finalization(plugin, job_specs) + + with patch( + "extensions.business.cybersec.red_mesh.services.finalization.ensure_rulebook_assessment", + return_value={"status": "ok", "artifact_cid": "QmRulebook", "pass_nr": 1}, + ) as ensure_mock: + PentesterApi01Plugin._maybe_finalize_pass(plugin) + + ensure_mock.assert_called_once_with(plugin, job_specs["job_id"], pass_nr=1) + self.assertEqual(job_specs["job_status"], "RUNNING") + self.assertIsNotNone(job_specs["next_pass_at"]) + plugin._build_job_archive.assert_not_called() + def test_aggregated_report_write_failure(self): """R1FS fails for aggregated → pass finalization skipped, no partial state.""" PentesterApi01Plugin = self._get_plugin_class() @@ -3042,6 +3262,23 @@ def test_archive_written_to_r1fs(self): self.assertIn("ui_aggregate", archive_dict) self.assertIn("total_open_ports", archive_dict["ui_aggregate"]) + def test_duplicate_archive_finalization_reuses_existing_cid(self): + """Retrying finalization after prune is harmless and writes no artifact.""" + Plugin = self._get_plugin_class() + plugin, job_specs, _, _ = self._build_archive_plugin() + plugin.chainstore_hget.return_value = { + "job_id": "test-job", + "job_status": "FINALIZED", + "launcher": "launcher-node", + "job_cid": "QmExistingArchive", + } + + result = Plugin._build_job_archive(plugin, "test-job", job_specs) + + self.assertEqual(result, "QmExistingArchive") + plugin.r1fs.add_json.assert_not_called() + plugin.chainstore_hset.assert_not_called() + def test_archive_ui_aggregate_includes_graybox_summary(self): """Archive UI aggregate preserves graybox scan metadata and scenario counts.""" Plugin = self._get_plugin_class() @@ -3404,6 +3641,16 @@ def _build_plugin(self, jobs_dict): plugin._get_job_from_cstore = lambda job_id: Plugin._get_job_from_cstore(plugin, job_id) return plugin + def test_get_report_does_not_pin_retrieved_cid(self): + Plugin = self._get_plugin_class() + plugin = self._build_plugin({}) + plugin.r1fs.get_json.return_value = {"artifact_kind": "review_submission"} + + result = Plugin.get_report(plugin, "QmReportCID") + + self.assertEqual(result["report"]["artifact_kind"], "review_submission") + plugin.r1fs.get_json.assert_called_once_with("QmReportCID", pin=False) + def test_get_job_archive_finalized(self): """get_job_archive for finalized job returns archive with matching job_id.""" Plugin = self._get_plugin_class() @@ -3510,8 +3757,9 @@ def test_get_job_archive_running(self): self.assertEqual(result["error"], "not_available") def test_manual_structured_analysis_backfills_legacy_fields(self): - """Manual structured analysis updates the pass report for get_analysis compatibility.""" + """Postponed manual analysis updates the pass report for compatibility.""" Plugin = self._get_plugin_class() + from extensions.business.cybersec.red_mesh.pentester_api_01 import _ManualAnalysisOutcome job_specs = self._build_running_job("job-llm", pass_count=1) for worker in job_specs["workers"].values(): worker["finished"] = True @@ -3525,6 +3773,8 @@ def test_manual_structured_analysis_backfills_legacy_fields(self): "AUTO_ANALYSIS_TYPE": "security_assessment", } plugin.cfg_llm_agent_api_port = 8080 + plugin.cfg_llm_agent_api_host = "127.0.0.1" + plugin.cfg_request_timeout = 120 plugin.r1fs = MagicMock() plugin.r1fs.get_json.return_value = { "pass_nr": 1, @@ -3546,23 +3796,41 @@ def test_manual_structured_analysis_backfills_legacy_fields(self): }) plugin._get_job_config = MagicMock(return_value={"target": "example.com"}) plugin._compute_risk_and_findings = MagicMock(return_value=({"score": 0, "breakdown": {}}, [])) + plugin._collect_bounded_manual_analysis_reports = ( + lambda workers: Plugin._collect_bounded_manual_analysis_reports(plugin, workers) + ) + sections = { + "executive_headline": "Manual structured headline", + "overall_posture": "Manual structured posture", + "recommendation_summary": ["Review internet exposure"], + "conclusion": "Manual structured conclusion", + } + future = MagicMock() + future.done.return_value = True + future.result.return_value = _ManualAnalysisOutcome(sections=sections, failed=False) + plugin._manual_analysis_executor = MagicMock() + plugin._manual_analysis_executor.submit.return_value = future + plugin._manual_analysis_state = None + plugin.time.return_value = 100.0 + plugin.create_postponed_request.return_value = "postponed" - def _structured_success(*_args, **_kwargs): - plugin._last_structured_llm_failed = False - return { - "executive_headline": "Manual structured headline", - "overall_posture": "Manual structured posture", - "recommendation_summary": ["Review internet exposure"], - "conclusion": "Manual structured conclusion", - } - - plugin._run_structured_report_sections = MagicMock(side_effect=_structured_success) + def _write_job(_owner, _job_id, persisted, **_kwargs): + job_specs["pass_reports"] = deepcopy(persisted["pass_reports"]) + return persisted - result = Plugin.analyze_job(plugin, job_id="job-llm") + with patch.object(Plugin, "_write_job_record", side_effect=_write_job) as write_job: + postponed = Plugin.analyze_job( + plugin, + job_id="job-llm", + ) + self.assertEqual(postponed, "postponed") + pending_id = plugin._manual_analysis_state["pending_id"] + result = Plugin.solve_postponed_analyze_job(plugin, pending_id) updated_pass = plugin.r1fs.add_json.call_args[0][0] + persisted_job = write_job.call_args[0][2] self.assertEqual(result["analysis_type"], "structured_report_sections") - self.assertEqual(job_specs["pass_reports"][-1]["report_cid"], "QmUpdatedPass") + self.assertEqual(persisted_job["pass_reports"][-1]["report_cid"], "QmUpdatedPass") self.assertEqual(updated_pass["quick_summary"], "Manual structured headline") self.assertIn("Manual structured posture", updated_pass["llm_analysis"]) self.assertIn("Review internet exposure", updated_pass["llm_analysis"]) @@ -3656,17 +3924,21 @@ def test_write_job_record_bumps_revision(self): self.assertEqual(running.job_revision, 3) plugin._log_audit_event.assert_not_called() - def test_job_write_guarantees_report_detection_only_mode(self): - """RedMesh exposes detection-only semantics when chainstore lacks CAS.""" + def test_job_write_guarantees_report_launcher_guard_mode(self): + """Lifecycle guards are explicit without claiming chainstore CAS.""" Plugin = self._get_plugin_class() plugin = self._build_plugin({}) - self.assertFalse(Plugin._supports_guarded_job_writes(plugin)) + self.assertTrue(Plugin._supports_guarded_job_writes(plugin)) self.assertEqual(Plugin._get_job_write_guarantees(plugin), { - "mode": "detection_only", - "guarded_writes": False, + "mode": "launcher_single_writer", + "guarded_writes": True, "stale_write_detection": True, + "terminal_monotonicity": True, "job_revision": True, + "atomic_compare_and_swap": False, + "distributed_lease": False, + "duplicate_owner_exclusion": "operational", }) def test_write_job_record_logs_stale_write(self): @@ -3687,9 +3959,78 @@ def test_write_job_record_logs_stale_write(self): "expected_revision": 3, "current_revision": 5, "context": "close_job", - "write_mode": "detection_only", + "write_mode": "launcher_single_writer", }) + def test_shared_job_write_rejects_non_launcher(self): + Plugin = self._get_plugin_class() + current = { + "job_id": "job-1", + "job_status": "RUNNING", + "launcher": "launcher-node", + "job_revision": 2, + } + plugin = self._build_plugin({"job-1": current}) + plugin.ee_addr = "worker-node" + plugin.chainstore_hset = MagicMock() + plugin._log_audit_event = MagicMock() + plugin.P = MagicMock() + + result = Plugin._write_job_record( + plugin, "job-1", dict(current), context="stix_export", + ) + + self.assertIsNone(result) + plugin.chainstore_hset.assert_not_called() + plugin._log_audit_event.assert_called_once() + + def test_terminal_write_cannot_regress(self): + Plugin = self._get_plugin_class() + current = { + "job_id": "job-1", + "job_status": "FINALIZED", + "launcher": "launcher-node", + "job_cid": "cid-final", + "job_revision": 4, + } + plugin = self._build_plugin({"job-1": current}) + plugin.chainstore_hset = MagicMock() + plugin._log_audit_event = MagicMock() + plugin.P = MagicMock() + + result = Plugin._write_job_record( + plugin, + "job-1", + {**current, "job_status": "RUNNING", "job_cid": None}, + context="finalize_collecting", + ) + + self.assertEqual(result, current) + plugin.chainstore_hset.assert_not_called() + plugin._log_audit_event.assert_called_once() + + def test_duplicate_terminal_archive_write_is_idempotent(self): + Plugin = self._get_plugin_class() + current = { + "job_id": "job-1", + "job_status": "FINALIZED", + "launcher": "launcher-node", + "job_cid": "cid-final", + "job_revision": 4, + } + plugin = self._build_plugin({"job-1": current}) + plugin.chainstore_hset = MagicMock() + plugin._log_audit_event = MagicMock() + plugin.P = MagicMock() + + result = Plugin._write_job_record( + plugin, "job-1", dict(current), context="archive_prune", + ) + + self.assertEqual(result, current) + plugin.chainstore_hset.assert_not_called() + plugin._log_audit_event.assert_not_called() + def test_get_job_config_resolves_secret_ref_for_runtime(self): """Runtime config loading resolves secret_ref into inline credentials.""" Plugin = self._get_plugin_class() @@ -3760,7 +4101,7 @@ def test_get_job_config_fails_closed_for_malformed_secret_payload(self): ) self.assertEqual(len(plugin.r1fs.get_json.call_args_list), 2) - def test_mark_worker_terminal_error_sets_common_fields(self): + def test_mark_worker_terminal_error_publishes_live_terminal_state(self): Plugin = self._get_plugin_class() plugin = self._build_plugin({}) job_specs = { @@ -3768,92 +4109,27 @@ def test_mark_worker_terminal_error_sets_common_fields(self): "workers": {"worker-a": {"start_port": 443, "end_port": 443}}, } - with patch.object(Plugin, "_write_job_record", return_value=job_specs) as write: - Plugin._mark_worker_terminal_error( - plugin, - job_specs, - "worker-a", - "secret_resolution_failed", - "Failed to resolve graybox secret_ref", - context="test_terminal", - ) - - worker = job_specs["workers"]["worker-a"] - self.assertTrue(worker["finished"]) - self.assertEqual(worker["terminal_reason"], "secret_resolution_failed") - self.assertIn("secret_ref", worker["error"]) - write.assert_called_once() - - def test_mark_worker_terminal_error_merges_against_current_record(self): - """B8: concurrent terminal writes must merge by worker key, not overwrite.""" - Plugin = self._get_plugin_class() - # Current record in CStore has worker-A already terminal (written by - # worker A's concurrent failure). - current_record = { - "job_id": "job-concurrent", - "job_status": "RUNNING", - "job_pass": 1, - "run_mode": "SINGLEPASS", - "launcher": "launcher-node", - "target": "example.com", - "scan_type": "webapp", - "target_url": "https://example.com/app", - "start_port": 443, - "end_port": 443, - "date_created": 1000000.0, - "job_config_cid": "QmConfig", - "workers": { - "worker-A": { - "start_port": 443, "end_port": 443, - "finished": True, - "terminal_reason": "assignment_validation_failed", - "error": "A error", - }, - "worker-B": {"start_port": 443, "end_port": 443, "finished": False}, - }, - "timeline": [], - "pass_reports": [], - "job_revision": 7, - } - plugin = self._build_plugin({"job-concurrent": current_record}) - - # Worker-B's stale local snapshot doesn't know about A's terminal flag. - stale_snapshot = { - "job_id": "job-concurrent", - "workers": { - "worker-A": {"start_port": 443, "end_port": 443, "finished": False}, - "worker-B": {"start_port": 443, "end_port": 443, "finished": False}, - }, - } - - captured = {} - - def _capture(self_plugin, job_id, job_specs, expected_revision=None, context=""): - captured["job_id"] = job_id - captured["job_specs"] = dict(job_specs) - captured["context"] = context - return job_specs - - with patch.object(Plugin, "_write_job_record", side_effect=_capture): - Plugin._mark_worker_terminal_error( - plugin, - stale_snapshot, - "worker-B", - "launch_failed", - "B error", - context="b_terminal", - ) + plugin.time.return_value = 100.0 + Plugin._mark_worker_terminal_error( + plugin, + job_specs, + "worker-a", + "secret_resolution_failed", + "Failed to resolve graybox secret_ref", + context="test_terminal", + ) - persisted_workers = captured["job_specs"]["workers"] - # A's pre-existing terminal data survived the B write. - self.assertTrue(persisted_workers["worker-A"]["finished"]) - self.assertEqual(persisted_workers["worker-A"]["terminal_reason"], "assignment_validation_failed") - self.assertEqual(persisted_workers["worker-A"]["error"], "A error") - # B's terminal patch is applied. - self.assertTrue(persisted_workers["worker-B"]["finished"]) - self.assertEqual(persisted_workers["worker-B"]["terminal_reason"], "launch_failed") + live_call = next( + call for call in plugin.chainstore_hset.call_args_list + if call.kwargs.get("hkey") == "test-instance:live" + ) + live = live_call.kwargs["value"] + self.assertTrue(live["finished"]) + self.assertEqual(live["error_class"], "secret_resolution_failed") + self.assertNotIn("error", live) + self.assertNotIn("error_message", live) - def test_maybe_launch_jobs_secret_resolution_failure_marks_terminal(self): + def test_maybe_launch_jobs_secret_resolution_failure_publishes_terminal_live(self): Plugin = self._get_plugin_class() assignments, error = build_graybox_worker_assignments(["launcher-node"]) self.assertIsNone(error) @@ -3898,10 +4174,33 @@ def test_maybe_launch_jobs_secret_resolution_failure_marks_terminal(self): with patch.object(Plugin, "_write_job_record", return_value=job_specs) as write: Plugin._maybe_launch_jobs(plugin) - self.assertTrue(worker_entry["finished"]) - self.assertEqual(worker_entry["terminal_reason"], "secret_resolution_failed") - self.assertIn("secret_ref", worker_entry["error"]) - write.assert_called_once() + write.assert_not_called() + live_call = next( + call for call in plugin.chainstore_hset.call_args_list + if call.kwargs.get("hkey") == "test-instance:live" + ) + self.assertTrue(live_call.kwargs["value"]["finished"]) + self.assertEqual(live_call.kwargs["value"]["error_class"], "secret_resolution_failed") + + def test_worker_background_paths_have_no_shared_lifecycle_writer_calls(self): + """Structural guard: worker/background paths cannot call the shared writer.""" + Plugin = self._get_plugin_class() + functions = ( + Plugin._maybe_launch_model_test_jobs, + Plugin._maybe_close_model_test_jobs, + Plugin._maybe_stop_canceled_jobs, + Plugin._close_job, + Plugin._mark_worker_terminal_error, + ) + for function in functions: + tree = ast.parse(textwrap.dedent(inspect.getsource(function))) + writer_calls = [ + node for node in ast.walk(tree) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "_write_job_record" + ] + self.assertEqual(writer_calls, [], function.__name__) def test_get_job_data_running_last_5(self): """Running job with 8 passes returns last 5 refs only.""" @@ -4525,6 +4824,99 @@ def test_get_job_triage_not_found(self): self.assertEqual(result["audit"], []) +class TestModelTestingEndpointAuth(unittest.TestCase): + """Protected Model Testing endpoints validate the backend bearer token.""" + + @classmethod + def setUpClass(cls): + mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + cls.Plugin = PentesterApi01Plugin + + def test_missing_backend_token_configuration_fails_closed(self): + plugin = MagicMock() + with patch.dict("os.environ", {}, clear=True), patch( + "extensions.business.cybersec.red_mesh.pentester_api_01.launch_model_test" + ) as launch: + result = self.Plugin.launch_model_test(plugin, "presented-token") + + self.assertEqual(result["status_code"], 401) + self.assertEqual(result["error_class"], "backend_auth_unavailable") + self.assertNotIn("presented-token", str(result)) + launch.assert_not_called() + + def test_invalid_backend_token_is_forbidden_without_leak(self): + plugin = MagicMock() + expected = "expected-backend-token-material-32-bytes" + with patch.dict("os.environ", {"REDMESH_BACKEND_TOKEN": expected}), patch( + "extensions.business.cybersec.red_mesh.pentester_api_01.launch_model_test" + ) as launch: + result = self.Plugin.launch_model_test(plugin, "invalid-presented-token") + + self.assertEqual(result["status_code"], 403) + self.assertEqual(result["error_class"], "backend_auth_invalid") + self.assertNotIn(expected, str(result)) + self.assertNotIn("invalid-presented-token", str(result)) + launch.assert_not_called() + + def test_short_backend_token_configuration_fails_closed(self): + plugin = MagicMock() + with patch.dict("os.environ", {"REDMESH_BACKEND_TOKEN": "short-test-token"}), patch( + "extensions.business.cybersec.red_mesh.pentester_api_01.launch_model_test" + ) as launch: + result = self.Plugin.launch_model_test(plugin, "short-test-token") + + self.assertEqual(result["status_code"], 401) + self.assertEqual(result["error_class"], "backend_auth_unavailable") + self.assertNotIn("short-test-token", str(result)) + launch.assert_not_called() + + def test_empty_presented_token_is_unauthorized(self): + plugin = MagicMock() + expected = "expected-backend-token-material-32-bytes" + with patch.dict("os.environ", {"REDMESH_BACKEND_TOKEN": expected}), patch( + "extensions.business.cybersec.red_mesh.pentester_api_01.launch_model_test" + ) as launch: + result = self.Plugin.launch_model_test(plugin, "") + + self.assertEqual(result["status_code"], 401) + self.assertEqual(result["error_class"], "backend_auth_required") + self.assertNotIn(expected, str(result)) + launch.assert_not_called() + + def test_authenticated_launch_forwards_navigator_actor_assertion(self): + plugin = MagicMock() + token = "valid-backend-token-material-at-least-32-bytes" + with patch.dict("os.environ", {"REDMESH_BACKEND_TOKEN": token}), patch( + "extensions.business.cybersec.red_mesh.pentester_api_01.launch_model_test", + return_value={"status": "ok"}, + ) as launch: + result = self.Plugin.launch_model_test( + plugin, + token, + created_by_id="navigator-user-123", + ) + + self.assertEqual(result, {"status": "ok"}) + self.assertEqual(launch.call_args.kwargs["created_by_id"], "navigator-user-123") + + def test_authenticated_preflight_forwards_navigator_actor_assertion(self): + plugin = MagicMock() + token = "valid-backend-token-material-at-least-32-bytes" + with patch.dict("os.environ", {"REDMESH_BACKEND_TOKEN": token}), patch( + "extensions.business.cybersec.red_mesh.pentester_api_01.preflight_model_test_provider", + return_value={"status": "ok"}, + ) as preflight: + result = self.Plugin.preflight_model_test_provider( + plugin, + token, + created_by_id="navigator-user-123", + ) + + self.assertEqual(result, {"status": "ok"}) + self.assertEqual(preflight.call_args.kwargs["created_by_id"], "navigator-user-123") + + class TestPhase2AuditCounting(unittest.TestCase): """Phase 2: audit counts include graybox findings.""" diff --git a/extensions/business/cybersec/red_mesh/tests/test_api_operation_queue.py b/extensions/business/cybersec/red_mesh/tests/test_api_operation_queue.py deleted file mode 100644 index fb10938af..000000000 --- a/extensions/business/cybersec/red_mesh/tests/test_api_operation_queue.py +++ /dev/null @@ -1,729 +0,0 @@ -import hashlib -import copy -import json -import sys -import types -import unittest - -if "pymisp" not in sys.modules: - pymisp_stub = types.ModuleType("pymisp") - pymisp_stub.MISPEvent = object - pymisp_stub.MISPObject = object - pymisp_stub.MISPAttribute = object - pymisp_stub.PyMISP = object - sys.modules["pymisp"] = pymisp_stub - -from extensions.business.cybersec.red_mesh.services.api_operations import ( - ApiOperationRepository, - cancel_api_operation, - create_analyze_job_operation, - get_api_operation_result, - get_api_operation_status, - maybe_start_api_operation_worker, -) -from extensions.business.cybersec.red_mesh.services.config import ( - DEFAULT_API_OPERATIONS_CONFIG, - get_api_operations_config, -) - - -def _token_hash(token: str) -> str: - return hashlib.sha256(token.encode("utf-8")).hexdigest() - - -def _join_worker(owner): - worker = getattr(owner, "_api_operation_worker_thread", None) - if worker: - worker.join(timeout=2) - - -def _structured_sections(owner): - owner.llm_calls += 1 - owner._last_structured_llm_failed = False - return { - "executive_headline": "High priority HTTPS exposure", - "overall_posture": "The exposed service should be reviewed.", - "recommendation_summary": ["Review TLS and authentication controls."], - "strategic_roadmap": {}, - "attack_chain_narratives": [], - "coverage_gaps": [], - "conclusion": "Remediation is practical.", - } - - -class _Owner: - def __init__(self, *, token_hashes=None, max_queue_global=32, max_queue_per_job=1): - self.cfg_instance_id = "test-instance" - self.ee_addr = "node-a" - self.cfg_llm_agent = {"ENABLED": True, "TIMEOUT": 5} - self.cfg_llm_agent_api_port = 8080 - self.cfg_llm_api_retries = 2 - self.cfg_api_operations = { - "ENABLED": True, - "TOKEN_HASHES": token_hashes if token_hashes is not None else [_token_hash("good-token")], - "TOKEN_ENV": "", - "HMAC_SECRET": "operation-hmac-secret", - "HMAC_SECRET_ENV": "", - "MAX_QUEUE_GLOBAL": max_queue_global, - "MAX_QUEUE_PER_JOB": max_queue_per_job, - "MAX_QUEUE_PER_ACTOR": 8, - "POLL_AFTER_MS": 500, - } - self._store = {} - self._jobs = {} - self.r1fs = _R1FS() - self.audit_events = [] - self.timeline_events = [] - self.llm_calls = 0 - - def add_job(self, job_id="job-1"): - self.r1fs.objects["QmPassReport"] = { - "pass_nr": 1, - "aggregated_report_cid": "QmAggregatedReport", - "worker_reports": {}, - "risk_score": 12, - } - self._jobs[job_id] = { - "job_id": job_id, - "job_revision": 3, - "workers": { - "node-a": { - "finished": True, - "report_cid": "QmWorkerReport", - }, - }, - "pass_reports": [ - {"pass_nr": 1, "report_cid": "QmPassReport"}, - ], - } - return self._jobs[job_id] - - def _get_job_from_cstore(self, job_id): - return self._jobs.get(job_id) - - def chainstore_hget(self, *, hkey, key): - return self._store.get(hkey, {}).get(key) - - def chainstore_hgetall(self, *, hkey): - return dict(self._store.get(hkey, {})) - - def chainstore_hset(self, *, hkey, key, value): - self._store.setdefault(hkey, {}) - if value is None: - self._store[hkey].pop(key, None) - else: - self._store[hkey][key] = value - - def _log_audit_event(self, event, payload): - self.audit_events.append((event, payload)) - - def _collect_node_reports(self, workers): - return {"node-a": {"open_ports": [443], "service_info": {"443": {"name": "https"}}}} - - def _get_aggregated_report(self, node_reports): - return {"open_ports": [443], "service_info": {"443": {"name": "https"}}} - - def _compute_risk_and_findings(self, aggregated_report): - return {"score": 12, "breakdown": {"findings_score": 12}}, [ - {"severity": "HIGH", "port": 443, "category": "tls"} - ] - - def _get_job_config(self, job_specs): - return {"engagement": {"name": "test engagement"}} - - def _run_structured_report_sections(self, **kwargs): - return _structured_sections(self) - - def _emit_timeline_event(self, job_specs, event_type, message, actor_type="system", meta=None): - event = { - "event_type": event_type, - "message": message, - "actor_type": actor_type, - "meta": dict(meta or {}), - } - self.timeline_events.append(event) - job_specs.setdefault("timeline", []).append(event) - - def _write_job_record(self, job_id, job_specs, expected_revision=None, context=""): - current = self._jobs.get(job_id, {}) - job_specs["job_revision"] = int(current.get("job_revision", 0) or 0) + 1 - job_specs["write_context"] = context - self._jobs[job_id] = job_specs - return job_specs - - def P(self, *args, **kwargs): - return None - - -class _R1FS: - def __init__(self): - self.objects = {} - self.added_payloads = [] - - def get_json(self, cid): - payload = self.objects.get(cid) - return copy.deepcopy(payload) if isinstance(payload, dict) else payload - - def add_json(self, payload, show_logs=False): - self.added_payloads.append(copy.deepcopy(payload)) - cid = f"QmGeneratedPassReport{len(self.added_payloads)}" - self.objects[cid] = copy.deepcopy(payload) - return cid - - -class TestApiOperationConfig(unittest.TestCase): - - def test_default_api_operations_are_disabled(self): - owner = _Owner() - owner.cfg_api_operations = None - - cfg = get_api_operations_config(owner) - - self.assertFalse(cfg["ENABLED"]) - self.assertEqual(DEFAULT_API_OPERATIONS_CONFIG["TOKEN_ENV"], "REDMESH_API_OPERATION_TOKEN") - - def test_api_operation_config_bounds_values(self): - owner = _Owner() - owner.cfg_api_operations.update({ - "MAX_QUEUE_GLOBAL": 0, - "MAX_QUEUE_PER_ACTOR": -1, - "MAX_IDEMPOTENCY_KEY_LENGTH": 2, - "POLL_AFTER_MS": 1, - }) - - cfg = get_api_operations_config(owner) - - self.assertEqual(cfg["MAX_QUEUE_GLOBAL"], DEFAULT_API_OPERATIONS_CONFIG["MAX_QUEUE_GLOBAL"]) - self.assertEqual(cfg["MAX_QUEUE_PER_ACTOR"], DEFAULT_API_OPERATIONS_CONFIG["MAX_QUEUE_PER_ACTOR"]) - self.assertEqual( - cfg["MAX_IDEMPOTENCY_KEY_LENGTH"], - DEFAULT_API_OPERATIONS_CONFIG["MAX_IDEMPOTENCY_KEY_LENGTH"], - ) - self.assertEqual(cfg["POLL_AFTER_MS"], DEFAULT_API_OPERATIONS_CONFIG["POLL_AFTER_MS"]) - - -class TestApiOperationAdmission(unittest.TestCase): - - def test_create_requires_configured_server_actor(self): - owner = _Owner(token_hashes=[]) - owner.add_job() - - result = create_analyze_job_operation(owner, "good-token", "job-1") - - self.assertEqual(result["error"], "operation_auth_not_configured") - - def test_create_rejects_unauthorized_token(self): - owner = _Owner() - owner.add_job() - - result = create_analyze_job_operation(owner, "bad-token", "job-1") - - self.assertEqual(result["error"], "operation_auth_denied") - - def test_create_returns_sanitized_queued_operation(self): - owner = _Owner() - owner.add_job() - - result = create_analyze_job_operation( - owner, - "good-token", - "job-1", - focus_areas=["web", "web", "network"], - idempotency_key="idem-secret", - ) - - self.assertEqual(result["status"], "accepted") - operation = result["operation"] - self.assertEqual(operation["state"], "queued") - self.assertEqual(operation["operation_type"], "analyze_job") - self.assertNotIn("actor_hash", operation) - self.assertNotIn("request_fingerprint", operation) - - serialized_store = json.dumps(owner._store, sort_keys=True) - self.assertNotIn("good-token", serialized_store) - self.assertNotIn("idem-secret", serialized_store) - - def test_same_idempotency_key_replays_same_operation(self): - owner = _Owner() - owner.add_job() - - first = create_analyze_job_operation(owner, "good-token", "job-1", idempotency_key="idem-1") - second = create_analyze_job_operation(owner, "good-token", "job-1", idempotency_key="idem-1") - - self.assertTrue(second["idempotent_replay"]) - self.assertEqual(first["operation"]["operation_id"], second["operation"]["operation_id"]) - - def test_same_idempotency_key_different_request_conflicts_without_enqueue(self): - owner = _Owner(max_queue_per_job=2) - owner.add_job() - - first = create_analyze_job_operation(owner, "good-token", "job-1", idempotency_key="idem-1") - before_count = len(owner._store["test-instance:api_operations"]) - conflict = create_analyze_job_operation( - owner, - "good-token", - "job-1", - focus_areas=["web"], - idempotency_key="idem-1", - ) - - self.assertEqual(first["status"], "accepted") - self.assertEqual(conflict["error"], "idempotency_conflict") - self.assertEqual(len(owner._store["test-instance:api_operations"]), before_count) - - def test_create_rejects_invalid_analysis_type_and_focus_area(self): - owner = _Owner() - owner.add_job() - - invalid_type = create_analyze_job_operation( - owner, - "good-token", - "job-1", - analysis_type="legacy_raw_summary", - ) - invalid_focus = create_analyze_job_operation( - owner, - "good-token", - "job-1", - focus_areas=["custom prompt"], - ) - - self.assertEqual(invalid_type["error"], "invalid_analysis_type") - self.assertEqual(invalid_focus["error"], "invalid_focus_area") - - def test_same_idempotency_key_different_actor_does_not_replay_or_conflict(self): - owner = _Owner(token_hashes=[_token_hash("good-token"), _token_hash("other-token")]) - owner.add_job("job-1") - owner.add_job("job-2") - - first = create_analyze_job_operation(owner, "good-token", "job-1", idempotency_key="idem-1") - second = create_analyze_job_operation(owner, "other-token", "job-2", idempotency_key="idem-1") - - self.assertEqual(first["status"], "accepted") - self.assertEqual(second["status"], "accepted") - self.assertNotEqual(first["operation"]["operation_id"], second["operation"]["operation_id"]) - - def test_queue_full_returns_backpressure_without_operation_record(self): - owner = _Owner(max_queue_global=1) - owner.add_job("job-1") - owner.add_job("job-2") - - first = create_analyze_job_operation(owner, "good-token", "job-1") - before_count = len(owner._store["test-instance:api_operations"]) - second = create_analyze_job_operation(owner, "good-token", "job-2") - - self.assertEqual(first["status"], "accepted") - self.assertEqual(second["error"], "operation_backpressure") - self.assertTrue(second["retryable"]) - self.assertEqual(len(owner._store["test-instance:api_operations"]), before_count) - - def test_expired_queued_operation_does_not_consume_backpressure(self): - owner = _Owner(max_queue_global=1) - owner.add_job("job-1") - owner.add_job("job-2") - first = create_analyze_job_operation(owner, "good-token", "job-1") - repo = ApiOperationRepository(owner) - raw = repo.get_operation(first["operation"]["operation_id"]) - raw["expires_at"] = "2000-01-01T00:00:00Z" - repo.put_operation(raw, expected_revision=raw["revision"], context="test_expire") - - second = create_analyze_job_operation(owner, "good-token", "job-2") - status = get_api_operation_status(owner, "good-token", first["operation"]["operation_id"]) - - self.assertEqual(second["status"], "accepted") - self.assertEqual(status["operation"]["state"], "expired") - - -class TestApiOperationAccessAndState(unittest.TestCase): - - def test_cross_actor_status_and_unknown_status_are_indistinguishable(self): - owner = _Owner(token_hashes=[_token_hash("good-token"), _token_hash("other-token")]) - owner.add_job() - created = create_analyze_job_operation(owner, "good-token", "job-1") - operation_id = created["operation"]["operation_id"] - - foreign = get_api_operation_status(owner, "other-token", operation_id) - unknown = get_api_operation_status(owner, "other-token", "op_missing") - - self.assertEqual(foreign, unknown) - self.assertEqual(foreign["error"], "operation_not_found") - - def test_status_requires_continued_job_visibility(self): - owner = _Owner() - owner.add_job() - created = create_analyze_job_operation(owner, "good-token", "job-1") - owner._jobs.pop("job-1") - - status = get_api_operation_status(owner, "good-token", created["operation"]["operation_id"]) - - self.assertEqual(status["error"], "operation_not_found") - - def test_cancel_queued_operation_prevents_worker_start(self): - owner = _Owner() - owner.add_job() - created = create_analyze_job_operation(owner, "good-token", "job-1") - - canceled = cancel_api_operation(owner, "good-token", created["operation"]["operation_id"]) - - self.assertEqual(canceled["operation"]["state"], "canceled") - self.assertEqual(canceled["operation"]["phase"], "canceled") - self.assertEqual(canceled["operation"]["cancel"]["side_effects"], "none") - - def test_status_response_does_not_expose_raw_cids_or_diagnostics(self): - owner = _Owner() - owner.add_job() - created = create_analyze_job_operation(owner, "good-token", "job-1") - repo = ApiOperationRepository(owner) - raw = repo.get_operation(created["operation"]["operation_id"]) - raw["state"] = "succeeded" - raw["result_ref"] = "QmInternalResult" - raw["result_public"] = { - "kind": "redmesh_analyze_job_result", - "handle": "opaque-handle", - "pass_nr": 1, - "summary": { - "llm_report_sections_available": True, - "nested": { - "url": "https://provider.example/internal", - "note": "token abc", - }, - }, - "report_cid": "QmShouldNotLeak", - } - raw["failure"] = { - "failure_class": "llm_provider_error", - "retryable": False, - "short_message": "Provider failed", - "details": "secret diagnostic", - "provider_url": "https://provider.example", - } - repo.put_operation(raw, expected_revision=raw["revision"], context="test") - - status = get_api_operation_status(owner, "good-token", raw["operation_id"]) - serialized = json.dumps(status, sort_keys=True) - - self.assertIn("opaque-handle", serialized) - self.assertNotIn("QmInternalResult", serialized) - self.assertNotIn("QmShouldNotLeak", serialized) - self.assertNotIn("secret diagnostic", serialized) - self.assertNotIn("provider.example", serialized) - self.assertNotIn("token abc", serialized) - - def test_raw_cid_is_rejected_by_operation_result_endpoint(self): - owner = _Owner() - - result = get_api_operation_result(owner, "good-token", "Qmabcdefghijklmnopqrstuvwx") - - self.assertEqual(result["error"], "invalid_result_handle") - - def test_worker_executes_analyze_job_and_exposes_opaque_result(self): - owner = _Owner() - owner.add_job() - created = create_analyze_job_operation(owner, "good-token", "job-1") - - started = maybe_start_api_operation_worker(owner) - _join_worker(owner) - - self.assertTrue(started) - self.assertEqual(owner.llm_calls, 1) - status = get_api_operation_status(owner, "good-token", created["operation"]["operation_id"]) - self.assertEqual(status["operation"]["state"], "succeeded") - self.assertEqual(status["operation"]["phase"], "succeeded") - handle = status["operation"]["result"]["handle"] - self.assertTrue(handle.startswith("opres_")) - - latest_ref = owner._jobs["job-1"]["pass_reports"][-1] - self.assertNotEqual(latest_ref["report_cid"], "QmPassReport") - updated_pass = owner.r1fs.get_json(latest_ref["report_cid"]) - self.assertEqual(updated_pass["llm_operation_id"], created["operation"]["operation_id"]) - self.assertIn("llm_report_sections", updated_pass) - self.assertIn("llm_analysis", updated_pass) - self.assertEqual(owner.timeline_events[0]["meta"], {"pass_nr": 1}) - - result = get_api_operation_result(owner, "good-token", handle) - serialized_status = json.dumps(status, sort_keys=True) - serialized_result = json.dumps(result, sort_keys=True) - self.assertEqual(result["operation_id"], created["operation"]["operation_id"]) - self.assertEqual(result["result"]["handle"], handle) - self.assertNotIn("QmPassReport", serialized_status) - self.assertNotIn(latest_ref["report_cid"], serialized_status) - self.assertNotIn(latest_ref["report_cid"], serialized_result) - - def test_worker_aborts_if_job_changes_during_llm_wait(self): - owner = _Owner() - owner.add_job() - created = create_analyze_job_operation(owner, "good-token", "job-1") - - def _llm_changes_job(**kwargs): - owner._jobs["job-1"]["job_revision"] += 1 - owner._jobs["job-1"]["pass_reports"][-1]["report_cid"] = "QmConcurrentPassReport" - owner.r1fs.objects["QmConcurrentPassReport"] = {"pass_nr": 1, "aggregated_report_cid": "QmOther"} - return _structured_sections(owner) - - owner._run_structured_report_sections = _llm_changes_job - started = maybe_start_api_operation_worker(owner) - _join_worker(owner) - - status = get_api_operation_status(owner, "good-token", created["operation"]["operation_id"]) - self.assertTrue(started) - self.assertEqual(status["operation"]["state"], "failed") - self.assertEqual(status["operation"]["phase"], "job_changed") - self.assertEqual(status["operation"]["failure"]["failure_class"], "job_changed") - self.assertEqual(owner._jobs["job-1"]["pass_reports"][-1]["report_cid"], "QmConcurrentPassReport") - self.assertEqual(owner.r1fs.added_payloads, []) - - def test_worker_aborts_if_job_revision_changes_during_llm_wait(self): - owner = _Owner() - owner.add_job() - created = create_analyze_job_operation(owner, "good-token", "job-1") - - def _llm_changes_job_revision_only(**kwargs): - owner._jobs["job-1"]["job_revision"] += 1 - owner._jobs["job-1"]["unrelated_update"] = "export status changed" - return _structured_sections(owner) - - owner._run_structured_report_sections = _llm_changes_job_revision_only - started = maybe_start_api_operation_worker(owner) - _join_worker(owner) - - status = get_api_operation_status(owner, "good-token", created["operation"]["operation_id"]) - self.assertTrue(started) - self.assertEqual(status["operation"]["state"], "failed") - self.assertEqual(status["operation"]["phase"], "job_changed") - self.assertEqual(owner._jobs["job-1"]["pass_reports"][-1]["report_cid"], "QmPassReport") - self.assertEqual(owner._jobs["job-1"]["unrelated_update"], "export status changed") - self.assertEqual(owner.r1fs.added_payloads, []) - - def test_worker_cancel_during_llm_wait_avoids_side_effects(self): - owner = _Owner() - owner.add_job() - created = create_analyze_job_operation(owner, "good-token", "job-1") - operation_id = created["operation"]["operation_id"] - - def _llm_cancels(**kwargs): - cancel_api_operation(owner, "good-token", operation_id, reason="operator stop") - return _structured_sections(owner) - - owner._run_structured_report_sections = _llm_cancels - started = maybe_start_api_operation_worker(owner) - _join_worker(owner) - - status = get_api_operation_status(owner, "good-token", operation_id) - self.assertTrue(started) - self.assertEqual(status["operation"]["state"], "canceled") - self.assertEqual(status["operation"]["cancel"]["side_effects"], "none") - self.assertEqual(owner._jobs["job-1"]["pass_reports"][-1]["report_cid"], "QmPassReport") - self.assertEqual(owner.r1fs.added_payloads, []) - - def test_worker_expired_during_llm_wait_avoids_side_effects(self): - owner = _Owner() - owner.add_job() - created = create_analyze_job_operation(owner, "good-token", "job-1") - operation_id = created["operation"]["operation_id"] - - def _llm_expires_operation(**kwargs): - repo = ApiOperationRepository(owner) - raw = repo.get_operation(operation_id) - raw["expires_at"] = "2000-01-01T00:00:00Z" - repo.put_operation(raw, expected_revision=raw["revision"], context="test_expire_running") - return _structured_sections(owner) - - owner._run_structured_report_sections = _llm_expires_operation - started = maybe_start_api_operation_worker(owner) - _join_worker(owner) - - status = get_api_operation_status(owner, "good-token", operation_id) - self.assertTrue(started) - self.assertEqual(status["operation"]["state"], "expired") - self.assertEqual(status["operation"]["phase"], "expired") - self.assertEqual(owner._jobs["job-1"]["pass_reports"][-1]["report_cid"], "QmPassReport") - self.assertEqual(owner.r1fs.added_payloads, []) - - def test_worker_recovers_expired_running_lease_after_restart(self): - owner = _Owner() - owner.add_job() - created = create_analyze_job_operation(owner, "good-token", "job-1") - operation_id = created["operation"]["operation_id"] - repo = ApiOperationRepository(owner) - raw = repo.get_operation(operation_id) - raw.update({ - "state": "running", - "phase": "llm_pending", - "attempt": 1, - "lease": { - "owner_node": owner.ee_addr, - "token": "stale-worker-token", - "acquired_at": "2000-01-01T00:00:00Z", - "heartbeat_at": "2000-01-01T00:00:00Z", - "expires_at": "2000-01-01T00:00:01Z", - }, - }) - repo.put_operation(raw, expected_revision=raw["revision"], context="test_stale_running") - - started = maybe_start_api_operation_worker(owner) - _join_worker(owner) - - status = get_api_operation_status(owner, "good-token", operation_id) - self.assertTrue(started) - self.assertEqual(status["operation"]["state"], "succeeded") - self.assertEqual(status["operation"]["attempt"], 2) - self.assertEqual(owner.llm_calls, 1) - - def test_worker_finalizes_expired_cancel_requested_lease_after_restart(self): - owner = _Owner() - owner.add_job() - created = create_analyze_job_operation(owner, "good-token", "job-1") - operation_id = created["operation"]["operation_id"] - repo = ApiOperationRepository(owner) - raw = repo.get_operation(operation_id) - raw.update({ - "state": "cancel_requested", - "phase": "cancel_requested", - "attempt": 1, - "cancel": { - "requested": True, - "requested_at": "2000-01-01T00:00:00Z", - "side_effects": "none", - }, - "lease": { - "owner_node": owner.ee_addr, - "token": "stale-worker-token", - "acquired_at": "2000-01-01T00:00:00Z", - "heartbeat_at": "2000-01-01T00:00:00Z", - "expires_at": "2000-01-01T00:00:01Z", - }, - }) - repo.put_operation(raw, expected_revision=raw["revision"], context="test_stale_cancel") - - started = maybe_start_api_operation_worker(owner) - status = get_api_operation_status(owner, "good-token", operation_id) - - self.assertFalse(started) - self.assertEqual(status["operation"]["state"], "canceled") - self.assertEqual(status["operation"]["cancel"]["side_effects"], "unknown_after_restart") - - def test_operation_ttl_takes_precedence_over_stale_running_lease_recovery(self): - owner = _Owner() - owner.add_job() - created = create_analyze_job_operation(owner, "good-token", "job-1") - operation_id = created["operation"]["operation_id"] - repo = ApiOperationRepository(owner) - raw = repo.get_operation(operation_id) - raw.update({ - "state": "running", - "phase": "llm_pending", - "attempt": 1, - "expires_at": "2000-01-01T00:00:00Z", - "lease": { - "owner_node": owner.ee_addr, - "token": "stale-worker-token", - "acquired_at": "2000-01-01T00:00:00Z", - "heartbeat_at": "2000-01-01T00:00:00Z", - "expires_at": "2000-01-01T00:00:01Z", - }, - }) - repo.put_operation(raw, expected_revision=raw["revision"], context="test_expired_running") - - started = maybe_start_api_operation_worker(owner) - status = get_api_operation_status(owner, "good-token", operation_id) - - self.assertFalse(started) - self.assertEqual(status["operation"]["state"], "expired") - self.assertEqual(owner.llm_calls, 0) - - def test_expired_running_lease_does_not_consume_backpressure(self): - owner = _Owner(max_queue_global=1) - owner.add_job("job-1") - owner.add_job("job-2") - created = create_analyze_job_operation(owner, "good-token", "job-1") - repo = ApiOperationRepository(owner) - raw = repo.get_operation(created["operation"]["operation_id"]) - raw.update({ - "state": "running", - "phase": "llm_pending", - "attempt": raw["max_attempts"], - "lease": { - "owner_node": owner.ee_addr, - "token": "stale-worker-token", - "acquired_at": "2000-01-01T00:00:00Z", - "heartbeat_at": "2000-01-01T00:00:00Z", - "expires_at": "2000-01-01T00:00:01Z", - }, - }) - repo.put_operation(raw, expected_revision=raw["revision"], context="test_stale_backpressure") - - second = create_analyze_job_operation(owner, "good-token", "job-2") - - self.assertEqual(second["status"], "accepted") - - def test_worker_cancel_after_r1fs_write_does_not_update_job_record(self): - owner = _Owner() - owner.add_job() - created = create_analyze_job_operation(owner, "good-token", "job-1") - operation_id = created["operation"]["operation_id"] - original_add_json = owner.r1fs.add_json - - def _cancel_after_add_json(payload, show_logs=False): - cid = original_add_json(payload, show_logs=show_logs) - cancel_api_operation(owner, "good-token", operation_id, reason="late stop") - return cid - - owner.r1fs.add_json = _cancel_after_add_json - started = maybe_start_api_operation_worker(owner) - _join_worker(owner) - - status = get_api_operation_status(owner, "good-token", operation_id) - self.assertTrue(started) - self.assertEqual(status["operation"]["state"], "canceled") - self.assertEqual(status["operation"]["cancel"]["side_effects"], "result_artifact_written") - self.assertEqual(owner._jobs["job-1"]["pass_reports"][-1]["report_cid"], "QmPassReport") - self.assertEqual(len(owner.r1fs.added_payloads), 1) - - def test_worker_failure_does_not_expose_provider_diagnostics_or_token_values(self): - owner = _Owner() - owner.add_job() - created = create_analyze_job_operation(owner, "good-token", "job-1") - operation_id = created["operation"]["operation_id"] - - def _llm_raises(**kwargs): - raise RuntimeError("OpenAI provider returned sk-liveverysecretvalue via https://provider.example") - - owner._run_structured_report_sections = _llm_raises - started = maybe_start_api_operation_worker(owner) - _join_worker(owner) - - status = get_api_operation_status(owner, "good-token", operation_id) - serialized = json.dumps(status, sort_keys=True) - self.assertTrue(started) - self.assertEqual(status["operation"]["state"], "failed") - self.assertEqual(status["operation"]["failure"]["failure_class"], "operation_failed") - self.assertEqual(status["operation"]["failure"]["short_message"], "Operation failed") - self.assertNotIn("OpenAI", serialized) - self.assertNotIn("provider.example", serialized) - self.assertNotIn("sk-liveverysecretvalue", serialized) - - def test_operation_row_stale_revision_is_audit_logged(self): - owner = _Owner() - repo = ApiOperationRepository(owner) - operation = { - "operation_id": "op_1", - "state": "queued", - "phase": "queued", - "revision": 0, - } - stored = repo.put_operation(operation, expected_revision=0, context="first") - stale = dict(stored) - stale["revision"] = 0 - stale["state"] = "running" - - repo.put_operation(stale, expected_revision=0, context="stale") - - current = repo.get_operation("op_1") - self.assertEqual(owner.audit_events[0][0], "api_operation_stale_write_detected") - self.assertEqual(owner.audit_events[0][1]["operation_id"], "op_1") - self.assertEqual(owner.audit_events[0][1]["write_mode"], "detection_only") - self.assertEqual(current["state"], "queued") - - -if __name__ == "__main__": - unittest.main() diff --git a/extensions/business/cybersec/red_mesh/tests/test_api_operation_removal.py b/extensions/business/cybersec/red_mesh/tests/test_api_operation_removal.py new file mode 100644 index 000000000..3ba5e2297 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_api_operation_removal.py @@ -0,0 +1,32 @@ +import unittest +from pathlib import Path + +from .conftest import mock_plugin_modules + + +class TestApiOperationRemoval(unittest.TestCase): + + def test_operation_surface_is_absent_and_supported_endpoints_remain(self): + mock_plugin_modules() + from extensions.business.cybersec.red_mesh import services + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + + redmesh_root = Path(__file__).resolve().parents[1] + self.assertFalse((redmesh_root / "services" / "api_operations.py").exists()) + self.assertNotIn("API_OPERATIONS", PentesterApi01Plugin.CONFIG) + + for name in ( + "DEFAULT_API_OPERATIONS_CONFIG", + "get_api_operations_config", + "create_analyze_job_operation", + "get_api_operation_status", + "cancel_api_operation", + "get_api_operation_result", + "maybe_start_api_operation_worker", + ): + self.assertFalse(hasattr(services, name), name) + self.assertFalse(hasattr(PentesterApi01Plugin, name), name) + + self.assertTrue(callable(PentesterApi01Plugin.analyze_job)) + self.assertTrue(callable(PentesterApi01Plugin.solve_postponed_analyze_job)) + self.assertTrue(callable(PentesterApi01Plugin.preflight_model_test_provider)) diff --git a/extensions/business/cybersec/red_mesh/tests/test_integration.py b/extensions/business/cybersec/red_mesh/tests/test_integration.py index 649ae7a5a..14d705628 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_integration.py +++ b/extensions/business/cybersec/red_mesh/tests/test_integration.py @@ -567,14 +567,17 @@ def test_publish_live_progress_zero_interval_uses_default(self): self.assertEqual(Plugin._get_progress_publish_interval(plugin), 30.0) plugin.chainstore_hset.assert_not_called() - def test_job_write_guarantees_are_detection_only(self): - """Mutable job writes explicitly advertise detection-only semantics.""" + def test_job_write_guarantees_are_launcher_guarded_without_cas(self): + """Mutable lifecycle writes advertise ownership guards, not atomic CAS.""" Plugin = self._get_plugin_class() plugin = MagicMock() - self.assertFalse(Plugin._supports_guarded_job_writes(plugin)) - self.assertEqual(Plugin._get_job_write_guarantees(plugin)["mode"], "detection_only") - self.assertFalse(Plugin._get_job_write_guarantees(plugin)["guarded_writes"]) + self.assertTrue(Plugin._supports_guarded_job_writes(plugin)) + guarantees = Plugin._get_job_write_guarantees(plugin) + self.assertEqual(guarantees["mode"], "launcher_single_writer") + self.assertTrue(guarantees["guarded_writes"]) + self.assertFalse(guarantees["atomic_compare_and_swap"]) + self.assertFalse(guarantees["distributed_lease"]) def test_live_hsync_due_uses_fixed_config_interval(self): """Launcher live-hsync schedule uses the normalized fixed interval.""" @@ -1610,11 +1613,14 @@ def test_purge_finalized_collects_all_cids(self): } plugin.r1fs.get_json.return_value = archive plugin.r1fs.delete_file.return_value = True - plugin.chainstore_hgetall.side_effect = [ - {}, - {"job-1:f-1": {"job_id": "job-1", "finding_id": "f-1", "status": "accepted_risk"}}, - {"job-1:f-1": [{"job_id": "job-1", "finding_id": "f-1", "status": "accepted_risk", "timestamp": 1.0}]}, - ] + plugin.chainstore_hgetall.side_effect = lambda *, hkey: { + "test-instance:triage": { + "job-1:f-1": {"job_id": "job-1", "finding_id": "f-1", "status": "accepted_risk"}, + }, + "test-instance:triage:audit": { + "job-1:f-1": [{"job_id": "job-1", "finding_id": "f-1", "status": "accepted_risk", "timestamp": 1.0}], + }, + }.get(hkey, {}) # Normalize returns the specs as-is plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) @@ -1654,11 +1660,14 @@ def test_purge_finalized_no_pass_report_cids(self): plugin.chainstore_hget.return_value = job_specs plugin.r1fs.get_json.return_value = {"passes": []} plugin.r1fs.delete_file.return_value = True - plugin.chainstore_hgetall.side_effect = [ - {}, - {"job-1:f-1": {"job_id": "job-1", "finding_id": "f-1", "status": "accepted_risk"}}, - {"job-1:f-1": [{"job_id": "job-1", "finding_id": "f-1", "status": "accepted_risk", "timestamp": 1.0}]}, - ] + plugin.chainstore_hgetall.side_effect = lambda *, hkey: { + "test-instance:triage": { + "job-1:f-1": {"job_id": "job-1", "finding_id": "f-1", "status": "accepted_risk"}, + }, + "test-instance:triage:audit": { + "job-1:f-1": [{"job_id": "job-1", "finding_id": "f-1", "status": "accepted_risk", "timestamp": 1.0}], + }, + }.get(hkey, {}) plugin._normalize_job_record = MagicMock(return_value=("job-1", job_specs)) result = Plugin.purge_job(plugin, "job-1") @@ -2107,6 +2116,29 @@ def test_status_error_falls_back_to_force_purge(self): deleted_cids = {c.args[0] for c in plugin.r1fs.delete_file.call_args_list} self.assertEqual(deleted_cids, {"cid-x"}) + def test_bulk_purge_preserves_foreign_launcher_jobs_without_force_fallback(self): + Plugin = self._get_plugin_class() + jobs = { + "job-foreign": { + "job_id": "job-foreign", + "job_status": "FINALIZED", + "job_cid": "cid-foreign", + "launcher": "node-B", + }, + } + plugin = self._make_plugin(jobs) + plugin.r1fs = MagicMock() + + result = Plugin.purge_all_redmesh_data(plugin, confirm=True) + + self.assertEqual(result["status"], "partial") + self.assertEqual(result["jobs_failed"], 1) + self.assertEqual(result["jobs_force_purged"], 0) + self.assertIn("job-foreign", plugin._hashes["test-instance"]) + plugin.purge_job.assert_not_called() + plugin.stop_and_delete_job.assert_not_called() + plugin.r1fs.delete_file.assert_not_called() + def test_partial_status_preserves_state_no_force_purge(self): """status='partial' is the retry contract — state preserved, no force-purge.""" Plugin = self._get_plugin_class() @@ -2130,6 +2162,162 @@ def test_partial_status_preserves_state_no_force_purge(self): # no force-purge R1FS calls plugin.r1fs.delete_file.assert_not_called() + def test_purge_all_deletes_orphan_submission_cids_before_sweeping_registry(self): + Plugin = self._get_plugin_class() + plugin = self._make_plugin({}) + submission_hkey = "test-instance:rulebook_review:submissions" + plugin._hashes[submission_hkey] = { + "orphan-job:nis2.eu_baseline.v1": { + "contract_version": "1.0.0", + "latest_revision": 1, + "submissions": [{"revision": 1, "cid": "cid-orphan-submission"}], + }, + } + plugin.r1fs = MagicMock() + plugin.r1fs.delete_file.return_value = True + plugin.r1fs.get_json.return_value = {"artifact_kind": "review_submission"} + + result = Plugin.purge_all_redmesh_data(plugin, confirm=True) + + self.assertEqual(result["status"], "success") + self.assertEqual(result["cids_deleted"], 1) + plugin.r1fs.delete_file.assert_called_once() + plugin.r1fs.get_json.assert_not_called() + self.assertEqual(plugin._hashes[submission_hkey], {}) + + def test_purge_all_protects_orphan_cid_referenced_by_failed_job_record(self): + Plugin = self._get_plugin_class() + jobs = { + "job-partial": { + "job_id": "job-partial", + "job_status": "RUNNING", + "legacy_submission_cid": "cid-shared-submission", + }, + } + plugin = self._make_plugin(jobs) + submission_hkey = "test-instance:rulebook_review:submissions" + orphan_key = "orphan-job:nis2.eu_baseline.v1" + plugin._hashes[submission_hkey] = { + orphan_key: {"submissions": [{"revision": 1, "cid": "cid-shared-submission"}]}, + } + plugin.stop_and_delete_job.return_value = { + "status": "partial", + "cids_deleted": 0, + "cids_failed": 1, + } + plugin.r1fs = MagicMock() + + result = Plugin.purge_all_redmesh_data(plugin, confirm=True) + + self.assertEqual(result["status"], "partial") + self.assertIn(orphan_key, plugin._hashes[submission_hkey]) + plugin.r1fs.delete_file.assert_not_called() + + def test_force_purge_retains_rows_when_submission_cid_is_shared(self): + Plugin = self._get_plugin_class() + jobs = {"job-bad": {"job_id": "job-bad", "job_status": "legacy"}} + plugin = self._make_plugin(jobs) + submission_hkey = "test-instance:rulebook_review:submissions" + plugin._hashes[submission_hkey] = { + "job-bad:nis2.eu_baseline.v1": { + "submissions": [{"revision": 1, "cid": "cid-shared-submission"}], + }, + "orphan-peer:nis2.eu_baseline.v1": { + "submissions": [{"revision": 1, "cid": "cid-shared-submission"}], + }, + } + plugin.r1fs = MagicMock() + plugin.r1fs.delete_file.return_value = True + plugin.stop_and_delete_job.side_effect = RuntimeError("legacy parse failure") + + result = Plugin.purge_all_redmesh_data(plugin, confirm=True) + + self.assertEqual(result["status"], "partial") + self.assertIn("job-bad", plugin._hashes["test-instance"]) + self.assertIn("job-bad:nis2.eu_baseline.v1", plugin._hashes[submission_hkey]) + self.assertIn("orphan-peer:nis2.eu_baseline.v1", plugin._hashes[submission_hkey]) + self.assertNotIn("cid-shared-submission", {call.args[0] for call in plugin.r1fs.delete_file.call_args_list}) + + def test_force_purge_retains_rows_when_submission_registry_read_fails(self): + from extensions.business.cybersec.red_mesh.services.control import _force_purge_job + + jobs = {"job-bad": {"job_id": "job-bad", "job_status": "legacy"}} + plugin = self._make_plugin(jobs) + submission_hkey = "test-instance:rulebook_review:submissions" + submission_key = "job-bad:nis2.eu_baseline.v1" + plugin._hashes[submission_hkey] = { + submission_key: {"submissions": [{"revision": 1, "cid": "cid-retained-submission"}]}, + } + original_hgetall = plugin.chainstore_hgetall.side_effect + + def _fail_submission_read(*, hkey): + if hkey == submission_hkey: + raise RuntimeError("submission registry unavailable") + return original_hgetall(hkey=hkey) + + plugin.chainstore_hgetall.side_effect = _fail_submission_read + plugin.r1fs = MagicMock() + errors = [] + + cids_deleted, cids_failed = _force_purge_job(plugin, "job-bad", jobs["job-bad"], errors) + + self.assertEqual((cids_deleted, cids_failed), (0, 1)) + self.assertIn("job-bad", plugin._hashes["test-instance"]) + self.assertIn(submission_key, plugin._hashes[submission_hkey]) + plugin.r1fs.delete_file.assert_not_called() + + def test_force_purge_retains_rows_when_shared_cid_discovery_fails(self): + from extensions.business.cybersec.red_mesh.services.control import _force_purge_job + + jobs = {"job-bad": {"job_id": "job-bad", "job_status": "legacy"}} + plugin = self._make_plugin(jobs) + submission_hkey = "test-instance:rulebook_review:submissions" + submission_key = "job-bad:nis2.eu_baseline.v1" + plugin._hashes[submission_hkey] = { + submission_key: {"submissions": [{"revision": 1, "cid": "cid-retained-submission"}]}, + } + original_hgetall = plugin.chainstore_hgetall.side_effect + + def _fail_job_registry_read(*, hkey): + if hkey == "test-instance": + raise RuntimeError("job registry unavailable") + return original_hgetall(hkey=hkey) + + plugin.chainstore_hgetall.side_effect = _fail_job_registry_read + plugin.r1fs = MagicMock() + errors = [] + + cids_deleted, cids_failed = _force_purge_job(plugin, "job-bad", jobs["job-bad"], errors) + + self.assertEqual((cids_deleted, cids_failed), (0, 1)) + self.assertIn("job-bad", plugin._hashes["test-instance"]) + self.assertIn(submission_key, plugin._hashes[submission_hkey]) + plugin.r1fs.delete_file.assert_not_called() + + def test_force_purge_clears_rows_after_relay_acknowledges_formal_cid_delete(self): + Plugin = self._get_plugin_class() + jobs = {"job-bad": {"job_id": "job-bad", "job_status": "legacy"}} + plugin = self._make_plugin(jobs) + submission_hkey = "test-instance:rulebook_review:submissions" + submission_key = "job-bad:nis2.eu_baseline.v1" + plugin._hashes[submission_hkey] = { + submission_key: { + "submissions": [{"revision": 1, "cid": "cid-retained-submission"}], + }, + } + plugin.r1fs = MagicMock() + plugin.r1fs.delete_file.return_value = True + plugin.r1fs.get_json.return_value = {"artifact_kind": "review_submission"} + plugin.stop_and_delete_job.side_effect = RuntimeError("legacy parse failure") + + result = Plugin.purge_all_redmesh_data(plugin, confirm=True) + + self.assertEqual(result["status"], "partial") + self.assertEqual(result["cids_failed"], 0) + self.assertNotIn("job-bad", plugin._hashes["test-instance"]) + self.assertNotIn(submission_key, plugin._hashes[submission_hkey]) + plugin.r1fs.get_json.assert_not_called() + def test_confirm_required(self): """Endpoint refuses to purge without confirm=True.""" Plugin = self._get_plugin_class() diff --git a/extensions/business/cybersec/red_mesh/tests/test_model_testing.py b/extensions/business/cybersec/red_mesh/tests/test_model_testing.py index 97f68218c..5b92efced 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_model_testing.py +++ b/extensions/business/cybersec/red_mesh/tests/test_model_testing.py @@ -934,17 +934,19 @@ def resolver(hostname, port, type=socket.SOCK_STREAM): self.assertIsNotNone(err) self.assertEqual(err["error_class"], "forbidden_destination") - def test_duplicate_credential_sources_fail_closed(self): + def test_credential_ref_with_secret_payload_is_rejected_without_identifier_leak(self): + credential_ref = "model_provider/operator/user-123/provider-a" _, err = validate_model_provider_credentials( { - "credential_ref": "model_provider/operator/user-123/provider-a", + "credential_ref": credential_ref, }, {"api_key": "secret"}, role="tested_model", created_by_id="user-123", ) - self.assertEqual(err["error_class"], "duplicate_credential_source") + self.assertEqual(err["error_class"], "credential_unavailable") + self.assertNotIn(credential_ref, str(err)) def test_inline_credential_fields_in_provider_config_fail_closed(self): owner = _owner(cfg_model_testing={"ENABLED": True}) @@ -1014,6 +1016,20 @@ def test_invalid_credential_refs_use_same_sanitized_error(self): self.assertEqual(err["error_class"], "credential_unavailable") self.assertNotIn(ref, str(err)) + def test_launch_rejects_validly_shaped_credential_ref_before_persistence(self): + owner = _owner(cfg_model_testing={"ENABLED": True}) + credential_ref = "model_provider/operator/user-123/provider-a" + kwargs = _valid_launch_kwargs() + kwargs["tested_model"] = _provider(credential_ref=credential_ref) + kwargs["tested_model_secret_payload"] = None + + result = launch_model_test(owner, **kwargs) + + self.assertEqual(result["error_class"], "credential_unavailable") + self.assertNotIn(credential_ref, str(result)) + owner.r1fs.add_json.assert_not_called() + owner.chainstore_hset.assert_not_called() + def test_preflight_model_test_provider_accepts_valid_remote_provider(self): owner = _owner(cfg_model_testing={"ENABLED": True}) secret = "sentinel-model-api-key" @@ -1064,17 +1080,19 @@ def test_preflight_model_test_provider_returns_sanitized_provider_auth_failure(s def test_preflight_model_test_provider_requires_api_key_payload(self): owner = _owner(cfg_model_testing={"ENABLED": True}) + credential_ref = "model_provider/operator/user-123/provider-a" result = preflight_model_test_provider( owner, created_by_id="user-123", - tested_model=_provider(credential_ref="model_provider/operator/user-123/provider-a"), + tested_model=_provider(credential_ref=credential_ref), tested_model_secret_payload=None, ) self.assertFalse(result["ok"]) self.assertEqual(result["error_class"], "credential_unavailable") self.assertIn("requires an API key", result["message"]) + self.assertNotIn(credential_ref, str(result)) owner.r1fs.add_json.assert_not_called() owner.chainstore_hset.assert_not_called() @@ -1379,6 +1397,11 @@ def test_model_test_finalization_requires_end_attestation_for_success(self): "job_type": "model_test", "blockchain_attestation_enabled": True, } + plugin.r1fs.add_json.return_value = "QmArchiveCID" + plugin.r1fs.get_json.side_effect = [ + {"job_type": "model_test", "blockchain_attestation_enabled": True}, + {"job_id": "job-123"}, + ] plugin._submit_redmesh_test_attestation = MagicMock(return_value=None) job_specs = { "job_id": "job-123", @@ -1409,15 +1432,17 @@ def test_model_test_finalization_requires_end_attestation_for_success(self): "QmWorkerResult", ) - self.assertFalse(result) + self.assertTrue(result) plugin._submit_redmesh_test_attestation.assert_called_once() - plugin.r1fs.add_json.assert_not_called() - plugin._write_job_record.assert_called_with( - "job-123", - job_specs, - context="model_test_attestation_failed", - ) - self.assertEqual(job_specs["job_status"], "FAILED") + archive = plugin.r1fs.add_json.call_args.args[0] + terminal_events = [event for event in archive["timeline"] if event["type"] in {"finalized", "failed", "canceled"}] + self.assertEqual(len(terminal_events), 1) + self.assertEqual(terminal_events[0]["type"], "failed") + self.assertEqual(terminal_events[0]["meta"]["overall_status"], "complete") + self.assertEqual(terminal_events[0]["meta"]["error_class"], "finalization_failed") + stub = plugin._write_job_record.call_args.args[1] + self.assertEqual(stub["job_status"], "FAILED") + self.assertEqual(stub["failure_class"], "attestation_failed") self.assertEqual(job_specs["failure_class"], "attestation_failed") def test_model_test_finalization_end_attestation_exception_marks_failed(self): @@ -1434,6 +1459,11 @@ def test_model_test_finalization_end_attestation_exception_marks_failed(self): "job_type": "model_test", "blockchain_attestation_enabled": True, } + plugin.r1fs.add_json.return_value = "QmArchiveCID" + plugin.r1fs.get_json.side_effect = [ + {"job_type": "model_test", "blockchain_attestation_enabled": True}, + {"job_id": "job-123"}, + ] plugin._submit_redmesh_test_attestation = MagicMock(side_effect=RuntimeError("chain offline")) job_specs = { "job_id": "job-123", @@ -1464,9 +1494,11 @@ def test_model_test_finalization_end_attestation_exception_marks_failed(self): "QmWorkerResult", ) - self.assertFalse(result) - plugin.r1fs.add_json.assert_not_called() - self.assertEqual(job_specs["job_status"], "FAILED") + self.assertTrue(result) + archive = plugin.r1fs.add_json.call_args.args[0] + self.assertEqual(archive["timeline"][-1]["type"], "failed") + stub = plugin._write_job_record.call_args.args[1] + self.assertEqual(stub["job_status"], "FAILED") self.assertEqual(job_specs["failure_class"], "attestation_failed") def test_model_test_finalization_stores_successful_end_attestation(self): @@ -1526,6 +1558,43 @@ def test_model_test_finalization_stores_successful_end_attestation(self): self.assertEqual(stub["job_status"], "FINALIZED") self.assertTrue(stub["blockchain_attestation_enabled"]) + def test_model_test_terminal_event_is_idempotent(self): + mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + + plugin = MagicMock() + plugin.ee_addr = "launcher-node" + plugin.ee_id = "Launcher" + plugin.time.return_value = 200.0 + job_specs = { + "job_id": "job-idempotent", + "launcher": "launcher-node", + "model_test_summary": { + "overall_status": "incomplete", + "cases_completed": 4, + "cases_total": 12, + }, + "model_test_node_selection": {"selected_execution_node": "worker-node"}, + "workers": {"worker-node": {}}, + "timeline": [], + } + + PentesterApi01Plugin._terminalize_model_test_job( + plugin, job_specs, "incomplete", "FINALIZED", + ) + PentesterApi01Plugin._terminalize_model_test_job( + plugin, job_specs, "incomplete", "FINALIZED", + ) + + terminal_events = [event for event in job_specs["timeline"] if event["type"] == "finalized"] + self.assertEqual(len(terminal_events), 1) + self.assertEqual(terminal_events[0]["meta"], { + "overall_status": "incomplete", + "selected_execution_node": "worker-node", + "cases_completed": 4, + "cases_total": 12, + }) + def test_model_test_finalization_records_raw_evidence_capture_failed_when_requested_without_artifact(self): mock_plugin_modules() from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin @@ -1568,9 +1637,9 @@ def test_model_test_finalization_records_raw_evidence_capture_failed_when_reques "job-raw", job_specs, { - "status": "completed", - "model_test_results": {"overall_status": "completed", "cases": []}, - "model_test_summary": {"overall_status": "completed"}, + "status": "incomplete", + "model_test_results": {"overall_status": "incomplete", "cases": []}, + "model_test_summary": {"overall_status": "incomplete"}, }, "QmWorkerResult", ) @@ -1588,8 +1657,10 @@ def test_model_test_finalization_records_raw_evidence_capture_failed_when_reques self.assertEqual(stub["model_test_raw_evidence"]["status"], RAW_EVIDENCE_STATUS_CAPTURE_FAILED) self.assertEqual(stub["model_test_raw_evidence"]["error_class"], RAW_EVIDENCE_ERROR_CAPTURE_UNAVAILABLE) event_types = [event["type"] for event in archive_payload["timeline"]] - self.assertIn("completed", event_types) - self.assertIn("finalized", event_types) + self.assertEqual(event_types.count("finalized"), 1) + terminal_event = next(event for event in archive_payload["timeline"] if event["type"] == "finalized") + self.assertEqual(terminal_event["meta"]["overall_status"], "incomplete") + self.assertEqual(archive_payload["model_test_summary"]["overall_status"], "incomplete") def test_model_test_finalization_stores_requested_raw_evidence_in_restricted_lane(self): mock_plugin_modules() @@ -1677,8 +1748,9 @@ def add_json(payload, show_logs=False, secret=None): self.assertNotIn("raw prompt secret", str(archive_payload)) self.assertNotIn("raw answer secret", str(archive_payload)) event_types = [event["type"] for event in archive_payload["timeline"]] - self.assertIn("completed", event_types) - self.assertIn("finalized", event_types) + self.assertEqual(event_types.count("finalized"), 1) + terminal_event = next(event for event in archive_payload["timeline"] if event["type"] == "finalized") + self.assertEqual(terminal_event["meta"]["overall_status"], "complete") raw_metadata_write = next( call.kwargs["value"] @@ -1870,11 +1942,6 @@ def _raw_evidence_endpoint_plugin(self): plugin = MagicMock() plugin.cfg_instance_id = "instance" - plugin.cfg_api_operations = { - "ENABLED": True, - "TOKEN_HASHES": [hashlib.sha256(b"backend-token").hexdigest()], - "HMAC_SECRET": "unit-test-hmac-secret", - } job_specs = { "job_id": "job-raw", "job_type": "model_test", @@ -2041,6 +2108,21 @@ def test_falsey_non_list_selected_peers_rejected(self): class TestModelTestingPersistenceContracts(unittest.TestCase): + def test_legacy_completed_status_remains_readable(self): + from extensions.business.cybersec.red_mesh.model_test_sanitization import ( + sanitize_model_test_results, + sanitize_model_test_summary, + ) + + self.assertEqual( + sanitize_model_test_summary({"overall_status": "completed"})["overall_status"], + "completed", + ) + self.assertEqual( + sanitize_model_test_results({"overall_status": "completed"})["overall_status"], + "completed", + ) + def test_model_test_artifact_serializers_strip_raw_payload_fields(self): from extensions.business.cybersec.red_mesh.model_testing.artifacts import ( ModelTestArchive, @@ -2480,6 +2562,39 @@ def test_model_test_listing_preserves_summary_and_node_selection(self): self.assertEqual(jobs["job-1"]["model_test_summary"]["overall_status"], "queued") self.assertEqual(jobs["job-1"]["model_test_node_selection"]["selected_execution_node"], "node-a") + def test_local_listing_includes_sanitized_model_test_job(self): + from extensions.business.cybersec.red_mesh.services.query import list_local_jobs + + job_specs = { + "job_id": "job-1", + "job_status": "RUNNING", + "job_type": "model_test", + "scan_type": "model_test", + "model_test_summary": {"overall_status": "queued"}, + "model_test_node_selection": {"selected_execution_node": "node-a"}, + } + worker = MagicMock() + worker.state = { + "model_test_summary": { + "overall_status": "running", + "error_message": "raw provider exception secret-token", + }, + "error_message": "raw worker exception secret-token", + } + owner = _owner() + owner.scan_jobs = {} + owner.model_test_jobs = {"job-1": worker} + owner.chainstore_hget = MagicMock() + owner.chainstore_hget.return_value = job_specs + + jobs = list_local_jobs(owner) + + self.assertEqual(jobs["job-1"]["job_type"], "model_test") + self.assertEqual(jobs["job-1"]["task_kind"], "model_test") + self.assertEqual(jobs["job-1"]["model_test_summary"]["overall_status"], "running") + self.assertNotIn("error_message", str(jobs["job-1"])) + self.assertNotIn("secret-token", str(jobs["job-1"])) + def test_finalized_cstore_model_preserves_model_test_fields(self): finalized = CStoreJobFinalized( job_id="job-1", @@ -2653,7 +2768,7 @@ def chat(self, messages, *, max_tokens, temperature): worker = selected.model_test_jobs["job-1"] self.assertIsInstance(worker, ModelTestWorker) worker.thread.join(timeout=1) - self.assertEqual(worker.state["model_test_summary"]["overall_status"], "completed") + self.assertEqual(worker.state["model_test_summary"]["overall_status"], "complete") self.assertEqual(worker.state["model_test_summary"]["evaluated_cases"], 12) self.assertEqual(len(worker.state["model_test_results"]["cases"]), 12) self.assertEqual(worker.state["model_test_results"]["cases"][0]["status"], "evaluated") @@ -2805,6 +2920,100 @@ def test_stop_monitoring_marks_selected_model_test_worker_cancel_requested(self) self.assertEqual(job_specs["model_test_summary"]["overall_status"], "cancel_requested") self.assertNotIn("launcher-node", job_specs["workers"]) + def test_stop_monitoring_rejects_foreign_launcher_without_mutation(self): + from extensions.business.cybersec.red_mesh.services.control import stop_monitoring + + stored = { + "job_id": "job-1", + "job_status": "RUNNING", + "job_type": "model_test", + "run_mode": "SINGLEPASS", + "launcher": "launcher-node", + "workers": {"node-a": {"worker_type": "model_test", "finished": False}}, + } + local_worker = MagicMock() + owner = _owner( + ee_addr="worker-node", + chainstore_hget=MagicMock(return_value=deepcopy(stored)), + ) + owner.scan_jobs = {} + owner.model_test_jobs = {"job-1": local_worker} + owner._normalize_job_record = MagicMock( + side_effect=lambda key, specs: (key, deepcopy(specs)), + ) + owner._emit_timeline_event = MagicMock() + owner._log_audit_event = MagicMock() + owner.P = MagicMock() + + result = stop_monitoring(owner, "job-1", stop_type="HARD") + + self.assertEqual(result["error"], "job_launcher_mismatch") + self.assertEqual(result["status_code"], 409) + self.assertEqual(stored["job_status"], "RUNNING") + owner.chainstore_hset.assert_not_called() + owner._emit_timeline_event.assert_not_called() + local_worker.stop.assert_not_called() + + def test_stop_and_delete_rejects_foreign_launcher_before_side_effects(self): + from extensions.business.cybersec.red_mesh.services.control import stop_and_delete_job + + stored = { + "job_id": "job-1", + "job_status": "RUNNING", + "job_type": "model_test", + "launcher": "launcher-node", + "workers": {"node-a": {"worker_type": "model_test", "finished": False}}, + } + local_worker = MagicMock() + owner = _owner( + ee_addr="worker-node", + chainstore_hget=MagicMock(return_value=deepcopy(stored)), + ) + owner.scan_jobs = {} + owner.model_test_jobs = {"job-1": local_worker} + owner._normalize_job_record = MagicMock( + side_effect=lambda key, specs: (key, deepcopy(specs)), + ) + owner._log_audit_event = MagicMock() + owner.P = MagicMock() + owner.purge_job = MagicMock() + + result = stop_and_delete_job(owner, "job-1") + + self.assertEqual(result["error"], "job_launcher_mismatch") + self.assertEqual(result["status_code"], 409) + owner.chainstore_hset.assert_not_called() + owner.purge_job.assert_not_called() + local_worker.stop.assert_not_called() + + def test_purge_rejects_foreign_launcher_before_artifact_deletion(self): + from extensions.business.cybersec.red_mesh.services.control import _purge_job_locked + + stored = { + "job_id": "job-1", + "job_status": "FINALIZED", + "launcher": "launcher-node", + "job_cid": "cid-archive", + "workers": {}, + } + owner = _owner( + ee_addr="worker-node", + chainstore_hget=MagicMock(return_value=deepcopy(stored)), + ) + owner._normalize_job_record = MagicMock( + side_effect=lambda key, specs: (key, deepcopy(specs)), + ) + owner._log_audit_event = MagicMock() + owner.P = MagicMock() + owner.r1fs.delete = MagicMock() + + result = _purge_job_locked(owner, "job-1") + + self.assertEqual(result["error"], "job_launcher_mismatch") + self.assertEqual(result["status_code"], 409) + owner.r1fs.delete.assert_not_called() + owner.chainstore_hset.assert_not_called() + def test_maybe_stop_canceled_jobs_stops_active_model_test_worker(self): mock_plugin_modules() from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin @@ -2887,7 +3096,7 @@ def test_close_model_test_worker_writes_result_and_removes_tracking(self): "done": True, "canceled": False, "model_test_results": { - "overall_status": "completed", + "overall_status": "complete", "cases": [ { "case_id": "cbrn-chemical-001", @@ -2905,7 +3114,7 @@ def test_close_model_test_worker_writes_result_and_removes_tracking(self): ], }, "model_test_summary": { - "overall_status": "completed", + "overall_status": "complete", "cases_total": 12, "cases_completed": 12, "evaluated_cases": 12, @@ -3008,43 +3217,19 @@ def get_json(cid): PentesterApi01Plugin._maybe_close_model_test_jobs(plugin) self.assertEqual(plugin.model_test_jobs, {}) - self.assertEqual(plugin.r1fs.add_json.call_count, 2) + self.assertEqual(plugin.r1fs.add_json.call_count, 1) stored_result = stored_artifacts[0] self.assertEqual(stored_result["schema_version"], "model_test_worker_result_v1") self.assertEqual(stored_result["job_id"], "job-1") self.assertEqual(stored_result["worker_addr"], "node-a") - self.assertEqual(stored_result["status"], "completed") - self.assertEqual(stored_result["model_test_summary"]["overall_status"], "completed") + self.assertEqual(stored_result["status"], "complete") + self.assertEqual(stored_result["model_test_summary"]["overall_status"], "complete") self.assertEqual(stored_result["model_test_results"]["cases"][0]["status"], "evaluated") - stored_archive = stored_artifacts[1] - self.assertEqual(stored_archive["schema_version"], "model_test_archive_v1") - self.assertEqual(stored_archive["job_id"], "job-1") - self.assertEqual(stored_archive["job_type"], "model_test") - self.assertEqual(stored_archive["job_config"]["job_id"], "job-1") - self.assertNotIn("model_provider_secret_ref", stored_archive["job_config"]) - self.assertNotIn("model_provider_secret_store_key_id", stored_archive["job_config"]) - self.assertEqual(stored_archive["model_test_results"]["overall_status"], "completed") - self.assertEqual(stored_archive["model_test_results"]["cases"][0]["case_id"], "cbrn-chemical-001") - self.assertEqual(stored_archive["model_test_summary"]["overall_status"], "completed") - self.assertEqual(stored_archive["model_test_node_selection"]["selected_execution_node"], "node-a") - self.assertEqual(stored_archive["ui_aggregate"]["scan_type"], "model_test") - self.assertEqual(stored_archive["ui_aggregate"]["finding_count"], 0) - self.assertEqual(stored_archive["duration"], 10.0) - self.assertEqual(len(written_records), 1) - persisted_specs = written_records[0][1] - self.assertEqual(written_records[0][2], "model_test_archive_prune") - self.assertEqual(persisted_specs["job_status"], "FINALIZED") - self.assertEqual(persisted_specs["job_type"], "model_test") - self.assertEqual(persisted_specs["scan_type"], "model_test") - self.assertEqual(persisted_specs["job_cid"], "cid-archive") - self.assertEqual(persisted_specs["job_config_cid"], "cid-config") - self.assertEqual(persisted_specs["model_test_summary"]["overall_status"], "completed") - self.assertEqual(persisted_specs["model_test_node_selection"]["selected_execution_node"], "node-a") - self.assertNotIn("workers", persisted_specs) + self.assertEqual(written_records, []) plugin._publish_model_test_progress.assert_called_once() _, _, progress_specs = plugin._publish_model_test_progress.call_args.args[:3] self.assertEqual(progress_specs["workers"]["node-a"]["report_cid"], "cid-result") - self.assertEqual(progress_specs["model_test_summary"]["overall_status"], "completed") + self.assertEqual(progress_specs["model_test_summary"]["overall_status"], "complete") self.assertEqual(plugin.scan_jobs, {}) def test_finished_model_test_job_recovery_finalizes_stale_running_record(self): @@ -3081,10 +3266,9 @@ def test_finished_model_test_job_recovery_finalizes_stale_running_record(self): "worker_type": "model_test", "start_port": 0, "end_port": 0, - "finished": True, + "finished": False, "canceled": False, - "model_test_worker_status": "finished", - "report_cid": "cid-result", + "model_test_worker_status": "assigned", "assignment_revision": 1, "assigned_at": 123.0, }, @@ -3137,10 +3321,31 @@ def test_finished_model_test_job_recovery_finalizes_stale_running_record(self): }, } plugin = MagicMock() - plugin.ee_addr = "node-a" + plugin.ee_addr = "launcher-node" plugin.cfg_instance_id = "instance" plugin.time.return_value = 130.0 - plugin.chainstore_hgetall.return_value = {"job-stale": job_specs} + terminal_live = { + "job-stale:node-a": { + "job_id": "job-stale", + "worker_addr": "node-a", + "pass_nr": 1, + "assignment_revision_seen": 1, + "progress": 100.0, + "phase": "done", + "ports_scanned": 0, + "ports_total": 0, + "open_ports_found": [], + "completed_tests": [], + "updated_at": 130.0, + "finished": True, + "report_cid": "cid-result", + "scan_type": "model_test", + "job_type": "model_test", + }, + } + plugin.chainstore_hgetall.side_effect = lambda hkey: ( + terminal_live if hkey == "instance:live" else {"job-stale": job_specs} + ) plugin._normalize_job_record.side_effect = lambda key, specs, migrate=False: (key, specs) written_records = [] stored_artifacts = [] @@ -3174,16 +3379,40 @@ def get_json(cid): stored_archive = stored_artifacts[0] self.assertEqual(stored_archive["schema_version"], "model_test_archive_v1") self.assertEqual(stored_archive["job_id"], "job-stale") - self.assertEqual(stored_archive["model_test_summary"]["overall_status"], "completed") + self.assertEqual(stored_archive["model_test_summary"]["overall_status"], "complete") self.assertEqual(stored_archive["model_test_results"]["cases"][0]["case_id"], "cbrn-chemical-001") self.assertEqual(len(written_records), 1) persisted_specs = written_records[0][1] self.assertEqual(written_records[0][2], "model_test_archive_prune") self.assertEqual(persisted_specs["job_status"], "FINALIZED") self.assertEqual(persisted_specs["job_cid"], "cid-archive") - self.assertEqual(persisted_specs["model_test_summary"]["overall_status"], "completed") + self.assertEqual(persisted_specs["model_test_summary"]["overall_status"], "complete") self.assertNotIn("workers", persisted_specs) + def test_non_launcher_cannot_finalize_model_test_job(self): + mock_plugin_modules() + from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin + + plugin = MagicMock() + plugin.ee_addr = "worker-node" + plugin.r1fs = MagicMock() + job_specs = { + "job_id": "job-owned", + "launcher": "launcher-node", + "job_config_cid": "cid-config", + } + + finalized = PentesterApi01Plugin._finalize_model_test_job( + plugin, + "job-owned", + job_specs, + {"status": "complete", "model_test_results": {}, "model_test_summary": {}}, + "cid-result", + ) + + self.assertFalse(finalized) + plugin.r1fs.add_json.assert_not_called() + def test_finished_model_test_recovery_skips_failed_attestation_record(self): mock_plugin_modules() from extensions.business.cybersec.red_mesh.pentester_api_01 import PentesterApi01Plugin @@ -3408,8 +3637,16 @@ def get_json(cid): self.assertEqual(stored_artifacts[1]["schema_version"], "model_test_archive_v1") self.assertEqual(stored_artifacts[1]["model_test_summary"]["cases_completed"], 4) self.assertEqual(stored_artifacts[1]["model_test_results"]["cases"][0]["case_id"], "case-1") + terminal_events = [ + event for event in stored_artifacts[1]["timeline"] + if event["type"] in {"finalized", "failed", "canceled"} + ] + self.assertEqual(len(terminal_events), 1) + self.assertEqual(terminal_events[0]["type"], "failed") + self.assertEqual(terminal_events[0]["meta"]["overall_status"], "failed") + self.assertEqual(terminal_events[0]["meta"]["error_class"], MODEL_TEST_ERROR_WORKER_LOST) persisted_specs = written_records[-1][1] - self.assertEqual(persisted_specs["job_status"], "STOPPED") + self.assertEqual(persisted_specs["job_status"], "FAILED") self.assertEqual(persisted_specs["model_test_summary"]["error_class"], MODEL_TEST_ERROR_WORKER_LOST) self.assertEqual(persisted_specs["model_test_summary"]["cases_completed"], 4) self.assertEqual(persisted_specs["job_cid"], "cid-archive") @@ -3519,6 +3756,14 @@ def get_json(cid): self.assertEqual(failed, []) self.assertEqual(stored_artifacts[0]["status"], "canceled") self.assertEqual(stored_artifacts[0]["error_class"], MODEL_TEST_ERROR_CANCELED_BY_USER) + terminal_events = [ + event for event in stored_artifacts[1]["timeline"] + if event["type"] in {"finalized", "failed", "canceled"} + ] + self.assertEqual(len(terminal_events), 1) + self.assertEqual(terminal_events[0]["type"], "canceled") + self.assertEqual(terminal_events[0]["meta"]["overall_status"], "canceled") + self.assertNotIn("stopped", [event["type"] for event in stored_artifacts[1]["timeline"]]) persisted_specs = written_records[-1][1] self.assertEqual(persisted_specs["job_status"], "STOPPED") self.assertEqual(persisted_specs["model_test_summary"]["overall_status"], "canceled") diff --git a/extensions/business/cybersec/red_mesh/tests/test_postponed_analyze.py b/extensions/business/cybersec/red_mesh/tests/test_postponed_analyze.py new file mode 100644 index 000000000..d9f2d5c89 --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_postponed_analyze.py @@ -0,0 +1,714 @@ +import importlib.util +import json +import threading +import time +import unittest +from concurrent.futures import ThreadPoolExecutor +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from unittest.mock import MagicMock, patch + +from .conftest import mock_plugin_modules + + +mock_plugin_modules() + +from extensions.business.cybersec.red_mesh.llm_input_builder import build_llm_input +from extensions.business.cybersec.red_mesh.pentester_api_01 import ( + PentesterApi01Plugin, + _ManualAnalysisOutcome, + _ManualAnalysisWork, + _ManualAnalysisWorker, + _bounded_provider_post, +) + + +SECRET_SENTINEL = "credential-sentinel-private-value" + + +def _valid_remote_sections(): + return { + "executive_headline": "Material exposure requires executive attention.", + "background_draft": "Authorized external assessment.", + "overall_posture": "The engagement identified high-severity issues requiring remediation.", + "recommendation_summary": ["Prioritize the verified high-severity findings."], + "strategic_roadmap": { + "near_term": ["Address verified exposure."], + "mid_term": ["Add regression coverage."], + "long_term": ["Adopt continuous assurance."], + }, + "attack_chain_narratives": ["Initial exposure could enable privilege escalation."], + "coverage_gaps": ["Social engineering was outside scope."], + "conclusion": "Retest after remediation.", + } + + +class TestPostponedAnalyze(unittest.TestCase): + + def test_busy_request_does_not_prepare_or_queue_work(self): + plugin = MagicMock() + plugin._manual_analysis_state = { + "future": MagicMock(), + "discard_result": False, + } + result = PentesterApi01Plugin.analyze_job(plugin, job_id="job-1") + self.assertEqual(result["error"], "analysis_busy") + self.assertEqual(result["status_code"], 409) + self.assertTrue(result["retryable"]) + plugin._get_job_from_cstore.assert_not_called() + + def test_pending_solver_uses_only_opaque_key(self): + plugin = MagicMock() + future = MagicMock() + future.done.return_value = False + plugin._manual_analysis_state = { + "pending_id": "opaque-key", + "future": future, + "job_id": "job-1", + "deadline_monotonic": 200.0, + "next_check_monotonic": 0.0, + "discard_result": False, + } + plugin.create_postponed_request.return_value = "postponed" + + with patch( + "extensions.business.cybersec.red_mesh.pentester_api_01.time.monotonic", + return_value=100.0, + ): + result = PentesterApi01Plugin.solve_postponed_analyze_job(plugin, "opaque-key") + + self.assertEqual(result, "postponed") + kwargs = plugin.create_postponed_request.call_args.kwargs + self.assertEqual(kwargs["method_kwargs"], {"pending_id": "opaque-key"}) + self.assertNotIn("job-1", str(kwargs["method_kwargs"])) + + def test_solver_does_not_poll_future_before_next_check(self): + plugin = MagicMock() + future = MagicMock() + plugin._manual_analysis_state = { + "pending_id": "opaque-key", + "future": future, + "job_id": "job-1", + "deadline_monotonic": 200.0, + "next_check_monotonic": 100.1, + "discard_result": False, + } + plugin.create_postponed_request.return_value = "postponed" + + with patch( + "extensions.business.cybersec.red_mesh.pentester_api_01.time.monotonic", + return_value=100.0, + ): + result = PentesterApi01Plugin.solve_postponed_analyze_job(plugin, "opaque-key") + + self.assertEqual(result, "postponed") + future.done.assert_not_called() + + def test_deadline_returns_timeout_and_holds_slot_until_worker_drains(self): + plugin = MagicMock() + future = MagicMock() + future.done.return_value = False + plugin._manual_analysis_state = { + "pending_id": "opaque-key", + "future": future, + "job_id": "job-1", + "deadline_monotonic": 99.0, + "next_check_monotonic": 0.0, + "discard_result": False, + } + + with patch( + "extensions.business.cybersec.red_mesh.pentester_api_01.time.monotonic", + return_value=100.0, + ): + result = PentesterApi01Plugin.solve_postponed_analyze_job(plugin, "opaque-key") + + self.assertEqual(result["error"], "analysis_timeout") + self.assertEqual(result["status_code"], 504) + self.assertTrue(plugin._manual_analysis_state["discard_result"]) + self.assertIsNotNone(plugin._manual_analysis_state) + + future.done.return_value = True + future.result.return_value = _ManualAnalysisOutcome(sections={}, failed=False) + PentesterApi01Plugin._discard_drained_manual_analysis(plugin) + self.assertIsNone(plugin._manual_analysis_state) + + def test_late_completed_outcome_is_not_finalized(self): + plugin = MagicMock() + future = MagicMock() + future.done.return_value = True + future.result.return_value = _ManualAnalysisOutcome( + sections=_valid_remote_sections(), + failed=False, + deadline_exceeded=True, + ) + plugin._manual_analysis_state = { + "pending_id": "opaque-key", + "future": future, + "job_id": "job-1", + "deadline_monotonic": 99.0, + "next_check_monotonic": 0.0, + "discard_result": False, + } + + with patch( + "extensions.business.cybersec.red_mesh.pentester_api_01.time.monotonic", + return_value=100.0, + ), patch.object( + PentesterApi01Plugin, + "_finalize_manual_analysis", + ) as finalize: + result = PentesterApi01Plugin.solve_postponed_analyze_job( + plugin, + "opaque-key", + ) + + self.assertEqual(result["error"], "analysis_timeout") + self.assertEqual(result["status_code"], 504) + finalize.assert_not_called() + self.assertIsNone(plugin._manual_analysis_state) + + def test_request_shape_bounds_are_explicit(self): + invalid_type = PentesterApi01Plugin._validate_manual_analysis_request( + "job-1", + "x" * 65, + [], + ) + invalid_focus_count = PentesterApi01Plugin._validate_manual_analysis_request( + "job-1", + "", + ["web"] * 9, + ) + invalid_focus_value = PentesterApi01Plugin._validate_manual_analysis_request( + "job-1", + "", + ["x" * 65], + ) + + self.assertEqual(invalid_type["error"], "invalid_analysis_type") + self.assertEqual(invalid_focus_count["error"], "invalid_focus_areas") + self.assertEqual(invalid_focus_value["error"], "invalid_focus_areas") + + def test_executor_start_failure_releases_unused_slot(self): + plugin = PentesterApi01Plugin.__new__(PentesterApi01Plugin) + plugin._manual_analysis_state = None + plugin._manual_analysis_executor = MagicMock() + plugin._manual_analysis_executor.submit.side_effect = RuntimeError( + "credential-sentinel start failure" + ) + state = { + "pending_id": "opaque-key", + "job_id": "job-1", + "work": object(), + "discard_result": False, + } + + with patch.object( + PentesterApi01Plugin, + "_prepare_manual_analysis", + return_value=(state, None), + ): + result = PentesterApi01Plugin.analyze_job( + plugin, + job_id="job-1", + ) + + self.assertEqual(result["error"], "analysis_executor_failed") + self.assertIsNone(plugin._manual_analysis_state) + self.assertNotIn("credential-sentinel", str(result)) + + def test_manual_and_automatic_analysis_share_one_bounded_executor(self): + plugin = PentesterApi01Plugin.__new__(PentesterApi01Plugin) + executor = ThreadPoolExecutor(max_workers=1) + plugin._manual_analysis_executor = executor + plugin._manual_analysis_state = None + plugin.create_postponed_request = MagicMock(return_value="postponed") + automatic_started = threading.Event() + release_automatic = threading.Event() + + def _automatic_work(): + automatic_started.set() + release_automatic.wait(timeout=2) + return {"executive_headline": "automatic"} + + automatic_future = executor.submit(_automatic_work) + plugin._automatic_analysis_state = {"future": automatic_future} + self.assertTrue(automatic_started.wait(timeout=1)) + state = { + "pending_id": "opaque-key", + "job_id": "job-1", + "work": object(), + "discard_result": False, + } + + try: + with patch.object( + PentesterApi01Plugin, + "_prepare_manual_analysis", + return_value=(state, None), + ), patch( + "extensions.business.cybersec.red_mesh.pentester_api_01._run_manual_analysis_worker", + return_value=_ManualAnalysisOutcome(sections={}, failed=False), + ): + result = PentesterApi01Plugin.analyze_job(plugin, job_id="job-1") + manual_future = plugin._manual_analysis_state["future"] + self.assertEqual(result, "postponed") + self.assertFalse(manual_future.done()) + release_automatic.set() + automatic_future.result(timeout=1) + self.assertIsInstance(manual_future.result(timeout=1), _ManualAnalysisOutcome) + finally: + release_automatic.set() + executor.shutdown(wait=True, cancel_futures=True) + + def test_completion_releases_slot_for_retry(self): + plugin = MagicMock() + future = MagicMock() + future.done.return_value = True + future.result.return_value = _ManualAnalysisOutcome( + sections=_valid_remote_sections(), + failed=False, + ) + plugin._manual_analysis_state = { + "pending_id": "opaque-key", + "future": future, + "job_id": "job-1", + "deadline_monotonic": 200.0, + "next_check_monotonic": 0.0, + "discard_result": False, + } + + with patch( + "extensions.business.cybersec.red_mesh.pentester_api_01.time.monotonic", + return_value=100.0, + ), patch.object( + PentesterApi01Plugin, + "_finalize_manual_analysis", + return_value={"job_id": "job-1"}, + ): + result = PentesterApi01Plugin.solve_postponed_analyze_job( + plugin, + "opaque-key", + ) + + self.assertEqual(result, {"job_id": "job-1"}) + self.assertIsNone(plugin._manual_analysis_state) + + def test_stale_completion_does_not_write_artifacts(self): + plugin = MagicMock() + plugin._get_job_from_cstore.return_value = { + "job_id": "job-1", + "job_revision": 8, + "pass_reports": [{"pass_nr": 1, "report_cid": "QmCurrent"}], + } + state = { + "job_id": "job-1", + "job_revision": 7, + "pass_nr": 1, + "report_cid": "QmExpected", + "target": "example.test", + "num_workers": 1, + } + + result = PentesterApi01Plugin._finalize_manual_analysis( + plugin, + state, + _ManualAnalysisOutcome(sections=_valid_remote_sections(), failed=False), + ) + + self.assertEqual(result["error"], "analysis_state_changed") + self.assertEqual(result["status_code"], 409) + plugin.r1fs.get_json.assert_not_called() + plugin.r1fs.add_json.assert_not_called() + + def test_persistence_failure_is_typed_and_does_not_expose_artifact(self): + current_job = { + "job_id": "job-1", + "job_revision": 7, + "pass_reports": [{"pass_nr": 1, "report_cid": "QmExpected"}], + } + plugin = MagicMock() + plugin._get_job_from_cstore.return_value = current_job + plugin.r1fs.get_json.return_value = {"pass_nr": 1} + plugin.r1fs.add_json.return_value = None + state = { + "job_id": "job-1", + "job_revision": 7, + "pass_nr": 1, + "report_cid": "QmExpected", + "target": "example.test", + "num_workers": 1, + } + + result = PentesterApi01Plugin._finalize_manual_analysis( + plugin, + state, + _ManualAnalysisOutcome(sections=_valid_remote_sections(), failed=False), + ) + + self.assertEqual(result["error"], "analysis_persistence_failed") + self.assertEqual(result["status_code"], 503) + self.assertNotIn("QmExpected", str(result)) + self.assertNotIn("QmExpected", str(plugin.P.call_args_list)) + + def test_manual_finalization_rejects_detected_prewrite_race(self): + current_job = { + "job_id": "job-1", + "job_revision": 7, + "pass_reports": [{"pass_nr": 1, "report_cid": "QmExpected"}], + } + plugin = MagicMock() + plugin._get_job_from_cstore.return_value = current_job + plugin.r1fs.get_json.return_value = {"pass_nr": 1} + plugin.r1fs.add_json.return_value = "QmUnreferenced" + state = { + "job_id": "job-1", + "job_revision": 7, + "pass_nr": 1, + "report_cid": "QmExpected", + "target": "example.test", + "num_workers": 1, + } + + with patch.object( + PentesterApi01Plugin, + "_write_job_record", + return_value=None, + ) as write: + result = PentesterApi01Plugin._finalize_manual_analysis( + plugin, + state, + _ManualAnalysisOutcome(sections=_valid_remote_sections(), failed=False), + ) + + self.assertEqual(result["error"], "analysis_state_changed") + self.assertEqual(result["status_code"], 409) + self.assertNotIn("QmUnreferenced", str(result)) + self.assertTrue(write.call_args.kwargs["reject_stale"]) + + def test_guarded_job_write_refuses_detected_stale_revision(self): + plugin = MagicMock() + repository = MagicMock() + repository.get_job.return_value = { + "job_id": "job-1", + "job_revision": 8, + } + with patch.object( + PentesterApi01Plugin, + "_get_job_state_repository", + return_value=repository, + ): + result = PentesterApi01Plugin._write_job_record( + plugin, + "job-1", + {"job_id": "job-1", "job_revision": 7}, + expected_revision=7, + context="manual_llm_update", + reject_stale=True, + ) + + self.assertIsNone(result) + repository.put_job.assert_not_called() + + def test_guarded_job_write_does_not_resurrect_deleted_job(self): + plugin = MagicMock() + repository = MagicMock() + repository.get_job.return_value = None + with patch.object( + PentesterApi01Plugin, + "_get_job_state_repository", + return_value=repository, + ): + result = PentesterApi01Plugin._write_job_record( + plugin, + "job-1", + {"job_id": "job-1", "job_revision": 7}, + expected_revision=7, + context="manual_llm_update", + reject_stale=True, + ) + + self.assertIsNone(result) + repository.put_job.assert_not_called() + plugin._log_audit_event.assert_called_once() + + def test_admission_exception_is_sanitized(self): + plugin = PentesterApi01Plugin.__new__(PentesterApi01Plugin) + plugin._manual_analysis_state = None + with patch.object( + PentesterApi01Plugin, + "_prepare_manual_analysis", + side_effect=RuntimeError(f"provider exploded with {SECRET_SENTINEL}"), + ): + result = PentesterApi01Plugin.analyze_job( + plugin, + job_id="job-1", + ) + + self.assertEqual(result["error"], "analysis_executor_failed") + self.assertNotIn(SECRET_SENTINEL, str(result)) + + def test_shutdown_cancels_pending_work_before_base_close(self): + plugin = PentesterApi01Plugin.__new__(PentesterApi01Plugin) + future = MagicMock() + automatic_future = MagicMock() + executor = MagicMock() + plugin._manual_analysis_state = {"future": future} + plugin._automatic_analysis_state = {"future": automatic_future} + plugin._manual_analysis_executor = executor + + PentesterApi01Plugin.on_close(plugin) + + future.cancel.assert_called_once_with() + automatic_future.cancel.assert_called_once_with() + executor.shutdown.assert_called_once_with(wait=False, cancel_futures=True) + self.assertIsNone(plugin._manual_analysis_state) + self.assertIsNone(plugin._automatic_analysis_state) + self.assertTrue(plugin._base_closed) + + def test_worker_uses_prepared_input_without_plugin_or_secret_state(self): + prepared = build_llm_input( + findings=[{ + "severity": "HIGH", + "title": "Authorization bypass", + "description": "Structured description", + "raw_response": "credential-sentinel-private-body", + }], + aggregated_report={"open_ports": [443], "service_info": {"443": {}}}, + engagement={"client_name": "Example"}, + ) + work = _ManualAnalysisWork( + llm_input=prepared, + llm_config={ + "MODEL": "deepseek-chat", + "PROVIDER": "remote", + "PROMPT_PROFILE": "remote_rich_v1", + "LOCAL_PROMPT_PROFILE": "local_cybersecqwen_quota_v1", + "REMOTE_PROMPT_PROFILE": "remote_rich_v1", + "STRUCTURED_MAX_FINDINGS": 6, + "STRUCTURED_MAX_TOKENS": 1024, + "STRUCTURED_TEMPERATURE": 0, + }, + api_host="127.0.0.1", + api_port=8080, + deadline_monotonic=time.monotonic() + 5, + ) + response_payload = { + "result": { + "choices": [{ + "message": {"content": json.dumps(_valid_remote_sections())}, + }], + }, + } + + with patch( + "extensions.business.cybersec.red_mesh.pentester_api_01._bounded_provider_post", + return_value=json.dumps(response_payload).encode("utf-8"), + ) as post: + outcome = _ManualAnalysisWorker(work).run() + + self.assertFalse(outcome.failed) + self.assertIsNotNone(outcome.sections) + serialized_payload = json.dumps(post.call_args.args[1]) + self.assertNotIn("credential-sentinel-private-body", serialized_payload) + self.assertNotIn(SECRET_SENTINEL, serialized_payload) + self.assertFalse(any("plugin" in name for name in work.__dataclass_fields__)) + provider_timeout = post.call_args.args[2] + self.assertGreater(provider_timeout, 0) + self.assertLessEqual(provider_timeout, 30) + self.assertEqual(post.call_args.args[3], 2 * 1024 * 1024) + + def test_provider_body_and_exception_details_are_not_returned(self): + prepared = build_llm_input(findings=[], aggregated_report={}, engagement={}) + work = _ManualAnalysisWork( + llm_input=prepared, + llm_config={ + "MODEL": "deepseek-chat", + "PROVIDER": "remote", + "PROMPT_PROFILE": "remote_rich_v1", + "LOCAL_PROMPT_PROFILE": "local_cybersecqwen_quota_v1", + "REMOTE_PROMPT_PROFILE": "remote_rich_v1", + "STRUCTURED_MAX_FINDINGS": 6, + "STRUCTURED_MAX_TOKENS": 1024, + "STRUCTURED_TEMPERATURE": 0, + }, + api_host="private.provider.internal", + api_port=8080, + deadline_monotonic=time.monotonic() + 5, + ) + with patch( + "extensions.business.cybersec.red_mesh.pentester_api_01._bounded_provider_post", + side_effect=RuntimeError( + "credential-sentinel provider body https://private.provider.internal" + ), + ): + outcome = _ManualAnalysisWorker(work).run() + + public = str(outcome) + self.assertTrue(outcome.failed) + self.assertNotIn("credential-sentinel", public) + self.assertNotIn("private.provider.internal", public) + self.assertNotIn("provider body", public) + + def test_invalid_structured_result_returns_sanitized_failed_sections(self): + prepared = build_llm_input(findings=[], aggregated_report={}, engagement={}) + work = _ManualAnalysisWork( + llm_input=prepared, + llm_config={ + "MODEL": "deepseek-chat", + "PROVIDER": "remote", + "PROMPT_PROFILE": "remote_rich_v1", + "LOCAL_PROMPT_PROFILE": "local_cybersecqwen_quota_v1", + "REMOTE_PROMPT_PROFILE": "remote_rich_v1", + "STRUCTURED_MAX_FINDINGS": 6, + "STRUCTURED_MAX_TOKENS": 1024, + "STRUCTURED_TEMPERATURE": 0, + }, + api_host="private.provider.internal", + api_port=8080, + deadline_monotonic=time.monotonic() + 5, + ) + response_payload = { + "result": { + "choices": [{ + "message": {"content": "not-json credential-sentinel provider body"}, + }], + }, + } + + with patch( + "extensions.business.cybersec.red_mesh.pentester_api_01._bounded_provider_post", + return_value=json.dumps(response_payload).encode("utf-8"), + ): + outcome = _ManualAnalysisWorker(work).run() + + self.assertTrue(outcome.failed) + self.assertTrue(outcome.sections["error"]) + self.assertNotIn("credential-sentinel", str(outcome)) + self.assertNotIn("private.provider.internal", str(outcome)) + + def test_oversized_provider_response_failure_is_sanitized(self): + prepared = build_llm_input(findings=[], aggregated_report={}, engagement={}) + work = _ManualAnalysisWork( + llm_input=prepared, + llm_config={ + "MODEL": "deepseek-chat", + "PROVIDER": "remote", + "PROMPT_PROFILE": "remote_rich_v1", + "REMOTE_PROMPT_PROFILE": "remote_rich_v1", + "STRUCTURED_MAX_FINDINGS": 6, + "STRUCTURED_MAX_TOKENS": 1024, + }, + api_host="private.provider.internal", + api_port=8080, + deadline_monotonic=time.monotonic() + 5, + ) + with patch( + "extensions.business.cybersec.red_mesh.pentester_api_01._bounded_provider_post", + side_effect=RuntimeError( + "private.provider.internal returned an oversized credential-sentinel body" + ), + ): + outcome = _ManualAnalysisWorker(work).run() + + self.assertTrue(outcome.failed) + self.assertNotIn("private.provider.internal", str(outcome)) + self.assertNotIn("credential-sentinel", str(outcome)) + + @unittest.skipUnless( + importlib.util.find_spec("aiohttp") is not None, + "aiohttp is unavailable in the host test environment", + ) + def test_provider_transport_enforces_total_wall_timeout_while_body_trickles(self): + class _TrickleHandler(BaseHTTPRequestHandler): + def log_message(self, *_args): + return + + def do_POST(self): + body_length = int(self.headers.get("Content-Length", "0")) + self.rfile.read(body_length) + self.send_response(200) + self.send_header("Content-Length", "100") + self.end_headers() + try: + for _ in range(100): + self.wfile.write(b"x") + self.wfile.flush() + time.sleep(0.05) + except (BrokenPipeError, ConnectionResetError): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), _TrickleHandler) + server.daemon_threads = True + server_thread = threading.Thread(target=server.serve_forever, daemon=True) + server_thread.start() + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + + started = time.monotonic() + with self.assertRaises(TimeoutError): + _bounded_provider_post( + f"http://127.0.0.1:{server.server_port}/chat", + {"messages": []}, + 0.2, + 1024, + ) + elapsed = time.monotonic() - started + + # Keep a wide scheduler margin while still proving the 0.2-second client + # deadline ends well before the server's five-second trickle completes. + self.assertLess(elapsed, 2.0) + + @unittest.skipUnless( + importlib.util.find_spec("aiohttp") is not None, + "aiohttp is unavailable in the host test environment", + ) + def test_provider_transport_rejects_declared_and_streamed_oversize_bodies(self): + class _OversizeHandler(BaseHTTPRequestHandler): + def log_message(self, *_args): + return + + def do_POST(self): + body_length = int(self.headers.get("Content-Length", "0")) + self.rfile.read(body_length) + self.send_response(200) + if self.path == "/declared": + self.send_header("Content-Length", "2048") + else: + self.send_header("Connection", "close") + self.close_connection = True + self.end_headers() + if self.path != "/declared": + self.wfile.write(b"x" * 2048) + self.wfile.flush() + + server = ThreadingHTTPServer(("127.0.0.1", 0), _OversizeHandler) + server.daemon_threads = True + server_thread = threading.Thread(target=server.serve_forever, daemon=True) + server_thread.start() + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + + for path in ("declared", "streamed"): + with self.subTest(path=path), self.assertRaisesRegex( + RuntimeError, + "response is too large", + ): + _bounded_provider_post( + f"http://127.0.0.1:{server.server_port}/{path}", + {"messages": []}, + 2.0, + 1024, + ) + + def test_prepared_input_rejects_wrong_type(self): + from extensions.business.cybersec.red_mesh.services.llm_structured import generate_exec_summary + + with self.assertRaises(TypeError): + generate_exec_summary( + llm_call=lambda *_args: "", + prepared_input={"not": "trusted"}, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/extensions/business/cybersec/red_mesh/tests/test_postponed_analyze_native_ipc.py b/extensions/business/cybersec/red_mesh/tests/test_postponed_analyze_native_ipc.py new file mode 100644 index 000000000..409d7a73a --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_postponed_analyze_native_ipc.py @@ -0,0 +1,526 @@ +import asyncio +import importlib.util +import inspect +import json +import os +import queue +import shutil +import socket +import subprocess +import sys +import tempfile +import threading +import time +import types +import unittest +import urllib.error +import urllib.request +from collections import deque +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from unittest.mock import patch + +REPO_ROOT = Path(__file__).resolve().parents[5] +FRAMEWORK_PACKAGE = REPO_ROOT / "naeural_core" / "naeural_core" +IPC_MANAGER_PATH = FRAMEWORK_PACKAGE / "utils" / "uvicorn_fast_api_ipc_manager.py" +FASTAPI_PLUGIN_PATH = ( + FRAMEWORK_PACKAGE / "business" / "default" / "web_app" / "fast_api_web_app.py" +) +FASTAPI_UTILS_PATH = FRAMEWORK_PACKAGE / "utils" / "fastapi_utils.py" +NATIVE_RUNTIME_SOURCE_AVAILABLE = all( + path.is_file() + for path in ( + IPC_MANAGER_PATH, + FASTAPI_PLUGIN_PATH, + FASTAPI_UTILS_PATH, + ) +) + + +def _load_module(name, path): + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Could not load {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _load_native_runtime(): + """Load the real scheduler with only its unrelated base dependencies stubbed.""" + module_names = ( + "naeural_core", + "naeural_core.business", + "naeural_core.business.base", + "naeural_core.business.base.web_app", + "naeural_core.business.base.web_app.base_web_app_plugin", + "naeural_core.utils", + "naeural_core.utils.fastapi_utils", + "naeural_core.utils.uvicorn_fast_api_ipc_manager", + ) + previous = {name: sys.modules.get(name) for name in module_names} + try: + for name in module_names: + if name not in { + "naeural_core.business.base.web_app.base_web_app_plugin", + "naeural_core.utils.fastapi_utils", + "naeural_core.utils.uvicorn_fast_api_ipc_manager", + }: + sys.modules[name] = types.ModuleType(name) + + class _BaseWebAppPlugin: + CONFIG = {"VALIDATION_RULES": {}} + + def _process(self): + return None + + def on_close(self): + return None + + base_module = types.ModuleType( + "naeural_core.business.base.web_app.base_web_app_plugin" + ) + base_module.BaseWebAppPlugin = _BaseWebAppPlugin + sys.modules[base_module.__name__] = base_module + + fastapi_utils = _load_module("_rm040_fastapi_utils", FASTAPI_UTILS_PATH) + sys.modules["naeural_core.utils.fastapi_utils"] = fastapi_utils + ipc_stub = types.ModuleType("naeural_core.utils.uvicorn_fast_api_ipc_manager") + ipc_stub.get_server_manager = lambda _auth: None + sys.modules[ipc_stub.__name__] = ipc_stub + runtime = _load_module("_rm040_fastapi_runtime", FASTAPI_PLUGIN_PATH) + return runtime.FastApiWebAppPlugin, fastapi_utils.PostponedRequest + finally: + for name, module in previous.items(): + if module is None: + sys.modules.pop(name, None) + else: + sys.modules[name] = module + + +if NATIVE_RUNTIME_SOURCE_AVAILABLE: + NativeFastApiPlugin, NativePostponedRequest = _load_native_runtime() + ipc_manager = _load_module("_rm040_ipc_manager", IPC_MANAGER_PATH) +else: + # Edge source checkouts do not vendor the Ratio1 runtime. The integration + # fixture is exercised when a sibling runtime source checkout is available. + NativeFastApiPlugin, NativePostponedRequest = object, None + ipc_manager = None + +from .conftest import mock_plugin_modules + + +mock_plugin_modules() + +from extensions.business.cybersec.red_mesh.pentester_api_01 import ( + PentesterApi01Plugin, + _ManualAnalysisOutcome, +) + + +class _SchedulerHarness(NativeFastApiPlugin): + def P(self, *args, **_kwargs): + if hasattr(self, "_test_messages") and args: + self._test_messages.append(str(args[0])) + return None + + def on_response(self, _method, _response): + return None + + def get_additional_fastapi_data(self): + return {} + + def get_process_budget_s(self): + return None + + def get_max_incoming_per_loop(self): + return 10 + + def get_max_postponed_per_loop(self): + return 10 + + def _maybe_log_profile_stats(self): + return None + + +class _Owner: + def __init__(self, worker): + self._manual_analysis_state = None + self._manual_analysis_executor = ThreadPoolExecutor( + max_workers=1, + thread_name_prefix="rm040-native-test", + ) + self._worker = worker + self.solve_postponed_analyze_job = ( + lambda pending_id: PentesterApi01Plugin.solve_postponed_analyze_job( + self, + pending_id, + ) + ) + + def time(self): + return time.monotonic() + + def create_postponed_request(self, solver_method, method_kwargs=None): + return NativePostponedRequest(solver_method, dict(method_kwargs or {})) + + +@unittest.skipUnless( + NATIVE_RUNTIME_SOURCE_AVAILABLE, + "native Ratio1 runtime source fixture is unavailable", +) +class TestPostponedAnalyzeNativeIpc(unittest.TestCase): + @staticmethod + def _find_free_port(): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + @staticmethod + def _descriptor(name, method, parameters, *, require_token=False): + return { + "name": name, + "method": method, + "args": [str(parameter) for parameter in parameters], + "params": [parameter.name for parameter in parameters], + "endpoint_doc": "", + "require_token": require_token, + "has_kwargs": False, + "streaming_type": None, + "chunk_size": 1024 * 1024, + } + + def _render_server(self, destination, manager_port, manager_auth): + from jinja2 import Environment, FileSystemLoader + + analyze_parameters = list( + inspect.signature(PentesterApi01Plugin.analyze_job).parameters.values() + )[1:] + status_parameters = [ + inspect.Parameter("job_id", inspect.Parameter.POSITIONAL_OR_KEYWORD), + ] + endpoints = [ + self._descriptor( + "analyze_job", + "post", + analyze_parameters, + ), + self._descriptor("get_job_status", "get", status_parameters), + ] + template_dir = FRAMEWORK_PACKAGE / "business" / "base" / "uvicorn_templates" + rendered = Environment(loader=FileSystemLoader(str(template_dir))).get_template( + "basic_server.j2" + ).render( + additional_fastapi_data={}, + manager_port=manager_port, + manager_auth=repr(manager_auth), + request_timeout=120, + api_title=repr("RM-041 native postponed test"), + api_summary=repr("RM-041"), + api_description=repr("RM-041"), + api_version=repr("0.0.0-test"), + static_directory="assets", + debug_web_app=False, + debug_timings=False, + debug_timings_steps=False, + default_route=None, + profile_rate=0, + profile_log_per_request=False, + node_comm_params=endpoints, + html_files=[], + ) + destination.mkdir(parents=True, exist_ok=True) + (destination / "assets").mkdir() + (destination / "main.py").write_text(rendered, encoding="utf-8") + temp_utils = destination / "naeural_core" / "utils" + temp_utils.mkdir(parents=True) + (temp_utils.parent / "__init__.py").write_text("", encoding="utf-8") + (temp_utils / "__init__.py").write_text("", encoding="utf-8") + shutil.copy2(IPC_MANAGER_PATH, temp_utils / IPC_MANAGER_PATH.name) + + @staticmethod + def _request(port, method, path, *, token="", payload=None, timeout=5): + headers = {} + if token: + headers["Authorization"] = f"Bearer {token}" + body = None + if payload is not None: + headers["Content-Type"] = "application/json" + body = json.dumps(payload).encode("utf-8") + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=body, + headers=headers, + method=method, + ) + started = time.monotonic() + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + return ( + response.status, + json.loads(response.read().decode("utf-8")), + time.monotonic() - started, + ) + except urllib.error.HTTPError as exc: + return ( + exc.code, + json.loads(exc.read().decode("utf-8")), + time.monotonic() - started, + ) + + def test_ipc_timeout_drops_waiter_without_canceling_plugin_work(self): + manager_auth = b"rm040-native-timeout" + manager = ipc_manager.get_server_manager(manager_auth) + self.addCleanup(manager.shutdown) + _, manager_port = manager.address + server_queue = manager.get_server_queue() + client_queue = manager.get_client_queue() + comms = ipc_manager.UvicornPluginComms( + port=manager_port, + auth=manager_auth, + timeout_s=0.05, + additional_fastapi_data={}, + ) + + loop = asyncio.new_event_loop() + try: + result = loop.run_until_complete( + comms.call_plugin("analyze_job", "job-1") + ) + + self.assertEqual(result["status_code"], 504) + self.assertEqual(comms._commands, {}) + request = server_queue.get(timeout=1) + self.assertEqual(request["value"][:2], ("analyze_job", "job-1")) + with self.assertRaises(queue.Empty): + server_queue.get(timeout=0.05) + + client_queue.put({ + "id": request["id"], + "value": {"result": {"job_id": "job-1"}}, + }) + loop.run_until_complete(asyncio.sleep(0.1)) + self.assertEqual(comms._commands, {}) + finally: + comms._stop_reader.set() + if comms._reader_thread is not None: + comms._reader_thread.join(timeout=1) + loop.close() + + def test_real_postponed_scheduler_keeps_status_responsive(self): + manager_auth = b"rm040-native-ipc" + manager = ipc_manager.get_server_manager(manager_auth) + self.addCleanup(manager.shutdown) + _, manager_port = manager.address + server_queue = manager.get_server_queue() + client_queue = manager.get_client_queue() + + worker_entered = threading.Event() + release_worker = threading.Event() + + def _blocking_worker(_work): + worker_entered.set() + if not release_worker.wait(timeout=10): + raise TimeoutError("native test worker was not released") + return _ManualAnalysisOutcome( + sections={"executive_headline": "done"}, + failed=False, + ) + + owner = _Owner(_blocking_worker) + self.addCleanup(owner._manual_analysis_executor.shutdown, wait=False) + harness = _SchedulerHarness.__new__(_SchedulerHarness) + harness._endpoints = { + "analyze_job": lambda job_id, analysis_type="", focus_areas=None: ( + PentesterApi01Plugin.analyze_job( + owner, + job_id, + analysis_type, + focus_areas, + ) + ), + "get_job_status": lambda job_id: {"status": "ok", "job_id": job_id}, + } + harness._incoming_requests = deque() + harness.postponed_requests = deque() + harness._incoming_lock = threading.Lock() + harness._client_queue = client_queue + harness._stop_request_monitor = threading.Event() + harness._stop_request_monitor.set() + harness._request_monitor_thread = None + harness._profile_stats = {} + harness.cfg_log_requests = False + harness.cfg_response_format = "WRAPPED" + harness.cfg_fair_scheduling = True + harness._test_messages = [] + + stop_dispatcher = threading.Event() + + def _dispatch(): + while not stop_dispatcher.is_set(): + try: + request = server_queue.get_nowait() + except queue.Empty: + request = None + if request is not None: + with harness._incoming_lock: + harness._incoming_requests.append(request) + harness._process() + time.sleep(0.005) + + dispatcher = threading.Thread(target=_dispatch, daemon=True) + dispatcher.start() + + def _stop_dispatcher(): + stop_dispatcher.set() + release_worker.set() + dispatcher.join(timeout=2) + + self.addCleanup(_stop_dispatcher) + + state = { + "pending_id": "unused", + "job_id": "job-1", + "job_revision": 1, + "pass_nr": 1, + "report_cid": "QmExpected", + "target": "example.test", + "num_workers": 1, + "deadline_monotonic": time.monotonic() + 4, + "next_check_monotonic": 0.0, + "discard_result": False, + "work": object(), + } + final_response = { + "job_id": "job-1", + "target": "example.test", + "num_workers": 1, + "pass_nr": 1, + "analysis_type": "structured_report_sections", + "llm_failed": False, + "llm_report_sections": {"executive_headline": "done"}, + } + + with tempfile.TemporaryDirectory(prefix="rm040-native-") as temp_dir: + app_dir = Path(temp_dir) + self._render_server(app_dir, manager_port, manager_auth) + port = self._find_free_port() + process = subprocess.Popen( + [ + sys.executable, + "-m", + "uvicorn", + "--app-dir", + str(app_dir), + "main:app", + "--host", + "127.0.0.1", + "--port", + str(port), + ], + cwd=app_dir, + env={**os.environ, "PYTHONPATH": str(app_dir)}, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + try: + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + try: + with socket.create_connection(("127.0.0.1", port), timeout=0.2): + break + except OSError: + time.sleep(0.05) + else: + stderr = (process.stderr.read() or b"").decode("utf-8", errors="replace") + self.fail(f"Generated Uvicorn server did not start:\n{stderr}") + + analysis_result = [] + + def _request_analysis(): + analysis_result.append(self._request( + port, + "POST", + "/analyze_job", + payload={"job_id": "job-1"}, + )) + + def _prepare(_plugin, job_id): + if job_id == "explode": + raise RuntimeError("native admission failure") + return dict(state), None + + with patch.object( + PentesterApi01Plugin, + "_prepare_manual_analysis", + side_effect=_prepare, + ), patch.object( + PentesterApi01Plugin, + "_finalize_manual_analysis", + return_value=final_response, + ), patch( + "extensions.business.cybersec.red_mesh.pentester_api_01._run_manual_analysis_worker", + side_effect=_blocking_worker, + ): + failed_status, failed_body, failed_elapsed = self._request( + port, + "POST", + "/analyze_job", + payload={"job_id": "explode"}, + ) + self.assertEqual(failed_status, 503, failed_body) + self.assertLess(failed_elapsed, 1.0) + self.assertEqual( + failed_body["detail"]["error"], + "analysis_executor_failed", + ) + analysis_thread = threading.Thread(target=_request_analysis, daemon=True) + analysis_thread.start() + self.assertTrue(worker_entered.wait(timeout=2)) + + status, body, elapsed = self._request( + port, + "GET", + "/get_job_status?job_id=job-1", + ) + self.assertEqual(status, 200, body) + self.assertLess(elapsed, 1.0) + self.assertEqual(body["result"]["status"], "ok") + + busy_status, busy_body, busy_elapsed = self._request( + port, + "POST", + "/analyze_job", + payload={"job_id": "job-1"}, + ) + self.assertEqual(busy_status, 409, busy_body) + self.assertLess(busy_elapsed, 1.0) + self.assertEqual(busy_body["detail"]["error"], "analysis_busy") + + release_worker.set() + analysis_thread.join(timeout=3) + self.assertFalse(analysis_thread.is_alive()) + + self.assertEqual(analysis_result[0][0], 200, analysis_result) + result_body = analysis_result[0][1] + self.assertEqual( + result_body["result"]["analysis_type"], + "structured_report_sections", + ) + self.assertNotIn("operation_id", json.dumps(result_body)) + finally: + release_worker.set() + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + if process.stdout: + process.stdout.close() + if process.stderr: + process.stderr.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/extensions/business/cybersec/red_mesh/tests/test_reconciliation.py b/extensions/business/cybersec/red_mesh/tests/test_reconciliation.py index bb916ebff..12fda53b0 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_reconciliation.py +++ b/extensions/business/cybersec/red_mesh/tests/test_reconciliation.py @@ -521,6 +521,32 @@ def test_reconcile_workers_from_live_requires_report_cid(self): self.assertNotIn("report_cid", job_specs["workers"]["worker-A"]) owner._write_job_record.assert_not_called() + def test_reconcile_workers_from_terminal_error_stops_job(self): + job_specs = { + "job_id": "job-1", + "job_status": "RUNNING", + "job_pass": 2, + "launcher": "launcher-A", + "workers": { + "worker-A": {"start_port": 1, "end_port": 10, "assignment_revision": 3}, + }, + } + live_payloads = self._terminal_live_payload(cid=None) + live_payloads["job-1:worker-A"]["phase"] = "failed" + live_payloads["job-1:worker-A"]["error_class"] = "launch_failed" + owner, _repo = self._make_live_reconcile_owner(job_specs, live_payloads) + + changed = reconcile_workers_from_live(owner, "job-1") + + self.assertTrue(changed) + self.assertEqual(job_specs["job_status"], "STOPPED") + worker = job_specs["workers"]["worker-A"] + self.assertTrue(worker["finished"]) + self.assertEqual(worker["terminal_reason"], "launch_failed") + owner._write_job_record.assert_called_once_with( + "job-1", job_specs, context="reconcile_from_live", + ) + def test_reconcile_workers_from_live_skips_canceled_and_unreachable(self): job_specs = { "job_id": "job-1", diff --git a/extensions/business/cybersec/red_mesh/tests/test_repositories.py b/extensions/business/cybersec/red_mesh/tests/test_repositories.py index fffb55b3b..5000e7741 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_repositories.py +++ b/extensions/business/cybersec/red_mesh/tests/test_repositories.py @@ -301,6 +301,21 @@ def test_artifact_repository_delete_is_guarded_on_empty_cid(self): self.assertFalse(repo.delete("")) owner.r1fs.delete_file.assert_not_called() + def test_artifact_repository_purge_removes_remote_pin_and_local_cache(self): + owner = self._make_owner() + repo = ArtifactRepository(owner) + + repo.delete("QmCID", show_logs=True, purge=True) + + owner.r1fs.delete_file.assert_called_once_with( + "QmCID", + unpin_remote=True, + run_gc=True, + cleanup_local_files=True, + show_logs=True, + raise_on_error=False, + ) + def test_artifact_repository_supports_typed_models(self): owner = self._make_owner() repo = ArtifactRepository(owner) diff --git a/extensions/business/cybersec/red_mesh/tests/test_rulebook_assessment.py b/extensions/business/cybersec/red_mesh/tests/test_rulebook_assessment.py new file mode 100644 index 000000000..438585bfa --- /dev/null +++ b/extensions/business/cybersec/red_mesh/tests/test_rulebook_assessment.py @@ -0,0 +1,1132 @@ +import json +import sys +import time +import types +import unittest +from concurrent.futures import ThreadPoolExecutor +from unittest.mock import MagicMock + + +def _install_pymisp_stub(): + if "pymisp" in sys.modules: + return + pymisp_stub = types.ModuleType("pymisp") + pymisp_stub.MISPEvent = MagicMock + pymisp_stub.MISPObject = MagicMock + pymisp_stub.MISPAttribute = MagicMock + pymisp_stub.PyMISP = MagicMock + sys.modules["pymisp"] = pymisp_stub + + +_install_pymisp_stub() + +from extensions.business.cybersec.red_mesh.services.control import purge_job +from extensions.business.cybersec.red_mesh.models import ( + RULEBOOK_ASSESSMENT_SCHEMA_VERSION, + RULEBOOK_SUBMISSION_CONTRACT_VERSION, + RulebookPendingSubmission, + RulebookSubmissionReference, + RulebookSubmissionRegistry, +) +from extensions.business.cybersec.red_mesh.repositories import JobStateRepository +from extensions.business.cybersec.red_mesh.services.rulebook_assessment import ( + DEFAULT_RULEBOOK_PROFILE_ID, + build_rulebook_assessment, + ensure_rulebook_assessment, + generate_rulebook_assessment, + get_rulebook_assessment_status, + get_rulebook_review, + reopen_rulebook_review, + save_rulebook_review_draft, + submit_rulebook_review, + update_rulebook_review, +) +from extensions.business.cybersec.red_mesh.services.triage import update_finding_triage + + +def _sample_findings(): + return [ + { + "finding_id": "finding-auth-1", + "severity": "HIGH", + "title": "Authentication bypass on app.example.test", + "description": "The endpoint leaks token=supersecret for 10.0.0.4.", + "evidence": {"raw_response": "password=supersecret", "credential_ref": "secret://graybox"}, + "remediation": "Require authentication and validate sessions.", + "cwe_id": "CWE-306", + "owasp_id": "A01:2021", + "probe": "_web_test_auth_bypass", + "category": "auth", + "status": "vulnerable", + }, + { + "finding_id": "finding-crypto-1", + "severity": "LOW", + "title": "TLS certificate expires soon on app.example.test", + "description": "Certificate metadata needs review.", + "cwe_id": "CWE-295", + "probe": "_tls_certificate_check", + "category": "tls", + "status": "vulnerable", + }, + ] + + +def _sample_pass_report(): + return { + "pass_nr": 3, + "date_started": 1770000000.0, + "date_completed": 1770000300.0, + "aggregated_report_cid": "agg-cid", + "risk_score": 75, + "quick_summary": "Issues were observed on app.example.test.", + "findings": _sample_findings(), + } + + +def _sample_archive(): + return { + "job_id": "job-1", + "job_config": { + "target": "https://app.example.test/login", + "target_url": "https://app.example.test/login", + "scan_type": "webapp", + "task_name": "Application scan", + "start_port": 443, + "end_port": 443, + "secret_ref": "secret://graybox", + }, + "passes": [_sample_pass_report()], + "ui_aggregate": {}, + "duration": 300.0, + "date_created": 1770000000.0, + "date_completed": 1770000300.0, + "soc_event_status": {"last_status": "sent"}, + } + + +def _sample_job_specs(**overrides): + payload = { + "job_id": "job-1", + "job_status": "FINALIZED", + "scan_type": "webapp", + "target": "https://app.example.test/login", + "target_url": "https://app.example.test/login", + "job_cid": "archive-cid", + "job_config_cid": "config-cid", + "date_created": 1770000000.0, + "date_completed": 1770000300.0, + "run_mode": "SINGLEPASS", + "launcher": "operator", + "start_port": 443, + "end_port": 443, + } + payload.update(overrides) + return payload + + +def _complete_review_answers(): + return { + "nis2.risk.risk_treatment_reviewed": {"value": "yes"}, + "nis2.incident.incident_process": {"value": "yes"}, + "nis2.bcm.business_continuity": {"value": "yes"}, + "nis2.supply.supplier_risk_reviewed": {"value": "yes"}, + "nis2.access.access_controls_reviewed": {"value": "yes"}, + "nis2.crypto.crypto_policy_reviewed": {"value": "yes"}, + "nis2.effectiveness.assessment_reviewed": {"value": "yes"}, + "nis2.reporting.reporting_process": {"value": "yes"}, + } + + +class _FakeArtifactRepo: + def __init__(self, owner, archive=None, aggregated=None): + self.owner = owner + self.archive = archive or _sample_archive() + self.aggregated = aggregated or {"open_ports": [443]} + self.deleted = [] + + def get_archive(self, job_specs): + return self.archive + + def get_json(self, cid): + if cid == "archive-cid": + return self.archive + if cid == "agg-cid": + return self.aggregated + if cid == "config-cid": + return self.archive.get("job_config", {}) + return self.owner.artifacts.get(cid) + + def get_job_config(self, job_specs): + return self.archive.get("job_config", {}) + + def get_pass_report(self, report_cid): + return self.owner.artifacts.get(report_cid) + + def put_json(self, payload, *, show_logs=False): + return self.owner.r1fs.add_json(payload, show_logs=show_logs) + + def delete(self, cid, *, show_logs=False, raise_on_error=False, purge=False): + self.deleted.append(cid) + self.owner.artifacts.pop(cid, None) + return True + + +class _Owner: + cfg_instance_id = "tenant-a" + + def __init__(self, job_specs=None, archive=None): + self.job_specs = job_specs or _sample_job_specs() + self.archive = archive or _sample_archive() + self.artifacts = {} + self.records = {} + self.messages = [] + self.audit_events = [] + self.artifact_repo = _FakeArtifactRepo(self, archive=self.archive) + self.r1fs = MagicMock() + self.r1fs.add_json.side_effect = self._add_json + + def _add_json(self, payload, show_logs=False): + cid = f"QmRulebook{len(self.artifacts) + 1}" + self.artifacts[cid] = payload + return cid + + def P(self, msg, **kwargs): + self.messages.append(msg) + + def time(self): + return 1770000400.0 + + def _get_job_from_cstore(self, job_id): + return self.job_specs if job_id == "job-1" else None + + def _get_artifact_repository(self): + return self.artifact_repo + + def _write_job_record(self, job_id, updated, context=""): + self.job_specs = updated + self.records[(self.cfg_instance_id, job_id)] = updated + return updated + + def _normalize_job_record(self, job_id, raw): + return job_id, raw + + def _log_audit_event(self, event_type, payload): + self.audit_events.append((event_type, payload)) + + def chainstore_hget(self, hkey, key): + return self.records.get((hkey, key)) + + def chainstore_hset(self, hkey, key, value): + self.records[(hkey, key)] = value + return True + + def chainstore_hgetall(self, hkey): + return { + key: value + for (row_hkey, key), value in self.records.items() + if row_hkey == hkey and value is not None + } + + +class TestRulebookAssessment(unittest.TestCase): + + def test_generate_persists_artifact_metadata_and_redacts_sensitive_values(self): + owner = _Owner() + + result = generate_rulebook_assessment(owner, "job-1") + + self.assertEqual(result["status"], "ok") + self.assertEqual(result["artifact_cid"], "QmRulebook1") + self.assertEqual( + owner.job_specs["rulebook_assessments"][DEFAULT_RULEBOOK_PROFILE_ID]["artifact_cid"], + "QmRulebook1", + ) + meta = owner.job_specs["rulebook_assessments"][DEFAULT_RULEBOOK_PROFILE_ID] + self.assertTrue(meta["auto_enabled"]) + self.assertEqual(meta["run_state"], "succeeded") + self.assertEqual(meta["latest_pass_nr"], 3) + self.assertEqual(meta["history"], []) + assessment = owner.artifacts["QmRulebook1"] + self.assertEqual(assessment["schema"], "redmesh.rulebook_assessment.v1") + self.assertEqual(assessment["schema_version"], "1.1.0") + self.assertEqual(assessment["artifact_kind"], "generated_assessment") + self.assertEqual(assessment["profile"]["profile_id"], DEFAULT_RULEBOOK_PROFILE_ID) + self.assertGreater(assessment["status_counts"]["gap"], 0) + + serialized = json.dumps(assessment, sort_keys=True).lower() + self.assertNotIn("app.example.test", serialized) + self.assertNotIn("10.0.0.4", serialized) + self.assertNotIn("supersecret", serialized) + self.assertNotIn("credential_ref", serialized) + self.assertNotIn("secret://graybox", serialized) + self.assertNotIn("non_compliant", serialized) + self.assertNotIn("compliant", serialized) + self.assertIn("target:", serialized) + + status = get_rulebook_assessment_status(owner, "job-1") + self.assertTrue(status["found"]) + self.assertTrue(status["generated"]) + self.assertEqual(status["artifact_cid"], "QmRulebook1") + self.assertEqual(status["run_state"], "succeeded") + + def test_persisted_generated_assessment_excludes_mutable_draft_review(self): + owner = _Owner() + update_rulebook_review( + owner, + "job-1", + answers={"nis2.bcm.business_continuity": {"value": "yes", "note": "private draft comment"}}, + reviewer="draft-reviewer", + review_state="draft", + ) + + preview = generate_rulebook_assessment(owner, "job-1", persist=False) + persisted = generate_rulebook_assessment(owner, "job-1", persist=True) + + self.assertEqual( + preview["assessment"]["checks"][2]["review_answer"]["note"], + "private draft comment", + ) + artifact = owner.artifacts[persisted["artifact_cid"]] + self.assertEqual(artifact["review_state"], {"review_state": "draft", "answers": {}}) + serialized = json.dumps(artifact, sort_keys=True) + self.assertNotIn("private draft comment", serialized) + self.assertNotIn("draft-reviewer", serialized) + + def test_submission_models_and_registry_round_trip(self): + reference = RulebookSubmissionReference( + revision=1, + cid="QmSubmission1", + submitted_at=1770000400.0, + actor="alice", + pass_nr=3, + profile_id=DEFAULT_RULEBOOK_PROFILE_ID, + profile_version="1.0.0", + schema_version=RULEBOOK_ASSESSMENT_SCHEMA_VERSION, + review_revision=2, + idempotency_key="submission-1", + fingerprint="a" * 64, + ) + pending = RulebookPendingSubmission( + target_revision=2, + expected_review_revision=3, + expected_pass_nr=3, + expected_profile_version="1.0.0", + actor="alice", + idempotency_key="submission-2", + fingerprint="b" * 64, + ) + registry = RulebookSubmissionRegistry( + submissions=[reference], + pending=pending, + ) + + payload = registry.to_dict() + restored = RulebookSubmissionRegistry.from_dict(payload).to_dict() + + self.assertEqual(restored["contract_version"], RULEBOOK_SUBMISSION_CONTRACT_VERSION) + self.assertEqual(restored["latest_revision"], 1) + self.assertEqual(restored["submissions"][0]["cid"], "QmSubmission1") + self.assertEqual(restored["pending"]["target_revision"], 2) + + def test_submission_registry_uses_dedicated_cstore_hash(self): + owner = _Owner() + repo = JobStateRepository(owner) + registry = RulebookSubmissionRegistry(submissions=[]) + + stored = repo.put_rulebook_submission_registry("job-1", DEFAULT_RULEBOOK_PROFILE_ID, registry) + loaded = repo.get_rulebook_submission_registry_model("job-1", DEFAULT_RULEBOOK_PROFILE_ID) + + key = f"job-1:{DEFAULT_RULEBOOK_PROFILE_ID}" + self.assertEqual( + owner.records[(f"{owner.cfg_instance_id}:rulebook_review:submissions", key)], + stored, + ) + self.assertEqual(loaded.to_dict(), stored) + + def test_save_draft_is_revisioned_and_never_writes_r1fs(self): + owner = _Owner() + + saved = save_rulebook_review_draft( + owner, + "job-1", + answers={"nis2.bcm.business_continuity": {"value": "yes", "note": "Plan reviewed."}}, + actor="alice", + expected_review_revision=0, + ) + stale = save_rulebook_review_draft( + owner, + "job-1", + answers={}, + actor="alice", + expected_review_revision=0, + ) + + self.assertEqual(saved["status"], "ok") + self.assertEqual(saved["review"]["review_revision"], 1) + self.assertEqual(saved["effective_review_state"], "draft") + self.assertEqual(stale["error"], "review_revision_conflict") + self.assertEqual(owner.r1fs.add_json.call_count, 0) + + def test_submit_persists_complete_snapshot_and_replays_idempotently(self): + owner = _Owner() + saved = save_rulebook_review_draft( + owner, + "job-1", + answers=_complete_review_answers(), + actor="alice", + expected_review_revision=0, + ) + + submitted = submit_rulebook_review( + owner, + "job-1", + expected_review_revision=saved["review_revision"], + expected_pass_nr=3, + expected_profile_version="1.0.0", + idempotency_key="submission-1", + actor="alice", + ) + replay = submit_rulebook_review( + owner, + "job-1", + expected_review_revision=saved["review_revision"], + expected_pass_nr=3, + expected_profile_version="1.0.0", + idempotency_key="submission-1", + actor="alice", + ) + conflict = submit_rulebook_review( + owner, + "job-1", + expected_review_revision=saved["review_revision"], + expected_pass_nr=3, + expected_profile_version="1.0.0", + idempotency_key="submission-1", + actor="mallory", + ) + + self.assertEqual(submitted["effective_review_state"], "submitted") + self.assertEqual(submitted["submission"]["revision"], 1) + self.assertEqual(replay["submission"]["cid"], submitted["submission"]["cid"]) + self.assertTrue(replay["idempotent_replay"]) + self.assertEqual(conflict["error"], "submission_idempotency_conflict") + self.assertEqual(owner.r1fs.add_json.call_count, 1) + snapshot = owner.artifacts[submitted["submission"]["cid"]] + self.assertEqual(snapshot["schema_version"], "1.1.0") + self.assertEqual(snapshot["artifact_kind"], "review_submission") + self.assertEqual(snapshot["submission"]["revision"], 1) + self.assertEqual(snapshot["review_state"]["review_state"], "submitted") + + def test_committed_idempotency_replay_conflicts_when_assessment_inputs_change(self): + owner = _Owner() + saved = save_rulebook_review_draft( + owner, "job-1", answers=_complete_review_answers(), actor="alice", expected_review_revision=0, + ) + submitted = submit_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], expected_pass_nr=3, + expected_profile_version="1.0.0", idempotency_key="submission-triage", actor="alice", + ) + owner.records[(f"{owner.cfg_instance_id}:triage", "job-1:finding-auth-1")] = { + "job_id": "job-1", + "finding_id": "finding-auth-1", + "status": "false_positive", + "actor": "analyst", + "updated_at": owner.time(), + } + + replay = submit_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], expected_pass_nr=3, + expected_profile_version="1.0.0", idempotency_key="submission-triage", actor="alice", + ) + + self.assertEqual(submitted["status"], "ok") + self.assertEqual(replay["error"], "submission_idempotency_conflict") + self.assertEqual(owner.r1fs.add_json.call_count, 1) + + def test_submission_requires_complete_answers_and_comments(self): + owner = _Owner() + answers = _complete_review_answers() + answers["nis2.reporting.reporting_process"] = {"value": "unknown", "note": ""} + saved = save_rulebook_review_draft( + owner, + "job-1", + answers=answers, + actor="alice", + expected_review_revision=0, + ) + + result = submit_rulebook_review( + owner, + "job-1", + expected_review_revision=saved["review_revision"], + expected_pass_nr=3, + expected_profile_version="1.0.0", + idempotency_key="submission-comments", + actor="alice", + ) + + self.assertEqual(result["error"], "submission_comments_required") + self.assertEqual( + result["comment_required_question_ids"], + ["nis2.reporting.reporting_process"], + ) + self.assertEqual(owner.r1fs.add_json.call_count, 0) + + def test_r1fs_failure_keeps_draft_and_same_key_retry_recovers(self): + owner = _Owner() + saved = save_rulebook_review_draft( + owner, + "job-1", + answers=_complete_review_answers(), + actor="alice", + expected_review_revision=0, + ) + owner.r1fs.add_json.side_effect = None + owner.r1fs.add_json.return_value = None + + failed = submit_rulebook_review( + owner, + "job-1", + expected_review_revision=saved["review_revision"], + expected_pass_nr=3, + expected_profile_version="1.0.0", + idempotency_key="submission-retry", + actor="alice", + ) + visible = get_rulebook_review(owner, "job-1") + owner.r1fs.add_json.side_effect = owner._add_json + recovered = submit_rulebook_review( + owner, + "job-1", + expected_review_revision=saved["review_revision"], + expected_pass_nr=3, + expected_profile_version="1.0.0", + idempotency_key="submission-retry", + actor="alice", + ) + + self.assertEqual(failed["error"], "submission_persist_failed") + self.assertEqual(visible["effective_review_state"], "draft") + self.assertEqual(visible["submission_operation_state"], "failed") + self.assertEqual(recovered["effective_review_state"], "submitted") + self.assertEqual(recovered["submission"]["revision"], 1) + + def test_pending_submission_fences_finding_triage_until_retry(self): + owner = _Owner() + saved = save_rulebook_review_draft( + owner, "job-1", answers=_complete_review_answers(), actor="alice", expected_review_revision=0, + ) + owner.r1fs.add_json.side_effect = None + owner.r1fs.add_json.return_value = None + failed = submit_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], expected_pass_nr=3, + expected_profile_version="1.0.0", idempotency_key="pending-triage", actor="alice", + ) + + triage = update_finding_triage( + owner, "job-1", "finding-auth-1", "false_positive", actor="analyst", + ) + + self.assertEqual(failed["error"], "submission_persist_failed") + self.assertEqual(triage["error"], "submission_in_progress") + self.assertIsNone(owner.records.get((f"{owner.cfg_instance_id}:triage", "job-1:finding-auth-1"))) + + def test_partial_final_registry_write_recovers_without_second_r1fs_write(self): + owner = _Owner() + saved = save_rulebook_review_draft( + owner, + "job-1", + answers=_complete_review_answers(), + actor="alice", + expected_review_revision=0, + ) + original_hset = owner.chainstore_hset + failed_once = {"value": False} + + def fail_final_registry_write(hkey, key, value): + if ( + hkey.endswith(":rulebook_review:submissions") + and isinstance(value, dict) + and value.get("submissions") + and not value.get("pending") + and not failed_once["value"] + ): + failed_once["value"] = True + raise RuntimeError("simulated final registry failure") + return original_hset(hkey, key, value) + + owner.chainstore_hset = fail_final_registry_write + failed = submit_rulebook_review( + owner, + "job-1", + expected_review_revision=saved["review_revision"], + expected_pass_nr=3, + expected_profile_version="1.0.0", + idempotency_key="submission-partial", + actor="alice", + ) + owner.chainstore_hset = original_hset + recovered = submit_rulebook_review( + owner, + "job-1", + expected_review_revision=saved["review_revision"], + expected_pass_nr=3, + expected_profile_version="1.0.0", + idempotency_key="submission-partial", + actor="alice", + ) + + self.assertEqual(failed["error"], "submission_record_failed") + self.assertEqual(recovered["effective_review_state"], "submitted") + self.assertEqual(owner.r1fs.add_json.call_count, 1) + + def test_concurrent_duplicate_submission_allocates_one_revision(self): + owner = _Owner() + saved = save_rulebook_review_draft( + owner, + "job-1", + answers=_complete_review_answers(), + actor="alice", + expected_review_revision=0, + ) + + def submit(): + return submit_rulebook_review( + owner, + "job-1", + expected_review_revision=saved["review_revision"], + expected_pass_nr=3, + expected_profile_version="1.0.0", + idempotency_key="submission-concurrent", + actor="alice", + ) + + with ThreadPoolExecutor(max_workers=2) as executor: + results = list(executor.map(lambda _: submit(), range(2))) + + self.assertEqual([result["status"] for result in results], ["ok", "ok"]) + self.assertEqual(owner.r1fs.add_json.call_count, 1) + self.assertEqual(len(get_rulebook_review(owner, "job-1")["submissions"]), 1) + + def test_submit_rejects_stale_revision_pass_and_profile(self): + owner = _Owner() + saved = save_rulebook_review_draft( + owner, + "job-1", + answers=_complete_review_answers(), + actor="alice", + expected_review_revision=0, + ) + + stale_revision = submit_rulebook_review( + owner, "job-1", expected_review_revision=0, expected_pass_nr=3, + expected_profile_version="1.0.0", idempotency_key="stale-revision", actor="alice", + ) + stale_pass = submit_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], expected_pass_nr=2, + expected_profile_version="1.0.0", idempotency_key="stale-pass", actor="alice", + ) + stale_profile = submit_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], expected_pass_nr=3, + expected_profile_version="0.9.0", idempotency_key="stale-profile", actor="alice", + ) + + self.assertEqual(stale_revision["error"], "review_revision_conflict") + self.assertEqual(stale_pass["error"], "submission_pass_stale") + self.assertEqual(stale_profile["error"], "submission_profile_stale") + + def test_reopen_and_resubmit_preserve_both_revisions(self): + owner = _Owner() + saved = save_rulebook_review_draft( + owner, "job-1", answers=_complete_review_answers(), actor="alice", expected_review_revision=0, + ) + first = submit_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], expected_pass_nr=3, + expected_profile_version="1.0.0", idempotency_key="revision-1", actor="alice", + ) + reopened = reopen_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], + idempotency_key="reopen-revision-1", actor="alice", + ) + second = submit_rulebook_review( + owner, "job-1", expected_review_revision=reopened["review_revision"], expected_pass_nr=3, + expected_profile_version="1.0.0", idempotency_key="revision-2", actor="alice", + ) + + self.assertEqual(first["submission"]["revision"], 1) + self.assertEqual(reopened["effective_review_state"], "draft") + self.assertEqual(reopened["review_revision"], 2) + self.assertEqual(second["submission"]["revision"], 2) + history = get_rulebook_review(owner, "job-1")["submissions"] + self.assertEqual([item["revision"] for item in history], [2, 1]) + self.assertNotEqual(history[0]["cid"], history[1]["cid"]) + + def test_reopen_replays_same_operation_key_after_response_loss(self): + owner = _Owner() + saved = save_rulebook_review_draft( + owner, "job-1", answers=_complete_review_answers(), actor="alice", expected_review_revision=0, + ) + submit_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], expected_pass_nr=3, + expected_profile_version="1.0.0", idempotency_key="reopen-source", actor="alice", + ) + + first = reopen_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], + idempotency_key="reopen-operation", actor="alice", + ) + replay = reopen_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], + idempotency_key="reopen-operation", actor="alice", + ) + conflict = reopen_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], + idempotency_key="reopen-operation", actor="mallory", + ) + + self.assertEqual(first["review_revision"], 2) + self.assertTrue(replay["idempotent_replay"]) + self.assertEqual(replay["review_revision"], 2) + self.assertEqual(conflict["error"], "reopen_idempotency_conflict") + + def test_legacy_write_remains_compatible_after_native_review_is_reopened(self): + owner = _Owner() + saved = save_rulebook_review_draft( + owner, "job-1", answers=_complete_review_answers(), actor="alice", expected_review_revision=0, + ) + submit_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], expected_pass_nr=3, + expected_profile_version="1.0.0", idempotency_key="rollback-source", actor="alice", + ) + reopened = reopen_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], + idempotency_key="rollback-reopen", actor="alice", + ) + + legacy_saved = update_rulebook_review( + owner, + "job-1", + answers={"nis2.bcm.business_continuity": {"value": "yes", "note": "edited after rollback"}}, + reviewer="legacy-navigator", + review_state="draft", + ) + legacy_reviewed = update_rulebook_review( + owner, + "job-1", + answers={}, + reviewer="legacy-navigator", + review_state="reviewed", + ) + + self.assertEqual(reopened["effective_review_state"], "draft") + self.assertEqual(legacy_saved["status"], "ok") + self.assertEqual(legacy_saved["review"]["review_revision"], 3) + self.assertEqual(legacy_reviewed["error"], "review_already_submitted") + self.assertEqual(len(get_rulebook_review(owner, "job-1")["submissions"]), 1) + + def test_two_revision_submission_smoke_retrieves_then_purges_every_snapshot(self): + owner = _Owner(job_specs=_sample_job_specs()) + owner.records[(owner.cfg_instance_id, "job-1")] = owner.job_specs + review_key = f"job-1:{DEFAULT_RULEBOOK_PROFILE_ID}" + + saved = save_rulebook_review_draft( + owner, "job-1", answers=_complete_review_answers(), actor="alice", expected_review_revision=0, + ) + first = submit_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], expected_pass_nr=3, + expected_profile_version="1.0.0", idempotency_key="smoke-revision-1", actor="alice", + ) + reopened = reopen_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], + idempotency_key="smoke-reopen-revision-1", actor="alice", + ) + second = submit_rulebook_review( + owner, "job-1", expected_review_revision=reopened["review_revision"], expected_pass_nr=3, + expected_profile_version="1.0.0", idempotency_key="smoke-revision-2", actor="bob", + ) + + first_cid = first["submission"]["cid"] + second_cid = second["submission"]["cid"] + self.assertEqual(owner.artifact_repo.get_json(first_cid)["submission"]["revision"], 1) + self.assertEqual(owner.artifact_repo.get_json(second_cid)["submission"]["revision"], 2) + self.assertEqual( + [item["cid"] for item in get_rulebook_review(owner, "job-1")["submissions"]], + [second_cid, first_cid], + ) + + result = purge_job(owner, "job-1") + + self.assertEqual(result["status"], "success") + self.assertIsNone(owner.artifact_repo.get_json(first_cid)) + self.assertIsNone(owner.artifact_repo.get_json(second_cid)) + self.assertIn(first_cid, owner.artifact_repo.deleted) + self.assertIn(second_cid, owner.artifact_repo.deleted) + self.assertIsNone(owner.records[(f"{owner.cfg_instance_id}:rulebook_review", review_key)]) + self.assertIsNone(owner.records[(f"{owner.cfg_instance_id}:rulebook_review:audit", review_key)]) + self.assertIsNone(owner.records[(f"{owner.cfg_instance_id}:rulebook_review:submissions", review_key)]) + + def test_newer_scan_evidence_marks_prior_submission_stale(self): + owner = _Owner() + saved = save_rulebook_review_draft( + owner, "job-1", answers=_complete_review_answers(), actor="alice", expected_review_revision=0, + ) + submit_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], expected_pass_nr=3, + expected_profile_version="1.0.0", idempotency_key="stale-after-pass", actor="alice", + ) + owner.archive["passes"].append({**_sample_pass_report(), "pass_nr": 4}) + + current = get_rulebook_review(owner, "job-1") + + self.assertTrue(current["submissions"][0]["stale"]) + self.assertEqual(current["submissions"][0]["stale_reasons"], ["newer_scan_pass"]) + + def test_formal_submission_redacts_sensitive_review_comments(self): + owner = _Owner() + answers = _complete_review_answers() + answers["nis2.bcm.business_continuity"] = { + "value": "unknown", + "note": "password=supersecret observed at 10.0.0.4", + } + saved = save_rulebook_review_draft( + owner, "job-1", answers=answers, actor="alice", expected_review_revision=0, + ) + submitted = submit_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], expected_pass_nr=3, + expected_profile_version="1.0.0", idempotency_key="redacted-submission", actor="alice", + ) + + serialized = json.dumps(owner.artifacts[submitted["submission"]["cid"]], sort_keys=True).lower() + self.assertNotIn("supersecret", serialized) + self.assertNotIn("10.0.0.4", serialized) + self.assertIn("", serialized) + self.assertIn("ip:", serialized) + + def test_formal_submission_redacts_bearer_jwt_pem_and_unlabelled_tokens(self): + owner = _Owner() + evidence_cid = "QmbuqxraU9uNEYcwiKnMZacSNrRwaGpUXctewuiL5HNF94" + evidence_sha256 = "0123456789abcdef" * 4 + answers = _complete_review_answers() + answers["nis2.bcm.business_continuity"] = { + "value": "unknown", + "note": ( + "Authorization: Bearer sk_live_1234567890abcdef " + "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.signature1234 " + "-----BEGIN PRIVATE KEY-----\nprivatekeymaterial123456\n-----END PRIVATE KEY----- " + "a9b8c7d6e5f4a3b2c1d0e9f8a7b6c5d4 " + "AKIA1234567890ABCDEF glpat-1234567890abcdefghij " + f"evidence {evidence_cid} sha256:{evidence_sha256}" + ), + } + saved = save_rulebook_review_draft( + owner, "job-1", answers=answers, actor="alice", expected_review_revision=0, + ) + submitted = submit_rulebook_review( + owner, "job-1", expected_review_revision=saved["review_revision"], expected_pass_nr=3, + expected_profile_version="1.0.0", idempotency_key="credential-redaction", actor="alice", + ) + + serialized = json.dumps(owner.artifacts[submitted["submission"]["cid"]], sort_keys=True) + self.assertNotIn("sk_live_1234567890abcdef", serialized) + self.assertNotIn("eyJhbGciOiJIUzI1NiJ9", serialized) + self.assertNotIn("privatekeymaterial123456", serialized) + self.assertNotIn("a9b8c7d6e5f4a3b2c1d0e9f8a7b6c5d4", serialized) + self.assertNotIn("AKIA1234567890ABCDEF", serialized) + self.assertNotIn("glpat-1234567890abcdefghij", serialized) + self.assertIn(evidence_cid, serialized) + self.assertIn(evidence_sha256, serialized) + self.assertIn("", serialized) + + def test_future_registry_contract_returns_stable_upgrade_error_for_writes(self): + owner = _Owner() + hkey = f"{owner.cfg_instance_id}:rulebook_review:submissions" + owner.records[(hkey, f"job-1:{DEFAULT_RULEBOOK_PROFILE_ID}")] = { + "contract_version": "2.0.0", + "latest_revision": 0, + "submissions": [], + } + + saved = save_rulebook_review_draft( + owner, "job-1", answers={}, actor="alice", expected_review_revision=0, + ) + submitted = submit_rulebook_review( + owner, "job-1", expected_review_revision=0, expected_pass_nr=3, + expected_profile_version="1.0.0", idempotency_key="future-submit", actor="alice", + ) + reopened = reopen_rulebook_review( + owner, "job-1", expected_review_revision=0, + idempotency_key="future-reopen", actor="alice", + ) + + self.assertEqual(saved["error"], "submission_contract_unsupported") + self.assertEqual(submitted["error"], "submission_contract_unsupported") + self.assertEqual(reopened["error"], "submission_contract_unsupported") + + def test_legacy_reviewed_records_surface_revision_zero_or_migration(self): + owner = _Owner() + updated = update_rulebook_review( + owner, + "job-1", + answers=_complete_review_answers(), + reviewer="legacy-reviewer", + review_state="reviewed", + ) + migration = get_rulebook_review(owner, "job-1") + owner.job_specs["rulebook_assessments"] = { + DEFAULT_RULEBOOK_PROFILE_ID: { + "artifact_cid": "QmLegacyReviewed", + "pass_nr": 3, + "profile_version": "1.0.0", + "schema_version": "1.0.0", + }, + } + referenced = get_rulebook_review(owner, "job-1") + + self.assertEqual(updated["review"]["review_state"], "reviewed") + self.assertTrue(migration["migration_submission_required"]) + self.assertEqual(migration["effective_review_state"], "draft") + self.assertEqual(referenced["effective_review_state"], "submitted") + self.assertEqual(referenced["submissions"][0]["revision"], 0) + self.assertTrue(referenced["submissions"][0]["legacy"]) + + def test_ensure_is_idempotent_for_existing_same_pass_and_force_regenerates(self): + owner = _Owner() + + first = ensure_rulebook_assessment(owner, "job-1") + second = ensure_rulebook_assessment(owner, "job-1") + forced = generate_rulebook_assessment(owner, "job-1", force=True) + + self.assertEqual(first["artifact_cid"], "QmRulebook1") + self.assertEqual(second["artifact_cid"], "QmRulebook1") + self.assertTrue(second["cached"]) + self.assertEqual(forced["artifact_cid"], "QmRulebook2") + self.assertEqual(owner.r1fs.add_json.call_count, 2) + + meta = owner.job_specs["rulebook_assessments"][DEFAULT_RULEBOOK_PROFILE_ID] + self.assertEqual(meta["artifact_cid"], "QmRulebook2") + self.assertEqual(meta["history"][0]["artifact_cid"], "QmRulebook1") + + def test_ensure_replaces_legacy_same_pass_artifact_before_reuse(self): + owner = _Owner(job_specs=_sample_job_specs(rulebook_assessments={ + DEFAULT_RULEBOOK_PROFILE_ID: { + "artifact_cid": "QmLegacyRulebook", + "pass_nr": 3, + "latest_pass_nr": 3, + "run_state": "succeeded", + "schema_version": "1.0.0", + }, + })) + owner.artifacts["QmLegacyRulebook"] = { + "schema": "redmesh.rulebook_assessment.v1", + "schema_version": "1.0.0", + "review_state": {"answers": {"q": {"note": "legacy draft"}}}, + } + + result = ensure_rulebook_assessment(owner, "job-1") + + self.assertFalse(result.get("cached", False)) + self.assertNotEqual(result["artifact_cid"], "QmLegacyRulebook") + self.assertEqual(owner.artifacts[result["artifact_cid"]]["artifact_kind"], "generated_assessment") + self.assertNotIn("legacy draft", json.dumps(owner.artifacts[result["artifact_cid"]])) + + def test_running_job_with_completed_pass_report_is_eligible(self): + owner = _Owner(job_specs=_sample_job_specs( + job_status="RUNNING", + job_cid="", + pass_reports=[{"pass_nr": 3, "report_cid": "pass-cid", "risk_score": 75}], + )) + owner.artifacts["pass-cid"] = _sample_pass_report() + + result = build_rulebook_assessment(owner, "job-1") + + self.assertEqual(result["status"], "ok") + self.assertEqual(result["pass_nr"], 3) + self.assertEqual(result["assessment"]["scan_context"]["job_status"], "RUNNING") + + def test_artifact_write_failure_persists_failed_status_without_generated_flag(self): + owner = _Owner() + owner.r1fs.add_json.side_effect = None + owner.r1fs.add_json.return_value = None + + result = generate_rulebook_assessment(owner, "job-1") + + self.assertEqual(result["status"], "error") + self.assertEqual(result["error"], "artifact_write_failed") + meta = owner.job_specs["rulebook_assessments"][DEFAULT_RULEBOOK_PROFILE_ID] + self.assertEqual(meta["run_state"], "failed") + self.assertTrue(meta["auto_enabled"]) + self.assertNotIn("artifact_cid", meta) + status = get_rulebook_assessment_status(owner, "job-1") + self.assertFalse(status["generated"]) + self.assertEqual(status["run_state"], "failed") + self.assertEqual(status["last_error"]["error"], "artifact_write_failed") + + def test_reviewer_answer_cannot_hide_automated_gap_but_can_support_manual_check(self): + owner = _Owner() + + review = update_rulebook_review( + owner, + "job-1", + answers={ + "nis2.access.access_controls_reviewed": {"value": "yes", "note": "Reviewed."}, + "nis2.bcm.business_continuity": {"value": "yes", "note": "Plan exists."}, + }, + reviewer="alice", + review_state="reviewed", + ) + self.assertEqual(review["status"], "ok") + self.assertEqual(review["audit"][-1]["changed_question_ids"], [ + "nis2.access.access_controls_reviewed", + "nis2.bcm.business_continuity", + ]) + + result = build_rulebook_assessment(owner, "job-1") + checks = {check["check_id"]: check for check in result["assessment"]["checks"]} + + self.assertEqual(checks["NIS2-21-ACCESS-001"]["automated_status"], "gap") + self.assertEqual(checks["NIS2-21-ACCESS-001"]["status"], "gap") + self.assertEqual(checks["NIS2-21-ACCESS-001"]["source"], "mixed") + self.assertEqual(checks["NIS2-21-BCM-001"]["automated_status"], "not_observable") + self.assertEqual(checks["NIS2-21-BCM-001"]["status"], "supported") + self.assertEqual(checks["NIS2-21-BCM-001"]["source"], "reviewer") + + review_payload = get_rulebook_review(owner, "job-1") + self.assertTrue(review_payload["found"]) + self.assertEqual(review_payload["review"]["review_state"], "reviewed") + + def test_invalid_review_answer_is_rejected(self): + owner = _Owner() + + result = update_rulebook_review( + owner, + "job-1", + answers={"nis2.unknown": {"value": "yes"}}, + ) + + self.assertEqual(result["status"], "error") + self.assertEqual(result["error"], "invalid_review_answer") + + def test_ineligible_jobs_fail_cleanly(self): + model_owner = _Owner(job_specs=_sample_job_specs(job_type="model_test", scan_type="model_test")) + running_owner = _Owner(job_specs=_sample_job_specs(job_status="RUNNING", job_cid="")) + missing_pass_owner = _Owner(archive={**_sample_archive(), "passes": []}) + + self.assertEqual( + build_rulebook_assessment(model_owner, "job-1")["error"], + "model_test_not_supported", + ) + self.assertEqual( + build_rulebook_assessment(running_owner, "job-1")["error"], + "no_completed_passes", + ) + self.assertEqual( + update_rulebook_review(running_owner, "job-1", answers={})["error"], + "job_not_finalized", + ) + self.assertEqual( + build_rulebook_assessment(missing_pass_owner, "job-1")["error"], + "no_completed_passes", + ) + + def test_purge_removes_rulebook_artifact_and_review_rows(self): + owner = _Owner(job_specs=_sample_job_specs( + job_cid="", + rulebook_assessments={ + DEFAULT_RULEBOOK_PROFILE_ID: { + "artifact_cid": "QmRulebookAssessment", + "history": [{"artifact_cid": "QmRulebookAssessmentOld"}], + }, + }, + )) + owner.records[(owner.cfg_instance_id, "job-1")] = owner.job_specs + review_key = f"job-1:{DEFAULT_RULEBOOK_PROFILE_ID}" + owner.records[(f"{owner.cfg_instance_id}:rulebook_review", review_key)] = {"job_id": "job-1"} + owner.records[(f"{owner.cfg_instance_id}:rulebook_review:audit", review_key)] = [{"job_id": "job-1"}] + owner.records[(f"{owner.cfg_instance_id}:rulebook_review:submissions", review_key)] = { + "contract_version": "1.0.0", + "latest_revision": 2, + "submissions": [ + {"revision": 1, "cid": "QmSubmission1"}, + {"revision": 2, "cid": "QmSubmission2"}, + ], + "pending": {"cid": "QmPendingSubmission"}, + } + + result = purge_job(owner, "job-1") + + self.assertEqual(result["status"], "success") + self.assertIn("QmRulebookAssessment", owner.artifact_repo.deleted) + self.assertIn("QmRulebookAssessmentOld", owner.artifact_repo.deleted) + self.assertIn("QmSubmission1", owner.artifact_repo.deleted) + self.assertIn("QmSubmission2", owner.artifact_repo.deleted) + self.assertIn("QmPendingSubmission", owner.artifact_repo.deleted) + self.assertIsNone(owner.records[(f"{owner.cfg_instance_id}:rulebook_review", review_key)]) + self.assertIsNone(owner.records[(f"{owner.cfg_instance_id}:rulebook_review:audit", review_key)]) + self.assertIsNone(owner.records[(f"{owner.cfg_instance_id}:rulebook_review:submissions", review_key)]) + + def test_purge_fails_closed_for_submission_cid_shared_with_another_job(self): + owner = _Owner(job_specs=_sample_job_specs(job_cid="")) + owner.records[(owner.cfg_instance_id, "job-1")] = owner.job_specs + hkey = f"{owner.cfg_instance_id}:rulebook_review:submissions" + owner.records[(hkey, f"job-1:{DEFAULT_RULEBOOK_PROFILE_ID}")] = { + "submissions": [{"revision": 1, "cid": "QmSharedSubmission"}], + } + owner.records[(hkey, f"job-2:{DEFAULT_RULEBOOK_PROFILE_ID}")] = { + "submissions": [{"revision": 1, "cid": "QmSharedSubmission"}], + } + + result = purge_job(owner, "job-1") + + self.assertEqual(result["status"], "partial") + self.assertEqual(result["cids_deleted"], 0) + self.assertNotIn("QmSharedSubmission", owner.artifact_repo.deleted) + self.assertIsNotNone(owner.records[(owner.cfg_instance_id, "job-1")]) + + def test_purge_clears_cstore_after_relay_acknowledges_formal_cid_delete(self): + owner = _Owner(job_specs=_sample_job_specs(job_cid="")) + owner.records[(owner.cfg_instance_id, "job-1")] = owner.job_specs + hkey = f"{owner.cfg_instance_id}:rulebook_review:submissions" + review_key = f"job-1:{DEFAULT_RULEBOOK_PROFILE_ID}" + owner.records[(hkey, review_key)] = { + "contract_version": "1.0.0", + "latest_revision": 1, + "submissions": [{"revision": 1, "cid": "QmRetainedSubmission"}], + } + owner.artifacts["QmRetainedSubmission"] = {"artifact_kind": "review_submission"} + owner.artifact_repo.delete = MagicMock(return_value=True) + owner.artifact_repo.get_json = MagicMock(return_value={"artifact_kind": "review_submission"}) + + result = purge_job(owner, "job-1") + + self.assertEqual(result["status"], "success") + owner.artifact_repo.delete.assert_any_call( + "QmRetainedSubmission", show_logs=True, raise_on_error=False, purge=True, + ) + self.assertIsNone(owner.records[(owner.cfg_instance_id, "job-1")]) + self.assertIsNone(owner.records[(hkey, review_key)]) + owner.artifact_repo.get_json.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/extensions/business/cybersec/red_mesh/tests/test_state_machine.py b/extensions/business/cybersec/red_mesh/tests/test_state_machine.py index 3c392b59d..374b83f43 100644 --- a/extensions/business/cybersec/red_mesh/tests/test_state_machine.py +++ b/extensions/business/cybersec/red_mesh/tests/test_state_machine.py @@ -47,6 +47,13 @@ def test_allows_analyzing_retry_to_collecting(self): self.assertEqual(job_specs["job_status"], JOB_STATUS_COLLECTING) + def test_allows_soft_stop_to_be_scheduled_during_analysis(self): + job_specs = {"job_status": JOB_STATUS_ANALYZING} + + set_job_status(job_specs, JOB_STATUS_SCHEDULED_FOR_STOP) + + self.assertEqual(job_specs["job_status"], JOB_STATUS_SCHEDULED_FOR_STOP) + def test_allows_finalizing_retry_to_collecting(self): job_specs = {"job_status": JOB_STATUS_FINALIZING} diff --git a/extensions/business/deeploy/deeploy_manager_api.py b/extensions/business/deeploy/deeploy_manager_api.py index 283747e95..5dc1eef2b 100644 --- a/extensions/business/deeploy/deeploy_manager_api.py +++ b/extensions/business/deeploy/deeploy_manager_api.py @@ -21,7 +21,6 @@ DEEPLOY_APP_COMMAND_REQUEST, DEEPLOY_GET_ORACLE_JOB_DETAILS_REQUEST, DEEPLOY_GET_R1FS_JOB_PIPELINE_REQUEST, DEEPLOY_NODE_SPECS_REQUEST, DEEPLOY_PLUGIN_DATA, JOB_APP_TYPES, JOB_APP_TYPES_ALL, DEEPLOY_GET_PREFERRED_NODES_REQUEST, DEEPLOY_SAVE_PREFERRED_NODES_REQUEST, - CONTAINERIZED_APPS_SIGNATURES, ) @@ -656,6 +655,33 @@ def save_preferred_nodes( return response + def _validate_service_update_plugin_instance_ids(self, inputs, job_app_type): + """ + Require stable plugin identity for every managed-service update entry. + + This runs before legacy ID backfill and after request/persisted job-type + reconciliation. Native updates remain free to submit new no-ID plugins. + """ + if job_app_type != JOB_APP_TYPES.SERVICE: + return True + + plugins_array = inputs.get(DEEPLOY_KEYS.PLUGINS) + missing_indexes = [] + for index, plugin_entry in enumerate(plugins_array or []): + if not isinstance(plugin_entry, dict): + continue + instance_id = plugin_entry.get(DEEPLOY_KEYS.PLUGIN_INSTANCE_ID) + if not instance_id or not str(instance_id).strip(): + missing_indexes.append(index) + + if missing_indexes: + raise ValueError( + f"{DEEPLOY_ERRORS.PLUGINS3}: Service update plugins must include instance_id " + f"after identity resolution. Missing for plugin indexes {missing_indexes}." + ) + return True + + def _process_pipeline_request( self, request: dict, @@ -668,7 +694,8 @@ def _process_pipeline_request( Parameters ---------- request : dict - The request dictionary + The request dictionary. Updates must include a valid `job_app_type`; + service updates must also include `instance_id` for every plugin. is_create : bool True for create operations, False for update operations async_mode : bool @@ -683,8 +710,25 @@ def _process_pipeline_request( self.__ensure_eth_balance() request_type = "create pipeline" if is_create else "update pipeline" sender, inputs = self.deeploy_verify_and_get_inputs(request, request_type=request_type) - normalized_request = self._normalize_plugins_input(self.deepcopy(request)) + normalized_request = self._normalize_plugins_input( + self.deepcopy(request), + preserve_legacy_instance_id=not is_create, + ) self._sync_normalized_plugins_input(inputs, normalized_request) + submitted_job_app_type = inputs.get(DEEPLOY_KEYS.JOB_APP_TYPE, None) + if not is_create: + if not isinstance(submitted_job_app_type, str) or not submitted_job_app_type.strip(): + raise ValueError( + f"{DEEPLOY_ERRORS.REQUEST3}. job_app_type is required for update requests." + ) + job_app_type = submitted_job_app_type.strip().lower() + if job_app_type not in JOB_APP_TYPES_ALL: + raise ValueError( + f"{DEEPLOY_ERRORS.REQUEST3}. Invalid job_app_type '{submitted_job_app_type}'. " + f"Expected one of {JOB_APP_TYPES_ALL}." + ) + else: + job_app_type = submitted_job_app_type auth_result = self.deeploy_get_auth_result(inputs) job_id = inputs.get(DEEPLOY_KEYS.JOB_ID, None) is_confirmable_job = inputs.chainstore_response @@ -702,18 +746,17 @@ def _process_pipeline_request( app_alias = inputs.app_alias app_type = inputs.pipeline_input_type - job_app_type = inputs.get(DEEPLOY_KEYS.JOB_APP_TYPE, None) - has_request_job_app_type = bool(job_app_type) - if job_app_type: - job_app_type = str(job_app_type).lower() - if job_app_type not in JOB_APP_TYPES_ALL: - raise ValueError(f"Invalid job_app_type '{job_app_type}'. Expected one of {JOB_APP_TYPES_ALL}.") - else: - plugins_for_detection = self.deeploy_prepare_plugins(inputs) - job_app_type = self.deeploy_detect_job_app_type(plugins_for_detection) - if job_app_type not in JOB_APP_TYPES_ALL: - job_app_type = JOB_APP_TYPES.NATIVE - self.P(f"Detected job app type: {job_app_type}") + if is_create: + if job_app_type: + job_app_type = str(job_app_type).lower() + if job_app_type not in JOB_APP_TYPES_ALL: + raise ValueError(f"Invalid job_app_type '{job_app_type}'. Expected one of {JOB_APP_TYPES_ALL}.") + else: + plugins_for_detection = self.deeploy_prepare_plugins(inputs) + job_app_type = self.deeploy_detect_job_app_type(plugins_for_detection) + if job_app_type not in JOB_APP_TYPES_ALL: + job_app_type = JOB_APP_TYPES.NATIVE + self.P(f"Resolved job app type: {job_app_type}") # persist job type so downstream mixins can adjust validations (e.g. native app resource checks) inputs[DEEPLOY_KEYS.JOB_APP_TYPE] = job_app_type inputs.job_app_type = job_app_type @@ -777,9 +820,8 @@ def _process_pipeline_request( current_nodes = pipeline_context["nodes"] deeploy_specs_for_update = pipeline_context["deeploy_specs"] self.P( - "Discovered plugin instances: {}".format( - self.json_dumps(self._redact_per_node_config_for_log(discovered_plugin_instances)) - ) + f"Discovered {len(discovered_plugin_instances)} live plugin instance record(s) " + f"for update job_id={job_id}, app_id={app_id}." ) requested_nodes = inputs.get(DEEPLOY_KEYS.TARGET_NODES, None) @@ -833,15 +875,6 @@ def _process_pipeline_request( except Exception as exc: self.Pd(f"Unable to read previous pipeline CID for job {job_id}: {exc}", color='y') - # Ensure plugin IDs are preserved for existing instances before any destructive action. - self._ensure_plugin_instance_ids( - inputs, - discovered_plugin_instances=discovered_plugin_instances, - owner=auth_result[DEEPLOY_KEYS.ESCROW_OWNER], - app_id=app_id, - job_id=job_id, - ) - if deeploy_specs_for_update is not None and not isinstance(deeploy_specs_for_update, dict): msg = ( f"{DEEPLOY_ERRORS.REQUEST3}. Unexpected 'deeploy_specs' payload type " @@ -862,7 +895,6 @@ def _process_pipeline_request( existing_job_app_type = existing_job_app_type.lower() if ( existing_job_app_type in JOB_APP_TYPES_ALL - and has_request_job_app_type and job_app_type != existing_job_app_type ): msg = ( @@ -873,43 +905,30 @@ def _process_pipeline_request( plugins_array = inputs.get(DEEPLOY_KEYS.PLUGINS) if isinstance(plugins_array, list): - materialized_plugins = self._materialize_update_plugins_for_redeploy( + # The submitted plugins array is the complete desired replacement. + # Discovery is used only for identity safety and drift diagnostics. + self._validate_plugins_array(plugins_array) + + self._validate_service_update_plugin_instance_ids(inputs, job_app_type) + if job_app_type != JOB_APP_TYPES.SERVICE: + # Legacy non-service updates may still identify an existing plugin by + # exact identity matching. Service identity is request-authoritative. + self._ensure_plugin_instance_ids( inputs, - discovered_plugin_instances, + discovered_plugin_instances=discovered_plugin_instances, + owner=auth_result[DEEPLOY_KEYS.ESCROW_OWNER], + app_id=app_id, + job_id=job_id, ) - inputs[DEEPLOY_KEYS.PLUGINS] = materialized_plugins - inputs.plugins = materialized_plugins - # Validate the exact replacement payload, including omitted live plugins - # that were materialized from discovery, before any payment/node/delete work. - self._validate_plugins_array(materialized_plugins) - - if not has_request_job_app_type: - replacement_job_app_type = deeploy_specs_payload.get(DEEPLOY_KEYS.JOB_APP_TYPE) - if isinstance(replacement_job_app_type, str): - replacement_job_app_type = replacement_job_app_type.lower() - if replacement_job_app_type in JOB_APP_TYPES_ALL: - job_app_type = replacement_job_app_type - else: - plugins_for_detection = self.deeploy_prepare_plugins(inputs) - job_app_type = self.deeploy_detect_job_app_type(plugins_for_detection) - has_containerized_replacement = any( - isinstance(plugin_entry, dict) and - isinstance(plugin_entry.get(DEEPLOY_KEYS.PLUGIN_SIGNATURE), str) and - plugin_entry.get(DEEPLOY_KEYS.PLUGIN_SIGNATURE).upper() in CONTAINERIZED_APPS_SIGNATURES - for plugin_entry in materialized_plugins - ) - if job_app_type == JOB_APP_TYPES.NATIVE and has_containerized_replacement: - msg = ( - f"{DEEPLOY_ERRORS.REQUEST3}. Update request omitted job_app_type and the live " - "deeploy_specs do not identify the replacement job_app_type for a containerized " - "replacement payload. Provide job_app_type explicitly." - ) - raise ValueError(msg) - if job_app_type not in JOB_APP_TYPES_ALL: - job_app_type = JOB_APP_TYPES.NATIVE - inputs[DEEPLOY_KEYS.JOB_APP_TYPE] = job_app_type - inputs.job_app_type = job_app_type - self.P(f"Detected replacement job app type: {job_app_type}") + self._validate_update_plugin_identities( + inputs, + discovered_plugin_instances=discovered_plugin_instances, + ) + self._warn_on_live_plugin_config_drift( + discovered_plugin_instances, + job_id=job_id, + app_id=app_id, + ) is_valid = self.deeploy_check_payment_and_job_owner(inputs, auth_result[DEEPLOY_KEYS.ESCROW_OWNER], is_create=is_create, debug=self.cfg_deeploy_verbose > 1) if not is_valid: @@ -1523,6 +1542,10 @@ def update_pipeline( job_id : int The job ID from blockchain + job_app_type : str + Required for every update. Must be one of generic, native, service, or stack. + The submitted value is authoritative and is not inferred from plugin configuration. + pipeline_params : dict, optional Additional pipeline-level parameters forwarded to the data capture thread. `null` falls back to `{}`. The provided keys are merged into the pipeline configuration at the top level. @@ -1538,16 +1561,21 @@ def update_pipeline( **Plugin instances:** plugins : list - Array of plugin instance configurations. Each object represents ONE plugin instance: + Complete desired replacement set. Each object represents ONE plugin instance: - plugin_signature : str (required) - instance_id : str (required when updating an existing plugin instance) + - Required on every submitted service plugin and never inferred by the backend - **instance-specific parameters** (payload merged into the instance configuration) - Omit instance_id to attach a brand new plugin instance; supported for native apps only + - Omit a live plugin from this array to remove it from the replacement deployment **Legacy format:** plugin_signature : str The signature of the single plugin. Legacy payloads without the plugins array are normalized internally; any deprecated `app_params` field is ignored in responses. + instance_id : str + Required at the top level for legacy service updates. The value is copied into the + normalized plugin and validated against the running instance. Returns ------- @@ -1558,7 +1586,7 @@ def update_pipeline( ----- - Existing pipelines are stopped and redeployed in place; requests must reference the active node set. - Updates are applied to existing plugin instances on the same nodes - - For multi-plugin pipelines, all plugins are updated with new configurations + - The plugins array is a full replacement, not a partial patch; omitted live plugins are removed - Resource validation applies the same as create operations - The simplified plugins array format is the same as create_pipeline - New plugin instances can be introduced by omitting `instance_id` (native job type only) diff --git a/extensions/business/deeploy/deeploy_mixin.py b/extensions/business/deeploy/deeploy_mixin.py index 6959dd677..b66728a0a 100644 --- a/extensions/business/deeploy/deeploy_mixin.py +++ b/extensions/business/deeploy/deeploy_mixin.py @@ -1894,13 +1894,20 @@ def _validate_send_instance_command_request(self, inputs): return - def _normalize_plugins_input(self, request: dict): + def _normalize_plugins_input( + self, + request: dict, + *, + preserve_legacy_instance_id: bool = False, + ): """ Normalize plugin input to always use the plugins array format. Converts legacy single-plugin format (plugin_signature + app_params) to new multi-plugin format. Args: request (dict): The request dictionary + preserve_legacy_instance_id (bool): Copy a canonical top-level identity into a + synthesized legacy plugin. Update processing enables this; create processing does not. Returns: dict: Request with normalized plugins array (simple format: each object is a plugin instance) @@ -1948,6 +1955,17 @@ def _normalize_plugins_input(self, request: dict): DEEPLOY_KEYS.PLUGIN_SIGNATURE: plugin_signature, **app_params } + if preserve_legacy_instance_id and DEEPLOY_KEYS.PLUGIN_INSTANCE_ID in request: + top_level_instance_id = request.get(DEEPLOY_KEYS.PLUGIN_INSTANCE_ID) + if ( + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID in app_params + and app_params.get(DEEPLOY_KEYS.PLUGIN_INSTANCE_ID) != top_level_instance_id + ): + raise ValueError( + f"{DEEPLOY_ERRORS.REQUEST3}. Conflicting legacy instance_id values were provided " + "at the top level and in app_params." + ) + plugin_instance[DEEPLOY_KEYS.PLUGIN_INSTANCE_ID] = top_level_instance_id request_key, request_config = self._get_request_per_node_config(request) if request_key is not None: existing_keys = [ @@ -2586,206 +2604,211 @@ def _remove_consumed_new_plugin_config(new_plugin_configs, plugin_config): return True return False - def _materialize_update_plugins_for_redeploy(self, inputs, discovered_plugin_instances): + def _validate_update_plugin_identities(self, inputs, discovered_plugin_instances): """ - Build a full plugin request array for delete/redeploy updates. + Validate explicit requested plugin IDs against discovered live identities. - Update preflight can reconstruct omitted live plugin configs from discovery. - The post-delete create path only sees inputs, so partial update requests must - be expanded before deletion to keep the deployed replacement equivalent to - the validated update payload. + Plugins without an ID are new instances. Legacy existing plugins have their + IDs backfilled before this method is called. """ - requested_by_instance_id, requested_by_signature, new_plugin_configs = self._organize_requested_plugins(inputs) - materialized_plugins = [] + requested_by_instance_id, _, _ = self._organize_requested_plugins(inputs) + discovered_by_instance_id = self.defaultdict(list) + for plugin in discovered_plugin_instances or []: + instance_id = plugin.get(DEEPLOY_PLUGIN_DATA.INSTANCE_ID) + if instance_id: + discovered_by_instance_id[str(instance_id)].append(plugin) - instance_id_key = self.ct.BIZ_PLUGIN_DATA.INSTANCE_ID - chainstore_response_key = self.ct.BIZ_PLUGIN_DATA.CHAINSTORE_RESPONSE_KEY - chainstore_peers_key = self.ct.BIZ_PLUGIN_DATA.CHAINSTORE_PEERS - discovered_records = [] - discovered_by_key = {} - config_occurrences_by_node = {} - nameless_record_counts = self.defaultdict(int) - - def get_plugin_name_from_conf(discovered_plugin, extracted_config): - return ( - extracted_config.get(DEEPLOY_KEYS.PLUGIN_NAME) - or discovered_plugin.get(DEEPLOY_KEYS.PLUGIN_NAME) + for instance_id, requested_plugin in requested_by_instance_id.items(): + discovered = discovered_by_instance_id.get(instance_id, []) + if not discovered: + raise ValueError( + f"{DEEPLOY_ERRORS.PLUGINS3}: Unknown plugin instance_id(s) in update request: " + f"{[instance_id]}" + ) + + requested_signature = ( + requested_plugin.get(DEEPLOY_KEYS.PLUGIN_SIGNATURE) + or requested_plugin.get("signature") ) + normalized_requested_signature = ( + requested_signature.upper() + if isinstance(requested_signature, str) + else requested_signature + ) + observed_signatures = { + signature.upper() if isinstance(signature, str) else signature + for signature in ( + plugin.get(DEEPLOY_PLUGIN_DATA.PLUGIN_SIGNATURE) + for plugin in discovered + ) + if signature + } + if observed_signatures != {normalized_requested_signature}: + raise ValueError( + f"{DEEPLOY_ERRORS.PLUGINS3}: Plugin instance_id '{instance_id}' cannot be reused " + f"with signature '{requested_signature}'." + ) + return True - def get_discovered_materialization_key(discovered_plugin, signature, instance_id, extracted_config): - normalized_sig = signature.upper() if isinstance(signature, str) else signature - if instance_id: - return (normalized_sig, "instance_id", str(instance_id)) + @staticmethod + def _bounded_drift_log_values(values, max_items=8, max_length=80): + bounded = [] + for value in sorted({str(item) for item in values if item is not None}): + cleaned = "".join(char if char.isprintable() else "?" for char in value) + if len(cleaned) > max_length: + cleaned = cleaned[:max_length - 3] + "..." + bounded.append(cleaned) + if len(bounded) >= max_items: + break + return bounded - plugin_name = get_plugin_name_from_conf(discovered_plugin, extracted_config) - if plugin_name: - return (normalized_sig, "plugin_name", str(plugin_name)) + @staticmethod + def _bounded_drift_log_message(message, max_length=2000): + if len(message) <= max_length: + return message + suffix = "...[truncated]" + return message[:max_length - len(suffix)] + suffix + + @classmethod + def _differing_config_paths(cls, configs, max_paths=20, max_depth=8): + if len(configs) < 2: + return [] - config_hash = compact_canonical_sha256(extracted_config) - node = discovered_plugin.get(DEEPLOY_PLUGIN_DATA.NODE, "") - occurrence_bucket = (node, normalized_sig, config_hash) - occurrence_idx = config_occurrences_by_node.get(occurrence_bucket, 0) - config_occurrences_by_node[occurrence_bucket] = occurrence_idx + 1 - return (normalized_sig, "config", config_hash, occurrence_idx) + missing = object() + differing_paths = [] - for plugin in discovered_plugin_instances: - signature = plugin.get(DEEPLOY_PLUGIN_DATA.PLUGIN_SIGNATURE) - if not signature: - continue + def add_path(parts): + if len(differing_paths) >= max_paths: + return + if not parts: + differing_paths.append("$") + return + formatted = "$" + for part in parts: + if isinstance(part, int): + formatted += f"[{part}]" + continue + segment = "".join(char if char.isprintable() else "?" for char in str(part)) + if len(segment) > 64: + segment = segment[:61] + "..." + formatted += f".{segment}" + differing_paths.append(formatted) + + def walk(values, parts, depth): + if len(differing_paths) >= max_paths: + return + first = values[0] + if all(value == first for value in values[1:]): + return + if depth >= max_depth or missing in values: + add_path(parts) + return + if all(isinstance(value, dict) for value in values): + keys = sorted( + {key for value in values for key in value}, + key=lambda key: str(key), + ) + for key in keys: + walk([value.get(key, missing) for value in values], parts + [key], depth + 1) + return + if all(isinstance(value, list) for value in values): + lengths = {len(value) for value in values} + if len(lengths) != 1: + add_path(parts) + return + for idx in range(len(values[0])): + walk([value[idx] for value in values], parts + [idx], depth + 1) + return + add_path(parts) - normalized_signature = signature.upper() if isinstance(signature, str) else signature + walk(configs, [], 0) + return differing_paths + + def _warn_on_live_plugin_config_drift( + self, + discovered_plugin_instances, + job_id=None, + app_id=None, + max_groups=10, + ): + """ + Warn about differing live replica configs without logging config values. + + Discovery is an oracle signal only. Drift never changes or blocks the + request-authoritative replacement payload. + """ + instance_id_key = self.ct.BIZ_PLUGIN_DATA.INSTANCE_ID + chainstore_response_key = self.ct.BIZ_PLUGIN_DATA.CHAINSTORE_RESPONSE_KEY + chainstore_peers_key = self.ct.BIZ_PLUGIN_DATA.CHAINSTORE_PEERS + grouped = self.defaultdict(list) + + for plugin in discovered_plugin_instances or []: + signature = plugin.get(DEEPLOY_PLUGIN_DATA.PLUGIN_SIGNATURE) instance_id = plugin.get(DEEPLOY_PLUGIN_DATA.INSTANCE_ID) - if instance_id: - instance_id = str(instance_id) + if not instance_id: + continue extracted_config = self._extract_discovered_plugin_conf( plugin, instance_id_key=instance_id_key, chainstore_response_key=chainstore_response_key, chainstore_peers_key=chainstore_peers_key, ) - plugin_name = get_plugin_name_from_conf(plugin, extracted_config) - config_hash = compact_canonical_sha256(extracted_config) - materialization_key = get_discovered_materialization_key( - plugin, - signature, - instance_id, - extracted_config, - ) - - compatibility_key = (normalized_signature, str(plugin_name or ""), config_hash) - existing_record = discovered_by_key.get(materialization_key) - if existing_record is not None: - if existing_record["compatibility_key"] != compatibility_key: - raise ValueError( - f"{DEEPLOY_ERRORS.PLUGINS3}: Corrupt live discovery for plugin identity " - f"{materialization_key}. Incompatible plugin instances were reported." - ) - continue - - record = { - "key": materialization_key, - "compatibility_key": compatibility_key, + grouped[str(instance_id)].append({ + "node": plugin.get(DEEPLOY_PLUGIN_DATA.NODE), "signature": signature, - "normalized_signature": normalized_signature, - "instance_id": instance_id, - "plugin_name": str(plugin_name) if plugin_name else None, + "plugin_name": ( + extracted_config.get(DEEPLOY_KEYS.PLUGIN_NAME) + or plugin.get(DEEPLOY_KEYS.PLUGIN_NAME) + ), "config": extracted_config, - "config_hash": config_hash, - } - discovered_by_key[materialization_key] = record - discovered_records.append(record) + }) - for record in discovered_records: - if record["instance_id"] or record["plugin_name"]: + warned_groups = 0 + drift_group_count = 0 + for instance_id, replicas in sorted(grouped.items()): + differing_paths = self._differing_config_paths( + [replica["config"] for replica in replicas], + ) + if not differing_paths: + continue + drift_group_count += 1 + if warned_groups >= max_groups: continue - nameless_match_key = (record["normalized_signature"], record["config_hash"]) - nameless_record_counts[nameless_match_key] += 1 - - def candidate_has_instance_id(candidate): - return bool( - candidate.get(DEEPLOY_KEYS.PLUGIN_INSTANCE_ID) - or candidate.get("instance_id") - or candidate.get(instance_id_key) + warned_groups += 1 + nodes = self._bounded_drift_log_values(replica["node"] for replica in replicas) + signatures = self._bounded_drift_log_values(replica["signature"] for replica in replicas) + plugin_names = self._bounded_drift_log_values(replica["plugin_name"] for replica in replicas) + safe_instance_id = self._bounded_drift_log_values([instance_id], max_items=1)[0] + safe_job_id = self._bounded_drift_log_values([job_id], max_items=1) + safe_app_id = self._bounded_drift_log_values([app_id], max_items=1) + warning = ( + "Deeploy live replica config drift: " + f"job_id={safe_job_id[0] if safe_job_id else None}, " + f"app_id={safe_app_id[0] if safe_app_id else None}, " + f"plugin_instance_id={safe_instance_id}, " + f"nodes={nodes}, observed_signatures={signatures}, " + f"observed_plugin_names={plugin_names}, differing_paths={differing_paths}. " + "The submitted update remains authoritative." ) - - def consume_signature_candidate(record): - candidate_list = requested_by_signature.get(record["normalized_signature"], []) - if not candidate_list: - return None - - if record["plugin_name"]: - candidates = [ - candidate for candidate in candidate_list - if not candidate_has_instance_id(candidate) - and candidate.get(DEEPLOY_KEYS.PLUGIN_NAME) == record["plugin_name"] - ] - if len(candidates) > 1: - raise ValueError( - f"{DEEPLOY_ERRORS.PLUGINS3}: Ambiguous update request for plugin_name " - f"'{record['plugin_name']}'." - ) - else: - match_key = (record["normalized_signature"], record["config_hash"]) - candidates = [] - for candidate in candidate_list: - if candidate_has_instance_id(candidate) or candidate.get(DEEPLOY_KEYS.PLUGIN_NAME): - continue - requested_conf = self._extract_plugin_request_conf( - candidate, - instance_id_key=instance_id_key, - chainstore_response_key=chainstore_response_key, - chainstore_peers_key=chainstore_peers_key, - ) - if self._plugin_update_request_matches_identity( - requested_conf, - discovered_plugin_name=record["plugin_name"], - discovered_config_hash=record["config_hash"], - ): - candidates.append(candidate) - - if candidates and nameless_record_counts[match_key] > 1: - self._raise_ambiguous_plugin_update_match( - requested_name=None, - signature=record["signature"], - ) - - self._validate_single_plugin_update_match( - candidates, - requested_name=record["plugin_name"], - signature=record["signature"], + self.P( + self._bounded_drift_log_message(warning), + color='y', ) - if not candidates: - return None - - plugin_config = candidates[0] - for idx, candidate in enumerate(candidate_list): - if candidate is plugin_config: - candidate_list.pop(idx) - break - self._remove_consumed_new_plugin_config(new_plugin_configs, plugin_config) - return plugin_config - - for record in discovered_records: - instance_id = record["instance_id"] - plugin_config = None - - if instance_id: - plugin_config = requested_by_instance_id.pop(instance_id, None) - candidate_list = requested_by_signature.get(record["normalized_signature"], []) - if plugin_config and candidate_list: - for idx, candidate in enumerate(candidate_list): - if candidate is plugin_config: - candidate_list.pop(idx) - break - else: - plugin_config = consume_signature_candidate(record) - - if plugin_config: - plugin_entry = self.deepcopy(plugin_config) - plugin_entry.pop("signature", None) - else: - plugin_entry = self.deepcopy(record["config"]) - - plugin_entry[DEEPLOY_KEYS.PLUGIN_SIGNATURE] = record["signature"] - if instance_id: - plugin_entry[DEEPLOY_KEYS.PLUGIN_INSTANCE_ID] = instance_id - materialized_plugins.append(plugin_entry) - - if requested_by_instance_id: - missing_ids = list(requested_by_instance_id.keys()) - raise ValueError( - f"{DEEPLOY_ERRORS.PLUGINS3}: Unknown plugin instance_id(s) in update request: {missing_ids}" + if drift_group_count > max_groups: + safe_job_id = self._bounded_drift_log_values([job_id], max_items=1) + safe_app_id = self._bounded_drift_log_values([app_id], max_items=1) + summary = ( + "Additional Deeploy live replica drift groups omitted from logs: " + f"job_id={safe_job_id[0] if safe_job_id else None}, " + f"app_id={safe_app_id[0] if safe_app_id else None}, " + f"omitted_groups={drift_group_count - max_groups}." ) - - for plugin_config in new_plugin_configs: - plugin_entry = self.deepcopy(plugin_config) - if DEEPLOY_KEYS.PLUGIN_SIGNATURE not in plugin_entry and plugin_entry.get("signature"): - plugin_entry[DEEPLOY_KEYS.PLUGIN_SIGNATURE] = plugin_entry.get("signature") - plugin_entry.pop("signature", None) - materialized_plugins.append(plugin_entry) - - return materialized_plugins + self.P( + self._bounded_drift_log_message(summary), + color='y', + ) + return warned_groups def deeploy_check_payment_and_job_owner(self, inputs, owner, is_create, debug=False): """ diff --git a/extensions/business/deeploy/tests/test_update_requests.py b/extensions/business/deeploy/tests/test_update_requests.py index 36b7861a9..14d0ac29d 100644 --- a/extensions/business/deeploy/tests/test_update_requests.py +++ b/extensions/business/deeploy/tests/test_update_requests.py @@ -63,7 +63,7 @@ def _make_process_update_plugin(self, discovered_instances, nodes=None, deeploy_ DEEPLOY_KEYS.ERROR: str(exc), } plugin.deeploy_verify_and_get_inputs = lambda request, **kwargs: ("0xSender", make_inputs(**request)) - plugin._normalize_plugins_input = lambda request: request + plugin._normalize_plugins_input = lambda request, **kwargs: request plugin.deeploy_get_auth_result = lambda inputs: { DEEPLOY_KEYS.SENDER: "0xSender", DEEPLOY_KEYS.SENDER_ESCROW: "0xEscrow", @@ -78,7 +78,6 @@ def _make_process_update_plugin(self, discovered_instances, nodes=None, deeploy_ "deeploy_specs": deeploy_specs or {"job_id": 11}, } plugin._get_pipeline_from_cstore = lambda job_id: None - plugin._ensure_plugin_instance_ids = lambda *args, **kwargs: None plugin._check_nodes_availability = lambda inputs: nodes or ["node-1"] called = {"delete": 0, "deploy": 0, "deploy_kwargs": None, "queued": 0, "bc_update": 0} @@ -98,6 +97,200 @@ def check_and_deploy_pipelines(**kwargs): plugin._queue_pipeline_persistence = lambda state: called.__setitem__("queued", called["queued"] + 1) return plugin, called + def _make_four_replica_cockroach_update_fixture(self, plugin): + nodes = ["0xai_node_a", "0xai_node_b", "0xai_node_c", "0xai_node_d"] + instance_id = "CONTAINER_APP_3ab323" + runtime_config = { + plugin.ct.CONFIG_INSTANCE.K_INSTANCE_ID: instance_id, + "IMAGE": "ghcr.io/ratio1/deeploy-cockroachdb-service:main", + "CONTAINER_RESOURCES": {"cpu": 1, "memory": "2g", "storage": "0g"}, + "FIXED_SIZE_VOLUMES": { + "cockroach_data": {"SIZE": "8G", "MOUNTING_POINT": "/cockroach/cockroach-data"}, + }, + "ENV": { + "CRDB_DATABASE": "appdb", + "CRDB_USER": "app_user", + "CRDB_PASSWORD": "sanitized-password", + "CRDB_NODE_COUNT": "4", + "CRDB_HOSTNAMES": "roach1,roach2,roach3,roach4", + }, + "PER_NODE_TARGET_NODES": nodes, + } + discovered_instances = [ + { + DEEPLOY_PLUGIN_DATA.INSTANCE_ID: instance_id, + DEEPLOY_PLUGIN_DATA.PLUGIN_SIGNATURE: "CONTAINER_APP_RUNNER", + DEEPLOY_PLUGIN_DATA.NODE: node, + DEEPLOY_PLUGIN_DATA.PLUGIN_INSTANCE: { + "instance_conf": copy.deepcopy(runtime_config), + }, + } + for node in nodes + ] + request_plugin = make_plugin_entry( + "CONTAINER_APP_RUNNER", + instance_id=instance_id, + IMAGE=runtime_config["IMAGE"], + CONTAINER_RESOURCES=copy.deepcopy(runtime_config["CONTAINER_RESOURCES"]), + FIXED_SIZE_VOLUMES=copy.deepcopy(runtime_config["FIXED_SIZE_VOLUMES"]), + ENV=copy.deepcopy(runtime_config["ENV"]), + PER_NODE_CONFIG={ + "byNode": { + node: {"ENV": {"CF_TUNNEL_TOKEN": f"sanitized-token-{index + 1}"}} + for index, node in enumerate(nodes) + }, + }, + ) + return nodes, discovered_instances, request_plugin + + def _make_legacy_service_update_request( + self, + nodes, + request_plugin, + top_level_instance_id=None, + nested_instance_id=None, + ): + app_params = { + key: copy.deepcopy(value) + for key, value in request_plugin.items() + if key not in (DEEPLOY_KEYS.PLUGIN_SIGNATURE, DEEPLOY_KEYS.PLUGIN_INSTANCE_ID) + } + if nested_instance_id is not None: + app_params[DEEPLOY_KEYS.PLUGIN_INSTANCE_ID] = nested_instance_id + request = { + DEEPLOY_KEYS.APP_ID: "cockroachdb_422ce92", + DEEPLOY_KEYS.APP_ALIAS: "cockroachdb", + DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.JOB_APP_TYPE: JOB_APP_TYPES.SERVICE, + DEEPLOY_KEYS.PIPELINE_INPUT_TYPE: "void", + DEEPLOY_KEYS.CHAINSTORE_RESPONSE: False, + DEEPLOY_KEYS.TARGET_NODES: nodes, + DEEPLOY_KEYS.TARGET_NODES_COUNT: len(nodes), + DEEPLOY_KEYS.PLUGIN_SIGNATURE: request_plugin[DEEPLOY_KEYS.PLUGIN_SIGNATURE], + DEEPLOY_KEYS.APP_PARAMS: app_params, + } + if top_level_instance_id is not None: + request[DEEPLOY_KEYS.PLUGIN_INSTANCE_ID] = top_level_instance_id + return request + + def test_normalize_legacy_update_copies_top_level_instance_id(self): + plugin = make_deeploy_plugin() + request = { + DEEPLOY_KEYS.PLUGIN_SIGNATURE: "CONTAINER_APP_RUNNER", + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID: "CONTAINER_APP_3ab323", + DEEPLOY_KEYS.APP_PARAMS: {"IMAGE": "repo/app:2.0"}, + } + + normalized = plugin._normalize_plugins_input( + plugin.deepcopy(request), + preserve_legacy_instance_id=True, + ) + + self.assertEqual( + normalized[DEEPLOY_KEYS.PLUGINS][0][DEEPLOY_KEYS.PLUGIN_INSTANCE_ID], + "CONTAINER_APP_3ab323", + ) + + def test_normalize_legacy_identity_duplicate_and_create_compatibility(self): + plugin = make_deeploy_plugin() + matching_request = { + DEEPLOY_KEYS.PLUGIN_SIGNATURE: "CONTAINER_APP_RUNNER", + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID: "CONTAINER_APP_3ab323", + DEEPLOY_KEYS.APP_PARAMS: { + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID: "CONTAINER_APP_3ab323", + "IMAGE": "repo/app:2.0", + }, + } + + normalized_update = plugin._normalize_plugins_input( + plugin.deepcopy(matching_request), + preserve_legacy_instance_id=True, + ) + normalized_create = plugin._normalize_plugins_input(plugin.deepcopy({ + DEEPLOY_KEYS.PLUGIN_SIGNATURE: "CONTAINER_APP_RUNNER", + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID: "CONTAINER_APP_3ab323", + DEEPLOY_KEYS.APP_PARAMS: {"IMAGE": "repo/app:2.0"}, + })) + nested_only = plugin._normalize_plugins_input( + plugin.deepcopy({ + DEEPLOY_KEYS.PLUGIN_SIGNATURE: "CONTAINER_APP_RUNNER", + DEEPLOY_KEYS.APP_PARAMS: { + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID: "CONTAINER_APP_3ab323", + "IMAGE": "repo/app:2.0", + }, + }), + preserve_legacy_instance_id=True, + ) + modern_plugins = plugin._normalize_plugins_input( + plugin.deepcopy({ + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID: "IGNORED_TOP_LEVEL_ID", + DEEPLOY_KEYS.PLUGINS: [ + make_plugin_entry( + "CONTAINER_APP_RUNNER", + instance_id="CONTAINER_APP_3ab323", + IMAGE="repo/app:2.0", + ), + ], + }), + preserve_legacy_instance_id=True, + ) + + self.assertEqual( + normalized_update[DEEPLOY_KEYS.PLUGINS][0][DEEPLOY_KEYS.PLUGIN_INSTANCE_ID], + "CONTAINER_APP_3ab323", + ) + self.assertNotIn( + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID, + normalized_create[DEEPLOY_KEYS.PLUGINS][0], + ) + self.assertEqual( + nested_only[DEEPLOY_KEYS.PLUGINS][0][DEEPLOY_KEYS.PLUGIN_INSTANCE_ID], + "CONTAINER_APP_3ab323", + ) + self.assertEqual( + modern_plugins[DEEPLOY_KEYS.PLUGINS][0][DEEPLOY_KEYS.PLUGIN_INSTANCE_ID], + "CONTAINER_APP_3ab323", + ) + + def test_normalize_legacy_update_rejects_conflicting_instance_ids(self): + plugin = make_deeploy_plugin() + request = { + DEEPLOY_KEYS.PLUGIN_SIGNATURE: "CONTAINER_APP_RUNNER", + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID: "CONTAINER_APP_3ab323", + DEEPLOY_KEYS.APP_PARAMS: { + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID: "CONTAINER_APP_other", + "IMAGE": "repo/app:2.0", + }, + } + + with self.assertRaisesRegex(ValueError, DEEPLOY_ERRORS.REQUEST3): + plugin._normalize_plugins_input( + request, + preserve_legacy_instance_id=True, + ) + + def test_normalize_legacy_update_preserves_blank_and_null_identity_for_validation(self): + plugin = make_deeploy_plugin() + for submitted_instance_id in ("", None): + with self.subTest(submitted_instance_id=submitted_instance_id): + normalized = plugin._normalize_plugins_input( + { + DEEPLOY_KEYS.PLUGIN_SIGNATURE: "CONTAINER_APP_RUNNER", + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID: submitted_instance_id, + DEEPLOY_KEYS.APP_PARAMS: {"IMAGE": "repo/app:2.0"}, + }, + preserve_legacy_instance_id=True, + ) + + self.assertIn( + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID, + normalized[DEEPLOY_KEYS.PLUGINS][0], + ) + self.assertEqual( + normalized[DEEPLOY_KEYS.PLUGINS][0][DEEPLOY_KEYS.PLUGIN_INSTANCE_ID], + submitted_instance_id, + ) + def test_prepare_single_plugin_instance_update_uses_plugin_config_and_strips_signature_fields(self): plugin = make_deeploy_plugin() @@ -120,6 +313,421 @@ def test_prepare_single_plugin_instance_update_uses_plugin_config_and_strips_sig self.assertNotIn(DEEPLOY_KEYS.PLUGIN_SIGNATURE, instance) self.assertNotIn("signature", instance) + def test_process_service_update_preserves_four_replica_identity_and_storage(self): + fixture_plugin = make_deeploy_plugin() + nodes, discovered_instances, request_plugin = self._make_four_replica_cockroach_update_fixture(fixture_plugin) + plugin, called = self._make_process_update_plugin( + discovered_instances=discovered_instances, + nodes=nodes, + deeploy_specs={ + DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.JOB_APP_TYPE: JOB_APP_TYPES.SERVICE, + DEEPLOY_KEYS.CURRENT_TARGET_NODES: nodes, + }, + ) + + response = plugin._process_pipeline_request( + { + DEEPLOY_KEYS.APP_ID: "cockroachdb_422ce92", + DEEPLOY_KEYS.APP_ALIAS: "cockroachdb", + DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.JOB_APP_TYPE: " SERVICE ", + DEEPLOY_KEYS.PIPELINE_INPUT_TYPE: "void", + DEEPLOY_KEYS.CHAINSTORE_RESPONSE: False, + DEEPLOY_KEYS.TARGET_NODES: nodes, + DEEPLOY_KEYS.TARGET_NODES_COUNT: len(nodes), + DEEPLOY_KEYS.PLUGINS: [request_plugin], + }, + is_create=False, + async_mode=True, + ) + + self.assertEqual(response[DEEPLOY_KEYS.STATUS], DEEPLOY_STATUS.COMMAND_DELIVERED) + self.assertEqual(called["delete"], 1) + self.assertEqual(called["deploy"], 1) + self.assertEqual(called["deploy_kwargs"]["job_app_type"], JOB_APP_TYPES.SERVICE) + redeploy_inputs = called["deploy_kwargs"]["inputs"] + self.assertEqual(len(redeploy_inputs[DEEPLOY_KEYS.PLUGINS]), 1) + self.assertEqual( + redeploy_inputs[DEEPLOY_KEYS.PLUGINS][0][DEEPLOY_KEYS.PLUGIN_INSTANCE_ID], + "CONTAINER_APP_3ab323", + ) + self.assertEqual( + plugin._aggregate_container_resources(redeploy_inputs)["storage"], + "8192m", + ) + + prepared_plan = called["deploy_kwargs"]["prepared_create_deploy_plan"] + self.assertEqual(set(prepared_plan["node_plugins_by_addr"]), set(nodes)) + for node_plugins in prepared_plan["node_plugins_by_addr"].values(): + self.assertEqual(len(node_plugins), 1) + instance = node_plugins[0][plugin.ct.CONFIG_PLUGIN.K_INSTANCES][0] + self.assertEqual(instance[plugin.ct.CONFIG_INSTANCE.K_INSTANCE_ID], "CONTAINER_APP_3ab323") + self.assertEqual(instance["PER_NODE_TARGET_NODES"], nodes) + self.assertEqual(instance["CONTAINER_RESOURCES"]["storage"], "0g") + self.assertEqual(instance["FIXED_SIZE_VOLUMES"]["cockroach_data"]["SIZE"], "8G") + + def test_process_legacy_service_update_preserves_four_replica_identity_and_storage(self): + fixture_plugin = make_deeploy_plugin() + nodes, discovered_instances, request_plugin = self._make_four_replica_cockroach_update_fixture(fixture_plugin) + plugin, called = self._make_process_update_plugin( + discovered_instances=discovered_instances, + nodes=nodes, + deeploy_specs={ + DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.JOB_APP_TYPE: JOB_APP_TYPES.SERVICE, + DEEPLOY_KEYS.CURRENT_TARGET_NODES: nodes, + }, + ) + plugin._normalize_plugins_input = types.MethodType( + DeeployManagerApiPlugin._normalize_plugins_input, + plugin, + ) + request = self._make_legacy_service_update_request( + nodes, + request_plugin, + top_level_instance_id="CONTAINER_APP_3ab323", + ) + + response = plugin._process_pipeline_request( + request, + is_create=False, + async_mode=True, + ) + + self.assertEqual(response[DEEPLOY_KEYS.STATUS], DEEPLOY_STATUS.COMMAND_DELIVERED) + self.assertEqual(called["delete"], 1) + self.assertEqual(called["deploy"], 1) + redeploy_inputs = called["deploy_kwargs"]["inputs"] + self.assertEqual(len(redeploy_inputs[DEEPLOY_KEYS.PLUGINS]), 1) + self.assertEqual( + redeploy_inputs[DEEPLOY_KEYS.PLUGINS][0][DEEPLOY_KEYS.PLUGIN_INSTANCE_ID], + "CONTAINER_APP_3ab323", + ) + self.assertEqual(plugin._aggregate_container_resources(redeploy_inputs)["storage"], "8192m") + prepared_plan = called["deploy_kwargs"]["prepared_create_deploy_plan"] + self.assertEqual(set(prepared_plan["node_plugins_by_addr"]), set(nodes)) + prepared_ids = { + instance[plugin.ct.CONFIG_INSTANCE.K_INSTANCE_ID] + for node_plugins in prepared_plan["node_plugins_by_addr"].values() + for node_plugin in node_plugins + for instance in node_plugin[plugin.ct.CONFIG_PLUGIN.K_INSTANCES] + } + self.assertEqual(prepared_ids, {"CONTAINER_APP_3ab323"}) + + def test_process_legacy_service_update_without_identity_fails_before_side_effects(self): + fixture_plugin = make_deeploy_plugin() + nodes, discovered_instances, request_plugin = self._make_four_replica_cockroach_update_fixture(fixture_plugin) + plugin, called = self._make_process_update_plugin( + discovered_instances=discovered_instances, + nodes=nodes, + deeploy_specs={ + DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.JOB_APP_TYPE: JOB_APP_TYPES.SERVICE, + DEEPLOY_KEYS.CURRENT_TARGET_NODES: nodes, + }, + ) + plugin._normalize_plugins_input = types.MethodType( + DeeployManagerApiPlugin._normalize_plugins_input, + plugin, + ) + phase_calls = defaultdict(int) + plugin._ensure_plugin_instance_ids = ( + lambda *args, **kwargs: phase_calls.__setitem__("backfill", phase_calls["backfill"] + 1) + ) + plugin.deeploy_check_payment_and_job_owner = ( + lambda *args, **kwargs: phase_calls.__setitem__("payment", phase_calls["payment"] + 1) or True + ) + plugin._prepare_create_pipeline_deploy_plan = ( + lambda **kwargs: phase_calls.__setitem__("preparation", phase_calls["preparation"] + 1) or {} + ) + plugin._reset_chainstore_response_keys = ( + lambda *args, **kwargs: phase_calls.__setitem__("reset", phase_calls["reset"] + 1) + ) + request = self._make_legacy_service_update_request(nodes, request_plugin) + + response = plugin._process_pipeline_request( + request, + is_create=False, + async_mode=True, + ) + + self.assertEqual(response[DEEPLOY_KEYS.STATUS], "failed") + self.assertIn(DEEPLOY_ERRORS.PLUGINS3, response[DEEPLOY_KEYS.ERROR]) + self.assertEqual(dict(phase_calls), {}) + self.assertEqual(called["delete"], 0) + self.assertEqual(called["deploy"], 0) + self.assertEqual(called["queued"], 0) + + def test_process_legacy_service_update_rejects_conflicting_identity_before_discovery(self): + fixture_plugin = make_deeploy_plugin() + nodes, discovered_instances, request_plugin = self._make_four_replica_cockroach_update_fixture(fixture_plugin) + plugin, called = self._make_process_update_plugin( + discovered_instances=discovered_instances, + nodes=nodes, + deeploy_specs={ + DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.JOB_APP_TYPE: JOB_APP_TYPES.SERVICE, + DEEPLOY_KEYS.CURRENT_TARGET_NODES: nodes, + }, + ) + plugin._normalize_plugins_input = types.MethodType( + DeeployManagerApiPlugin._normalize_plugins_input, + plugin, + ) + phase_calls = defaultdict(int) + plugin._gather_running_pipeline_context = ( + lambda **kwargs: phase_calls.__setitem__("discovery", phase_calls["discovery"] + 1) or {} + ) + plugin.deeploy_check_payment_and_job_owner = ( + lambda *args, **kwargs: phase_calls.__setitem__("payment", phase_calls["payment"] + 1) or True + ) + plugin._prepare_create_pipeline_deploy_plan = ( + lambda **kwargs: phase_calls.__setitem__("preparation", phase_calls["preparation"] + 1) or {} + ) + plugin._reset_chainstore_response_keys = ( + lambda *args, **kwargs: phase_calls.__setitem__("reset", phase_calls["reset"] + 1) + ) + request = self._make_legacy_service_update_request( + nodes, + request_plugin, + top_level_instance_id="CONTAINER_APP_3ab323", + nested_instance_id="CONTAINER_APP_other", + ) + + response = plugin._process_pipeline_request( + request, + is_create=False, + async_mode=True, + ) + + self.assertEqual(response[DEEPLOY_KEYS.STATUS], "failed") + self.assertIn(DEEPLOY_ERRORS.REQUEST3, response[DEEPLOY_KEYS.ERROR]) + self.assertIn("Conflicting legacy instance_id", response[DEEPLOY_KEYS.ERROR]) + self.assertEqual(dict(phase_calls), {}) + self.assertEqual(called["delete"], 0) + self.assertEqual(called["deploy"], 0) + self.assertEqual(called["queued"], 0) + + def test_process_update_without_job_app_type_fails_before_discovery_or_side_effects(self): + for persisted_job_app_type in (None, JOB_APP_TYPES.SERVICE): + with self.subTest(persisted_job_app_type=persisted_job_app_type): + fixture_plugin = make_deeploy_plugin() + nodes, discovered_instances, request_plugin = self._make_four_replica_cockroach_update_fixture( + fixture_plugin + ) + request_plugin.pop(DEEPLOY_KEYS.PLUGIN_INSTANCE_ID) + request_plugin["IMAGE"] = "ghcr.io/ratio1/deeploy-cockroachdb-service:review-repro" + deeploy_specs = { + DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.CURRENT_TARGET_NODES: nodes, + } + if persisted_job_app_type is not None: + deeploy_specs[DEEPLOY_KEYS.JOB_APP_TYPE] = persisted_job_app_type + plugin, called = self._make_process_update_plugin( + discovered_instances=discovered_instances, + nodes=nodes, + deeploy_specs=deeploy_specs, + ) + phase_calls = defaultdict(int) + + def gather_context(**kwargs): + phase_calls["discovery"] += 1 + return { + "discovered_instances": discovered_instances, + "nodes": nodes, + "deeploy_specs": deeploy_specs, + } + + plugin._gather_running_pipeline_context = gather_context + plugin.deeploy_check_payment_and_job_owner = ( + lambda *args, **kwargs: phase_calls.__setitem__("payment", phase_calls["payment"] + 1) or True + ) + plugin._check_nodes_availability = ( + lambda inputs: phase_calls.__setitem__("nodes", phase_calls["nodes"] + 1) or nodes + ) + plugin._prepare_create_pipeline_deploy_plan = ( + lambda **kwargs: phase_calls.__setitem__("preparation", phase_calls["preparation"] + 1) + or {"enable_chainstore_response": False, "response_keys": {}, "node_plugins_by_addr": {}} + ) + plugin._reset_chainstore_response_keys = ( + lambda *args, **kwargs: phase_calls.__setitem__("reset", phase_calls["reset"] + 1) + ) + + response = plugin._process_pipeline_request( + { + DEEPLOY_KEYS.APP_ID: "cockroachdb_422ce92", + DEEPLOY_KEYS.APP_ALIAS: "cockroachdb", + DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.PIPELINE_INPUT_TYPE: "void", + DEEPLOY_KEYS.CHAINSTORE_RESPONSE: True, + DEEPLOY_KEYS.TARGET_NODES: nodes, + DEEPLOY_KEYS.TARGET_NODES_COUNT: len(nodes), + DEEPLOY_KEYS.PLUGINS: [request_plugin], + }, + is_create=False, + async_mode=True, + ) + + self.assertEqual(response[DEEPLOY_KEYS.STATUS], "failed") + self.assertIn(DEEPLOY_ERRORS.REQUEST3, response[DEEPLOY_KEYS.ERROR]) + self.assertIn("job_app_type is required for update requests", response[DEEPLOY_KEYS.ERROR]) + self.assertEqual(dict(phase_calls), {}) + self.assertEqual(called["delete"], 0) + self.assertEqual(called["deploy"], 0) + self.assertEqual(called["queued"], 0) + + def test_process_update_rejects_blank_and_invalid_job_app_type_before_discovery(self): + for submitted_job_app_type in (" ", "unsupported"): + with self.subTest(submitted_job_app_type=submitted_job_app_type): + plugin, called = self._make_process_update_plugin(discovered_instances=[]) + discovery_calls = [] + plugin._gather_running_pipeline_context = lambda **kwargs: discovery_calls.append(kwargs) or {} + + response = plugin._process_pipeline_request( + { + DEEPLOY_KEYS.APP_ID: "app-123", + DEEPLOY_KEYS.APP_ALIAS: "app", + DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.JOB_APP_TYPE: submitted_job_app_type, + DEEPLOY_KEYS.PIPELINE_INPUT_TYPE: "void", + DEEPLOY_KEYS.CHAINSTORE_RESPONSE: False, + DEEPLOY_KEYS.TARGET_NODES: ["node-1"], + DEEPLOY_KEYS.TARGET_NODES_COUNT: 1, + DEEPLOY_KEYS.PLUGINS: [ + make_plugin_entry( + "CONTAINER_APP_RUNNER", + IMAGE="repo/app:2.0", + CONTAINER_RESOURCES={"cpu": 1, "memory": "256m", "storage": "1g"}, + ), + ], + }, + is_create=False, + async_mode=True, + ) + + self.assertEqual(response[DEEPLOY_KEYS.STATUS], "failed") + self.assertIn(DEEPLOY_ERRORS.REQUEST3, response[DEEPLOY_KEYS.ERROR]) + self.assertIn("job_app_type", response[DEEPLOY_KEYS.ERROR]) + self.assertEqual(discovery_calls, []) + self.assertEqual(called["delete"], 0) + self.assertEqual(called["deploy"], 0) + + def test_process_create_without_job_app_type_keeps_inference(self): + plugin, called = self._make_process_update_plugin( + discovered_instances=[], + nodes=["node-1"], + ) + + response = plugin._process_pipeline_request( + { + DEEPLOY_KEYS.APP_ALIAS: "app", + DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.PIPELINE_INPUT_TYPE: "void", + DEEPLOY_KEYS.CHAINSTORE_RESPONSE: False, + DEEPLOY_KEYS.TARGET_NODES: ["node-1"], + DEEPLOY_KEYS.TARGET_NODES_COUNT: 1, + DEEPLOY_KEYS.PLUGINS: [ + make_plugin_entry( + "CONTAINER_APP_RUNNER", + IMAGE="repo/app:1.0", + CONTAINER_RESOURCES={"cpu": 1, "memory": "256m", "storage": "1g"}, + ), + ], + }, + is_create=True, + async_mode=True, + ) + + self.assertEqual(response[DEEPLOY_KEYS.STATUS], DEEPLOY_STATUS.COMMAND_DELIVERED) + self.assertEqual(called["deploy"], 1) + self.assertEqual(called["deploy_kwargs"]["job_app_type"], JOB_APP_TYPES.GENERIC) + self.assertEqual( + called["deploy_kwargs"]["inputs"][DEEPLOY_KEYS.JOB_APP_TYPE], + JOB_APP_TYPES.GENERIC, + ) + + def test_process_legacy_create_ignores_top_level_instance_id(self): + plugin, called = self._make_process_update_plugin( + discovered_instances=[], + nodes=["node-1"], + ) + plugin._normalize_plugins_input = types.MethodType( + DeeployManagerApiPlugin._normalize_plugins_input, + plugin, + ) + + response = plugin._process_pipeline_request( + { + DEEPLOY_KEYS.APP_ALIAS: "app", + DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.PIPELINE_INPUT_TYPE: "void", + DEEPLOY_KEYS.CHAINSTORE_RESPONSE: False, + DEEPLOY_KEYS.TARGET_NODES: ["node-1"], + DEEPLOY_KEYS.TARGET_NODES_COUNT: 1, + DEEPLOY_KEYS.PLUGIN_SIGNATURE: "CONTAINER_APP_RUNNER", + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID: "CALLER_SUPPLIED_CREATE_ID", + DEEPLOY_KEYS.APP_PARAMS: { + "IMAGE": "repo/app:1.0", + "CONTAINER_RESOURCES": {"cpu": 1, "memory": "256m", "storage": "1g"}, + }, + }, + is_create=True, + async_mode=True, + ) + + self.assertEqual(response[DEEPLOY_KEYS.STATUS], DEEPLOY_STATUS.COMMAND_DELIVERED) + self.assertEqual(called["deploy"], 1) + self.assertNotIn( + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID, + called["deploy_kwargs"]["inputs"][DEEPLOY_KEYS.PLUGINS][0], + ) + + def test_process_service_update_without_resolved_id_fails_before_side_effects(self): + fixture_plugin = make_deeploy_plugin() + nodes, discovered_instances, request_plugin = self._make_four_replica_cockroach_update_fixture(fixture_plugin) + request_plugin.pop(DEEPLOY_KEYS.PLUGIN_INSTANCE_ID) + request_plugin["IMAGE"] = "repo/reconfigured-service:latest" + plugin, called = self._make_process_update_plugin( + discovered_instances=discovered_instances, + nodes=nodes, + deeploy_specs={ + DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.JOB_APP_TYPE: JOB_APP_TYPES.SERVICE, + DEEPLOY_KEYS.CURRENT_TARGET_NODES: nodes, + }, + ) + payment_calls = [] + reset_calls = [] + backfill_calls = [] + plugin.deeploy_check_payment_and_job_owner = lambda *args, **kwargs: payment_calls.append(args) or True + plugin._reset_chainstore_response_keys = lambda *args, **kwargs: reset_calls.append(args) + plugin._ensure_plugin_instance_ids = lambda *args, **kwargs: backfill_calls.append(args) + + response = plugin._process_pipeline_request( + { + DEEPLOY_KEYS.APP_ID: "cockroachdb_422ce92", + DEEPLOY_KEYS.APP_ALIAS: "cockroachdb", + DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.JOB_APP_TYPE: JOB_APP_TYPES.SERVICE, + DEEPLOY_KEYS.PIPELINE_INPUT_TYPE: "void", + DEEPLOY_KEYS.CHAINSTORE_RESPONSE: True, + DEEPLOY_KEYS.TARGET_NODES: nodes, + DEEPLOY_KEYS.TARGET_NODES_COUNT: len(nodes), + DEEPLOY_KEYS.PLUGINS: [request_plugin], + }, + is_create=False, + async_mode=True, + ) + + self.assertIn(DEEPLOY_ERRORS.PLUGINS3, response[DEEPLOY_KEYS.ERROR]) + self.assertIn("Service update plugins must include instance_id", response[DEEPLOY_KEYS.ERROR]) + self.assertEqual(backfill_calls, []) + self.assertEqual(payment_calls, []) + self.assertEqual(reset_calls, []) + self.assertEqual(called["delete"], 0) + self.assertEqual(called["deploy"], 0) + def test_prepare_single_plugin_instance_update_falls_back_to_instance_conf(self): plugin = make_deeploy_plugin() fallback_instance = { @@ -821,7 +1429,7 @@ def test_process_update_rejects_duplicate_instance_ids_before_delete(self): self.assertEqual(called["delete"], 0) self.assertEqual(called["deploy"], 0) - def test_process_update_materializes_omitted_live_plugins_before_redeploy(self): + def test_process_update_omits_unrequested_live_plugins_from_redeploy(self): plugin, called = self._make_process_update_plugin( discovered_instances=[ { @@ -878,17 +1486,11 @@ def test_process_update_materializes_omitted_live_plugins_before_redeploy(self): self.assertEqual(called["deploy"], 1) redeploy_inputs = called["deploy_kwargs"]["inputs"] redeploy_plugins = redeploy_inputs[DEEPLOY_KEYS.PLUGINS] - self.assertEqual(len(redeploy_plugins), 2) + self.assertEqual(len(redeploy_plugins), 1) self.assertEqual( {entry[DEEPLOY_KEYS.PLUGIN_INSTANCE_ID] for entry in redeploy_plugins}, - {"api-instance", "worker-instance"}, - ) - worker = next( - entry for entry in redeploy_plugins - if entry[DEEPLOY_KEYS.PLUGIN_INSTANCE_ID] == "worker-instance" + {"api-instance"}, ) - self.assertEqual(worker["IMAGE"], "repo/worker:1.0") - self.assertEqual(worker[DEEPLOY_KEYS.PLUGIN_NAME], "worker") prepared_plan = called["deploy_kwargs"]["prepared_create_deploy_plan"] self.assertIsNotNone(prepared_plan) prepared_instances = [ @@ -898,7 +1500,7 @@ def test_process_update_materializes_omitted_live_plugins_before_redeploy(self): ] self.assertEqual( {instance[plugin.ct.CONFIG_INSTANCE.K_INSTANCE_ID] for instance in prepared_instances}, - {"api-instance", "worker-instance"}, + {"api-instance"}, ) prepared_api = next( instance for instance in prepared_instances @@ -906,7 +1508,7 @@ def test_process_update_materializes_omitted_live_plugins_before_redeploy(self): ) self.assertEqual(prepared_api["PROCESS_DELAY"], 10) - def test_process_update_validates_materialized_replacement_payload_before_delete(self): + def test_process_update_validates_requested_replacement_payload_before_delete(self): plugin, called = self._make_process_update_plugin( discovered_instances=[ { @@ -948,7 +1550,7 @@ def assert_full_replacement_payload(inputs, context): for entry in plugins ] validation_calls.append((context, plugin_ids, inputs.get(DEEPLOY_KEYS.JOB_APP_TYPE))) - self.assertEqual(set(plugin_ids), {"api-instance", "worker-instance"}) + self.assertEqual(set(plugin_ids), {"api-instance"}) self.assertEqual(inputs.get(DEEPLOY_KEYS.JOB_APP_TYPE), "stack") def check_payment_and_owner(inputs, *args, **kwargs): @@ -967,6 +1569,7 @@ def check_nodes_availability(inputs): DEEPLOY_KEYS.APP_ID: "app-123", DEEPLOY_KEYS.APP_ALIAS: "app", DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.JOB_APP_TYPE: "stack", DEEPLOY_KEYS.PIPELINE_INPUT_TYPE: "void", DEEPLOY_KEYS.CHAINSTORE_RESPONSE: False, DEEPLOY_KEYS.TARGET_NODES: ["node-1"], @@ -1026,6 +1629,16 @@ def test_process_update_uses_persisted_pipeline_when_all_old_nodes_are_offline(s }, ], }, + { + plugin.ct.CONFIG_PLUGIN.K_SIGNATURE: "CONTAINER_APP_RUNNER", + plugin.ct.CONFIG_PLUGIN.K_INSTANCES: [ + { + plugin.ct.CONFIG_INSTANCE.K_INSTANCE_ID: "omitted-invalid-instance", + DEEPLOY_KEYS.PLUGIN_NAME: "omitted-invalid", + "CONTAINER_RESOURCES": {"cpu": 1, "memory": "256m"}, + }, + ], + }, ], } plugin._check_nodes_availability = lambda inputs: ["new-node-1"] @@ -1062,6 +1675,11 @@ def test_process_update_uses_persisted_pipeline_when_all_old_nodes_are_offline(s self.assertEqual(called["bc_update"], 1) self.assertEqual(called["deploy_kwargs"]["new_nodes"], ["new-node-1"]) redeploy_plugins = called["deploy_kwargs"]["inputs"][DEEPLOY_KEYS.PLUGINS] + self.assertEqual(len(redeploy_plugins), 1) + self.assertEqual( + redeploy_plugins[0][DEEPLOY_KEYS.PLUGIN_INSTANCE_ID], + "current-instance", + ) self.assertEqual(redeploy_plugins[0]["IMAGE"], "repo/app:2.0") def test_process_update_rejects_job_app_type_change_before_payment_or_delete(self): @@ -1122,7 +1740,7 @@ def test_process_update_rejects_job_app_type_change_before_payment_or_delete(sel self.assertEqual(called["deploy"], 0) self.assertEqual(called["queued"], 0) - def test_process_update_rejects_invalid_materialized_omitted_plugin_before_payment_and_delete(self): + def test_process_update_ignores_invalid_omitted_live_plugin_config(self): plugin, called = self._make_process_update_plugin( discovered_instances=[ { @@ -1181,13 +1799,20 @@ def test_process_update_rejects_invalid_materialized_omitted_plugin_before_payme async_mode=True, ) - self.assertIn("'IMAGE' field is required", response[DEEPLOY_KEYS.ERROR]) - self.assertEqual(payment_calls, []) - self.assertEqual(node_calls, []) - self.assertEqual(called["delete"], 0) - self.assertEqual(called["deploy"], 0) + self.assertEqual(response[DEEPLOY_KEYS.STATUS], "command_delivered") + self.assertEqual(len(payment_calls), 1) + self.assertEqual(len(node_calls), 1) + self.assertEqual(called["delete"], 1) + self.assertEqual(called["deploy"], 1) + self.assertEqual( + [ + entry[DEEPLOY_KEYS.PLUGIN_INSTANCE_ID] + for entry in called["deploy_kwargs"]["inputs"][DEEPLOY_KEYS.PLUGINS] + ], + ["api-instance"], + ) - def test_process_update_rejects_ambiguous_containerized_replacement_type_before_delete(self): + def test_process_update_uses_explicit_type_with_requested_replacement_only(self): plugin, called = self._make_process_update_plugin( discovered_instances=[ { @@ -1223,6 +1848,7 @@ def test_process_update_rejects_ambiguous_containerized_replacement_type_before_ DEEPLOY_KEYS.APP_ID: "app-123", DEEPLOY_KEYS.APP_ALIAS: "app", DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.JOB_APP_TYPE: "generic", DEEPLOY_KEYS.PIPELINE_INPUT_TYPE: "void", DEEPLOY_KEYS.CHAINSTORE_RESPONSE: False, DEEPLOY_KEYS.TARGET_NODES: ["node-1"], @@ -1241,10 +1867,13 @@ def test_process_update_rejects_ambiguous_containerized_replacement_type_before_ async_mode=True, ) - self.assertIn("omitted job_app_type", response[DEEPLOY_KEYS.ERROR]) - self.assertIn("containerized replacement payload", response[DEEPLOY_KEYS.ERROR]) - self.assertEqual(called["delete"], 0) - self.assertEqual(called["deploy"], 0) + self.assertEqual(response[DEEPLOY_KEYS.STATUS], "command_delivered") + self.assertEqual(called["delete"], 1) + self.assertEqual(called["deploy"], 1) + self.assertEqual( + called["deploy_kwargs"]["inputs"][DEEPLOY_KEYS.PLUGINS][0]["IMAGE"], + "repo/api:2.0", + ) def test_process_update_does_not_append_consumed_no_id_update_as_new_plugin(self): plugin, called = self._make_process_update_plugin( @@ -1293,7 +1922,7 @@ def test_process_update_does_not_append_consumed_no_id_update_as_new_plugin(self self.assertEqual(redeploy_plugins[0][DEEPLOY_KEYS.PLUGIN_NAME], "legacy") self.assertEqual(redeploy_plugins[0]["PROCESS_DELAY"], 10) - def test_process_update_dedupes_nameless_no_id_legacy_plugin_across_nodes(self): + def test_process_update_does_not_restore_nameless_legacy_plugins_across_nodes(self): discovered_instances = [ { DEEPLOY_PLUGIN_DATA.PLUGIN_SIGNATURE: "A_SIMPLE_PLUGIN", @@ -1369,21 +1998,11 @@ def test_process_update_dedupes_nameless_no_id_legacy_plugin_across_nodes(self): self.assertEqual(called["deploy"], 1) redeploy_plugins = called["deploy_kwargs"]["inputs"][DEEPLOY_KEYS.PLUGINS] - self.assertEqual(len(redeploy_plugins), 2) - api = next( - entry for entry in redeploy_plugins - if entry.get(DEEPLOY_KEYS.PLUGIN_INSTANCE_ID) == "api-instance" - ) - legacy = next( - entry for entry in redeploy_plugins - if entry.get(DEEPLOY_KEYS.PLUGIN_INSTANCE_ID) != "api-instance" - ) + self.assertEqual(len(redeploy_plugins), 1) + api = redeploy_plugins[0] self.assertEqual(api["PROCESS_DELAY"], 10) - self.assertEqual(legacy["PROCESS_DELAY"], 5) - self.assertNotIn(DEEPLOY_KEYS.PLUGIN_INSTANCE_ID, legacy) - self.assertNotIn(DEEPLOY_KEYS.PLUGIN_NAME, legacy) - def test_process_update_preserves_same_node_nameless_no_id_legacy_multiplicity(self): + def test_process_update_does_not_restore_same_node_nameless_legacy_plugins(self): discovered_instances = [ { DEEPLOY_PLUGIN_DATA.PLUGIN_SIGNATURE: "A_SIMPLE_PLUGIN", @@ -1448,18 +2067,11 @@ def test_process_update_preserves_same_node_nameless_no_id_legacy_multiplicity(s self.assertEqual(called["deploy"], 1) redeploy_plugins = called["deploy_kwargs"]["inputs"][DEEPLOY_KEYS.PLUGINS] - self.assertEqual(len(redeploy_plugins), 3) - legacy_plugins = [ - entry for entry in redeploy_plugins - if entry.get(DEEPLOY_KEYS.PLUGIN_INSTANCE_ID) != "api-instance" - ] - self.assertEqual(len(legacy_plugins), 2) - for legacy in legacy_plugins: - self.assertEqual(legacy["PROCESS_DELAY"], 5) - self.assertNotIn(DEEPLOY_KEYS.PLUGIN_INSTANCE_ID, legacy) - self.assertNotIn(DEEPLOY_KEYS.PLUGIN_NAME, legacy) + self.assertEqual(len(redeploy_plugins), 1) + self.assertEqual(redeploy_plugins[0][DEEPLOY_KEYS.PLUGIN_INSTANCE_ID], "api-instance") + self.assertEqual(redeploy_plugins[0]["PROCESS_DELAY"], 10) - def test_process_update_materializes_one_logical_plugin_set_for_multinode_redeploy(self): + def test_process_update_uses_requested_plugin_set_for_multinode_redeploy(self): discovered_instances = [ { DEEPLOY_PLUGIN_DATA.INSTANCE_ID: "api-instance", @@ -1543,52 +2155,129 @@ def test_process_update_materializes_one_logical_plugin_set_for_multinode_redepl redeploy_inputs = called["deploy_kwargs"]["inputs"] redeploy_plugins = redeploy_inputs[DEEPLOY_KEYS.PLUGINS] - self.assertEqual(len(redeploy_plugins), 2) + self.assertEqual(len(redeploy_plugins), 1) self.assertEqual( {entry[DEEPLOY_KEYS.PLUGIN_INSTANCE_ID] for entry in redeploy_plugins}, - {"api-instance", "worker-instance"}, + {"api-instance"}, ) - api = next( - entry for entry in redeploy_plugins - if entry[DEEPLOY_KEYS.PLUGIN_INSTANCE_ID] == "api-instance" - ) - worker = next( - entry for entry in redeploy_plugins - if entry[DEEPLOY_KEYS.PLUGIN_INSTANCE_ID] == "worker-instance" - ) + api = redeploy_plugins[0] self.assertEqual(api["PROCESS_DELAY"], 10) - self.assertEqual(worker["IMAGE"], "repo/worker:1.0") - self.assertEqual(worker[DEEPLOY_KEYS.PLUGIN_NAME], "worker") - def test_process_update_rejects_corrupt_duplicate_live_instance_id_before_delete(self): + def test_process_update_warns_on_live_replica_drift_and_uses_requested_config(self): plugin, called = self._make_process_update_plugin( discovered_instances=[ { DEEPLOY_PLUGIN_DATA.INSTANCE_ID: "shared-instance", - DEEPLOY_PLUGIN_DATA.PLUGIN_SIGNATURE: "A_SIMPLE_PLUGIN", + DEEPLOY_PLUGIN_DATA.PLUGIN_SIGNATURE: "WORKER_APP_RUNNER", DEEPLOY_PLUGIN_DATA.NODE: "node-1", DEEPLOY_PLUGIN_DATA.PLUGIN_INSTANCE: { "instance_conf": { - DEEPLOY_KEYS.PLUGIN_NAME: "api", - "PROCESS_DELAY": 5, + DEEPLOY_KEYS.PLUGIN_NAME: "worker", + "IMAGE": "node:22", + "CONTAINER_RESOURCES": {"cpu": 1, "memory": "256m"}, + "ENV": {"API_TOKEN": "live-secret-a", "MODE": "replica-a"}, }, }, }, { DEEPLOY_PLUGIN_DATA.INSTANCE_ID: "shared-instance", - DEEPLOY_PLUGIN_DATA.PLUGIN_SIGNATURE: "A_SIMPLE_PLUGIN", + DEEPLOY_PLUGIN_DATA.PLUGIN_SIGNATURE: "WORKER_APP_RUNNER", DEEPLOY_PLUGIN_DATA.NODE: "node-2", DEEPLOY_PLUGIN_DATA.PLUGIN_INSTANCE: { "instance_conf": { DEEPLOY_KEYS.PLUGIN_NAME: "worker", - "PROCESS_DELAY": 10, + "IMAGE": "node:22", + "CONTAINER_RESOURCES": {"cpu": 1, "memory": "256m"}, + "ENV": {"API_TOKEN": "live-secret-b", "MODE": "replica-b"}, }, }, }, ], nodes=["node-1", "node-2"], ) + log_lines = [] + plugin.P = lambda message, **kwargs: log_lines.append(message) + + response = plugin._process_pipeline_request( + { + DEEPLOY_KEYS.APP_ID: "app-123", + DEEPLOY_KEYS.APP_ALIAS: "app", + DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.JOB_APP_TYPE: "generic", + DEEPLOY_KEYS.PIPELINE_INPUT_TYPE: "void", + DEEPLOY_KEYS.CHAINSTORE_RESPONSE: False, + DEEPLOY_KEYS.TARGET_NODES: ["node-1", "node-2"], + DEEPLOY_KEYS.TARGET_NODES_COUNT: 2, + DEEPLOY_KEYS.PLUGINS: [ + { + DEEPLOY_KEYS.PLUGIN_SIGNATURE: "WORKER_APP_RUNNER", + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID: "shared-instance", + DEEPLOY_KEYS.PLUGIN_NAME: "worker", + "IMAGE": "node:22", + "CONTAINER_RESOURCES": {"cpu": 1, "memory": "256m"}, + "ENV": {"API_TOKEN": "requested-secret", "MODE": "requested"}, + }, + ], + }, + is_create=False, + async_mode=True, + ) + + self.assertEqual(response[DEEPLOY_KEYS.STATUS], "command_delivered") + self.assertEqual(called["delete"], 1) + self.assertEqual(called["deploy"], 1) + prepared_plan = called["deploy_kwargs"]["prepared_create_deploy_plan"] + for node_plugins in prepared_plan["node_plugins_by_addr"].values(): + instance = node_plugins[0][plugin.ct.CONFIG_PLUGIN.K_INSTANCES][0] + self.assertEqual(instance["ENV"]["API_TOKEN"], "requested-secret") + self.assertEqual(instance["ENV"]["MODE"], "requested") + self.assertNotIn(instance["ENV"]["API_TOKEN"], {"live-secret-a", "live-secret-b"}) + self.assertNotIn(instance["ENV"]["MODE"], {"replica-a", "replica-b"}) + + drift_logs = [line for line in log_lines if "live replica config drift" in line] + self.assertEqual(len(drift_logs), 1) + warning = drift_logs[0] + self.assertIn("job_id=11", warning) + self.assertIn("app_id=app-123", warning) + self.assertIn("plugin_instance_id=shared-instance", warning) + self.assertIn("node-1", warning) + self.assertIn("node-2", warning) + self.assertIn("WORKER_APP_RUNNER", warning) + self.assertIn("worker", warning) + self.assertIn("$.ENV.API_TOKEN", warning) + self.assertIn("$.ENV.MODE", warning) + for value in ( + "live-secret-a", + "live-secret-b", + "requested-secret", + "replica-a", + "replica-b", + "requested", + ): + self.assertNotIn(value, warning) + + def test_process_update_emits_no_drift_warning_for_identical_replicas(self): + discovered_instances = [ + { + DEEPLOY_PLUGIN_DATA.INSTANCE_ID: "shared-instance", + DEEPLOY_PLUGIN_DATA.PLUGIN_SIGNATURE: "A_SIMPLE_PLUGIN", + DEEPLOY_PLUGIN_DATA.NODE: node, + DEEPLOY_PLUGIN_DATA.PLUGIN_INSTANCE: { + "instance_conf": { + DEEPLOY_KEYS.PLUGIN_NAME: "api", + "PROCESS_DELAY": 5, + }, + }, + } + for node in ("node-1", "node-2") + ] + plugin, called = self._make_process_update_plugin( + discovered_instances=discovered_instances, + nodes=["node-1", "node-2"], + ) + log_lines = [] + plugin.P = lambda message, **kwargs: log_lines.append(message) response = plugin._process_pipeline_request( { @@ -1605,7 +2294,51 @@ def test_process_update_rejects_corrupt_duplicate_live_instance_id_before_delete DEEPLOY_KEYS.PLUGIN_SIGNATURE: "A_SIMPLE_PLUGIN", DEEPLOY_KEYS.PLUGIN_INSTANCE_ID: "shared-instance", DEEPLOY_KEYS.PLUGIN_NAME: "api", - "PROCESS_DELAY": 5, + "PROCESS_DELAY": 10, + }, + ], + }, + is_create=False, + async_mode=True, + ) + + self.assertEqual(response[DEEPLOY_KEYS.STATUS], "command_delivered") + self.assertEqual(called["delete"], 1) + self.assertFalse(any("live replica config drift" in line for line in log_lines)) + + def test_process_update_rejects_instance_id_signature_mismatch_before_delete(self): + plugin, called = self._make_process_update_plugin( + discovered_instances=[ + { + DEEPLOY_PLUGIN_DATA.INSTANCE_ID: "shared-instance", + DEEPLOY_PLUGIN_DATA.PLUGIN_SIGNATURE: "A_SIMPLE_PLUGIN", + DEEPLOY_PLUGIN_DATA.NODE: "node-1", + DEEPLOY_PLUGIN_DATA.PLUGIN_INSTANCE: { + "instance_conf": { + DEEPLOY_KEYS.PLUGIN_NAME: "api", + "PROCESS_DELAY": 5, + }, + }, + }, + ], + ) + + response = plugin._process_pipeline_request( + { + DEEPLOY_KEYS.APP_ID: "app-123", + DEEPLOY_KEYS.APP_ALIAS: "app", + DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.JOB_APP_TYPE: "native", + DEEPLOY_KEYS.PIPELINE_INPUT_TYPE: "void", + DEEPLOY_KEYS.CHAINSTORE_RESPONSE: False, + DEEPLOY_KEYS.TARGET_NODES: ["node-1"], + DEEPLOY_KEYS.TARGET_NODES_COUNT: 1, + DEEPLOY_KEYS.PLUGINS: [ + { + DEEPLOY_KEYS.PLUGIN_SIGNATURE: "ANOTHER_SIMPLE_PLUGIN", + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID: "shared-instance", + DEEPLOY_KEYS.PLUGIN_NAME: "api", + "PROCESS_DELAY": 10, }, ], }, @@ -1613,14 +2346,109 @@ def test_process_update_rejects_corrupt_duplicate_live_instance_id_before_delete async_mode=True, ) - self.assertIn("Corrupt live discovery", response[DEEPLOY_KEYS.ERROR]) + self.assertIn("cannot be reused with signature", response[DEEPLOY_KEYS.ERROR]) self.assertEqual(called["delete"], 0) self.assertEqual(called["deploy"], 0) + def test_process_update_generates_id_for_new_plugin_without_id(self): + plugin, called = self._make_process_update_plugin( + discovered_instances=[ + { + DEEPLOY_PLUGIN_DATA.INSTANCE_ID: "existing-instance", + DEEPLOY_PLUGIN_DATA.PLUGIN_SIGNATURE: "A_SIMPLE_PLUGIN", + DEEPLOY_PLUGIN_DATA.NODE: "node-1", + DEEPLOY_PLUGIN_DATA.PLUGIN_INSTANCE: { + "instance_conf": { + DEEPLOY_KEYS.PLUGIN_NAME: "existing", + "PROCESS_DELAY": 5, + }, + }, + }, + ], + ) + + response = plugin._process_pipeline_request( + { + DEEPLOY_KEYS.APP_ID: "app-123", + DEEPLOY_KEYS.APP_ALIAS: "app", + DEEPLOY_KEYS.JOB_ID: 11, + DEEPLOY_KEYS.JOB_APP_TYPE: "native", + DEEPLOY_KEYS.PIPELINE_INPUT_TYPE: "void", + DEEPLOY_KEYS.CHAINSTORE_RESPONSE: False, + DEEPLOY_KEYS.TARGET_NODES: ["node-1"], + DEEPLOY_KEYS.TARGET_NODES_COUNT: 1, + DEEPLOY_KEYS.PLUGINS: [ + { + DEEPLOY_KEYS.PLUGIN_SIGNATURE: "A_SIMPLE_PLUGIN", + DEEPLOY_KEYS.PLUGIN_INSTANCE_ID: "existing-instance", + DEEPLOY_KEYS.PLUGIN_NAME: "existing", + "PROCESS_DELAY": 10, + }, + { + DEEPLOY_KEYS.PLUGIN_SIGNATURE: "A_SIMPLE_PLUGIN", + DEEPLOY_KEYS.PLUGIN_NAME: "new-plugin", + "PROCESS_DELAY": 15, + }, + ], + }, + is_create=False, + async_mode=True, + ) + + self.assertEqual(response[DEEPLOY_KEYS.STATUS], "command_delivered") + prepared_plan = called["deploy_kwargs"]["prepared_create_deploy_plan"] + instances = prepared_plan["node_plugins_by_addr"]["node-1"][0][plugin.ct.CONFIG_PLUGIN.K_INSTANCES] + by_name = {instance[DEEPLOY_KEYS.PLUGIN_NAME]: instance for instance in instances} + self.assertEqual( + by_name["existing"][plugin.ct.CONFIG_INSTANCE.K_INSTANCE_ID], + "existing-instance", + ) + self.assertEqual( + by_name["new-plugin"][plugin.ct.CONFIG_INSTANCE.K_INSTANCE_ID], + "A_SIMPLE_PLUG_xxxxxx", + ) + + def test_live_replica_drift_warning_is_bounded_and_value_free(self): + plugin = make_deeploy_plugin() + log_lines = [] + plugin.P = lambda message, **kwargs: log_lines.append(message) + long_segment = "X" * 200 + discovered_instances = [] + for idx in range(12): + for node_idx in range(2): + discovered_instances.append({ + DEEPLOY_PLUGIN_DATA.INSTANCE_ID: f"instance-{idx}", + DEEPLOY_PLUGIN_DATA.PLUGIN_SIGNATURE: "A_SIMPLE_PLUGIN", + DEEPLOY_PLUGIN_DATA.NODE: f"node-{idx}-{node_idx}\nforged", + DEEPLOY_PLUGIN_DATA.PLUGIN_INSTANCE: { + "instance_conf": { + DEEPLOY_KEYS.PLUGIN_NAME: f"plugin-{idx}", + f"{long_segment}-{idx}": f"secret-value-{idx}-{node_idx}", + }, + }, + }) + + warned = plugin._warn_on_live_plugin_config_drift( + discovered_instances, + job_id=11, + app_id="app-123", + ) + + self.assertEqual(warned, 10) + self.assertEqual(len(log_lines), 11) + self.assertTrue(all(len(line) <= 2000 for line in log_lines)) + self.assertTrue(all("\n" not in line for line in log_lines)) + self.assertIn("omitted_groups=2", log_lines[-1]) + joined = "\n".join(log_lines) + for idx in range(12): + self.assertNotIn(f"secret-value-{idx}-0", joined) + self.assertNotIn(f"secret-value-{idx}-1", joined) + def test_process_update_rejects_ambiguous_nameless_no_id_update_before_delete(self): plugin, called = self._make_process_update_plugin( discovered_instances=[ { + DEEPLOY_PLUGIN_DATA.INSTANCE_ID: "legacy-instance-a", DEEPLOY_PLUGIN_DATA.PLUGIN_SIGNATURE: "A_SIMPLE_PLUGIN", DEEPLOY_PLUGIN_DATA.NODE: "node-1", DEEPLOY_PLUGIN_DATA.PLUGIN_INSTANCE: { @@ -1630,6 +2458,7 @@ def test_process_update_rejects_ambiguous_nameless_no_id_update_before_delete(se }, }, { + DEEPLOY_PLUGIN_DATA.INSTANCE_ID: "legacy-instance-b", DEEPLOY_PLUGIN_DATA.PLUGIN_SIGNATURE: "A_SIMPLE_PLUGIN", DEEPLOY_PLUGIN_DATA.NODE: "node-1", DEEPLOY_PLUGIN_DATA.PLUGIN_INSTANCE: { diff --git a/requirements.txt b/requirements.txt index 2150c324a..18686e4e5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,7 @@ python-docx pdfplumber docker aiofiles +aiohttp paramiko pymisp # This has been moved to device.py additional_packages list for better compatibility with different devices. diff --git a/ver.py b/ver.py index 18ac0b5b5..c239b235f 100644 --- a/ver.py +++ b/ver.py @@ -1 +1 @@ -__VER__ = '2.10.390' +__VER__ = '2.10.400'