Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
163 changes: 86 additions & 77 deletions README.md

Large diffs are not rendered by default.

11 changes: 11 additions & 0 deletions data/processed/richer_sample/run_manifest.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{
"tool_version": "1.2.0",
"demo_id": "window",
"input_digest": "sha256:e8771283a83c146a2cffce5fe316c1408200b6d7717cbb6d074334aa891672f3",
"config_digest": "sha256:6bffef106b0fcdb04d54e3aac8fe5b4b800bf228ad03fe988861486691c76f65",
"artifact_schema_versions": {
"run_manifest": "run-manifest/v1",
"telemetry_summary": "telemetry-summary/v1"
},
"execution_mode": "synthetic-local"
}
67 changes: 34 additions & 33 deletions data/processed/richer_sample/summary.json
Original file line number Diff line number Diff line change
@@ -1,33 +1,34 @@
{
"input_path": "data/raw/richer_sample_events.jsonl",
"output_dir": "data/processed/richer_sample",
"normalized_event_count": 28,
"window_count": 24,
"feature_row_count": 24,
"alert_count": 8,
"triggered_rule_names": [
"high_error_rate",
"high_severity_spike",
"login_fail_burst",
"persistent_high_error",
"rare_event_repeat_malware_alert",
"rare_event_repeat_policy_denied"
],
"triggered_rule_counts": {
"high_error_rate": 2,
"high_severity_spike": 1,
"login_fail_burst": 1,
"persistent_high_error": 2,
"rare_event_repeat_malware_alert": 1,
"rare_event_repeat_policy_denied": 1
},
"cooldown_seconds": 120,
"generated_artifacts": [
"data/processed/richer_sample/features.csv",
"data/processed/richer_sample/alerts.csv",
"data/processed/richer_sample/summary.json",
"data/processed/richer_sample/event_count_timeline.png",
"data/processed/richer_sample/error_rate_timeline.png",
"data/processed/richer_sample/alerts_timeline.png"
]
}
{
"input_path": "data/raw/richer_sample_events.jsonl",
"output_dir": "data/processed/richer_sample",
"normalized_event_count": 28,
"window_count": 24,
"feature_row_count": 24,
"alert_count": 8,
"triggered_rule_names": [
"high_error_rate",
"high_severity_spike",
"login_fail_burst",
"persistent_high_error",
"rare_event_repeat_malware_alert",
"rare_event_repeat_policy_denied"
],
"triggered_rule_counts": {
"high_error_rate": 2,
"high_severity_spike": 1,
"login_fail_burst": 1,
"persistent_high_error": 2,
"rare_event_repeat_malware_alert": 1,
"rare_event_repeat_policy_denied": 1
},
"cooldown_seconds": 120,
"generated_artifacts": [
"data/processed/richer_sample/features.csv",
"data/processed/richer_sample/alerts.csv",
"data/processed/richer_sample/summary.json",
"data/processed/richer_sample/run_manifest.json",
"data/processed/richer_sample/event_count_timeline.png",
"data/processed/richer_sample/error_rate_timeline.png",
"data/processed/richer_sample/alerts_timeline.png"
]
}
11 changes: 11 additions & 0 deletions data/processed/run_manifest.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{
"tool_version": "1.2.0",
"demo_id": "window",
"input_digest": "sha256:c08795a6a3c361a3339414c5a8441fb74c58f027c96931dbdc0da28560e132ac",
"config_digest": "sha256:a6107a2a732e8f4bf03572e9e86697ce1daa0eed1871724d0b44ec677f924b37",
"artifact_schema_versions": {
"run_manifest": "run-manifest/v1",
"telemetry_summary": "telemetry-summary/v1"
},
"execution_mode": "synthetic-local"
}
67 changes: 34 additions & 33 deletions data/processed/summary.json
Original file line number Diff line number Diff line change
@@ -1,33 +1,34 @@
{
"input_path": "data/raw/sample_events.jsonl",
"output_dir": "data/processed",
"normalized_event_count": 41,
"window_count": 24,
"feature_row_count": 24,
"alert_count": 12,
"triggered_rule_names": [
"high_error_rate",
"high_severity_spike",
"login_fail_burst",
"persistent_high_error",
"rare_event_repeat_malware_alert",
"source_spread_spike"
],
"triggered_rule_counts": {
"high_error_rate": 3,
"high_severity_spike": 2,
"login_fail_burst": 2,
"persistent_high_error": 3,
"rare_event_repeat_malware_alert": 1,
"source_spread_spike": 1
},
"cooldown_seconds": 60,
"generated_artifacts": [
"data/processed/features.csv",
"data/processed/alerts.csv",
"data/processed/summary.json",
"data/processed/event_count_timeline.png",
"data/processed/error_rate_timeline.png",
"data/processed/alerts_timeline.png"
]
}
{
"input_path": "data/raw/sample_events.jsonl",
"output_dir": "data/processed",
"normalized_event_count": 41,
"window_count": 24,
"feature_row_count": 24,
"alert_count": 12,
"triggered_rule_names": [
"high_error_rate",
"high_severity_spike",
"login_fail_burst",
"persistent_high_error",
"rare_event_repeat_malware_alert",
"source_spread_spike"
],
"triggered_rule_counts": {
"high_error_rate": 3,
"high_severity_spike": 2,
"login_fail_burst": 2,
"persistent_high_error": 3,
"rare_event_repeat_malware_alert": 1,
"source_spread_spike": 1
},
"cooldown_seconds": 60,
"generated_artifacts": [
"data/processed/features.csv",
"data/processed/alerts.csv",
"data/processed/summary.json",
"data/processed/run_manifest.json",
"data/processed/event_count_timeline.png",
"data/processed/error_rate_timeline.png",
"data/processed/alerts_timeline.png"
]
}
196 changes: 99 additions & 97 deletions demos/ai-assisted-detection-demo/README.md
Original file line number Diff line number Diff line change
@@ -1,77 +1,78 @@
# AI-Assisted Detection Demo
This demo is part of `telemetry-lab` and is intentionally framed as a portfolio-grade security engineering prototype.
It demonstrates constrained AI-assisted case drafting for SOC-style workflows, not autonomous detection or response.
It combines deterministic detections with a tightly constrained LLM stage:
- the rules decide which activity is interesting
- the grouping logic decides which hits belong in the same case
- the LLM is limited to structured summaries, likely causes, uncertainty notes, and suggested next steps
The LLM does **not** make final incident decisions, modify rules, call tools, or execute response actions. Human verification is always required.
# AI-Assisted Detection Demo

This demo is part of `telemetry-lab` and is intentionally framed as a portfolio-grade security engineering prototype.

It demonstrates constrained AI-assisted case drafting for SOC-style workflows, not autonomous detection or response.

It combines deterministic detections with a tightly constrained LLM stage:

- the rules decide which activity is interesting
- the grouping logic decides which hits belong in the same case
- the LLM is limited to structured summaries, likely causes, uncertainty notes, and suggested next steps

The LLM does **not** make final incident decisions, modify rules, call tools, or execute response actions. Human verification is always required.

## Purpose

The goal is to show a credible bridge between deterministic telemetry analytics and safe analyst assistance.

This is not an autonomous SOC. It is a constrained drafting pipeline that keeps rule logic, ATT&CK mapping, case grouping, and evidence handling deterministic.

For a no-run reviewer pack, see [docs/ai-assisted-detection-examples.md](../../docs/ai-assisted-detection-examples.md).
## Pipeline
1. ingest sample auth, web, and process events from JSONL
2. normalize them into a shared internal schema
3. apply deterministic detection rules
4. group rule hits into cases by shared entities and time proximity
5. attach ATT&CK mappings from rule metadata
6. build a case bundle with raw evidence, rule hits, severity, and evidence highlights
7. pass the case bundle to a constrained local demo LLM adapter with strict instruction and data separation
8. require JSON-only output against a local schema
9. validate the response and reject invalid output
10. emit analyst-facing artifacts and audit traces
## Guardrails
- telemetry content is marked as untrusted data
- system instructions are separated from the evidence payload
- the response must pass local JSON schema validation
- the response must pass a semantic validation layer after schema validation
- `human_verification` is required and must be `required`
- no external tool use is allowed in the LLM stage
- no automated response actions are allowed
- forbidden action-taking or final-verdict language is rejected and recorded
- summaries are rejected if the returned `case_id` does not exactly match the input case bundle
- a prompt-injection-like sample event is included and treated as telemetry, not instruction
- rejected summaries are fail-closed: they do not enter `case_summaries.json`
- accepted and rejected outcomes are both recorded in `audit_traces.jsonl`
## Quick start
From the repository root:
```bash
python -m pip install -e .
python -m telemetry_window_demo.cli run-ai-demo
```
Generated artifacts are written to `demos/ai-assisted-detection-demo/artifacts/`.
## Demo inputs
- sample data: `data/raw/sample_security_events.jsonl`
- deterministic rules: `config/rules.yaml`
- structured output schema: `config/llm_case_output_schema.json`

## Pipeline

1. ingest sample auth, web, and process events from JSONL
2. normalize them into a shared internal schema
3. apply deterministic detection rules
4. group rule hits into cases by shared entities and time proximity
5. attach ATT&CK mappings from rule metadata
6. build a case bundle with raw evidence, rule hits, severity, and evidence highlights
7. pass the case bundle to a constrained local demo LLM adapter with strict instruction and data separation
8. require JSON-only output against a local schema
9. validate the response and reject invalid output
10. emit analyst-facing artifacts and audit traces

## Guardrails

- telemetry content is marked as untrusted data
- system instructions are separated from the evidence payload
- the response must pass local JSON schema validation
- the response must pass a semantic validation layer after schema validation
- `human_verification` is required and must be `required`
- no external tool use is allowed in the LLM stage
- no automated response actions are allowed
- forbidden action-taking or final-verdict language is rejected and recorded
- summaries are rejected if the returned `case_id` does not exactly match the input case bundle
- a prompt-injection-like sample event is included and treated as telemetry, not instruction
- rejected summaries are fail-closed: they do not enter `case_summaries.json`
- accepted and rejected outcomes are both recorded in `audit_traces.jsonl`

## Quick start

From the repository root:

```bash
python -m pip install -e ".[dev]"
telemetry-lab run ai-assisted
```

Generated artifacts are written to `demos/ai-assisted-detection-demo/artifacts/`.

## Demo inputs

- sample data: `data/raw/sample_security_events.jsonl`
- deterministic rules: `config/rules.yaml`
- structured output schema: `config/llm_case_output_schema.json`

## Expected artifacts

- `artifacts/rule_hits.json`
- `artifacts/case_bundles.json`
- `artifacts/case_summaries.json`
- `artifacts/case_report.md`
- `artifacts/audit_traces.jsonl`
- `artifacts/run_manifest.json`

## Expected run summary

Expand All @@ -85,33 +86,34 @@ The bundled sample run should report:
- `3` audit records

## Artifact semantics
- `rule_hits.json`: deterministic rule hits with rule metadata, ATT&CK mapping, entities, and evidence highlights
- `case_bundles.json`: grouped cases with severity, rule hits, ATT&CK mappings, raw evidence, and untrusted-data marking
- `case_summaries.json`: only accepted JSON summaries that passed schema and semantic validation

- `rule_hits.json`: deterministic rule hits with rule metadata, ATT&CK mapping, entities, and evidence highlights
- `case_bundles.json`: grouped cases with severity, rule hits, ATT&CK mappings, raw evidence, and untrusted-data marking
- `case_summaries.json`: only accepted JSON summaries that passed schema and semantic validation
- `case_report.md`: analyst-facing report with run counts, accepted summaries, and explicit notes for rejected case summaries
- `case_report.md`: includes a top-level run integrity section that surfaces rule/config degradation
- `audit_traces.jsonl`: stable per-record audit log for accepted and rejected paths, using `schema_version = ai-assisted-detection-audit/v1` and including `ts`, `case_id`, `validation_status`, `rejection_reason`, `rule_ids`, `prompt_input_digest`, `evidence_digest`, and bounded response excerpts

## Rejection behavior

- non-JSON or malformed JSON responses are rejected and recorded
- missing required fields or invalid enum values are rejected and recorded
- schema-valid summaries with the wrong `case_id` are rejected and recorded
- action-taking language is rejected
- final-verdict or confirmed-compromise language is rejected
- malformed rule or ATT&CK metadata is rejected before detection logic uses it

Rejected outputs do not become analyst summaries. Analysts can still inspect deterministic evidence through `case_bundles.json`, `case_report.md`, and `audit_traces.jsonl`.

- `audit_traces.jsonl`: stable per-record audit log for accepted and rejected paths, using `schema_version = ai-assisted-detection-audit/v1` and including `ts`, `case_id`, `validation_status`, `rejection_reason`, `rule_ids`, `prompt_input_digest`, `evidence_digest`, and bounded response excerpts
- `run_manifest.json`: synthetic-local run manifest with tool version, input/config digests, and artifact schema versions

## Rejection behavior

- non-JSON or malformed JSON responses are rejected and recorded
- missing required fields or invalid enum values are rejected and recorded
- schema-valid summaries with the wrong `case_id` are rejected and recorded
- action-taking language is rejected
- final-verdict or confirmed-compromise language is rejected
- malformed rule or ATT&CK metadata is rejected before detection logic uses it

Rejected outputs do not become analyst summaries. Analysts can still inspect deterministic evidence through `case_bundles.json`, `case_report.md`, and `audit_traces.jsonl`.

## Reviewer walkthrough

### Accepted summary path
Use the default sample run artifacts in `artifacts/case_summaries.json`, `artifacts/case_report.md`, and `artifacts/audit_traces.jsonl`.
Verify that `CASE-001` appears in all three places, that the `case_id` matches exactly, that `human_verification` is `required`, and that the audit record shows `validation_status = accepted` with `schema_version = ai-assisted-detection-audit/v1`.

Use the default sample run artifacts in `artifacts/case_summaries.json`, `artifacts/case_report.md`, and `artifacts/audit_traces.jsonl`.

Verify that `CASE-001` appears in all three places, that the `case_id` matches exactly, that `human_verification` is `required`, and that the audit record shows `validation_status = accepted` with `schema_version = ai-assisted-detection-audit/v1`.

### Rejected summary path

Run:
Expand All @@ -121,9 +123,9 @@ pytest tests/test_ai_assisted_detection_demo.py -k "audit_traces_capture_accepte
```

Then inspect the `case_report.md`, `case_summaries.json`, and `audit_traces.jsonl` files under `.pytest-artifacts-ai-demo-rejections/test_*/artifacts/`.
Verify that the rejected case is absent from `case_summaries.json`, appears in `case_report.md` as `Summary status: rejected`, and has an audit record with `validation_status = rejected` plus a concrete `rejection_reason` such as `missing_required_fields`, `semantic_validation_failed`, or `case_id_mismatch`.

Verify that the rejected case is absent from `case_summaries.json`, appears in `case_report.md` as `Summary status: rejected`, and has an audit record with `validation_status = rejected` plus a concrete `rejection_reason` such as `missing_required_fields`, `semantic_validation_failed`, or `case_id_mismatch`.

### Degraded coverage path

Run:
Expand All @@ -133,15 +135,15 @@ pytest tests/test_ai_assisted_detection_demo.py -k malformed_attack_metadata_is_
```

Then inspect the generated `case_report.md` and `audit_traces.jsonl` files under `.pytest-artifacts-ai-demo-degraded/test_*/artifacts/`.
Verify that `case_report.md` exposes `## Run Integrity`, `coverage_degraded: yes`, and the rejected rule id, and that `audit_traces.jsonl` contains a global rejection record with `case_id = null` and `rejection_reason = rule_metadata_validation_failed`.
## Limitations
- the LLM stage is a constrained local demo adapter, not a production model integration
- detections are intentionally small and rule-based
- grouping is simple and optimized for readability over recall
- sample telemetry is synthetic and limited in volume
- there is no ticketing, SOAR, sandboxing, or live data ingestion
- artifacts are for analyst review only and do not represent final incident disposition
- rejection logic is intentionally conservative and favors fail-closed behavior over model flexibility

Verify that `case_report.md` exposes `## Run Integrity`, `coverage_degraded: yes`, and the rejected rule id, and that `audit_traces.jsonl` contains a global rejection record with `case_id = null` and `rejection_reason = rule_metadata_validation_failed`.

## Limitations

- the LLM stage is a constrained local demo adapter, not a production model integration
- detections are intentionally small and rule-based
- grouping is simple and optimized for readability over recall
- sample telemetry is synthetic and limited in volume
- there is no ticketing, SOAR, sandboxing, or live data ingestion
- artifacts are for analyst review only and do not represent final incident disposition
- rejection logic is intentionally conservative and favors fail-closed behavior over model flexibility
Loading
Loading