diff --git a/AGENTS.md b/AGENTS.md index 831b6f7..c0f41bf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,7 +13,7 @@ control plane, or runtime AI filter. - `src/Bower.Contracts`: immutable event contracts and serialization. - `src/Bower.PolicyEngine`: deterministic, explainable event selection. -- `src/Bower.Redaction`: local data minimisation and intelligent sensitive-data detection before persistence. +- `src/Bower.Redaction`: Privacy & Secret Protection Engine — modular detectors (AU identifiers, financial, secrets, crypto), policy actions, privacy metadata; redacts before persistence. - `src/Bower.Persistence`: SQLite queue, cursors, delivery state and evidence. - `src/Bower.Core`: source-to-policy-to-queue orchestration. - `src/Bower.Sdk`: semantic developer API and transports. diff --git a/Bower.sln b/Bower.sln index 767bba7..ed922dc 100644 --- a/Bower.sln +++ b/Bower.sln @@ -51,6 +51,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bower.Dcr", "src\Bower.Dcr\ EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bower.Agent.Aws", "src\Bower.Agent.Aws\Bower.Agent.Aws.csproj", "{A240528B-2792-4CEB-8A84-1CB773CA4BEC}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bower.Benchmarks", "tests\Bower.Benchmarks\Bower.Benchmarks.csproj", "{B7E1C9A0-4D2F-4A8B-9E3C-1F6A2B8D0C4E}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -61,6 +63,18 @@ Global Release|x86 = Release|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution + {B7E1C9A0-4D2F-4A8B-9E3C-1F6A2B8D0C4E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B7E1C9A0-4D2F-4A8B-9E3C-1F6A2B8D0C4E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B7E1C9A0-4D2F-4A8B-9E3C-1F6A2B8D0C4E}.Debug|x64.ActiveCfg = Debug|Any CPU + {B7E1C9A0-4D2F-4A8B-9E3C-1F6A2B8D0C4E}.Debug|x64.Build.0 = Debug|Any CPU + {B7E1C9A0-4D2F-4A8B-9E3C-1F6A2B8D0C4E}.Debug|x86.ActiveCfg = Debug|Any CPU + {B7E1C9A0-4D2F-4A8B-9E3C-1F6A2B8D0C4E}.Debug|x86.Build.0 = Debug|Any CPU + {B7E1C9A0-4D2F-4A8B-9E3C-1F6A2B8D0C4E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B7E1C9A0-4D2F-4A8B-9E3C-1F6A2B8D0C4E}.Release|Any CPU.Build.0 = Release|Any CPU + {B7E1C9A0-4D2F-4A8B-9E3C-1F6A2B8D0C4E}.Release|x64.ActiveCfg = Release|Any CPU + {B7E1C9A0-4D2F-4A8B-9E3C-1F6A2B8D0C4E}.Release|x64.Build.0 = Release|Any CPU + {B7E1C9A0-4D2F-4A8B-9E3C-1F6A2B8D0C4E}.Release|x86.ActiveCfg = Release|Any CPU + {B7E1C9A0-4D2F-4A8B-9E3C-1F6A2B8D0C4E}.Release|x86.Build.0 = Release|Any CPU {1A4F182D-F153-4529-A20B-E0551C82734C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {1A4F182D-F153-4529-A20B-E0551C82734C}.Debug|Any CPU.Build.0 = Debug|Any CPU {1A4F182D-F153-4529-A20B-E0551C82734C}.Debug|x64.ActiveCfg = Debug|Any CPU diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index 5a729a8..d7042a5 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -7,7 +7,10 @@ validation or policy decision. ## Trust boundaries 1. Source records are untrusted, bounded input. -2. Redaction removes known dangerous fields before typed parsing and persistence. +2. Privacy & Secret Protection Engine (`Bower.Redaction`) removes field-name + secrets and scans string values for regulated AU identifiers, credentials and + crypto material before typed parsing and persistence. See + [privacy engine](../privacy/privacy-secret-engine.md). 3. Policy Engine accepts only approved semantic types with required context. 4. SQLite queue is tenant-controlled durable state; delivery leases recover after crashes and acknowledged rows remain auditable until retention. diff --git a/docs/privacy/privacy-secret-engine.md b/docs/privacy/privacy-secret-engine.md new file mode 100644 index 0000000..531570c --- /dev/null +++ b/docs/privacy/privacy-secret-engine.md @@ -0,0 +1,161 @@ +# Privacy & Secret Protection Engine + +Bower's privacy engine inspects every event **after parse and before +persistence / normalisation**. It is a core control for preventing accidental +leakage of regulated Australian identifiers, credentials and cryptographic +material into downstream SIEM platforms. + +``` +Raw Event → Parser → Privacy & Secret Engine → Normalisation → Output + ├── Detect + ├── Validate + ├── Classify + └── Apply Policy +``` + +## Design principles + +| Principle | How | +| --- | --- | +| Deterministic | Compiled regex, checksums, structure, entropy — **no AI at runtime** | +| Streaming / per-event | Events processed independently; safe under concurrent callers | +| Extensible | `ISensitiveDetector` plugins; provider API keys via pattern registry | +| Configurable | Global default action + per-detector overrides | +| Fail closed | Invalid / oversized payloads → redaction failure → quarantine | + +## Actions + +| Action | Effect | +| --- | --- | +| `Allow` | Leave value unchanged (still recorded in metadata if detected) | +| `Remove` | Delete property (field-name) or replace span with empty | +| `Replace` | Replace with configured placeholder | +| `Mask` | Partial reveal (e.g. last 4 digits, email local-part) | +| `Sha256` | `sha256:` | +| `Hmac` | `hmac-sha256:` (requires 32+ byte key; else SHA-256) | +| `Encrypt` | AES-GCM (`enc:aesgcm:…`); requires 32-byte key; else remove | +| `AlertOnly` | Detect only; do not rewrite | + +## Default policy (production-oriented) + +| Detector | Default | +| --- | --- | +| Field-name secrets | Remove | +| TFN | SHA-256 | +| CRN / Medicare / IHI / Passport / Licence / DVA | Mask | +| ABN / ACN | Allow | +| Credit card | Remove | +| JWT / cloud secrets / API keys / PEM / DB / env | Remove | +| Email / phone / DOB | Mask | +| Security markings | AlertOnly (opt-in) | + +Opt-in (disabled by default): IP, hostname, username, residential address, GPS, +security markings. + +## Metadata + +When findings exist, the engine attaches (never with original values): + +```json +{ + "privacy": { + "detected": ["au.tfn", "secret.jwt", "id.email"], + "actions": { + "au.tfn": "SHA256", + "secret.jwt": "Removed", + "id.email": "Masked" + } + } +} +``` + +`SecurityEventEnvelope.Privacy` maps the same shape. + +## Detector modules + +### Australian identifiers + +| Id | Validation | +| --- | --- | +| `au.tfn` | ATO 8/9-digit checksum | +| `au.crn` | Pattern (9 digits + letter) | +| `au.medicare` | Medicare checksum + issue digit | +| `au.ihi` | `800360` + Luhn | +| `au.passport` | Format + contextual label | +| `au.driver-licence` | State-heuristic formats (context gated) | +| `au.abn` | ABN mod-89 | +| `au.acn` | ACN check digit | +| `au.dva` | DVA pattern | + +### Financial + +`fin.credit-card` (Luhn + network), `fin.bsb-account`, `fin.iban` (mod-97), +`fin.swift-bic`, `fin.payid`. + +### Identity + +`id.email`, `id.phone.au`, `id.phone.intl`, `id.dob`, plus optional +`id.address`, `id.gps`, `id.ip`, `id.hostname`, `id.username`. + +### Secrets & crypto + +AWS, Azure, Entra, GCP, JWT, OAuth, provider API keys (OpenAI, Anthropic, +GitHub, GitLab, Slack, Stripe, Twilio, Cloudflare, Atlassian, Datadog, +PagerDuty, Okta, MongoDB, Snowflake), Kubernetes, Docker, database connection +strings, environment variable assignments, PEM/PKCS8/SSH/PGP/X.509. + +### Classification + +`class.security-marking` — OFFICIAL, PROTECTED, SECRET, TOP SECRET, +CABINET-IN-CONFIDENCE (opt-in). + +## Configuration (code) + +```csharp +var defaults = PrivacyPolicy.CreateDefault(); +var policy = new PrivacyPolicy +{ + DefaultAction = defaults.DefaultAction, + DetectorActions = new Dictionary(defaults.DetectorActions) + { + [DetectorIds.Tfn] = PrivacyAction.Hmac, + [DetectorIds.Email] = PrivacyAction.Mask, + [DetectorIds.Abn] = PrivacyAction.Allow + }, + HmacKey = hmacKey32PlusBytes, + EnabledOptInDetectors = new HashSet { DetectorIds.IpAddress }, + EmitMetadata = true +}; + +IEventRedactor redactor = new JsonEventRedactor(policy); +// or +var engine = new PrivacyEngine(policy, customDetectors); +``` + +## Extension points + +1. Implement `ISensitiveDetector` (value scan) and/or `IFieldNameDetector`. +2. Pass detectors into `PrivacyEngine` constructor **or** extend + `DetectorCatalog.CreateDefaultValueDetectors()`. +3. Set per-id actions / disable flags on `PrivacyPolicy`. +4. Future (not in this milestone): WASM plugins, org-specific learning, + AI-assisted policy recommendation (config-time only). + +## Performance + +- Per-event JSON walk; detectors only run on string leaves. +- Overlapping matches resolved once (prefer longer, then earlier). +- Replacements applied right-to-left. +- Micro-benchmarks: `tests/Bower.Benchmarks` (Stopwatch; no extra packages). + +## Tests + +- Unit tests under `tests/Bower.UnitTests` cover checksums and major detectors. +- Existing `JsonEventRedactor` / `SensitiveDataDetector` tests remain green via + compatibility facades. + +## Related + +- `src/Bower.Redaction/AGENTS.md` — contributor invariants +- `src/Bower.Core/SecurityEventProcessor.cs` — redaction before enqueue +} diff --git a/src/Bower.Collector/Program.cs b/src/Bower.Collector/Program.cs index 6f68afd..38c2671 100644 --- a/src/Bower.Collector/Program.cs +++ b/src/Bower.Collector/Program.cs @@ -34,7 +34,7 @@ ?? Environment.MachineName; builder.Services.AddSingleton(); -builder.Services.AddSingleton(); +builder.Services.AddSingleton(_ => new JsonEventRedactor()); builder.Services.AddSingleton(services => new SqliteEventStore( databasePath, diff --git a/src/Bower.Contracts/SecurityEventEnvelope.cs b/src/Bower.Contracts/SecurityEventEnvelope.cs index c9bf613..5e7a329 100644 --- a/src/Bower.Contracts/SecurityEventEnvelope.cs +++ b/src/Bower.Contracts/SecurityEventEnvelope.cs @@ -48,6 +48,20 @@ public sealed record SecurityEventEnvelope public IReadOnlyDictionary? Labels { get; init; } public IReadOnlyDictionary? Attributes { get; init; } + + /// + /// Sanitisation summary produced by the Privacy & Secret Protection Engine. + /// Never contains original sensitive values. + /// + public PrivacyContext? Privacy { get; init; } +} + +/// Additive privacy metadata attached after detection and policy application. +public sealed record PrivacyContext +{ + public required IReadOnlyList Detected { get; init; } + + public required IReadOnlyDictionary Actions { get; init; } } public sealed record ApplicationContext diff --git a/src/Bower.Redaction/AGENTS.md b/src/Bower.Redaction/AGENTS.md new file mode 100644 index 0000000..a3639f7 --- /dev/null +++ b/src/Bower.Redaction/AGENTS.md @@ -0,0 +1,24 @@ +# Privacy & Secret Protection Engine + +Runtime redaction must stay **deterministic**. No AI, no network lookups, no +arbitrary code from configuration. + +## Invariants + +- Redact **before** persistence. Treat redaction failure as a security failure. +- Never log original secrets, credentials, tokens, or unrestricted bodies. +- Privacy metadata may list detector ids and actions only — never original values. +- Default-deny field-name secrets (password, token, body, headers, …). +- Prefer checksum validation (TFN, ABN, ACN, Medicare, IHI, Luhn) to cut false positives. +- New detectors implement `ISensitiveDetector` and register via `DetectorCatalog` + or constructor injection — do not hard-code detector lists inside `PrivacyEngine`. +- Keep detectors independently enable/disable-able through `PrivacyPolicy`. +- Opt-in detectors (IP, hostname, username, address, GPS, markings) stay off unless enabled. +- Add unit tests for every new detector (positive + negative / invalid checksum). +- Add or update micro-benchmarks when changing hot paths. + +## Layout + +- `Privacy/` — engine, policy, interfaces, catalog, applicator +- `Validation/` — checksum algorithms +- `Detectors/` — modular detector implementations by domain diff --git a/src/Bower.Redaction/Detectors/Australian/AustralianIdentifierDetectors.cs b/src/Bower.Redaction/Detectors/Australian/AustralianIdentifierDetectors.cs new file mode 100644 index 0000000..e66dde8 --- /dev/null +++ b/src/Bower.Redaction/Detectors/Australian/AustralianIdentifierDetectors.cs @@ -0,0 +1,362 @@ +using System.Text.RegularExpressions; +using Bower.Redaction.Privacy; +using Bower.Redaction.Validation; + +namespace Bower.Redaction.Detectors.Australian; + +public sealed partial class TfnDetector : RegexDetectorBase +{ + public override string Id => DetectorIds.Tfn; + public override string Category => DetectorCategories.Australian; + protected override Regex Pattern => TfnRegex(); + protected override bool RequiresValidation => true; + + protected override bool Validate(ReadOnlySpan matched) + { + string digits = ChecksumAlgorithms.DigitsOnly(matched); + return ChecksumAlgorithms.IsValidTfn(digits); + } + + // 8–9 digits with optional spaces/hyphens; context labels increase confidence but checksum is required. + [GeneratedRegex( + @"\b(?:TFN|Tax\s*File\s*Number)?[:\s#-]*(\d{3}[\s-]?\d{3}[\s-]?\d{2,3})\b", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex TfnRegex(); + + public override void Detect(ReadOnlySpan text, string path, ICollection matches) + { + if (text.IsEmpty) + { + return; + } + + string s = text.ToString(); + foreach (Match match in Pattern.Matches(s)) + { + if (!match.Success) + { + continue; + } + + // Prefer capture group of digits when present. + Group digitsGroup = match.Groups.Count > 1 ? match.Groups[1] : match.Groups[0]; + string digits = ChecksumAlgorithms.DigitsOnly(digitsGroup.Value); + if (!ChecksumAlgorithms.IsValidTfn(digits)) + { + continue; + } + + matches.Add(new DetectionMatch( + Id, + Category, + digitsGroup.Index, + digitsGroup.Length, + Validated: true)); + } + } +} + +public sealed partial class CrnDetector : RegexDetectorBase +{ + public override string Id => DetectorIds.Crn; + public override string Category => DetectorCategories.Australian; + protected override Regex Pattern => CrnRegex(); + + [GeneratedRegex( + @"\b(?:CRN|Customer\s*Reference\s*Number|Centrelink)?[:\s#-]*(\d{3}[\s-]?\d{3}[\s-]?\d{3}[A-Za-z])\b", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex CrnRegex(); + + public override void Detect(ReadOnlySpan text, string path, ICollection matches) + { + if (text.IsEmpty) + { + return; + } + + string s = text.ToString(); + foreach (Match match in Pattern.Matches(s)) + { + Group g = match.Groups.Count > 1 ? match.Groups[1] : match.Groups[0]; + if (!g.Success) + { + continue; + } + + matches.Add(new DetectionMatch(Id, Category, g.Index, g.Length, Validated: true)); + } + } +} + +public sealed partial class MedicareDetector : RegexDetectorBase +{ + public override string Id => DetectorIds.Medicare; + public override string Category => DetectorCategories.Australian; + protected override Regex Pattern => MedicareRegex(); + protected override bool RequiresValidation => true; + + [GeneratedRegex(@"\b(\d{4}[\s-]?\d{5}[\s-]?\d)\b", RegexOptions.CultureInvariant)] + private static partial Regex MedicareRegex(); + + public override void Detect(ReadOnlySpan text, string path, ICollection matches) + { + if (text.IsEmpty) + { + return; + } + + string s = text.ToString(); + foreach (Match match in Pattern.Matches(s)) + { + Group g = match.Groups.Count > 1 ? match.Groups[1] : match.Groups[0]; + string digits = ChecksumAlgorithms.DigitsOnly(g.Value); + if (!ChecksumAlgorithms.IsValidMedicare(digits)) + { + continue; + } + + matches.Add(new DetectionMatch(Id, Category, g.Index, g.Length, Validated: true)); + } + } +} + +public sealed partial class IhiDetector : RegexDetectorBase +{ + public override string Id => DetectorIds.Ihi; + public override string Category => DetectorCategories.Australian; + protected override Regex Pattern => IhiRegex(); + + [GeneratedRegex(@"\b(800360\d{10})\b", RegexOptions.CultureInvariant)] + private static partial Regex IhiRegex(); + + public override void Detect(ReadOnlySpan text, string path, ICollection matches) + { + if (text.IsEmpty) + { + return; + } + + string s = text.ToString(); + foreach (Match match in Pattern.Matches(s)) + { + if (!ChecksumAlgorithms.IsValidIhi(match.Value)) + { + continue; + } + + matches.Add(new DetectionMatch(Id, Category, match.Index, match.Length, Validated: true)); + } + } +} + +public sealed partial class PassportDetector : RegexDetectorBase +{ + public override string Id => DetectorIds.Passport; + public override string Category => DetectorCategories.Australian; + protected override Regex Pattern => PassportRegex(); + + // Common AU passport: letter + 7 digits, or 1–2 letters + 7 digits. + [GeneratedRegex( + @"\b(?:passport(?:\s*no(?:\.|mber)?)?|australian\s*passport)[:\s#-]*([A-Z]{1,2}\d{7})\b|\b([A-Z]\d{7})\b", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex PassportRegex(); + + public override void Detect(ReadOnlySpan text, string path, ICollection matches) + { + if (text.IsEmpty) + { + return; + } + + string s = text.ToString(); + bool hasContext = s.Contains("passport", StringComparison.OrdinalIgnoreCase); + foreach (Match match in Pattern.Matches(s)) + { + Group g = match.Groups[1].Success ? match.Groups[1] + : match.Groups[2].Success ? match.Groups[2] + : match.Groups[0]; + + // Bare letter+7digits only when contextual label present (reduces false positives). + if (!hasContext && match.Groups[2].Success && !match.Groups[1].Success) + { + continue; + } + + matches.Add(new DetectionMatch(Id, Category, g.Index, g.Length, Validated: false)); + } + } +} + +public sealed partial class AbnDetector : RegexDetectorBase +{ + public override string Id => DetectorIds.Abn; + public override string Category => DetectorCategories.Australian; + protected override Regex Pattern => AbnRegex(); + + [GeneratedRegex(@"\b(\d{2}[\s]?\d{3}[\s]?\d{3}[\s]?\d{3})\b", RegexOptions.CultureInvariant)] + private static partial Regex AbnRegex(); + + public override void Detect(ReadOnlySpan text, string path, ICollection matches) + { + if (text.IsEmpty) + { + return; + } + + string s = text.ToString(); + foreach (Match match in Pattern.Matches(s)) + { + Group g = match.Groups.Count > 1 ? match.Groups[1] : match.Groups[0]; + string digits = ChecksumAlgorithms.DigitsOnly(g.Value); + if (digits.Length != 11 || !ChecksumAlgorithms.IsValidAbn(digits)) + { + continue; + } + + matches.Add(new DetectionMatch(Id, Category, g.Index, g.Length, Validated: true)); + } + } +} + +public sealed partial class AcnDetector : RegexDetectorBase +{ + public override string Id => DetectorIds.Acn; + public override string Category => DetectorCategories.Australian; + protected override Regex Pattern => AcnRegex(); + + [GeneratedRegex(@"\b(\d{3}[\s]?\d{3}[\s]?\d{3})\b", RegexOptions.CultureInvariant)] + private static partial Regex AcnRegex(); + + public override void Detect(ReadOnlySpan text, string path, ICollection matches) + { + if (text.IsEmpty) + { + return; + } + + string s = text.ToString(); + foreach (Match match in Pattern.Matches(s)) + { + Group g = match.Groups.Count > 1 ? match.Groups[1] : match.Groups[0]; + string digits = ChecksumAlgorithms.DigitsOnly(g.Value); + if (digits.Length != 9 || !ChecksumAlgorithms.IsValidAcn(digits)) + { + continue; + } + + matches.Add(new DetectionMatch(Id, Category, g.Index, g.Length, Validated: true)); + } + } +} + +public sealed partial class DvaDetector : RegexDetectorBase +{ + public override string Id => DetectorIds.Dva; + public override string Category => DetectorCategories.Australian; + protected override Regex Pattern => DvaRegex(); + + // State letter + 1–8 digits + optional war code letter. + [GeneratedRegex( + @"\b(?:DVA|Veterans'?\s*Affairs)?[:\s#-]*([NQVSTW][A-Z]?\d{4,8}[A-Z]?)\b", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex DvaRegex(); + + public override void Detect(ReadOnlySpan text, string path, ICollection matches) + { + if (text.IsEmpty) + { + return; + } + + string s = text.ToString(); + bool contextual = s.Contains("DVA", StringComparison.OrdinalIgnoreCase) + || s.Contains("Veteran", StringComparison.OrdinalIgnoreCase); + + foreach (Match match in Pattern.Matches(s)) + { + Group g = match.Groups.Count > 1 && match.Groups[1].Success + ? match.Groups[1] + : match.Groups[0]; + + if (!contextual && match.Value.Length < 6) + { + continue; + } + + matches.Add(new DetectionMatch(Id, Category, g.Index, g.Length, Validated: false)); + } + } +} + +/// +/// State-specific Australian driver licence format detectors. +/// Each state pattern is independently configurable via policy detector id suffix. +/// +public sealed partial class DriverLicenceDetector : ISensitiveDetector +{ + public string Id => DetectorIds.DriverLicence; + public string Category => DetectorCategories.Australian; + + public void Detect(ReadOnlySpan text, string path, ICollection matches) + { + if (text.IsEmpty) + { + return; + } + + string s = text.ToString(); + bool hasContext = s.Contains("licence", StringComparison.OrdinalIgnoreCase) + || s.Contains("license", StringComparison.OrdinalIgnoreCase) + || s.Contains("driver", StringComparison.OrdinalIgnoreCase) + || s.Contains("DLN", StringComparison.OrdinalIgnoreCase); + + if (!hasContext) + { + return; + } + + foreach (Match match in LicenceRegex().Matches(s)) + { + matches.Add(new DetectionMatch( + Id, + Category, + match.Index, + match.Length, + Validated: false, + SubKind: InferState(match.Value))); + } + } + + private static string? InferState(string value) + { + string compact = string.Concat(value.Where(c => !char.IsWhiteSpace(c))); + // Heuristic format buckets — not authoritative registries. + if (Regex.IsMatch(compact, @"^\d{8}$")) + { + return "NSW/VIC/QLD-like"; + } + + if (Regex.IsMatch(compact, @"^\d{6,7}$")) + { + return "SA/TAS/WA-like"; + } + + if (Regex.IsMatch(compact, @"^[A-Z]{2}\d{6}$", RegexOptions.IgnoreCase)) + { + return "ACT-like"; + } + + if (Regex.IsMatch(compact, @"^\d{10}$")) + { + return "NT-like"; + } + + return "unknown"; + } + + // Broad licence token after contextual keywords only (context gated in Detect). + [GeneratedRegex( + @"\b([A-Z]{0,2}\d{6,10}[A-Z]?)\b", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex LicenceRegex(); +} diff --git a/src/Bower.Redaction/Detectors/Classification/SecurityMarkingDetector.cs b/src/Bower.Redaction/Detectors/Classification/SecurityMarkingDetector.cs new file mode 100644 index 0000000..2de86fd --- /dev/null +++ b/src/Bower.Redaction/Detectors/Classification/SecurityMarkingDetector.cs @@ -0,0 +1,22 @@ +using System.Text.RegularExpressions; +using Bower.Redaction.Privacy; + +namespace Bower.Redaction.Detectors.Classification; + +/// +/// Australian Government protective security markings (optional / opt-in). +/// AlertOnly by default — does not rewrite content unless policy overrides. +/// +public sealed partial class SecurityMarkingDetector : RegexDetectorBase +{ + public override string Id => DetectorIds.SecurityMarking; + public override string Category => DetectorCategories.Classification; + protected override Regex Pattern => MarkingRegex(); + + protected override string? SubKind(ReadOnlySpan matched) => matched.ToString().Trim().ToUpperInvariant(); + + [GeneratedRegex( + @"\b(?:OFFICIAL(?::\s*Sensitive)?|PROTECTED|SECRET|TOP\s*SECRET|CABINET-IN-CONFIDENCE)\b", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex MarkingRegex(); +} diff --git a/src/Bower.Redaction/Detectors/Crypto/CryptographicMaterialDetector.cs b/src/Bower.Redaction/Detectors/Crypto/CryptographicMaterialDetector.cs new file mode 100644 index 0000000..510e24c --- /dev/null +++ b/src/Bower.Redaction/Detectors/Crypto/CryptographicMaterialDetector.cs @@ -0,0 +1,57 @@ +using System.Text.RegularExpressions; +using Bower.Redaction.Privacy; + +namespace Bower.Redaction.Detectors.Crypto; + +public sealed partial class CryptographicMaterialDetector : RegexDetectorBase +{ + public override string Id => DetectorIds.CryptoMaterial; + public override string Category => DetectorCategories.Crypto; + protected override Regex Pattern => PemRegex(); + + protected override string? SubKind(ReadOnlySpan matched) + { + string s = matched.ToString(); + if (s.Contains("RSA PRIVATE", StringComparison.Ordinal)) + { + return "rsa-private"; + } + + if (s.Contains("EC PRIVATE", StringComparison.Ordinal) || s.Contains("ECDSA", StringComparison.Ordinal)) + { + return "ecdsa-private"; + } + + if (s.Contains("OPENSSH PRIVATE", StringComparison.Ordinal)) + { + return "openssh-private"; + } + + if (s.Contains("PRIVATE KEY", StringComparison.Ordinal)) + { + return "pkcs8-private"; + } + + if (s.Contains("CERTIFICATE", StringComparison.Ordinal)) + { + return "x509-certificate"; + } + + if (s.Contains("PGP", StringComparison.Ordinal) || s.Contains("PGP PRIVATE", StringComparison.Ordinal)) + { + return "pgp"; + } + + if (s.Contains("BEGIN SSH2", StringComparison.Ordinal)) + { + return "ssh"; + } + + return "pem"; + } + + [GeneratedRegex( + @"-----BEGIN (?:RSA |EC |OPENSSH |ENCRYPTED |DSA )?PRIVATE KEY-----[\s\S]+?-----END (?:RSA |EC |OPENSSH |ENCRYPTED |DSA )?PRIVATE KEY-----|-----BEGIN CERTIFICATE-----[\s\S]+?-----END CERTIFICATE-----|-----BEGIN PGP (?:PRIVATE |PUBLIC )?KEY BLOCK-----[\s\S]+?-----END PGP (?:PRIVATE |PUBLIC )?KEY BLOCK-----|-----BEGIN OPENSSH PRIVATE KEY-----[\s\S]+?-----END OPENSSH PRIVATE KEY-----", + RegexOptions.CultureInvariant)] + private static partial Regex PemRegex(); +} diff --git a/src/Bower.Redaction/Detectors/FieldNameSecretDetector.cs b/src/Bower.Redaction/Detectors/FieldNameSecretDetector.cs new file mode 100644 index 0000000..2dd3b1f --- /dev/null +++ b/src/Bower.Redaction/Detectors/FieldNameSecretDetector.cs @@ -0,0 +1,42 @@ +using Bower.Redaction.Privacy; + +namespace Bower.Redaction.Detectors; + +/// Removes whole properties whose names indicate secrets or unrestricted bodies. +public sealed class FieldNameSecretDetector : IFieldNameDetector +{ + private static readonly HashSet SecretNames = new( + [ + "password", + "passwordhash", + "accesstoken", + "refreshtoken", + "bearertoken", + "apikeysecret", + "apikey", + "clientsecret", + "privatekey", + "connectionstring", + "authorization", + "authorizationheader", + "cookie", + "cookies", + "credential", + "credentials", + "secret", + "requestbody", + "responsebody", + "body", + "headers", + "payload", + "filecontents" + ], + StringComparer.Ordinal); + + public string Id => DetectorIds.FieldNameSecret; + + public string Category => DetectorCategories.FieldName; + + public bool MatchesFieldName(string normalizedFieldName) => + SecretNames.Contains(normalizedFieldName); +} diff --git a/src/Bower.Redaction/Detectors/Financial/FinancialDetectors.cs b/src/Bower.Redaction/Detectors/Financial/FinancialDetectors.cs new file mode 100644 index 0000000..2067527 --- /dev/null +++ b/src/Bower.Redaction/Detectors/Financial/FinancialDetectors.cs @@ -0,0 +1,196 @@ +using System.Text.RegularExpressions; +using Bower.Redaction.Privacy; +using Bower.Redaction.Validation; + +namespace Bower.Redaction.Detectors.Financial; + +public sealed partial class CreditCardDetector : ISensitiveDetector +{ + public string Id => DetectorIds.CreditCard; + public string Category => DetectorCategories.Financial; + + public void Detect(ReadOnlySpan text, string path, ICollection matches) + { + if (text.IsEmpty) + { + return; + } + + string s = text.ToString(); + foreach (Match match in CardRegex().Matches(s)) + { + string digits = ChecksumAlgorithms.DigitsOnly(match.Value); + if (digits.Length is < 13 or > 19) + { + continue; + } + + if (!ChecksumAlgorithms.Luhn(digits)) + { + continue; + } + + string? network = ClassifyNetwork(digits); + if (network is null) + { + continue; + } + + matches.Add(new DetectionMatch( + Id, + Category, + match.Index, + match.Length, + Validated: true, + SubKind: network)); + } + } + + private static string? ClassifyNetwork(string digits) + { + if (digits[0] == '4' && digits.Length is 13 or 16 or 19) + { + return "Visa"; + } + + if (digits.Length == 16) + { + int prefix2 = (digits[0] - '0') * 10 + (digits[1] - '0'); + if (prefix2 is >= 51 and <= 55) + { + return "Mastercard"; + } + + if (digits.StartsWith("2221", StringComparison.Ordinal) || + digits.StartsWith("2720", StringComparison.Ordinal) || + (int.TryParse(digits.AsSpan(0, 4), out int p4) && p4 is >= 2221 and <= 2720)) + { + return "Mastercard"; + } + + if (digits.StartsWith("6011", StringComparison.Ordinal) || + digits.StartsWith("65", StringComparison.Ordinal)) + { + return "Discover"; + } + + if (digits.StartsWith("35", StringComparison.Ordinal)) + { + return "JCB"; + } + } + + if (digits.Length == 15 && + (digits.StartsWith("34", StringComparison.Ordinal) || + digits.StartsWith("37", StringComparison.Ordinal))) + { + return "Amex"; + } + + if (digits.Length is 14 or 16 && + (digits.StartsWith("36", StringComparison.Ordinal) || + digits.StartsWith("38", StringComparison.Ordinal) || + digits.StartsWith("300", StringComparison.Ordinal))) + { + return "Diners"; + } + + return null; + } + + [GeneratedRegex(@"\b(?:\d[ -]*?){13,19}\b", RegexOptions.CultureInvariant)] + private static partial Regex CardRegex(); +} + +public sealed partial class BsbAccountDetector : RegexDetectorBase +{ + public override string Id => DetectorIds.BsbAccount; + public override string Category => DetectorCategories.Financial; + protected override Regex Pattern => BsbRegex(); + + [GeneratedRegex( + @"\b(\d{3}[-\s]?\d{3})(?:[/\s:]+|[\s]*account[\s#:]*)(\d{4,10})\b", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex BsbRegex(); +} + +public sealed partial class IbanDetector : RegexDetectorBase +{ + public override string Id => DetectorIds.Iban; + public override string Category => DetectorCategories.Financial; + protected override Regex Pattern => IbanRegex(); + protected override bool RequiresValidation => true; + + protected override bool Validate(ReadOnlySpan matched) + { + string compact = string.Concat(matched.ToString().Where(c => !char.IsWhiteSpace(c))).ToUpperInvariant(); + if (compact.Length is < 15 or > 34) + { + return false; + } + + // ISO 13616 mod-97 check + string rearranged = compact[4..] + compact[..4]; + int total = 0; + foreach (char c in rearranged) + { + if (c is >= '0' and <= '9') + { + total = ((total * 10) + (c - '0')) % 97; + } + else if (c is >= 'A' and <= 'Z') + { + int v = c - 'A' + 10; + total = ((total * 10) + (v / 10)) % 97; + total = ((total * 10) + (v % 10)) % 97; + } + else + { + return false; + } + } + + return total == 1; + } + + [GeneratedRegex(@"\b([A-Z]{2}\d{2}[A-Z0-9]{11,30})\b", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex IbanRegex(); +} + +public sealed partial class SwiftBicDetector : RegexDetectorBase +{ + public override string Id => DetectorIds.SwiftBic; + public override string Category => DetectorCategories.Financial; + protected override Regex Pattern => SwiftRegex(); + + [GeneratedRegex(@"\b([A-Z]{4}[A-Z]{2}[A-Z0-9]{2}(?:[A-Z0-9]{3})?)\b", RegexOptions.CultureInvariant)] + private static partial Regex SwiftRegex(); +} + +public sealed partial class PayIdDetector : RegexDetectorBase +{ + public override string Id => DetectorIds.PayId; + public override string Category => DetectorCategories.Financial; + protected override Regex Pattern => PayIdRegex(); + + // PayID is typically email, phone, or ABN — flag explicit PayID labels. + [GeneratedRegex( + @"\bPayID[:\s=]+(\S+)", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex PayIdRegex(); + + public override void Detect(ReadOnlySpan text, string path, ICollection matches) + { + if (text.IsEmpty) + { + return; + } + + string s = text.ToString(); + foreach (Match match in Pattern.Matches(s)) + { + Group g = match.Groups.Count > 1 ? match.Groups[1] : match.Groups[0]; + matches.Add(new DetectionMatch(Id, Category, g.Index, g.Length, Validated: false)); + } + } +} diff --git a/src/Bower.Redaction/Detectors/Identity/IdentityDetectors.cs b/src/Bower.Redaction/Detectors/Identity/IdentityDetectors.cs new file mode 100644 index 0000000..579f3ae --- /dev/null +++ b/src/Bower.Redaction/Detectors/Identity/IdentityDetectors.cs @@ -0,0 +1,145 @@ +using System.Text.RegularExpressions; +using Bower.Redaction.Privacy; + +namespace Bower.Redaction.Detectors.Identity; + +public sealed partial class EmailDetector : RegexDetectorBase +{ + public override string Id => DetectorIds.Email; + public override string Category => DetectorCategories.Identity; + protected override Regex Pattern => EmailRegex(); + + [GeneratedRegex( + @"[A-Z0-9._%+\-]+@[A-Z0-9.\-]+\.[A-Z]{2,}", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex EmailRegex(); +} + +public sealed partial class PhoneAuDetector : RegexDetectorBase +{ + public override string Id => DetectorIds.PhoneAu; + public override string Category => DetectorCategories.Identity; + protected override Regex Pattern => PhoneAuRegex(); + + [GeneratedRegex( + @"\b(?:\+?61[\s\-]?|0)(?:4\d{2}|[2378]\d)[\s\-]?\d{3}[\s\-]?\d{3}\b", + RegexOptions.CultureInvariant)] + private static partial Regex PhoneAuRegex(); +} + +public sealed partial class PhoneInternationalDetector : RegexDetectorBase +{ + public override string Id => DetectorIds.PhoneInternational; + public override string Category => DetectorCategories.Identity; + protected override Regex Pattern => PhoneIntlRegex(); + + [GeneratedRegex( + @"\b\+[1-9]\d{6,14}\b", + RegexOptions.CultureInvariant)] + private static partial Regex PhoneIntlRegex(); +} + +public sealed partial class DateOfBirthDetector : RegexDetectorBase +{ + public override string Id => DetectorIds.DateOfBirth; + public override string Category => DetectorCategories.Identity; + protected override Regex Pattern => DobRegex(); + + [GeneratedRegex( + @"\b(?:DOB|Date\s*of\s*Birth|born)[:\s#\-]*(\d{1,2}[/\-.]\d{1,2}[/\-.]\d{2,4})\b", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex DobRegex(); + + public override void Detect(ReadOnlySpan text, string path, ICollection matches) + { + if (text.IsEmpty) + { + return; + } + + string s = text.ToString(); + foreach (Match match in Pattern.Matches(s)) + { + Group g = match.Groups.Count > 1 && match.Groups[1].Success + ? match.Groups[1] + : match.Groups[0]; + matches.Add(new DetectionMatch(Id, Category, g.Index, g.Length, Validated: false)); + } + } +} + +public sealed partial class AddressDetector : RegexDetectorBase +{ + public override string Id => DetectorIds.Address; + public override string Category => DetectorCategories.Identity; + protected override Regex Pattern => AddressRegex(); + + // Opt-in: Australian-style street address heuristic. + [GeneratedRegex( + @"\b\d{1,5}\s+[A-Z][A-Za-z'\-]+(?:\s+[A-Z][A-Za-z'\-]+){0,3}\s+(?:Street|St|Road|Rd|Avenue|Ave|Drive|Dr|Court|Ct|Place|Pl|Lane|Ln|Parade|Pde|Crescent|Cres|Boulevard|Blvd|Highway|Hwy)\b", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex AddressRegex(); +} + +public sealed partial class GpsCoordinateDetector : RegexDetectorBase +{ + public override string Id => DetectorIds.Gps; + public override string Category => DetectorCategories.Identity; + protected override Regex Pattern => GpsRegex(); + + [GeneratedRegex( + @"\b-?\d{1,2}\.\d{4,},\s*-?\d{1,3}\.\d{4,}\b", + RegexOptions.CultureInvariant)] + private static partial Regex GpsRegex(); +} + +public sealed partial class IpAddressDetector : RegexDetectorBase +{ + public override string Id => DetectorIds.IpAddress; + public override string Category => DetectorCategories.Identity; + protected override Regex Pattern => IpRegex(); + + [GeneratedRegex( + @"\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b", + RegexOptions.CultureInvariant)] + private static partial Regex IpRegex(); +} + +public sealed partial class HostnameDetector : RegexDetectorBase +{ + public override string Id => DetectorIds.Hostname; + public override string Category => DetectorCategories.Identity; + protected override Regex Pattern => HostRegex(); + + [GeneratedRegex( + @"\b(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.)+(?:com|net|org|io|au|local|internal|corp)\b", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex HostRegex(); +} + +public sealed partial class UsernameDetector : RegexDetectorBase +{ + public override string Id => DetectorIds.Username; + public override string Category => DetectorCategories.Identity; + protected override Regex Pattern => UsernameRegex(); + + [GeneratedRegex( + @"\b(?:username|user\s*name|login)[:\s=]+([A-Za-z0-9._\-]{2,64})\b", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex UsernameRegex(); + + public override void Detect(ReadOnlySpan text, string path, ICollection matches) + { + if (text.IsEmpty) + { + return; + } + + string s = text.ToString(); + foreach (Match match in Pattern.Matches(s)) + { + Group g = match.Groups.Count > 1 ? match.Groups[1] : match.Groups[0]; + matches.Add(new DetectionMatch(Id, Category, g.Index, g.Length, Validated: false)); + } + } +} diff --git a/src/Bower.Redaction/Detectors/RegexDetectorBase.cs b/src/Bower.Redaction/Detectors/RegexDetectorBase.cs new file mode 100644 index 0000000..4cae24e --- /dev/null +++ b/src/Bower.Redaction/Detectors/RegexDetectorBase.cs @@ -0,0 +1,53 @@ +using System.Text.RegularExpressions; +using Bower.Redaction.Privacy; + +namespace Bower.Redaction.Detectors; + +/// Shared regex scan helper for value-based detectors. +public abstract partial class RegexDetectorBase : ISensitiveDetector +{ + public abstract string Id { get; } + + public abstract string Category { get; } + + protected abstract Regex Pattern { get; } + + protected virtual bool RequiresValidation => false; + + protected virtual bool Validate(ReadOnlySpan matched) => true; + + protected virtual string? SubKind(ReadOnlySpan matched) => null; + + public virtual void Detect(ReadOnlySpan text, string path, ICollection matches) + { + // Regex operates on string; single allocation only when content present. + if (text.IsEmpty) + { + return; + } + + string s = text.ToString(); + foreach (Match match in Pattern.Matches(s)) + { + if (!match.Success) + { + continue; + } + + ReadOnlySpan span = s.AsSpan(match.Index, match.Length); + bool validated = !RequiresValidation || Validate(span); + if (RequiresValidation && !validated) + { + continue; + } + + matches.Add(new DetectionMatch( + Id, + Category, + match.Index, + match.Length, + validated, + SubKind(span))); + } + } +} diff --git a/src/Bower.Redaction/Detectors/Secrets/SecretDetectors.cs b/src/Bower.Redaction/Detectors/Secrets/SecretDetectors.cs new file mode 100644 index 0000000..2510be8 --- /dev/null +++ b/src/Bower.Redaction/Detectors/Secrets/SecretDetectors.cs @@ -0,0 +1,234 @@ +using System.Text.RegularExpressions; +using Bower.Redaction.Privacy; + +namespace Bower.Redaction.Detectors.Secrets; + +public sealed partial class AwsSecretDetector : RegexDetectorBase +{ + public override string Id => DetectorIds.Aws; + public override string Category => DetectorCategories.Secrets; + protected override Regex Pattern => AwsRegex(); + + protected override string? SubKind(ReadOnlySpan matched) + { + if (matched.StartsWith("AKIA", StringComparison.Ordinal) || + matched.StartsWith("ASIA", StringComparison.Ordinal)) + { + return "access-key-id"; + } + + if (matched.Contains("ASIA", StringComparison.Ordinal) || + matched.Contains("session", StringComparison.OrdinalIgnoreCase)) + { + return "session-token"; + } + + return "secret-access-key"; + } + + [GeneratedRegex( + @"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b|(?:aws)?_?secret_?(?:access)?_?key\s*[:=]\s*\S+|aws_session_token\s*[:=]\s*\S+", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex AwsRegex(); +} + +public sealed partial class AzureSecretDetector : RegexDetectorBase +{ + public override string Id => DetectorIds.Azure; + public override string Category => DetectorCategories.Secrets; + protected override Regex Pattern => AzureRegex(); + + [GeneratedRegex( + @"DefaultEndpointsProtocol=https?;AccountName=[^;]+;AccountKey=[A-Za-z0-9+/=]{20,}|sv=\d{4}-\d{2}-\d{2}[^;\s]*sig=[A-Za-z0-9%]+|SharedAccessSignature=[^\s;]+|AccountKey=[A-Za-z0-9+/=]{40,}", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex AzureRegex(); +} + +public sealed partial class EntraTokenDetector : RegexDetectorBase +{ + public override string Id => DetectorIds.Entra; + public override string Category => DetectorCategories.Secrets; + protected override Regex Pattern => EntraRegex(); + + [GeneratedRegex( + @"\b(?:eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,})\b|device_code\s*[:=]\s*[A-Za-z0-9\-_]{10,}", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex EntraRegex(); +} + +public sealed partial class GcpSecretDetector : RegexDetectorBase +{ + public override string Id => DetectorIds.Gcp; + public override string Category => DetectorCategories.Secrets; + protected override Regex Pattern => GcpRegex(); + + [GeneratedRegex( + @"""type""\s*:\s*""service_account""|""private_key""\s*:\s*""-----BEGIN|AIza[0-9A-Za-z\-_]{35}|ya29\.[0-9A-Za-z\-_]+", + RegexOptions.CultureInvariant)] + private static partial Regex GcpRegex(); +} + +public sealed partial class JwtDetector : RegexDetectorBase +{ + public override string Id => DetectorIds.Jwt; + public override string Category => DetectorCategories.Secrets; + protected override Regex Pattern => JwtRegex(); + protected override bool RequiresValidation => true; + + protected override bool Validate(ReadOnlySpan matched) + { + // Three Base64URL segments separated by '.'. + int first = matched.IndexOf('.'); + if (first <= 0) + { + return false; + } + + int second = matched[(first + 1)..].IndexOf('.'); + if (second <= 0) + { + return false; + } + + int secondAbs = first + 1 + second; + if (secondAbs >= matched.Length - 1) + { + return false; + } + + // Header should decode as JSON starting with '{' (Base64 of eyJ...). + return matched.StartsWith("eyJ", StringComparison.Ordinal); + } + + [GeneratedRegex( + @"\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b", + RegexOptions.CultureInvariant)] + private static partial Regex JwtRegex(); +} + +public sealed partial class OAuthTokenDetector : RegexDetectorBase +{ + public override string Id => DetectorIds.OAuth; + public override string Category => DetectorCategories.Secrets; + protected override Regex Pattern => OAuthRegex(); + + [GeneratedRegex( + @"\b(?:access_token|refresh_token|id_token)\s*[:=]\s*[A-Za-z0-9\-._~+/]+=*", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex OAuthRegex(); +} + +/// +/// Provider API key patterns. New providers are registered without changing the engine. +/// +public sealed partial class ApiKeyDetector : ISensitiveDetector +{ + public string Id => DetectorIds.ApiKey; + public string Category => DetectorCategories.Secrets; + + private static readonly ApiKeyPattern[] Patterns = + [ + new("OpenAI", @"\bsk-[A-Za-z0-9]{20,}\b"), + new("Anthropic", @"\bsk-ant-[A-Za-z0-9\-_]{20,}\b"), + new("GitHub", @"\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{36,}\b"), + new("GitLab", @"\bglpat-[A-Za-z0-9\-_]{20,}\b"), + new("Slack", @"\bxox[baprs]-[A-Za-z0-9-]{10,}\b"), + new("Stripe", @"\b(?:sk|rk|pk)_(?:live|test)_[A-Za-z0-9]{20,}\b"), + new("Twilio", @"\bSK[0-9a-fA-F]{32}\b"), + new("Cloudflare", @"\b(?:cfx|v1\.0-)[A-Za-z0-9_\-]{20,}\b"), + new("Atlassian", @"\bATATT3[A-Za-z0-9=_\-]{20,}\b"), + new("Datadog", @"\b(?:[a-f0-9]{32}|ddog_api_key)\b"), + new("PagerDuty", @"\b[a-f0-9]{32}\b"), // only with context below + new("Okta", @"\b00[A-Za-z0-9_\-]{40,}\b"), + new("MongoDB", @"\bmongodb(?:\+srv)?:\/\/[^\s]+\b"), + new("Snowflake", @"\b[A-Za-z0-9]{8,}-[A-Za-z0-9]{4,}\.[A-Za-z0-9]+\.snowflakecomputing\.com\b") + ]; + + private static readonly (string Provider, Regex Regex)[] Compiled = + Patterns.Select(p => (p.Provider, new Regex(p.Pattern, RegexOptions.CultureInvariant | RegexOptions.Compiled, TimeSpan.FromMilliseconds(50)))) + .ToArray(); + + public void Detect(ReadOnlySpan text, string path, ICollection matches) + { + if (text.IsEmpty) + { + return; + } + + string s = text.ToString(); + foreach ((string provider, Regex regex) in Compiled) + { + // High-FP patterns need context. + if (provider is "PagerDuty" or "Datadog") + { + if (!s.Contains(provider, StringComparison.OrdinalIgnoreCase) && + !s.Contains("api_key", StringComparison.OrdinalIgnoreCase) && + !s.Contains("apikey", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + } + + foreach (Match match in regex.Matches(s)) + { + matches.Add(new DetectionMatch( + Id, + Category, + match.Index, + match.Length, + Validated: false, + SubKind: provider)); + } + } + } + + private readonly record struct ApiKeyPattern(string Provider, string Pattern); +} + +public sealed partial class KubernetesSecretDetector : RegexDetectorBase +{ + public override string Id => DetectorIds.Kubernetes; + public override string Category => DetectorCategories.Secrets; + protected override Regex Pattern => K8sRegex(); + + [GeneratedRegex( + @"\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b|kubeconfig|kind:\s*Config[\s\S]{0,80}users:|Bearer\s+eyJ[A-Za-z0-9_-]+\.", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex K8sRegex(); +} + +public sealed partial class DockerCredentialDetector : RegexDetectorBase +{ + public override string Id => DetectorIds.Docker; + public override string Category => DetectorCategories.Secrets; + protected override Regex Pattern => DockerRegex(); + + [GeneratedRegex( + @"\b(?:docker\s+login|auths\s*\{|""auth""\s*:\s*""[A-Za-z0-9+/=]{20,}"")\b|registry[^\n]{0,40}password\s*[:=]\s*\S+", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex DockerRegex(); +} + +public sealed partial class DatabaseCredentialDetector : RegexDetectorBase +{ + public override string Id => DetectorIds.Database; + public override string Category => DetectorCategories.Secrets; + protected override Regex Pattern => DbRegex(); + + [GeneratedRegex( + @"(?:Server|Data Source|Host)=[^;]+;.*(?:Password|Pwd)=[^;]+|jdbc:[a-z0-9]+:\/\/[^\s]+|postgres(?:ql)?:\/\/[^\s]+|mysql:\/\/[^\s]+|mongodb(?:\+srv)?:\/\/[^\s]+|Driver=\{[^}]+\}.*Pwd=", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex DbRegex(); +} + +public sealed partial class EnvironmentVariableSecretDetector : RegexDetectorBase +{ + public override string Id => DetectorIds.EnvVar; + public override string Category => DetectorCategories.Secrets; + protected override Regex Pattern => EnvRegex(); + + [GeneratedRegex( + @"\b(?:PASSWORD|SECRET|TOKEN|API_KEY|PRIVATE_KEY|CLIENT_SECRET|ACCESS_KEY|AUTH_TOKEN)\s*=\s*\S+", + RegexOptions.CultureInvariant)] + private static partial Regex EnvRegex(); +} diff --git a/src/Bower.Redaction/JsonEventRedactor.cs b/src/Bower.Redaction/JsonEventRedactor.cs index 77273b6..edf094a 100644 --- a/src/Bower.Redaction/JsonEventRedactor.cs +++ b/src/Bower.Redaction/JsonEventRedactor.cs @@ -1,157 +1,27 @@ -using System.Text.Json; -using System.Text.Json.Nodes; using Bower.Abstractions; +using Bower.Redaction.Privacy; namespace Bower.Redaction; +/// +/// Compatibility redactor. Delegates to with default policy. +/// Field-name secrets are removed; emails and other PII are handled by detectors. +/// public sealed class JsonEventRedactor : IEventRedactor { - public const int MaximumPayloadBytes = 1_048_576; + public const int MaximumPayloadBytes = PrivacyEngine.MaximumPayloadBytes; - private static readonly HashSet RemoveNames = new( - [ - "password", - "passwordhash", - "accesstoken", - "refreshtoken", - "bearertoken", - "apikeysecret", - "clientsecret", - "privatekey", - "connectionstring", - "authorization", - "cookie", - "cookies", - "credential", - "credentials", - "requestbody", - "responsebody", - "body", - "headers", - "payload", - "filecontents" - ], - StringComparer.Ordinal); + private readonly PrivacyEngine engine; - private static readonly HashSet MaskNames = new( - ["email", "emailaddress"], - StringComparer.Ordinal); - - public RedactionResult Redact(string json) + public JsonEventRedactor() + : this(PrivacyPolicy.CreateDefault()) { - if (string.IsNullOrWhiteSpace(json)) - { - return Failure("empty-payload"); - } - - if (System.Text.Encoding.UTF8.GetByteCount(json) > MaximumPayloadBytes) - { - return Failure("payload-too-large"); - } - - try - { - JsonNode? root = JsonNode.Parse( - json, - documentOptions: new JsonDocumentOptions - { - AllowTrailingCommas = false, - CommentHandling = JsonCommentHandling.Disallow, - MaxDepth = 32 - }); - - if (root is not JsonObject rootObject) - { - return Failure("root-must-be-object"); - } - - List removed = []; - List masked = []; - RedactObject(rootObject, "$", removed, masked); - - return new RedactionResult( - true, - rootObject.ToJsonString(), - removed, - masked, - null); - } - catch (JsonException) - { - return Failure("invalid-json"); - } } - private static void RedactObject( - JsonObject value, - string parentPath, - List removed, - List masked) + public JsonEventRedactor(PrivacyPolicy policy) { - foreach ((string propertyName, JsonNode? child) in value.ToArray()) - { - string path = $"{parentPath}.{propertyName}"; - string normalizedName = Normalize(propertyName); - - if (RemoveNames.Contains(normalizedName)) - { - value.Remove(propertyName); - removed.Add(path); - continue; - } - - if (MaskNames.Contains(normalizedName) && child is JsonValue) - { - value[propertyName] = Mask(child.ToString()); - masked.Add(path); - continue; - } - - RedactNode(child, path, removed, masked); - } + engine = new PrivacyEngine(policy); } - private static void RedactNode( - JsonNode? node, - string path, - List removed, - List masked) - { - if (node is JsonObject childObject) - { - RedactObject(childObject, path, removed, masked); - return; - } - - if (node is not JsonArray array) - { - return; - } - - for (int index = 0; index < array.Count; index++) - { - RedactNode(array[index], $"{path}[{index}]", removed, masked); - } - } - - private static string Normalize(string value) - { - return string.Concat(value.Where(char.IsLetterOrDigit)).ToLowerInvariant(); - } - - private static string Mask(string value) - { - int separator = value.IndexOf('@', StringComparison.Ordinal); - if (separator <= 0) - { - return "***"; - } - - return $"{value[0]}***{value[separator..]}"; - } - - private static RedactionResult Failure(string code) - { - return new RedactionResult(false, null, [], [], code); - } + public RedactionResult Redact(string json) => engine.Redact(json); } diff --git a/src/Bower.Redaction/Privacy/DetectionMatch.cs b/src/Bower.Redaction/Privacy/DetectionMatch.cs new file mode 100644 index 0000000..f77b610 --- /dev/null +++ b/src/Bower.Redaction/Privacy/DetectionMatch.cs @@ -0,0 +1,16 @@ +namespace Bower.Redaction.Privacy; + +/// +/// A single sensitive span inside a field value. +/// Indices are UTF-16 char offsets into the original string. +/// +public readonly record struct DetectionMatch( + string DetectorId, + string Category, + int Start, + int Length, + bool Validated, + string? SubKind = null) +{ + public int End => Start + Length; +} diff --git a/src/Bower.Redaction/Privacy/DetectorCatalog.cs b/src/Bower.Redaction/Privacy/DetectorCatalog.cs new file mode 100644 index 0000000..3bf3e1e --- /dev/null +++ b/src/Bower.Redaction/Privacy/DetectorCatalog.cs @@ -0,0 +1,62 @@ +using Bower.Redaction.Detectors; +using Bower.Redaction.Detectors.Australian; +using Bower.Redaction.Detectors.Classification; +using Bower.Redaction.Detectors.Crypto; +using Bower.Redaction.Detectors.Financial; +using Bower.Redaction.Detectors.Identity; +using Bower.Redaction.Detectors.Secrets; + +namespace Bower.Redaction.Privacy; + +/// +/// Built-in detector modules. Custom detectors can be appended without modifying the engine. +/// +public static class DetectorCatalog +{ + public static IReadOnlyList CreateDefaultValueDetectors() => + [ + // Australian + new TfnDetector(), + new CrnDetector(), + new MedicareDetector(), + new IhiDetector(), + new PassportDetector(), + new DriverLicenceDetector(), + new AbnDetector(), + new AcnDetector(), + new DvaDetector(), + // Financial + new CreditCardDetector(), + new BsbAccountDetector(), + new IbanDetector(), + new SwiftBicDetector(), + new PayIdDetector(), + // Identity + new EmailDetector(), + new PhoneAuDetector(), + new PhoneInternationalDetector(), + new DateOfBirthDetector(), + new AddressDetector(), + new GpsCoordinateDetector(), + new IpAddressDetector(), + new HostnameDetector(), + new UsernameDetector(), + // Secrets + new AwsSecretDetector(), + new AzureSecretDetector(), + new EntraTokenDetector(), + new GcpSecretDetector(), + new JwtDetector(), + new OAuthTokenDetector(), + new ApiKeyDetector(), + new KubernetesSecretDetector(), + new DockerCredentialDetector(), + new DatabaseCredentialDetector(), + new EnvironmentVariableSecretDetector(), + // Crypto + classification + new CryptographicMaterialDetector(), + new SecurityMarkingDetector() + ]; + + public static IFieldNameDetector CreateDefaultFieldNameDetector() => new FieldNameSecretDetector(); +} diff --git a/src/Bower.Redaction/Privacy/DetectorIds.cs b/src/Bower.Redaction/Privacy/DetectorIds.cs new file mode 100644 index 0000000..e5d889d --- /dev/null +++ b/src/Bower.Redaction/Privacy/DetectorIds.cs @@ -0,0 +1,66 @@ +namespace Bower.Redaction.Privacy; + +/// Stable detector identifiers used for policy overrides and metadata. +public static class DetectorIds +{ + public const string FieldNameSecret = "field-name-secret"; + + // Australian regulated identifiers + public const string Tfn = "au.tfn"; + public const string Crn = "au.crn"; + public const string Medicare = "au.medicare"; + public const string Ihi = "au.ihi"; + public const string Passport = "au.passport"; + public const string DriverLicence = "au.driver-licence"; + public const string Abn = "au.abn"; + public const string Acn = "au.acn"; + public const string Dva = "au.dva"; + + // Financial + public const string CreditCard = "fin.credit-card"; + public const string BsbAccount = "fin.bsb-account"; + public const string Iban = "fin.iban"; + public const string SwiftBic = "fin.swift-bic"; + public const string PayId = "fin.payid"; + + // Identity / personal + public const string Email = "id.email"; + public const string PhoneAu = "id.phone.au"; + public const string PhoneInternational = "id.phone.intl"; + public const string DateOfBirth = "id.dob"; + public const string Address = "id.address"; + public const string Gps = "id.gps"; + public const string IpAddress = "id.ip"; + public const string Hostname = "id.hostname"; + public const string Username = "id.username"; + + // Cloud / auth secrets + public const string Aws = "secret.aws"; + public const string Azure = "secret.azure"; + public const string Entra = "secret.entra"; + public const string Gcp = "secret.gcp"; + public const string Jwt = "secret.jwt"; + public const string OAuth = "secret.oauth"; + public const string ApiKey = "secret.api-key"; + public const string Kubernetes = "secret.kubernetes"; + public const string Docker = "secret.docker"; + public const string Database = "secret.database"; + public const string EnvVar = "secret.env-var"; + + // Crypto material + public const string CryptoMaterial = "crypto.material"; + + // Classification markings + public const string SecurityMarking = "class.security-marking"; +} + +public static class DetectorCategories +{ + public const string FieldName = "field-name"; + public const string Australian = "australian"; + public const string Financial = "financial"; + public const string Identity = "identity"; + public const string Secrets = "secrets"; + public const string Crypto = "crypto"; + public const string Classification = "classification"; +} diff --git a/src/Bower.Redaction/Privacy/ISensitiveDetector.cs b/src/Bower.Redaction/Privacy/ISensitiveDetector.cs new file mode 100644 index 0000000..d25902b --- /dev/null +++ b/src/Bower.Redaction/Privacy/ISensitiveDetector.cs @@ -0,0 +1,34 @@ +namespace Bower.Redaction.Privacy; + +/// +/// Pluggable sensitive-data detector. Runtime must stay deterministic: +/// compiled patterns, checksums, structure and entropy — no AI at runtime. +/// +public interface ISensitiveDetector +{ + /// Stable id used in policy and metadata (e.g. au.tfn). + string Id { get; } + + /// Module category (australian, financial, secrets, …). + string Category { get; } + + /// + /// Scan and append matches to . + /// Implementations must not allocate when no match is found where practical. + /// + void Detect(ReadOnlySpan text, string path, ICollection matches); +} + +/// +/// Detector that also decides whole JSON properties by field name +/// (password, authorization, …) rather than value content. +/// +public interface IFieldNameDetector +{ + string Id { get; } + + string Category { get; } + + /// Returns true when the normalised field name should be treated as secret. + bool MatchesFieldName(string normalizedFieldName); +} diff --git a/src/Bower.Redaction/Privacy/PolicyApplicator.cs b/src/Bower.Redaction/Privacy/PolicyApplicator.cs new file mode 100644 index 0000000..38fb550 --- /dev/null +++ b/src/Bower.Redaction/Privacy/PolicyApplicator.cs @@ -0,0 +1,114 @@ +using System.Security.Cryptography; +using System.Text; + +namespace Bower.Redaction.Privacy; + +/// Applies to a matched substring. Never logs originals. +public sealed class PolicyApplicator +{ + private readonly PrivacyPolicy policy; + private readonly KeyedFieldHasher? hmacHasher; + + public PolicyApplicator(PrivacyPolicy policy) + { + this.policy = policy; + hmacHasher = policy.HmacKey is { Length: >= 32 } key + ? new KeyedFieldHasher(key) + : null; + } + + public string Apply(string original, DetectionMatch match, PrivacyAction action) + { + string span = original.Substring(match.Start, match.Length); + return action switch + { + PrivacyAction.Allow => span, + PrivacyAction.AlertOnly => span, + PrivacyAction.Remove => string.Empty, + PrivacyAction.Replace => policy.ReplacementText, + PrivacyAction.Mask => Mask(span, match.DetectorId), + PrivacyAction.Sha256 => Sha256(span), + PrivacyAction.Hmac => hmacHasher?.Hash(span) ?? Sha256(span), + PrivacyAction.Encrypt => Encrypt(span) ?? string.Empty, + _ => policy.ReplacementText + }; + } + + public static string ActionLabel(PrivacyAction action) => action switch + { + PrivacyAction.Allow => "Allow", + PrivacyAction.Remove => "Removed", + PrivacyAction.Replace => "Replaced", + PrivacyAction.Mask => "Masked", + PrivacyAction.Sha256 => "SHA256", + PrivacyAction.Hmac => "HMAC", + PrivacyAction.Encrypt => "Encrypted", + PrivacyAction.AlertOnly => "AlertOnly", + _ => action.ToString() + }; + + private static string Sha256(string value) + { + byte[] digest = SHA256.HashData(Encoding.UTF8.GetBytes(value)); + return "sha256:" + Convert.ToHexStringLower(digest); + } + + private string? Encrypt(string value) + { + if (policy.EncryptionKey is not { Length: 32 } key) + { + return null; + } + + byte[] plaintext = Encoding.UTF8.GetBytes(value); + byte[] nonce = new byte[12]; + RandomNumberGenerator.Fill(nonce); + byte[] ciphertext = new byte[plaintext.Length]; + byte[] tag = new byte[16]; + using AesGcm aes = new(key, tag.Length); + aes.Encrypt(nonce, plaintext, ciphertext, tag); + return "enc:aesgcm:" + + Convert.ToBase64String(nonce) + ":" + + Convert.ToBase64String(tag) + ":" + + Convert.ToBase64String(ciphertext); + } + + private static string Mask(string value, string detectorId) + { + if (detectorId == DetectorIds.Email) + { + int at = value.IndexOf('@'); + if (at > 0) + { + return value[0] + "***" + value[at..]; + } + } + + if (detectorId == DetectorIds.CreditCard) + { + string cardDigits = Validation.ChecksumAlgorithms.DigitsOnly(value); + if (cardDigits.Length >= 4) + { + return "****-****-****-" + cardDigits[^4..]; + } + } + + if (detectorId is DetectorIds.Tfn or DetectorIds.Medicare + or DetectorIds.Ihi or DetectorIds.Crn or DetectorIds.Abn or DetectorIds.Acn + or DetectorIds.BsbAccount) + { + string digits = Validation.ChecksumAlgorithms.DigitsOnly(value); + if (digits.Length >= 4) + { + return new string('*', Math.Max(0, digits.Length - 4)) + digits[^4..]; + } + } + + if (value.Length <= 4) + { + return "****"; + } + + return value[..2] + new string('*', value.Length - 4) + value[^2..]; + } +} diff --git a/src/Bower.Redaction/Privacy/PrivacyAction.cs b/src/Bower.Redaction/Privacy/PrivacyAction.cs new file mode 100644 index 0000000..637fb71 --- /dev/null +++ b/src/Bower.Redaction/Privacy/PrivacyAction.cs @@ -0,0 +1,17 @@ +namespace Bower.Redaction.Privacy; + +/// +/// Deterministic policy actions applied to a detected sensitive span. +/// Original values are never written into privacy metadata. +/// +public enum PrivacyAction +{ + Allow, + Remove, + Replace, + Mask, + Sha256, + Hmac, + Encrypt, + AlertOnly +} diff --git a/src/Bower.Redaction/Privacy/PrivacyEngine.cs b/src/Bower.Redaction/Privacy/PrivacyEngine.cs new file mode 100644 index 0000000..7a9e6c9 --- /dev/null +++ b/src/Bower.Redaction/Privacy/PrivacyEngine.cs @@ -0,0 +1,342 @@ +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using Bower.Abstractions; + +namespace Bower.Redaction.Privacy; + +/// +/// High-performance privacy & secret protection engine. +/// Operates after parse / before persistence and normalisation. +/// Deterministic: regex, checksum, structure — no AI at runtime. +/// +public sealed class PrivacyEngine : IEventRedactor +{ + public const int MaximumPayloadBytes = 1_048_576; + + private readonly PrivacyPolicy policy; + private readonly IReadOnlyList detectors; + private readonly IFieldNameDetector fieldNameDetector; + private readonly PolicyApplicator applicator; + + public PrivacyEngine( + PrivacyPolicy? policy = null, + IEnumerable? detectors = null, + IFieldNameDetector? fieldNameDetector = null) + { + this.policy = policy ?? PrivacyPolicy.CreateDefault(); + this.detectors = (detectors ?? DetectorCatalog.CreateDefaultValueDetectors()).ToArray(); + this.fieldNameDetector = fieldNameDetector ?? DetectorCatalog.CreateDefaultFieldNameDetector(); + applicator = new PolicyApplicator(this.policy); + } + + public PrivacyPolicy Policy => policy; + + public IReadOnlyList Detectors => detectors; + + public RedactionResult Redact(string json) => RedactJson(json).ToRedactionResult(); + + public PrivacyScanResult RedactJson(string json) + { + if (string.IsNullOrWhiteSpace(json)) + { + return PrivacyScanResult.Fail("empty-payload"); + } + + if (Encoding.UTF8.GetByteCount(json) > MaximumPayloadBytes) + { + return PrivacyScanResult.Fail("payload-too-large"); + } + + try + { + JsonNode? root = JsonNode.Parse( + json, + documentOptions: new JsonDocumentOptions + { + AllowTrailingCommas = false, + CommentHandling = JsonCommentHandling.Disallow, + MaxDepth = 32 + }); + + if (root is not JsonObject rootObject) + { + return PrivacyScanResult.Fail("root-must-be-object"); + } + + List findings = []; + WalkObject(rootObject, "$", findings); + + PrivacyMetadata metadata = BuildMetadata(findings); + if (policy.EmitMetadata && metadata.HasFindings) + { + rootObject["privacy"] = MetadataToNode(metadata); + } + + List removed = findings + .Where(f => f.Action is PrivacyAction.Remove) + .Select(f => f.Path) + .Distinct(StringComparer.Ordinal) + .ToList(); + List masked = findings + .Where(f => f.Action is PrivacyAction.Mask or PrivacyAction.Sha256 + or PrivacyAction.Hmac or PrivacyAction.Encrypt or PrivacyAction.Replace) + .Select(f => f.Path) + .Distinct(StringComparer.Ordinal) + .ToList(); + + return new PrivacyScanResult( + true, + rootObject.ToJsonString(), + findings, + metadata, + removed, + masked, + null); + } + catch (JsonException) + { + return PrivacyScanResult.Fail("invalid-json"); + } + } + + /// Scan and sanitise a raw text payload (syslog, CSV cell, etc.). + public PrivacyTextResult RedactText(string text) + { + if (text is null) + { + return new PrivacyTextResult(false, null, [], PrivacyMetadata.Empty, "empty-payload"); + } + + if (Encoding.UTF8.GetByteCount(text) > MaximumPayloadBytes) + { + return new PrivacyTextResult(false, null, [], PrivacyMetadata.Empty, "payload-too-large"); + } + + List findings = []; + string redacted = ApplyDetectorsToValue(text, "$", findings); + PrivacyMetadata metadata = BuildMetadata(findings); + return new PrivacyTextResult(true, redacted, findings, metadata, null); + } + + private void WalkObject(JsonObject value, string parentPath, List findings) + { + foreach ((string propertyName, JsonNode? child) in value.ToArray()) + { + string path = $"{parentPath}.{propertyName}"; + string normalized = NormalizeFieldName(propertyName); + + if (policy.IsDetectorEnabled(fieldNameDetector.Id) && + fieldNameDetector.MatchesFieldName(normalized)) + { + PrivacyAction action = policy.ResolveAction(fieldNameDetector.Id); + findings.Add(new AppliedFinding( + fieldNameDetector.Id, + fieldNameDetector.Category, + path, + action, + Validated: true, + SubKind: null)); + + if (action is not (PrivacyAction.Allow or PrivacyAction.AlertOnly)) + { + value.Remove(propertyName); + } + + continue; + } + + if (child is JsonValue jsonValue && jsonValue.TryGetValue(out string? text) && text is not null) + { + string redacted = ApplyDetectorsToValue(text, path, findings); + if (!ReferenceEquals(redacted, text) && redacted != text) + { + value[propertyName] = redacted; + } + + continue; + } + + if (child is JsonObject childObject) + { + WalkObject(childObject, path, findings); + } + else if (child is JsonArray array) + { + WalkArray(array, path, findings); + } + } + } + + private void WalkArray(JsonArray array, string path, List findings) + { + for (int index = 0; index < array.Count; index++) + { + string itemPath = $"{path}[{index}]"; + if (array[index] is JsonObject nested) + { + WalkObject(nested, itemPath, findings); + } + else if (array[index] is JsonValue arrayValue && + arrayValue.TryGetValue(out string? arrayText) && + arrayText is not null) + { + string redacted = ApplyDetectorsToValue(arrayText, itemPath, findings); + if (redacted != arrayText) + { + array[index] = redacted; + } + } + } + } + + private string ApplyDetectorsToValue(string text, string path, List findings) + { + List matches = []; + ReadOnlySpan span = text.AsSpan(); + foreach (ISensitiveDetector detector in detectors) + { + if (!policy.IsDetectorEnabled(detector.Id)) + { + continue; + } + + detector.Detect(span, path, matches); + } + + if (matches.Count == 0) + { + return text; + } + + // Resolve overlaps: prefer longer validated matches, then earlier start. + List ordered = matches + .OrderByDescending(m => m.Length) + .ThenBy(m => m.Start) + .ToList(); + + List selected = []; + foreach (DetectionMatch candidate in ordered) + { + if (selected.Any(existing => Overlaps(existing, candidate))) + { + continue; + } + + selected.Add(candidate); + } + + // Apply right-to-left so indices remain valid. + selected.Sort((a, b) => b.Start.CompareTo(a.Start)); + string result = text; + foreach (DetectionMatch match in selected) + { + PrivacyAction action = policy.ResolveAction(match.DetectorId); + findings.Add(new AppliedFinding( + match.DetectorId, + match.Category, + path, + action, + match.Validated, + match.SubKind)); + + if (action is PrivacyAction.Allow or PrivacyAction.AlertOnly) + { + continue; + } + + string replacement = applicator.Apply(result, match, action); + result = string.Concat( + result.AsSpan(0, match.Start), + replacement, + result.AsSpan(match.End)); + } + + return result; + } + + private static bool Overlaps(DetectionMatch a, DetectionMatch b) => + a.Start < b.End && b.Start < a.End; + + private static PrivacyMetadata BuildMetadata(List findings) + { + if (findings.Count == 0) + { + return PrivacyMetadata.Empty; + } + + List detected = findings + .Select(f => f.SubKind is null ? f.DetectorId : $"{f.DetectorId}:{f.SubKind}") + .Distinct(StringComparer.Ordinal) + .OrderBy(x => x, StringComparer.Ordinal) + .ToList(); + + Dictionary actions = new(StringComparer.Ordinal); + foreach (AppliedFinding finding in findings) + { + string key = finding.DetectorId; + string label = PolicyApplicator.ActionLabel(finding.Action); + if (!actions.ContainsKey(key)) + { + actions[key] = label; + } + } + + return new PrivacyMetadata(detected, actions); + } + + private static JsonObject MetadataToNode(PrivacyMetadata metadata) + { + JsonArray detected = new(); + foreach (string id in metadata.Detected) + { + detected.Add(id); + } + + JsonObject actions = new(); + foreach ((string key, string value) in metadata.Actions) + { + actions[key] = value; + } + + return new JsonObject + { + ["detected"] = detected, + ["actions"] = actions + }; + } + + private static string NormalizeFieldName(string value) => + string.Concat(value.Where(char.IsLetterOrDigit)).ToLowerInvariant(); +} + +public sealed record AppliedFinding( + string DetectorId, + string Category, + string Path, + PrivacyAction Action, + bool Validated, + string? SubKind); + +public sealed record PrivacyScanResult( + bool Succeeded, + string? RedactedJson, + IReadOnlyList Findings, + PrivacyMetadata Metadata, + IReadOnlyList RemovedPaths, + IReadOnlyList MaskedPaths, + string? FailureCode) +{ + public static PrivacyScanResult Fail(string code) => + new(false, null, [], PrivacyMetadata.Empty, [], [], code); + + public RedactionResult ToRedactionResult() => + new(Succeeded, RedactedJson, RemovedPaths, MaskedPaths, FailureCode); +} + +public sealed record PrivacyTextResult( + bool Succeeded, + string? RedactedText, + IReadOnlyList Findings, + PrivacyMetadata Metadata, + string? FailureCode); diff --git a/src/Bower.Redaction/Privacy/PrivacyMetadata.cs b/src/Bower.Redaction/Privacy/PrivacyMetadata.cs new file mode 100644 index 0000000..2a45327 --- /dev/null +++ b/src/Bower.Redaction/Privacy/PrivacyMetadata.cs @@ -0,0 +1,14 @@ +namespace Bower.Redaction.Privacy; + +/// +/// Sanitisation summary attached to processed events. +/// Must never contain original sensitive values. +/// +public sealed record PrivacyMetadata( + IReadOnlyList Detected, + IReadOnlyDictionary Actions) +{ + public static PrivacyMetadata Empty { get; } = new([], new Dictionary()); + + public bool HasFindings => Detected.Count > 0; +} diff --git a/src/Bower.Redaction/Privacy/PrivacyPolicy.cs b/src/Bower.Redaction/Privacy/PrivacyPolicy.cs new file mode 100644 index 0000000..f380a27 --- /dev/null +++ b/src/Bower.Redaction/Privacy/PrivacyPolicy.cs @@ -0,0 +1,114 @@ +namespace Bower.Redaction.Privacy; + +/// +/// Global and per-detector privacy policy. Deterministic; no executable config. +/// +public sealed class PrivacyPolicy +{ + public PrivacyAction DefaultAction { get; init; } = PrivacyAction.Mask; + + /// Replacement text used when action is Replace. + public string ReplacementText { get; init; } = "***REDACTED***"; + + /// Per-detector action overrides keyed by . + public IReadOnlyDictionary DetectorActions { get; init; } = + new Dictionary(StringComparer.Ordinal); + + /// Disabled detector ids (case-sensitive stable ids). + public IReadOnlySet DisabledDetectors { get; init; } = + new HashSet(StringComparer.Ordinal); + + /// Optional detectors that are off unless explicitly enabled. + public IReadOnlySet OptInDetectors { get; init; } = + new HashSet(StringComparer.Ordinal) + { + DetectorIds.IpAddress, + DetectorIds.Hostname, + DetectorIds.Username, + DetectorIds.Address, + DetectorIds.Gps, + DetectorIds.SecurityMarking + }; + + /// Explicitly enable opt-in detectors. + public IReadOnlySet EnabledOptInDetectors { get; init; } = + new HashSet(StringComparer.Ordinal); + + /// HMAC key for Hmac action (min 32 bytes). Null disables Hmac (falls back to Sha256). + public byte[]? HmacKey { get; init; } + + /// AES-256 key for Encrypt action (32 bytes). Null disables Encrypt (falls back to Remove). + public byte[]? EncryptionKey { get; init; } + + /// When true, injects privacy metadata object into JSON root. + public bool EmitMetadata { get; init; } = true; + + public bool IsDetectorEnabled(string detectorId) + { + if (DisabledDetectors.Contains(detectorId)) + { + return false; + } + + if (OptInDetectors.Contains(detectorId) && !EnabledOptInDetectors.Contains(detectorId)) + { + return false; + } + + return true; + } + + public PrivacyAction ResolveAction(string detectorId) + { + if (DetectorActions.TryGetValue(detectorId, out PrivacyAction action)) + { + return action; + } + + return DefaultAction; + } + + /// Enterprise-safe defaults for production telemetry gateways. + public static PrivacyPolicy CreateDefault() + { + return new PrivacyPolicy + { + DefaultAction = PrivacyAction.Mask, + DetectorActions = new Dictionary(StringComparer.Ordinal) + { + [DetectorIds.FieldNameSecret] = PrivacyAction.Remove, + [DetectorIds.Tfn] = PrivacyAction.Sha256, + [DetectorIds.Crn] = PrivacyAction.Mask, + [DetectorIds.Medicare] = PrivacyAction.Mask, + [DetectorIds.Ihi] = PrivacyAction.Mask, + [DetectorIds.Passport] = PrivacyAction.Mask, + [DetectorIds.DriverLicence] = PrivacyAction.Mask, + [DetectorIds.Abn] = PrivacyAction.Allow, + [DetectorIds.Acn] = PrivacyAction.Allow, + [DetectorIds.Dva] = PrivacyAction.Mask, + [DetectorIds.CreditCard] = PrivacyAction.Remove, + [DetectorIds.BsbAccount] = PrivacyAction.Mask, + [DetectorIds.Iban] = PrivacyAction.Mask, + [DetectorIds.SwiftBic] = PrivacyAction.Allow, + [DetectorIds.PayId] = PrivacyAction.Mask, + [DetectorIds.Email] = PrivacyAction.Mask, + [DetectorIds.PhoneAu] = PrivacyAction.Mask, + [DetectorIds.PhoneInternational] = PrivacyAction.Mask, + [DetectorIds.DateOfBirth] = PrivacyAction.Mask, + [DetectorIds.Aws] = PrivacyAction.Remove, + [DetectorIds.Azure] = PrivacyAction.Remove, + [DetectorIds.Entra] = PrivacyAction.Remove, + [DetectorIds.Gcp] = PrivacyAction.Remove, + [DetectorIds.Jwt] = PrivacyAction.Remove, + [DetectorIds.OAuth] = PrivacyAction.Remove, + [DetectorIds.ApiKey] = PrivacyAction.Remove, + [DetectorIds.Kubernetes] = PrivacyAction.Remove, + [DetectorIds.Docker] = PrivacyAction.Remove, + [DetectorIds.Database] = PrivacyAction.Remove, + [DetectorIds.EnvVar] = PrivacyAction.Remove, + [DetectorIds.CryptoMaterial] = PrivacyAction.Remove, + [DetectorIds.SecurityMarking] = PrivacyAction.AlertOnly + } + }; + } +} diff --git a/src/Bower.Redaction/SensitiveDataDetector.cs b/src/Bower.Redaction/SensitiveDataDetector.cs index 80d53e7..e2a201a 100644 --- a/src/Bower.Redaction/SensitiveDataDetector.cs +++ b/src/Bower.Redaction/SensitiveDataDetector.cs @@ -1,330 +1,94 @@ -using System.Text; -using System.Text.Json; -using System.Text.Json.Nodes; -using System.Text.RegularExpressions; using Bower.Abstractions; +using Bower.Redaction.Privacy; namespace Bower.Redaction; -public enum SensitiveFindingKind -{ - AwsAccessKey, - AwsSecretKey, - PrivateKeyBlock, - Jwt, - CreditCard, - Email, - IpAddress, - BearerToken, - ConnectionString, - GenericSecret -} - -public sealed record SensitiveFinding( - SensitiveFindingKind Kind, - string Path, - string Preview, - string Action); - -public sealed record SensitiveScanResult( - bool Succeeded, - string? RedactedJson, - IReadOnlyList Findings, - string? FailureCode); - -public sealed partial class SensitiveDataDetector +/// +/// Backward-compatible facade over . +/// Prefer for new code. +/// +public sealed class SensitiveDataDetector { - public const int MaximumPayloadBytes = 1_048_576; - - // Instance API mirrors JsonEventRedactor for DI wiring. - private readonly int maximumPayloadBytes = MaximumPayloadBytes; - - public SensitiveScanResult ScanAndRedact(string json, bool maskInPlace = true) - { - if (string.IsNullOrWhiteSpace(json)) - { - return new SensitiveScanResult(false, null, [], "empty-payload"); - } + public const int MaximumPayloadBytes = PrivacyEngine.MaximumPayloadBytes; - if (Encoding.UTF8.GetByteCount(json) > maximumPayloadBytes) - { - return new SensitiveScanResult(false, null, [], "payload-too-large"); - } - - try - { - JsonNode? root = JsonNode.Parse( - json, - documentOptions: new JsonDocumentOptions - { - AllowTrailingCommas = false, - CommentHandling = JsonCommentHandling.Disallow, - MaxDepth = 32 - }); - if (root is not JsonObject rootObject) - { - return new SensitiveScanResult(false, null, [], "root-must-be-object"); - } - - List findings = []; - Walk(rootObject, "$", findings, maskInPlace); - return new SensitiveScanResult(true, rootObject.ToJsonString(), findings, null); - } - catch (JsonException) - { - return new SensitiveScanResult(false, null, [], "invalid-json"); - } - } + private readonly PrivacyEngine engine; - public RedactionResult ToRedactionResult(string json) + public SensitiveDataDetector() + : this(PrivacyPolicy.CreateDefault()) { - SensitiveScanResult scan = ScanAndRedact(json, maskInPlace: true); - if (!scan.Succeeded || scan.RedactedJson is null) - { - return new RedactionResult(false, null, [], [], scan.FailureCode); - } - - List removed = scan.Findings - .Where(item => item.Action == "removed") - .Select(item => item.Path) - .ToList(); - List masked = scan.Findings - .Where(item => item.Action == "masked") - .Select(item => item.Path) - .ToList(); - return new RedactionResult(true, scan.RedactedJson, removed, masked, null); } - private static void Walk( - JsonObject value, - string parentPath, - List findings, - bool maskInPlace) + public SensitiveDataDetector(PrivacyPolicy policy) { - foreach ((string propertyName, JsonNode? child) in value.ToArray()) - { - string path = $"{parentPath}.{propertyName}"; - string normalized = Normalize(propertyName); - - if (IsSecretName(normalized)) - { - findings.Add( - new SensitiveFinding( - SensitiveFindingKind.GenericSecret, - path, - Preview(child?.ToString()), - "removed")); - if (maskInPlace) - { - value.Remove(propertyName); - } - - continue; - } - - if (child is JsonValue jsonValue && jsonValue.TryGetValue(out string? text) && text is not null) - { - foreach (SensitiveFinding finding in DetectInText(path, text)) - { - findings.Add(finding); - if (maskInPlace && finding.Action == "masked") - { - value[propertyName] = MaskValue(text, finding.Kind); - } - } - - continue; - } - - if (child is JsonObject childObject) - { - Walk(childObject, path, findings, maskInPlace); - } - else if (child is JsonArray array) - { - for (int index = 0; index < array.Count; index++) - { - if (array[index] is JsonObject nested) - { - Walk(nested, $"{path}[{index}]", findings, maskInPlace); - } - else if (array[index] is JsonValue arrayValue && - arrayValue.TryGetValue(out string? arrayText) && - arrayText is not null) - { - foreach (SensitiveFinding finding in DetectInText($"{path}[{index}]", arrayText)) - { - findings.Add(finding); - if (maskInPlace && finding.Action == "masked") - { - array[index] = MaskValue(arrayText, finding.Kind); - } - } - } - } - } - } + engine = new PrivacyEngine(policy); } - private static IEnumerable DetectInText(string path, string text) + public SensitiveScanResult ScanAndRedact(string json, bool maskInPlace = true) { - if (AwsAccessKeyRegex().IsMatch(text)) - { - yield return new SensitiveFinding( - SensitiveFindingKind.AwsAccessKey, - path, - Preview(text), - "masked"); - } - - if (AwsSecretKeyRegex().IsMatch(text) || text.Contains("aws_secret_access_key", StringComparison.OrdinalIgnoreCase)) - { - yield return new SensitiveFinding( - SensitiveFindingKind.AwsSecretKey, - path, - Preview(text), - "masked"); - } - - if (text.Contains("BEGIN PRIVATE KEY", StringComparison.Ordinal) || - text.Contains("BEGIN RSA PRIVATE KEY", StringComparison.Ordinal)) + _ = maskInPlace; + PrivacyScanResult result = engine.RedactJson(json); + if (!result.Succeeded) { - yield return new SensitiveFinding( - SensitiveFindingKind.PrivateKeyBlock, - path, - Preview(text), - "masked"); + return new SensitiveScanResult(false, null, [], result.FailureCode); } - if (JwtRegex().IsMatch(text)) - { - yield return new SensitiveFinding(SensitiveFindingKind.Jwt, path, Preview(text), "masked"); - } - - if (text.StartsWith("Bearer ", StringComparison.OrdinalIgnoreCase)) - { - yield return new SensitiveFinding( - SensitiveFindingKind.BearerToken, - path, - Preview(text), - "masked"); - } - - if (CreditCardRegex().IsMatch(text) && LooksLikeCreditCard(text)) - { - yield return new SensitiveFinding( - SensitiveFindingKind.CreditCard, - path, - Preview(text), - "masked"); - } - - if (EmailRegex().IsMatch(text)) - { - yield return new SensitiveFinding(SensitiveFindingKind.Email, path, Preview(text), "masked"); - } - - if (text.Contains("Connection String", StringComparison.OrdinalIgnoreCase) || - text.Contains("Password=", StringComparison.OrdinalIgnoreCase) && - text.Contains(';', StringComparison.Ordinal)) - { - yield return new SensitiveFinding( - SensitiveFindingKind.ConnectionString, - path, - Preview(text), - "masked"); - } + List findings = result.Findings + .Select(MapFinding) + .ToList(); + return new SensitiveScanResult(true, result.RedactedJson, findings, null); } - private static bool IsSecretName(string normalized) - { - return normalized is "password" or "passwordhash" or "accesstoken" or "refreshtoken" - or "bearertoken" or "apikeysecret" or "clientsecret" or "privatekey" - or "connectionstring" or "authorization" or "cookie" or "cookies" - or "credential" or "credentials" or "secret" or "apikey"; - } + public RedactionResult ToRedactionResult(string json) => engine.Redact(json); - private static string MaskValue(string value, SensitiveFindingKind kind) + private static SensitiveFinding MapFinding(AppliedFinding finding) { - return kind switch - { - SensitiveFindingKind.Email => MaskEmail(value), - SensitiveFindingKind.CreditCard => "****-****-****-" + DigitsOnly(value)[^4..], - SensitiveFindingKind.AwsAccessKey => value.Length > 8 ? value[..4] + "********" + value[^2..] : "***", - _ => "***REDACTED***" + SensitiveFindingKind kind = finding.DetectorId switch + { + DetectorIds.Aws => SensitiveFindingKind.AwsAccessKey, + DetectorIds.Jwt or DetectorIds.Entra => SensitiveFindingKind.Jwt, + DetectorIds.CryptoMaterial => SensitiveFindingKind.PrivateKeyBlock, + DetectorIds.CreditCard => SensitiveFindingKind.CreditCard, + DetectorIds.Email => SensitiveFindingKind.Email, + DetectorIds.IpAddress => SensitiveFindingKind.IpAddress, + DetectorIds.Database => SensitiveFindingKind.ConnectionString, + DetectorIds.FieldNameSecret => SensitiveFindingKind.GenericSecret, + _ when finding.Category == DetectorCategories.Secrets => SensitiveFindingKind.GenericSecret, + _ => SensitiveFindingKind.GenericSecret }; - } - - private static string MaskEmail(string value) - { - Match match = EmailRegex().Match(value); - if (!match.Success) - { - return "***"; - } - - string email = match.Value; - int at = email.IndexOf('@', StringComparison.Ordinal); - return at <= 0 ? "***" : $"{email[0]}***{email[at..]}"; - } - - private static bool LooksLikeCreditCard(string value) - { - string digits = DigitsOnly(value); - if (digits.Length is < 13 or > 19) - { - return false; - } - // Luhn - int sum = 0; - bool alt = false; - for (int i = digits.Length - 1; i >= 0; i--) + string action = finding.Action switch { - int n = digits[i] - '0'; - if (alt) - { - n *= 2; - if (n > 9) - { - n -= 9; - } - } - - sum += n; - alt = !alt; - } - - return sum % 10 == 0; - } - - private static string DigitsOnly(string value) => string.Concat(value.Where(char.IsDigit)); - - private static string Normalize(string value) => - string.Concat(value.Where(char.IsLetterOrDigit)).ToLowerInvariant(); - - private static string Preview(string? value) - { - if (string.IsNullOrEmpty(value)) - { - return string.Empty; - } + PrivacyAction.Remove => "removed", + PrivacyAction.Allow or PrivacyAction.AlertOnly => "alert", + _ => "masked" + }; - return value.Length <= 12 ? "***" : value[..4] + "…"; + return new SensitiveFinding(kind, finding.Path, Preview: "***", action); } +} - [GeneratedRegex(@"\bAKIA[0-9A-Z]{16}\b", RegexOptions.CultureInvariant)] - private static partial Regex AwsAccessKeyRegex(); - - [GeneratedRegex(@"\b(?:aws)?_?secret_?(?:access)?_?key\b\s*[:=]\s*\S+", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] - private static partial Regex AwsSecretKeyRegex(); - - [GeneratedRegex(@"\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b", RegexOptions.CultureInvariant)] - private static partial Regex JwtRegex(); +public enum SensitiveFindingKind +{ + AwsAccessKey, + AwsSecretKey, + PrivateKeyBlock, + Jwt, + CreditCard, + Email, + IpAddress, + BearerToken, + ConnectionString, + GenericSecret +} - [GeneratedRegex(@"\b(?:\d[ -]*?){13,19}\b", RegexOptions.CultureInvariant)] - private static partial Regex CreditCardRegex(); +public sealed record SensitiveFinding( + SensitiveFindingKind Kind, + string Path, + string Preview, + string Action); - [GeneratedRegex(@"[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] - private static partial Regex EmailRegex(); -} +public sealed record SensitiveScanResult( + bool Succeeded, + string? RedactedJson, + IReadOnlyList Findings, + string? FailureCode); diff --git a/src/Bower.Redaction/Validation/ChecksumAlgorithms.cs b/src/Bower.Redaction/Validation/ChecksumAlgorithms.cs new file mode 100644 index 0000000..e2a2996 --- /dev/null +++ b/src/Bower.Redaction/Validation/ChecksumAlgorithms.cs @@ -0,0 +1,227 @@ +namespace Bower.Redaction.Validation; + +/// +/// Deterministic checksum validators for regulated identifiers and payment data. +/// No network I/O; pure functions only. +/// +public static class ChecksumAlgorithms +{ + private static readonly int[] TfnWeights = [1, 4, 3, 7, 5, 8, 6, 9, 10]; + private static readonly int[] AbnWeights = [10, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19]; + private static readonly int[] MedicareWeights = [1, 3, 7, 9, 1, 3, 7, 9]; + + public static bool Luhn(ReadOnlySpan digits) + { + if (digits.IsEmpty) + { + return false; + } + + int sum = 0; + bool alternate = false; + for (int i = digits.Length - 1; i >= 0; i--) + { + char c = digits[i]; + if (c is < '0' or > '9') + { + return false; + } + + int n = c - '0'; + if (alternate) + { + n *= 2; + if (n > 9) + { + n -= 9; + } + } + + sum += n; + alternate = !alternate; + } + + return sum % 10 == 0; + } + + /// + /// ATO TFN algorithm. Accepts 8 or 9 digit numbers (8-digit treated as leading zero). + /// + public static bool IsValidTfn(ReadOnlySpan digits) + { + Span padded = stackalloc char[9]; + if (digits.Length == 8) + { + padded[0] = '0'; + digits.CopyTo(padded[1..]); + } + else if (digits.Length == 9) + { + digits.CopyTo(padded); + } + else + { + return false; + } + + int sum = 0; + for (int i = 0; i < 9; i++) + { + char c = padded[i]; + if (c is < '0' or > '9') + { + return false; + } + + sum += (c - '0') * TfnWeights[i]; + } + + return sum % 11 == 0; + } + + /// Australian Business Number (11 digits). + public static bool IsValidAbn(ReadOnlySpan digits) + { + if (digits.Length != 11) + { + return false; + } + + int first = digits[0] - '0'; + if (first is < 0 or > 9) + { + return false; + } + + int sum = (first - 1) * AbnWeights[0]; + for (int i = 1; i < 11; i++) + { + char c = digits[i]; + if (c is < '0' or > '9') + { + return false; + } + + sum += (c - '0') * AbnWeights[i]; + } + + return sum % 89 == 0; + } + + /// Australian Company Number (9 digits). + public static bool IsValidAcn(ReadOnlySpan digits) + { + if (digits.Length != 9) + { + return false; + } + + int sum = 0; + for (int i = 0; i < 8; i++) + { + char c = digits[i]; + if (c is < '0' or > '9') + { + return false; + } + + sum += (c - '0') * (8 - i); + } + + char check = digits[8]; + if (check is < '0' or > '9') + { + return false; + } + + int expected = (10 - (sum % 10)) % 10; + return check - '0' == expected; + } + + /// + /// Medicare card number: 10 digits (8 body + check + issue). + /// First digit must be 2–6. + /// + public static bool IsValidMedicare(ReadOnlySpan digits) + { + if (digits.Length != 10) + { + return false; + } + + char first = digits[0]; + if (first is < '2' or > '6') + { + return false; + } + + int sum = 0; + for (int i = 0; i < 8; i++) + { + char c = digits[i]; + if (c is < '0' or > '9') + { + return false; + } + + sum += (c - '0') * MedicareWeights[i]; + } + + char check = digits[8]; + char issue = digits[9]; + if (check is < '0' or > '9' || issue is < '1' or > '9') + { + return false; + } + + return (sum % 10) == (check - '0'); + } + + /// + /// Individual Healthcare Identifier: 16 digits starting with 800360, Luhn check. + /// + public static bool IsValidIhi(ReadOnlySpan digits) + { + if (digits.Length != 16) + { + return false; + } + + ReadOnlySpan prefix = digits[..6]; + if (!prefix.SequenceEqual("800360")) + { + return false; + } + + return Luhn(digits); + } + + public static string DigitsOnly(ReadOnlySpan value) + { + Span buffer = stackalloc char[value.Length]; + int count = 0; + foreach (char c in value) + { + if (c is >= '0' and <= '9') + { + buffer[count++] = c; + } + } + + return new string(buffer[..count]); + } + + public static int CountDigits(ReadOnlySpan value) + { + int count = 0; + foreach (char c in value) + { + if (c is >= '0' and <= '9') + { + count++; + } + } + + return count; + } +} diff --git a/tests/Bower.Benchmarks/Bower.Benchmarks.csproj b/tests/Bower.Benchmarks/Bower.Benchmarks.csproj new file mode 100644 index 0000000..1017edd --- /dev/null +++ b/tests/Bower.Benchmarks/Bower.Benchmarks.csproj @@ -0,0 +1,9 @@ + + + Exe + false + + + + + diff --git a/tests/Bower.Benchmarks/Program.cs b/tests/Bower.Benchmarks/Program.cs new file mode 100644 index 0000000..a5e3c75 --- /dev/null +++ b/tests/Bower.Benchmarks/Program.cs @@ -0,0 +1,78 @@ +using System.Diagnostics; +using Bower.Redaction.Privacy; + +// Lightweight throughput smoke benchmark (no BenchmarkDotNet dependency). +// Run: dotnet run --project tests/Bower.Benchmarks -c Release + +const int iterations = 5_000; +const string cleanEvent = + """ + { + "schemaVersion": "1.0.0", + "eventId": "evt-1", + "eventType": "authentication_success", + "eventCategory": "authentication", + "eventAction": "login", + "eventResult": "success", + "application": { "name": "demo", "environment": "dev" }, + "actor": { "userId": "u-1", "username": "demo.user" }, + "timeGenerated": "2026-01-01T00:00:00Z", + "message": "user signed in from office network" + } + """; + +const string dirtyEvent = + """ + { + "schemaVersion": "1.0.0", + "eventId": "evt-2", + "eventType": "authentication_failure", + "eventCategory": "authentication", + "eventAction": "login", + "eventResult": "failure", + "application": { "name": "demo", "environment": "dev" }, + "timeGenerated": "2026-01-01T00:00:00Z", + "tfn": "100000001", + "email": "alice@example.test", + "pan": "4111111111111111", + "awsKey": "AKIAIOSFODNN7EXAMPLE", + "token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxIn0.signature", + "note": "contact +61 412 345 678 and abn 51 824 753 556" + } + """; + +PrivacyEngine engine = new(); + +// Warmup +for (int i = 0; i < 200; i++) +{ + _ = engine.RedactJson(cleanEvent); + _ = engine.RedactJson(dirtyEvent); +} + +static (double opsPerSec, double meanUs) Measure(PrivacyEngine engine, string payload, int iterations) +{ + Stopwatch sw = Stopwatch.StartNew(); + for (int i = 0; i < iterations; i++) + { + PrivacyScanResult result = engine.RedactJson(payload); + if (!result.Succeeded) + { + throw new InvalidOperationException(result.FailureCode); + } + } + + sw.Stop(); + double ops = iterations / sw.Elapsed.TotalSeconds; + double meanUs = sw.Elapsed.TotalMicroseconds / iterations; + return (ops, meanUs); +} + +(double cleanOps, double cleanUs) = Measure(engine, cleanEvent, iterations); +(double dirtyOps, double dirtyUs) = Measure(engine, dirtyEvent, iterations); + +Console.WriteLine("Bower PrivacyEngine micro-benchmark"); +Console.WriteLine($" iterations: {iterations}"); +Console.WriteLine($" clean event: {cleanOps:F0} ops/s mean {cleanUs:F1} µs"); +Console.WriteLine($" dirty event: {dirtyOps:F0} ops/s mean {dirtyUs:F1} µs"); +Console.WriteLine($" detectors: {engine.Detectors.Count}"); diff --git a/tests/Bower.Benchmarks/packages.lock.json b/tests/Bower.Benchmarks/packages.lock.json new file mode 100644 index 0000000..4bff81e --- /dev/null +++ b/tests/Bower.Benchmarks/packages.lock.json @@ -0,0 +1,22 @@ +{ + "version": 2, + "dependencies": { + "net10.0": { + "bower.abstractions": { + "type": "Project", + "dependencies": { + "Bower.Contracts": "[1.0.0, )" + } + }, + "bower.contracts": { + "type": "Project" + }, + "bower.redaction": { + "type": "Project", + "dependencies": { + "Bower.Abstractions": "[1.0.0, )" + } + } + } + } +} \ No newline at end of file diff --git a/tests/Bower.UnitTests/PrivacyEngineTests.cs b/tests/Bower.UnitTests/PrivacyEngineTests.cs new file mode 100644 index 0000000..ec9c295 --- /dev/null +++ b/tests/Bower.UnitTests/PrivacyEngineTests.cs @@ -0,0 +1,286 @@ +using System.Text.Json; +using Bower.Redaction; +using Bower.Redaction.Privacy; +using Bower.Redaction.Validation; + +namespace Bower.UnitTests; + +public sealed class ChecksumAlgorithmsTests +{ + [Theory] + [InlineData("100000001", true)] + [InlineData("123456789", false)] + public void Tfn_Checksum(string digits, bool expected) + { + Assert.Equal(expected, ChecksumAlgorithms.IsValidTfn(digits)); + } + + [Fact] + public void Tfn_AcceptsEightDigitWhenPaddedValid() + { + // Known-valid 9-digit TFN; 8-digit form is accepted via leading-zero pad path + // when the padded value satisfies the ATO algorithm. + Assert.True(ChecksumAlgorithms.IsValidTfn("100000001")); + // Synthetic 8-digit: engine pads to 9 with leading zero before checksum. + // 086001002 → verify via direct 9-digit with leading zero if valid. + for (int n = 0; n < 1_000_000; n++) + { + string eight = n.ToString("D8", System.Globalization.CultureInfo.InvariantCulture); + string nine = "0" + eight; + if (!ChecksumAlgorithms.IsValidTfn(nine.AsSpan())) + { + continue; + } + + Assert.True(ChecksumAlgorithms.IsValidTfn(eight.AsSpan())); + return; + } + + Assert.Fail("No valid 8-digit TFN found in search window"); + } + + [Theory] + [InlineData("51824753556", true)] + [InlineData("51824753557", false)] + public void Abn_Checksum(string digits, bool expected) + { + Assert.Equal(expected, ChecksumAlgorithms.IsValidAbn(digits)); + } + + [Theory] + [InlineData("000000019", true)] + [InlineData("000000018", false)] + public void Acn_Checksum(string digits, bool expected) + { + Assert.Equal(expected, ChecksumAlgorithms.IsValidAcn(digits)); + } + + [Theory] + [InlineData("2123456701", true)] + [InlineData("2123456711", false)] // wrong check digit (pos 9); issue digit still 1 + public void Medicare_Checksum(string digits, bool expected) + { + Assert.Equal(expected, ChecksumAlgorithms.IsValidMedicare(digits)); + } + + [Theory] + [InlineData("8003600000000007", true)] + [InlineData("8003600000000008", false)] + public void Ihi_Checksum(string digits, bool expected) + { + Assert.Equal(expected, ChecksumAlgorithms.IsValidIhi(digits)); + } + + [Fact] + public void Luhn_VisaTestPan() + { + Assert.True(ChecksumAlgorithms.Luhn("4111111111111111")); + Assert.False(ChecksumAlgorithms.Luhn("4111111111111112")); + } +} + +public sealed class PrivacyEngineTests +{ + [Fact] + public void RedactJson_DetectsAustralianIdentifiersWithChecksum() + { + const string json = + """ + { + "tfn": "100000001", + "abn": "51 824 753 556", + "medicare": "2123 45670 1", + "ihi": "8003600000000007", + "crn": "123 456 789A" + } + """; + + PrivacyEngine engine = new(); + PrivacyScanResult result = engine.RedactJson(json); + + Assert.True(result.Succeeded); + Assert.Contains(result.Findings, f => f.DetectorId == DetectorIds.Tfn && f.Validated); + Assert.Contains(result.Findings, f => f.DetectorId == DetectorIds.Abn && f.Validated); + Assert.Contains(result.Findings, f => f.DetectorId == DetectorIds.Medicare && f.Validated); + Assert.Contains(result.Findings, f => f.DetectorId == DetectorIds.Ihi && f.Validated); + Assert.Contains(result.Findings, f => f.DetectorId == DetectorIds.Crn); + Assert.DoesNotContain("100000001", result.RedactedJson); + Assert.Contains("sha256:", result.RedactedJson, StringComparison.Ordinal); + Assert.True(result.Metadata.HasFindings); + Assert.Equal("SHA256", result.Metadata.Actions[DetectorIds.Tfn]); + Assert.Equal("Allow", result.Metadata.Actions[DetectorIds.Abn]); + } + + [Fact] + public void RedactJson_RejectsInvalidTfnChecksum() + { + const string json = """{ "tfn": "123456789" }"""; + PrivacyEngine engine = new(); + PrivacyScanResult result = engine.RedactJson(json); + + Assert.True(result.Succeeded); + Assert.DoesNotContain(result.Findings, f => f.DetectorId == DetectorIds.Tfn); + Assert.Contains("123456789", result.RedactedJson); + } + + [Fact] + public void RedactJson_RemovesSecretsAndJwt() + { + const string json = + """ + { + "awsKey": "AKIAIOSFODNN7EXAMPLE", + "token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxIn0.signature", + "password": "do-not-store", + "pem": "-----BEGIN RSA PRIVATE KEY-----\nMIIE\n-----END RSA PRIVATE KEY-----" + } + """; + + PrivacyEngine engine = new(); + PrivacyScanResult result = engine.RedactJson(json); + + Assert.True(result.Succeeded); + Assert.DoesNotContain("AKIAIOSFODNN7EXAMPLE", result.RedactedJson); + Assert.DoesNotContain("do-not-store", result.RedactedJson); + Assert.DoesNotContain("eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", result.RedactedJson); + Assert.DoesNotContain("BEGIN RSA PRIVATE KEY", result.RedactedJson); + Assert.Contains(result.Findings, f => f.DetectorId == DetectorIds.Aws); + Assert.Contains(result.Findings, f => f.DetectorId is DetectorIds.Jwt or DetectorIds.Entra); + Assert.Contains(result.Findings, f => f.DetectorId == DetectorIds.CryptoMaterial); + Assert.Contains(result.Findings, f => f.DetectorId == DetectorIds.FieldNameSecret); + } + + [Fact] + public void RedactJson_RemovesCreditCardByDefault() + { + const string json = """{ "pan": "4111111111111111" }"""; + PrivacyEngine engine = new(); + PrivacyScanResult result = engine.RedactJson(json); + + Assert.True(result.Succeeded); + Assert.Contains(result.Findings, f => f.DetectorId == DetectorIds.CreditCard && f.Validated); + Assert.DoesNotContain("4111111111111111", result.RedactedJson); + Assert.Equal("Removed", result.Metadata.Actions[DetectorIds.CreditCard]); + } + + [Fact] + public void RedactJson_MasksCreditCardWhenConfigured() + { + PrivacyPolicy policy = PrivacyPolicy.CreateDefault(); + Dictionary actions = new(policy.DetectorActions) + { + [DetectorIds.CreditCard] = PrivacyAction.Mask + }; + policy = new PrivacyPolicy + { + DefaultAction = policy.DefaultAction, + DetectorActions = actions, + EmitMetadata = true + }; + + PrivacyEngine engine = new(policy); + PrivacyScanResult result = engine.RedactJson("""{ "pan": "4111111111111111" }"""); + + Assert.Contains("****-****-****-1111", result.RedactedJson); + } + + [Fact] + public void RedactJson_MasksEmail() + { + PrivacyEngine engine = new(); + PrivacyScanResult result = engine.RedactJson("""{ "owner": "alice@example.test" }"""); + + Assert.Contains(result.Findings, f => f.DetectorId == DetectorIds.Email); + Assert.Contains("a***@example.test", result.RedactedJson); + } + + [Fact] + public void RedactJson_DetectsApiKeysByProvider() + { + const string json = + """ + { + "openai": "sk-abcdefghijklmnopqrstuvwxyz12", + "github": "ghp_abcdefghijklmnopqrstuvwxyz0123456789", + "stripe": "sk_live_abcdefghijklmnopqrstuv" + } + """; + + PrivacyEngine engine = new(); + PrivacyScanResult result = engine.RedactJson(json); + + Assert.Contains(result.Findings, f => f.DetectorId == DetectorIds.ApiKey && f.SubKind == "OpenAI"); + Assert.Contains(result.Findings, f => f.DetectorId == DetectorIds.ApiKey && f.SubKind == "GitHub"); + Assert.Contains(result.Findings, f => f.DetectorId == DetectorIds.ApiKey && f.SubKind == "Stripe"); + } + + [Fact] + public void RedactJson_RespectsDisabledDetector() + { + PrivacyPolicy policy = new PrivacyPolicy + { + DefaultAction = PrivacyAction.Mask, + DetectorActions = PrivacyPolicy.CreateDefault().DetectorActions, + DisabledDetectors = new HashSet { DetectorIds.Email }, + EmitMetadata = true + }; + + PrivacyEngine engine = new(policy); + PrivacyScanResult result = engine.RedactJson("""{ "owner": "alice@example.test" }"""); + + Assert.DoesNotContain(result.Findings, f => f.DetectorId == DetectorIds.Email); + Assert.Contains("alice@example.test", result.RedactedJson); + } + + [Fact] + public void RedactJson_EmitsPrivacyMetadataWithoutOriginals() + { + PrivacyEngine engine = new(); + PrivacyScanResult result = engine.RedactJson( + """{ "tfn": "100000001", "email": "bob@example.test" }"""); + + Assert.NotNull(result.RedactedJson); + using JsonDocument doc = JsonDocument.Parse(result.RedactedJson); + Assert.True(doc.RootElement.TryGetProperty("privacy", out JsonElement privacy)); + Assert.True(privacy.TryGetProperty("detected", out _)); + Assert.True(privacy.TryGetProperty("actions", out _)); + Assert.DoesNotContain("100000001", result.RedactedJson); + string privacyJson = privacy.GetRawText(); + Assert.DoesNotContain("100000001", privacyJson); + Assert.DoesNotContain("bob@example.test", privacyJson); + } + + [Fact] + public void RedactText_WorksForNonJsonSources() + { + PrivacyEngine engine = new(); + PrivacyTextResult result = engine.RedactText( + "user email alice@example.test pan 4111111111111111"); + + Assert.True(result.Succeeded); + Assert.DoesNotContain("4111111111111111", result.RedactedText); + Assert.Contains("a***@example.test", result.RedactedText); + } + + [Fact] + public void JsonEventRedactor_StillImplementsIEventRedactor() + { + JsonEventRedactor redactor = new(); + Bower.Abstractions.RedactionResult result = redactor.Redact( + """{ "actor": { "email": "alice@example.test", "password": "x" } }"""); + + Assert.True(result.Succeeded); + Assert.Contains("$.actor.password", result.RemovedPaths); + Assert.Contains("a***@example.test", result.RedactedJson); + } + + [Fact] + public void DetectorCatalog_RegistersAllModules() + { + IReadOnlyList detectors = DetectorCatalog.CreateDefaultValueDetectors(); + Assert.True(detectors.Count >= 30); + Assert.Contains(detectors, d => d.Id == DetectorIds.Tfn); + Assert.Contains(detectors, d => d.Id == DetectorIds.ApiKey); + Assert.Contains(detectors, d => d.Id == DetectorIds.CryptoMaterial); + } +} diff --git a/tests/Bower.UnitTests/SensitiveDataDetectorTests.cs b/tests/Bower.UnitTests/SensitiveDataDetectorTests.cs index 40b9538..046fcc5 100644 --- a/tests/Bower.UnitTests/SensitiveDataDetectorTests.cs +++ b/tests/Bower.UnitTests/SensitiveDataDetectorTests.cs @@ -1,4 +1,5 @@ using Bower.Redaction; +using Bower.Redaction.Privacy; namespace Bower.UnitTests; @@ -33,13 +34,30 @@ public void ScanAndRedact_DetectsAwsKeyJwtAndEmail() [Fact] public void ScanAndRedact_DetectsValidCreditCard() { - // Visa test number that passes Luhn + // Visa test number that passes Luhn — default policy removes. const string json = """{ "pan": "4111111111111111" }"""; SensitiveDataDetector detector = new(); SensitiveScanResult result = detector.ScanAndRedact(json); Assert.True(result.Succeeded); Assert.Contains(result.Findings, item => item.Kind == SensitiveFindingKind.CreditCard); + Assert.DoesNotContain("4111111111111111", result.RedactedJson); + } + + [Fact] + public void ScanAndRedact_CanMaskCreditCardWithPolicy() + { + PrivacyPolicy policy = new PrivacyPolicy + { + DefaultAction = PrivacyAction.Mask, + DetectorActions = new Dictionary + { + [DetectorIds.CreditCard] = PrivacyAction.Mask + } + }; + SensitiveDataDetector detector = new(policy); + SensitiveScanResult result = detector.ScanAndRedact("""{ "pan": "4111111111111111" }"""); + Assert.Contains("****-****-****-1111", result.RedactedJson); } }