From 159f4b4f24719b2f2aa2a728cd45da622a890cdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Gro=C3=9Fmann?= Date: Thu, 20 Aug 2026 15:45:30 +0200 Subject: [PATCH] feat(secrets): add Dedupe to remove subsumed patterns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dedupe collapses a pattern list to its maximal elements: patterns whose matches are fully covered by another entry are dropped, equivalent spellings (e.g. **, **/* and */**) keep their first occurrence, and input order is preserved. Containment is decided as true language containment over identifiers, computed on canonical token sequences with an O(L^2) DP per pair; length, shape and literal-anchor filters skip most pairs. This is deliberately stricter than Pattern.Includes, which also treats a literal as covering '*' and '*' as covering '**'. Verified against a brute-force oracle: every pattern pair with up to three components, checked against all identifiers with up to nine components over a three-symbol alphabet. Signed-off-by: Johannes Großmann --- x/secrets/dedupe.go | 202 +++++++++++++++++++++++++++++++++++++++ x/secrets/dedupe_test.go | 143 +++++++++++++++++++++++++++ 2 files changed, 345 insertions(+) create mode 100644 x/secrets/dedupe.go create mode 100644 x/secrets/dedupe_test.go diff --git a/x/secrets/dedupe.go b/x/secrets/dedupe.go new file mode 100644 index 00000000..9f9fd029 --- /dev/null +++ b/x/secrets/dedupe.go @@ -0,0 +1,202 @@ +// Copyright 2026 Docker, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package secrets + +import "strings" + +type tokenKind uint8 + +const ( + tokenLit tokenKind = iota + tokenStar + tokenGap +) + +type token struct { + kind tokenKind + lit string +} + +// canonicalize tokenizes a pattern such that two patterns match the same +// identifiers iff their tokens are equal: wildcard runs collapse into '*'s +// followed by one '**', and a pure-'**' pattern becomes "*/**" since +// identifiers are never empty. +func canonicalize(s string) []token { + parts := split(s) + toks := make([]token, 0, len(parts)) + stars, gap := 0, false + flush := func() { + for range stars { + toks = append(toks, token{kind: tokenStar}) + } + if gap { + toks = append(toks, token{kind: tokenGap}) + } + stars, gap = 0, false + } + for _, part := range parts { + switch part { + case "*": + stars++ + case "**": + gap = true + default: + flush() + toks = append(toks, token{kind: tokenLit, lit: part}) + } + } + flush() + if len(toks) == 1 && toks[0].kind == tokenGap { + toks = []token{{kind: tokenStar}, {kind: tokenGap}} + } + return toks +} + +func canonicalKey(toks []token) string { + var sb strings.Builder + for i, t := range toks { + if i > 0 { + sb.WriteByte('/') + } + switch t.kind { + case tokenStar: + sb.WriteByte('*') + case tokenGap: + sb.WriteString("**") + case tokenLit: + sb.WriteString(t.lit) + } + } + return sb.String() +} + +// includes reports whether p matches every identifier q matches, for +// canonical tokens. The component alphabet is unbounded, so a literal in p +// never covers a '*' or '**' in q. +func includes(p, q []token) bool { + np, nq := len(p), len(q) + // dp[j] == "p[i:] includes q[j:]"; row i needs only row i+1 (prev). + prev := make([]bool, nq+1) + cur := make([]bool, nq+1) + prev[nq] = true + for i := np - 1; i >= 0; i-- { + cur[nq] = p[i].kind == tokenGap && prev[nq] + for j := nq - 1; j >= 0; j-- { + switch { + case p[i].kind == tokenGap: + cur[j] = prev[j] || cur[j+1] + case q[j].kind == tokenGap: + // q's gap may be empty or start with an unknown component. + cur[j] = p[i].kind == tokenStar && cur[j+1] && prev[j] + case p[i].kind == tokenStar: + cur[j] = prev[j+1] + default: + cur[j] = q[j].kind == tokenLit && p[i].lit == q[j].lit && prev[j+1] + } + } + prev, cur = cur, prev + } + return prev[0] +} + +// Dedupe removes every pattern whose matches are all covered by another +// entry, keeping the first of equivalent spellings and preserving input +// order. Containment is over identifiers (at least one component) and is +// stricter than [Pattern.Includes]. Worst case O(n²·L²). +func Dedupe(patterns []Pattern) []Pattern { + entries := make([]dedupeEntry, 0, len(patterns)) + seen := make(map[string]struct{}, len(patterns)) + for _, p := range patterns { + toks := canonicalize(p.String()) + key := canonicalKey(toks) + if _, dup := seen[key]; dup { + continue + } + seen[key] = struct{}{} + entries = append(entries, newDedupeEntry(p, toks)) + } + + result := make([]Pattern, 0, len(entries)) + for i := range entries { + redundant := false + for j := range entries { + if i == j || !subsumes(&entries[j], &entries[i]) { + continue + } + // Keep the earlier entry on (impossible) mutual inclusion. + if j < i || !includes(entries[i].toks, entries[j].toks) { + redundant = true + break + } + } + if !redundant { + result = append(result, entries[i].original) + } + } + return result +} + +type dedupeEntry struct { + original Pattern + toks []token + minLen int // non-gap token count: shortest matched identifier + hasWild bool + hasGaps bool + firstLit string // literal anchors; "" when that end is a wildcard + lastLit string +} + +func newDedupeEntry(p Pattern, toks []token) dedupeEntry { + e := dedupeEntry{original: p, toks: toks} + for _, t := range toks { + if t.kind != tokenLit { + e.hasWild = true + } + if t.kind == tokenGap { + e.hasGaps = true + } else { + e.minLen++ + } + } + if len(toks) > 0 { + e.firstLit = toks[0].lit + e.lastLit = toks[len(toks)-1].lit + } + return e +} + +// subsumes reports whether other matches every identifier e matches, with +// cheap filters before the containment check. +func subsumes(other, e *dedupeEntry) bool { + // An all-literal pattern only includes itself, and equivalent entries + // are already removed. + if !other.hasWild { + return false + } + if other.minLen > e.minLen { + return false + } + // Gap-free other has no slack: e must be gap-free with equal length. + if !other.hasGaps && (e.hasGaps || other.minLen != e.minLen) { + return false + } + if other.firstLit != "" && other.firstLit != e.firstLit { + return false + } + if other.lastLit != "" && other.lastLit != e.lastLit { + return false + } + return includes(other.toks, e.toks) +} diff --git a/x/secrets/dedupe_test.go b/x/secrets/dedupe_test.go new file mode 100644 index 00000000..9ba74f73 --- /dev/null +++ b/x/secrets/dedupe_test.go @@ -0,0 +1,143 @@ +// Copyright 2026 Docker, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package secrets + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDedupe(t *testing.T) { + tests := []struct { + name string + input []string + expected []string + }{ + { + name: "double star spans single star", + input: []string{"**", "a/*"}, + expected: []string{"**"}, + }, + { + name: "literal tail subsumed by star tail", + input: []string{"a/**", "b", "c/*/a", "c/*/*"}, + expected: []string{"a/**", "b", "c/*/*"}, + }, + { + name: "equivalent spellings keep first occurrence", + input: []string{"**/*", "*/**", "**"}, + expected: []string{"**/*"}, + }, + { + name: "star grid collapses to most general", + input: []string{"a/b", "a/*", "*/b", "*/*"}, + expected: []string{"*/*"}, + }, + { + name: "disjoint literals survive", + input: []string{"a", "b"}, + expected: []string{"a", "b"}, + }, + { + name: "overlap without containment keeps both", + input: []string{"a/**", "**/a"}, + expected: []string{"a/**", "**/a"}, + }, + { + name: "exact duplicates keep first occurrence", + input: []string{"a", "a", "b"}, + expected: []string{"a", "b"}, + }, + { + name: "prefix does not include sibling stars", + input: []string{"docker/proj1/**", "docker/*/mcp/*"}, + expected: []string{"docker/proj1/**", "docker/*/mcp/*"}, + }, + { + name: "chain collapses to maximum", + input: []string{"a/b/c", "a/b/*", "a/**", "**"}, + expected: []string{"**"}, + }, + { + name: "single pattern", + input: []string{"foo"}, + expected: []string{"foo"}, + }, + { + name: "empty list", + input: []string{}, + expected: []string{}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + input := make([]Pattern, 0, len(tc.input)) + for _, s := range tc.input { + input = append(input, MustParsePattern(s)) + } + result := Dedupe(input) + resultStrings := make([]string, 0, len(result)) + for _, p := range result { + resultStrings = append(resultStrings, p.String()) + } + assert.Equal(t, tc.expected, resultStrings) + }) + } +} + +func Test_includes(t *testing.T) { + tests := []struct { + p string + q string + expected bool + }{ + {"**", "a/*", true}, + {"*", "a", true}, + {"a", "*", false}, + {"**", "*", true}, + {"*", "**", false}, + {"**/*", "**", true}, + {"**", "**/*", true}, + {"**/*/*", "**", false}, + {"**", "**/*/*", true}, + {"*/**", "**/a", true}, + {"a/**", "**/a", false}, + {"c/*/*", "c/*/a", true}, + {"c/*/a", "c/*/*", false}, + {"docker/proj1/**", "docker/*/mcp/*", false}, + {"docker/**", "docker/**/mcp/**", true}, + {"docker/proj1/**", "docker/**/mcp/**", false}, + {"a/*/**/b", "a/**/x/b", true}, + {"a/**/b", "a/**/x/b", true}, + {"a/**/x/b", "a/*/**/b", false}, + {"**/a/**", "*/a/*", true}, + {"*/a/*", "**/a/**", false}, + {"**/a/*/*", "*/*/a/*/*", true}, + {"*/*/*/**", "*/**/a/*/*", true}, + {"**/a/*/*", "a/*/**", false}, + {"a/**/b/**", "a/**/b/**/b", true}, + {"a/**/b/**/b", "a/**/b/**", false}, + {"**/*/**/*/**", "*/**/*", true}, + {"*/**/*", "**/*/**/*/**", true}, + } + for _, tc := range tests { + t.Run(fmt.Sprintf("%s includes %s", tc.p, tc.q), func(t *testing.T) { + got := includes(canonicalize(tc.p), canonicalize(tc.q)) + assert.Equal(t, tc.expected, got, "%s includes %s", tc.p, tc.q) + }) + } +}