Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 3 additions & 17 deletions sdk/typescript/src/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -663,19 +663,6 @@ export class CodexSecurity {
`Shell-visible plugin root must be outside CODEX_HOME: ${canonicalShellPluginRoot}`,
);
}
const skillName = skillNameFor(normalized, mode);
const skillPath = join(shellPluginRoot, "skills", skillName, "SKILL.md");
const skillMetadata = await lstat(skillPath).catch(() => null);
if (
skillMetadata === null ||
!skillMetadata.isFile() ||
skillMetadata.isSymbolicLink()
) {
throw new IncompleteScanError(
`Installed plugin is missing scan skill: ${skillName}`,
);
}
checkOpen();
const expectation: ScanExpectation = {
repository: repo,
repositoryRevision: await (
Expand Down Expand Up @@ -870,10 +857,10 @@ export class CodexSecurity {
}
activeScan = { id: scanId, options: workbenchOptions };
checkOpen();
const basePrompt = scanPrompt(
const basePrompt = await scanPrompt(
shellPluginRoot,
normalized,
mode,
skillName,
scanId,
runtime.configPath !== undefined,
knowledgeBase !== null,
Expand Down Expand Up @@ -2134,7 +2121,6 @@ function trustedAccessWarning(
function scanPrompt(
target: NormalizedTarget,
mode: ScanMode,
skillName: string,
scanId: string,
hasConfigPath = false,
hasKnowledgeBase = false,
Expand All @@ -2145,7 +2131,7 @@ function scanPrompt(
"Run this Codex Security scan non-interactively.",
...(mode === "deep"
? [
`The SDK has already registered this scan. Call start_codex_security_deep_scan with ${JSON.stringify({ scanId })}; never pass targetPath or create another scan.`,
`The SDK has already registered this scan. Call start_codex_security_deep_scan with { scanId: ${JSON.stringify(scanId)} }; never pass targetPath or create another scan.`,
]
: skillName === "security-scan"
? [
Expand Down
322 changes: 65 additions & 257 deletions sdk/typescript/tests-ts/api.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2656,268 +2656,76 @@ describe("CodexSecurity orchestration", () => {
await client.close();
});

test("rejects a missing scan skill before registering a scan", async () => {
const root = await temporaryDirectory();
const repository = join(root, "repository");
const codexHome = join(root, "codex-home");
const pluginRoot = join(root, "plugin-without-skills");
const scanDir = join(root, "scan");
await mkdir(repository);
await mkdir(codexHome);
await mkdir(pluginRoot);
await mkdir(scanDir, { mode: 0o700 });
const runtime = preparedRuntime(codexHome);
const commands: string[] = [];
const client = new TestClient(
{},
{
environment: {},
prepareRuntime: async () => ({
...runtime,
plugin: {
...(runtime["plugin"] as Record<string, unknown>),
pluginRoot,
marketplaceRoot: pluginRoot,
installedRoot: pluginRoot,
},
}),
resolvePluginPython: async () => "/managed/python",
prepareOutputDir: async () => scanDir,
repositoryRevision: async () => "deadbeef",
runWorkbench: async (_options: unknown, args: readonly string[]) => {
commands.push(args[0]!);
return args[0] === "register-cli-scan"
? mockScanRegistration(args)
: {};
},
},
);

await expect(client.run(repository)).rejects.toThrow(
"Installed plugin is missing scan skill: security-scan",
);
expect(commands).toEqual([]);
await client.close();
});

test.each([
["standard without feedback", "standard", false],
["standard with feedback", "standard", true],
["deep without feedback", "deep", false],
["deep with feedback", "deep", true],
] as const)(
"uses the registered scan ID in %s",
async (_scenario, mode, withFeedback) => {
const root = await temporaryDirectory();
const repository = join(root, "repository");
const codexHome = join(root, "codex-home");
const scanDir = join(root, "scan");
const scanId = "123e4567-e89b-12d3-a456-426614174000";
await mkdir(repository);
await mkdir(codexHome);
await mkdir(scanDir, { mode: 0o700 });
let prompt = "";
const client = new TestClient(
{},
{
environment: {},
prepareRuntime: async () => preparedRuntime(codexHome),
resolvePluginPython: async () => "/managed/python",
prepareOutputDir: async () => scanDir,
repositoryRevision: async () => "deadbeef",
runWorkbench: async (_options: unknown, args: readonly string[]) => {
if (args[0] === "register-cli-scan") {
return { ...mockScanRegistration(args), scanId };
}
if (args[0] === "get-scan-feedback") {
return {
scanId,
targetId: "target_sha256_example",
falsePositives: withFeedback
? [{ reason: "The finding is no longer reproducible." }]
: [],
};
}
return {};
},
createCodex: () => ({
startThread: () => ({
id: null,
async runStreamed(input: string) {
prompt = input;
throw new Error("prompt captured");
},
}),
}),
},
);

await expect(client.run(repository, { mode })).rejects.toThrow(
"prompt captured",
);
expect(prompt).toContain(
`Use exactly "${scanId}" as the scan ID in the manifest, findings, and coverage.`,
);
expect(prompt).not.toContain("$CODEX_SECURITY_SCAN_ID");
if (mode === "deep") {
const deepScanArguments = prompt.match(
/start_codex_security_deep_scan with (\{[^\n]+\});/,
);
expect(deepScanArguments).not.toBeNull();
expect(JSON.parse(deepScanArguments![1]!)).toEqual({ scanId });
} else {
expect(prompt).not.toContain("start_codex_security_deep_scan");
}
expect(prompt.includes("false_positive_feedback.json")).toBe(
withFeedback,
);
await client.close();
},
);

test.each([
["semantic matching fails", "matcher", "matcher unavailable"],
["the repository index fails", "index", "index unavailable"],
["a cost limit still allows false-positive matching", "budget", undefined],
[
"dismissed history survives missing reviewer feedback",
"dismissed",
undefined,
],
] as const)(
"keeps a completed scan when %s",
async (_scenario, failure, warning) => {
const root = await temporaryDirectory();
const repository = join(root, "repository");
const codexHome = join(root, "codex-home");
const scanDir = join(root, "scan");
await mkdir(repository);
await mkdir(codexHome);
await mkdir(scanDir, { mode: 0o700 });
const current = {
findingId: "csf_852f90d6e1177502ff113d4a",
occurrenceId: "occ_e79cb19591e696572a1c22be",
};
const previous = {
findingId: "previous",
occurrenceId: "old",
scanId: "prior",
targetId: "target_sha256_example",
};
const falsePositive = {
findingId: "previous",
sourceScanId: "prior",
reason: "A reviewer confirmed this code is safe.",
};
const warnings: string[] = [];
const commands: (readonly string[])[] = [];
let modelCalled = false;
let matched = false;
const client = new TestClient(
{},
{
environment: {},
prepareRuntime: async () => preparedRuntime(codexHome),
resolvePluginPython: async () => "/managed/python",
prepareOutputDir: async () => scanDir,
repositoryRevision: async () => "deadbeef",
runWorkbench: async (_options: unknown, args: readonly string[]) => {
commands.push(args);
if (args[0] === "get-scan-feedback") {
return {
scanId: "scan_example_001",
targetId: "target_sha256_example",
falsePositives: failure === "budget" ? [falsePositive] : [],
};
}
if (args[0] === "list-unmatched-scan-pairs") {
return {
batches: [
{
afterScanId: "scan_example_001",
afterFindings: [current],
beforeScans: [{ scanId: "prior", findings: [previous] }],
},
],
};
}
if (args[0] === "list-global-findings") {
if (failure === "index") throw new Error("index unavailable");
if (failure === "dismissed") {
test("uses the registered scan ID in standard and deep prompts", async () => {
const scanId = "123e4567-e89b-12d3-a456-426614174000";

for (const mode of ["standard", "deep"] as const) {
for (const withFeedback of [false, true]) {
const root = await temporaryDirectory();
const repository = join(root, "repository");
const codexHome = join(root, "codex-home");
const scanDir = join(root, "scan");
await mkdir(repository);
await mkdir(codexHome);
await mkdir(scanDir, { mode: 0o700 });
let prompt = "";
const client = new TestClient(
{},
{
environment: {},
prepareRuntime: async () => preparedRuntime(codexHome),
resolvePluginPython: async () => "/managed/python",
prepareOutputDir: async () => scanDir,
repositoryRevision: async () => "deadbeef",
runWorkbench: async (
_options: unknown,
args: readonly string[],
) => {
if (args[0] === "register-cli-scan") {
return { ...mockScanRegistration(args), scanId };
}
if (args[0] === "get-scan-feedback") {
return {
findings: args.includes("--status")
? matched
? []
: [current]
: [{ ...previous, status: "closed" }, current],
scanId,
targetId: "target_sha256_example",
falsePositives: withFeedback
? [{ reason: "The finding is no longer reproducible." }]
: [],
};
}
return {
findings:
failure === "matcher"
? [previous]
: [{ findingId: "another-open-finding" }],
};
}
if (args[0] === "save-scan-comparison") matched = true;
return mockWorkbench(args);
},
async matchFindings() {
modelCalled = true;
if (failure === "matcher") throw new Error("matcher unavailable");
return {
matches: [
{
beforeOccurrenceIds: [previous.occurrenceId],
afterOccurrenceIds: [current.occurrenceId],
confidence: "high",
reason: "Same dismissed root cause.",
return {};
},
createCodex: () => ({
startThread: () => ({
id: null,
async runStreamed(input: string) {
prompt = input;
throw new Error("prompt captured");
},
],
uncertain: [],
};
},
createCodex: () => ({
startThread: () => ({
id: null,
async runStreamed() {
await copyCompletedScan(root);
return { events: completedEvents() };
},
}),
}),
}),
},
);
},
);

const result = await client.run(repository, {
...(failure === "budget" ? { maxCostUsd: 1 } : {}),
onWarning: (message) => warnings.push(message),
});
expect(result.threadId).toBe("thread-1");
expect(
result.repositoryFindings?.map(({ findingId }) => findingId),
).toEqual(
failure === "budget"
? ["another-open-finding"]
: failure === "dismissed"
? []
: undefined,
);
expect(warnings).toEqual(
warning === undefined
? []
: [`Could not update repository findings: ${warning}`],
);
expect(modelCalled).toBe(failure !== "index");
expect(commands.some(([command]) => command === "complete-scan")).toBe(
true,
);
expect(
commands.some(([command]) => command === "list-global-findings"),
).toBe(true);
await client.close();
},
);
await expect(client.run(repository, { mode })).rejects.toThrow(
"prompt captured",
);
expect(prompt).toContain(
`Use exactly "${scanId}" as the scan ID in the manifest, findings, and coverage.`,
);
expect(prompt).not.toContain("$CODEX_SECURITY_SCAN_ID");
if (mode === "deep") {
expect(prompt).toContain(
`start_codex_security_deep_scan with { scanId: "${scanId}" }`,
);
}
if (withFeedback) {
expect(prompt).toContain("false_positive_feedback.json");
}
await client.close();
}
}
});

test("rejects feedback from another scan or invalid reviewer feedback", async () => {
const scanId = "scan_example_001";
Expand Down Expand Up @@ -4795,7 +4603,7 @@ describe("CodexSecurity orchestration", () => {
);
expect(prompt).toContain("$codex-security:deep-security-scan");
expect(prompt).toContain(
'start_codex_security_deep_scan with {"scanId":"scan_example_001"}',
'start_codex_security_deep_scan with { scanId: "scan_example_001" }',
);
expect(prompt).not.toContain(
"This exhaustive scan authorizes the delegated-worker phases",
Expand Down