diff --git a/bun.lock b/bun.lock index 1c1557a68..632c7d31e 100644 --- a/bun.lock +++ b/bun.lock @@ -892,6 +892,7 @@ "@effect/atom-react": "catalog:", "@effect/vitest": "catalog:", "@executor-js/api": "workspace:*", + "@executor-js/emulate": "^0.7.6", "@executor-js/react": "workspace:*", "@types/node": "catalog:", "@types/react": "catalog:", @@ -1630,7 +1631,7 @@ "@executor-js/e2e": ["@executor-js/e2e@workspace:e2e"], - "@executor-js/emulate": ["@executor-js/emulate@0.7.5", "", { "dependencies": { "commander": "^14", "graphql": "^16.9.0", "jose": "^6", "picocolors": "^1.1.1", "yaml": "^2" }, "bin": { "emulate": "dist/index.js" } }, "sha512-tPq+XYyohEa4HjFFZpMK3hsKZEozXpudsWQCQfj1pwmLdiouz5a1cmH9RC1X9nYvx76sHPyR6GVIpAw+rFT3Sg=="], + "@executor-js/emulate": ["@executor-js/emulate@0.7.6", "", { "dependencies": { "commander": "^14", "graphql": "^16.9.0", "jose": "^6", "picocolors": "^1.1.1", "yaml": "^2" }, "bin": { "emulate": "dist/index.js" } }, "sha512-46e5NN3ZUxmFTXeRTs3AM1asHAGqSCMhfoN8cbEqmatWs9DGhqMBUmZk8T0jUytQ8E6r0yCYYdbJ+twLP5za4A=="], "@executor-js/example-all-plugins": ["@executor-js/example-all-plugins@workspace:examples/all-plugins"], diff --git a/packages/plugins/graphql/package.json b/packages/plugins/graphql/package.json index e9ef0f9ea..dea93f44e 100644 --- a/packages/plugins/graphql/package.json +++ b/packages/plugins/graphql/package.json @@ -71,6 +71,7 @@ "@effect/atom-react": "catalog:", "@effect/vitest": "catalog:", "@executor-js/api": "workspace:*", + "@executor-js/emulate": "^0.7.6", "@executor-js/react": "workspace:*", "@types/node": "catalog:", "@types/react": "catalog:", diff --git a/packages/plugins/graphql/src/sdk/invalid-operations-large-schema.test.ts b/packages/plugins/graphql/src/sdk/invalid-operations-large-schema.test.ts new file mode 100644 index 000000000..d0f956b3c --- /dev/null +++ b/packages/plugins/graphql/src/sdk/invalid-operations-large-schema.test.ts @@ -0,0 +1,255 @@ +// Reproduction for executor#1146: +// "GraphQL plugin generates invalid operations against large schemas +// (missing required nested args, composite fields without selections)". +// +// This drives the real product flow end-to-end against GitLab's REAL GraphQL +// schema, served by the GitLab emulator from @executor-js/emulate. The emulator +// loads GitLab's full published SDL (4000+ types) and stands it up with real +// graphql-js introspection and validation, so the plugin introspects the +// genuine GitLab type system, freezes one machine-built selection set per root +// field at connect time, and sends that operation string on every call. +// +// `createEmulator({ service: "gitlab" })` boots that server in-process here; the +// same service is hosted at https://gitlab.emulators.dev with zero setup. The +// emulator has no business-logic resolvers wired: against a schema this rich the +// generated operations are not valid GraphQL (the builder caps depth at 2, bails +// to "" on cycles, and never threads nested field arguments), so the server +// rejects them on VALIDATION before any field would resolve, and the synced tool +// fails on every call. +// +// `buildSelectionSet` (packages/plugins/graphql/src/sdk/plugin.ts) is the +// source: `if (depth > 2) return ""` and the `seen` cycle guard both make a +// nested selection empty, after which the parent still prints the composite +// field name bare (invalid for an object/connection type); nested required +// arguments are never emitted at all. +// +// The assertions encode the BUG as it stands today. When the selection-set +// builder is fixed (bound depth without emitting bare composites; thread nested +// required args, or omit fields it cannot satisfy) these calls should return +// ok:true, and these expectations should be flipped to assert success. + +import { createServer } from "node:net"; + +import { describe, it, expect } from "@effect/vitest"; +import { Effect } from "effect"; +import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/http"; +import { + buildClientSchema, + getIntrospectionQuery, + isNonNullType, + type GraphQLSchema, + type IntrospectionQuery, +} from "graphql"; + +import { + AuthTemplateSlug, + ConnectionName, + IntegrationSlug, + ToolAddress, + type ToolError, + createExecutor, +} from "@executor-js/sdk"; +import { makeTestConfig, memoryCredentialsPlugin } from "@executor-js/sdk/testing"; +import { createEmulator, type Emulator } from "@executor-js/emulate"; + +import { graphqlPlugin } from "./plugin"; + +// A free localhost port, so parallel test files never collide on a fixed one. +const availablePort = Effect.callback((resume) => { + const probe = createServer(); + probe.listen(0, "127.0.0.1", () => { + const address = probe.address(); + const port = typeof address === "object" && address ? address.port : 0; + probe.close(() => resume(Effect.succeed(port))); + }); +}); + +// The GitLab emulator, spawned in-process and torn down with the test scope. +// It serves GitLab's full real GraphQL schema at /api/graphql; the operations +// the plugin generates fail validation before any field would resolve. +const gitlabEmulator = Effect.acquireRelease( + Effect.gen(function* () { + const port = yield* availablePort; + return yield* Effect.promise(() => createEmulator({ service: "gitlab", port })); + }), + (emulator: Emulator) => Effect.promise(() => emulator.close()).pipe(Effect.ignore), +); + +const graphqlEndpoint = (emulator: Emulator): string => `${emulator.url}/api/graphql`; + +// Introspect the live emulator endpoint into an executable client schema, the +// same type system the plugin sees, so the sweep below can enumerate root query +// fields that take no required argument. +const introspectGitlabSchema = (endpoint: string): Effect.Effect => + Effect.gen(function* () { + const response = yield* HttpClient.execute( + HttpClientRequest.post(endpoint).pipe( + HttpClientRequest.bodyJsonUnsafe({ query: getIntrospectionQuery() }), + ), + ); + const body = (yield* response.json) as { + readonly data?: IntrospectionQuery; + readonly errors?: unknown; + }; + if (!body.data) { + return yield* Effect.die( + `gitlab emulator introspection failed: ${JSON.stringify(body.errors ?? body)}`, + ); + } + return buildClientSchema(body.data); + }).pipe(Effect.provide(FetchHttpClient.layer), Effect.orDie); + +// The two ways the generator produces an invalid operation, as graphql-js +// phrases them on the wire. +const COMPOSITE_WITHOUT_SELECTION = /must have a selection of subfields/; +const MISSING_REQUIRED_ARGUMENT = + /argument "[^"]+" of type "[^"]+" is required, but it was not provided\./; +const isInvalidOperationMessage = (message: string): boolean => + COMPOSITE_WITHOUT_SELECTION.test(message) || MISSING_REQUIRED_ARGUMENT.test(message); + +interface GraphqlErrorEntry { + readonly message?: string; +} +interface GraphqlErrorDetails { + readonly errors?: ReadonlyArray; +} + +// The upstream GraphQL errors ride in ToolError.details (typed Unknown at the +// core boundary). Narrow to the GraphQL error shape to read their messages. +const graphqlErrorMessages = (toolError: ToolError): readonly string[] => { + const details = toolError.details as GraphqlErrorDetails | undefined; + return (details?.errors ?? []).map((entry) => entry.message ?? ""); +}; + +const makeExecutor = () => + createExecutor( + makeTestConfig({ plugins: [memoryCredentialsPlugin(), graphqlPlugin()] as const }), + ); + +const toolAddr = (integration: string, connection: string, tool: string): ToolAddress => + ToolAddress.make(`tools.${integration}.org.${connection}.${tool}`); + +const createOrgConnection = ( + executor: Awaited> extends Effect.Effect ? A : never, + input: { readonly integration: string; readonly name: string }, +) => + executor.connections.create({ + owner: "org", + name: ConnectionName.make(input.name), + integration: IntegrationSlug.make(input.integration), + template: AuthTemplateSlug.make("none"), + value: "unused", + }); + +// Root query fields that take no required argument: the plugin can call these +// with `{}`, so any failure is the generated SELECTION's fault, not a missing +// top-level input. +const noRequiredArgQueryFields = (schema: GraphQLSchema): readonly string[] => { + const fields = schema.getQueryType()?.getFields() ?? {}; + return Object.keys(fields).filter((name) => + (fields[name]?.args ?? []).every((arg) => !isNonNullType(arg.type)), + ); +}; + +describe("graphqlPlugin invalid operations against GitLab's real schema (issue #1146)", () => { + // Headline: one real root field, both failure mechanisms from the issue in a + // single generated operation. + it.effect( + "query.currentUser: generated operation is rejected for bare composites and a dropped required arg", + () => + Effect.gen(function* () { + const emulator = yield* gitlabEmulator; + const executor = yield* makeExecutor(); + + yield* executor.graphql.addIntegration({ + endpoint: graphqlEndpoint(emulator), + slug: "gitlab", + }); + yield* createOrgConnection(executor, { integration: "gitlab", name: "main" }); + + // Introspection synced currentUser as a tool. + const tools = yield* executor.tools.list(); + expect( + tools.map((tool) => String(tool.name)), + "introspection produced a query.currentUser tool", + ).toContain("query.currentUser"); + + const result = yield* executor.execute(toolAddr("gitlab", "main", "query.currentUser"), {}); + + // BUG: a plain call fails. The upstream rejects the frozen operation. + expect(result, "the generated operation is rejected by the server").toMatchObject({ + ok: false, + error: { code: "graphql_errors" }, + }); + if (result.ok) return; // narrow for the type-checker; the assertion above already failed otherwise. + + const messages = graphqlErrorMessages(result.error); + + // Mechanism 1 (depth>2 cutoff + cycle guard): composite fields are + // emitted with no sub-selection, e.g. the `node` of a connection edge. + expect(messages, "a composite field is emitted with no sub-selection").toEqual( + expect.arrayContaining([expect.stringMatching(COMPOSITE_WITHOUT_SELECTION)]), + ); + + // Mechanism 2 (nested args never threaded): a selected field that + // requires an argument is emitted without it. + expect(messages, "a selected field's required argument is dropped").toEqual( + expect.arrayContaining([expect.stringMatching(MISSING_REQUIRED_ARGUMENT)]), + ); + }), + 60000, + ); + + // Systemic: the failure is not one unlucky field. Sweep every generated + // query.* tool that takes no required top-level argument and count how many + // produce an operation the server rejects as invalid. + it.effect( + "the generator emits invalid operations across many of the schema's root query fields", + () => + Effect.gen(function* () { + const emulator = yield* gitlabEmulator; + const executor = yield* makeExecutor(); + + yield* executor.graphql.addIntegration({ + endpoint: graphqlEndpoint(emulator), + slug: "gitlab", + }); + yield* createOrgConnection(executor, { integration: "gitlab", name: "main" }); + + const gitlabSchema = yield* introspectGitlabSchema(graphqlEndpoint(emulator)); + + const tools = yield* executor.tools.list(); + const generatedQueryTools = new Set( + tools.map((tool) => String(tool.name)).filter((name) => name.startsWith("query.")), + ); + const candidates = noRequiredArgQueryFields(gitlabSchema).filter((field) => + generatedQueryTools.has(`query.${field}`), + ); + expect( + candidates.length, + "the real schema yields many no-required-arg query tools to exercise", + ).toBeGreaterThan(20); + + const invalidOperationFields: string[] = []; + for (const field of candidates) { + const result = yield* executor.execute(toolAddr("gitlab", "main", `query.${field}`), {}); + if (result.ok) continue; + if (graphqlErrorMessages(result.error).some(isInvalidOperationMessage)) { + invalidOperationFields.push(field); + } + } + + // BUG: a large fraction of the generated query tools are dead on + // arrival. currentUser is one of them; it is not a lone outlier. + expect( + invalidOperationFields.length, + "many generated query tools emit operations the server rejects as invalid", + ).toBeGreaterThan(10); + expect( + invalidOperationFields, + "currentUser is among the invalid generated tools", + ).toContain("currentUser"); + }), + 60000, + ); +});