idek anymore

backend/package.json

@@ -26,14 +26,20 @@
         "typescript-eslint": "^8.50.1"
     },
     "dependencies": {
+        "@ai-sdk/google": "^3.0.10",
         "@ai-sdk/openai": "^3.0.0",
+        "@onkernel/ai-sdk": "^0.0.3",
+        "@onkernel/sdk": "^0.25.0",
+        "@openrouter/ai-sdk-provider": "^1.5.4",
         "@posthog/agent-toolkit": "^0.2.4",
         "@posthog/ai": "^7.3.0",
         "@trpc/server": "^11.0.0-rc.553",
         "@types/bcryptjs": "^3.0.0",
         "@vercel/sandbox": "^1.1.1",
+        "@workflow/ai": "^4.0.1-beta.51",
         "@workos-inc/node": "^7.82.0",
         "ai": "^6.0.0",
+        "ai-sdk-provider-gemini-cli": "^2.0.1",
         "axios": "^1.7.9",
         "cookie-parser": "^1.4.7",
         "cors": "^2.8.5",

backend/src/db/data-source.ts

@@ -14,6 +14,7 @@ import { WorkspaceTool } from './entities/WorkspaceTool';
 import { WorkerDefinitionTool } from './entities/WorkerDefinitionTool';
 import { seedDefaults } from './seed';
 import { SlackUserMapping } from './entities/SlackUserMapping';
+import { Environment } from './entities/Environment';
 
 const {
     PGHOST = 'localhost',
@@ -39,6 +40,7 @@ export const AppDataSource = new DataSource({
     synchronize: NODE_ENV !== 'production',
     entities: [
         Agent,
+        Environment,
         IntegrationConnection,
         IntegrationProvider,
         Message,

backend/src/db/entities/Agent.ts

@@ -1,6 +1,7 @@
 import { Entity, PrimaryGeneratedColumn, Column, ManyToOne, Index, OneToMany, CreateDateColumn, UpdateDateColumn } from 'typeorm';
 import type { Workspace } from './Workspace';
 import type { Message } from './Message';
+import type { Environment } from './Environment';
 
 export enum AgentStatus {
     PENDING = 'PENDING',
@@ -53,6 +54,15 @@ export class Agent {
     @Column({ type: 'boolean', default: false })
     isOrchestratorAgent!: boolean;
 
+    @Column({ nullable: true })
+    environmentId!: number | null;
+
+    @ManyToOne('Environment', {
+        onDelete: 'SET NULL',
+        nullable: true,
+    })
+    environment!: Environment | null;
+
     @CreateDateColumn({ default: 'now()' })
     createdAt!: Date;
 

backend/src/db/entities/Environment.ts

@@ -0,0 +1,40 @@
+import { Entity, PrimaryGeneratedColumn, Column, CreateDateColumn, UpdateDateColumn, Index, ManyToOne } from 'typeorm';
+import type { Organization } from './Organization';
+
+export interface EnvironmentFile {
+    path: string;
+    content: string;
+}
+
+@Entity()
+@Index(['name', 'organizationId'], { unique: true })
+export class Environment {
+    @PrimaryGeneratedColumn()
+    id!: number;
+
+    @Column({ length: 200 })
+    name!: string;
+
+    @Column()
+    organizationId!: number;
+
+    @ManyToOne('Organization', {
+        onDelete: 'CASCADE',
+    })
+    organization!: Organization;
+
+    @Column({ length: 500 })
+    githubRepositoryName!: string;
+
+    @Column({ type: 'text', nullable: true })
+    description!: string | null;
+
+    @Column({ type: 'jsonb', default: [] })
+    files!: EnvironmentFile[];
+
+    @CreateDateColumn()
+    createdAt!: Date;
+
+    @UpdateDateColumn()
+    updatedAt!: Date;
+}

backend/src/db/entities/Message.ts

@@ -47,7 +47,6 @@ export class Message {
     @Column({ type: 'int', default: 0 })
     costMicrodollars!: number;
 
-
     @Column({ type: 'int', default: 0 })
     sandboxDurationMs!: number;
 

backend/src/db/entities/ToolCall.ts

@@ -2,6 +2,11 @@ import { Entity, PrimaryGeneratedColumn, Column, ManyToOne, CreateDateColumn, In
 import type { Agent } from './Agent';
 import type { Message } from './Message';
 
+export type ToolCallImage = {
+    data: string;
+    mimeType: string;
+};
+
 @Entity()
 export class ToolCall {
     @PrimaryGeneratedColumn()
@@ -26,6 +31,9 @@ export class ToolCall {
     @Column({ type: 'text', default: '' })
     result!: string;
 
+    @Column({ type: 'jsonb', default: [] })
+    images!: ToolCallImage[];
+
     @Column({ length: 20, default: 'success' })
     status!: string;
 }

backend/src/payment/model-pricing.ts

@@ -17,6 +17,10 @@ export function getModelPricing(modelName: string): ModelPricing {
     const pricing = MODEL_PRICING[modelName];
 
     if (!pricing) {
+        return {
+            inputMicrodollarsPer1MTokens: 2_000_000, // $2.00
+            outputMicrodollarsPer1MTokens: 6_000_000, // $6.00
+        };
         throw new Error(`Unknown model pricing for "${modelName}"`);
     }
 

backend/src/providers/base.ts

@@ -1,6 +1,7 @@
 import { Agent } from '../db/entities/Agent';
 import { Workspace } from '../db/entities/Workspace';
 import type { SenderType, MessageImage } from '../db/entities/Message';
+import type { ToolCallImage } from '../db/entities/ToolCall';
 
 export type ProviderToolCall = {
     id: number;
@@ -9,7 +10,7 @@ export type ProviderToolCall = {
     arguments: Record<string, unknown>;
     result: string;
     status: string;
-    duration_ms: number | null;
+    images: ToolCallImage[];
 };
 
 export type ProviderMessage = {
@@ -33,6 +34,7 @@ export interface CloudProvider {
         model?: string | null;
         isOrchestratorAgent: boolean;
         images: MessageImage[];
+        environmentId?: number | null;
     }): Promise<Agent>;
     getMessages(agent: Agent): Promise<ProviderMessage[]>;
     sendMessage(agent: Agent, message: string, images: MessageImage[]): Promise<boolean>;

backend/src/providers/codee.ts

@@ -6,6 +6,7 @@ import { runAgentWorkflow, runOrchestratorAgentWorkflow } from '../workflows/age
 import { Message, type MessageImage } from '../db/entities/Message';
 import { emitStatus } from '../stream/events';
 import { ToolCall } from '../db/entities/ToolCall';
+import { WorkspaceTool } from '../db/entities/WorkspaceTool';
 import { In } from 'typeorm';
 
 export class CodeeProvider implements CloudProvider {
@@ -20,6 +21,7 @@ export class CodeeProvider implements CloudProvider {
         model,
         isOrchestratorAgent,
         images,
+        environmentId,
     }: {
         organizationId: number;
         workspace: Workspace;
@@ -30,6 +32,7 @@ export class CodeeProvider implements CloudProvider {
         model?: string | null;
         isOrchestratorAgent: boolean;
         images: MessageImage[];
+        environmentId?: number | null;
     }): Promise<Agent> {
         const agentRepository = AppDataSource.getRepository(Agent);
         const messageRepository = AppDataSource.getRepository(Message);
@@ -42,6 +45,7 @@ export class CodeeProvider implements CloudProvider {
             name: `Codee Agent${model ? ` (${model})` : ''}`,
             model: model || null,
             isOrchestratorAgent,
+            environmentId: environmentId || null,
         });
         await agentRepository.save(agent);
         agent.url = `http://localhost:5173/agent/${agent.id}`;
@@ -103,7 +107,7 @@ export class CodeeProvider implements CloudProvider {
                 arguments: toolCall.arguments,
                 result: toolCall.result,
                 status: toolCall.status,
-                duration_ms: toolCall.durationMs,
+                images: toolCall.images,
             });
             toolCallsByMessage.set(toolCall.message.id, list);
         }
@@ -131,9 +135,18 @@ export class CodeeProvider implements CloudProvider {
             console.error('Failed to emit status:', err);
         });
 
+        const workspaceToolRepository = AppDataSource.getRepository(WorkspaceTool);
+        const workspaceTools = await workspaceToolRepository.find({
+            where: { workspace: { id: agent.workspace.id } },
+            relations: ['tool'],
+        });
+        const toolSlugs = Array.from(new Set(workspaceTools.map((workspaceTool) => workspaceTool.tool.slugName)));
+
         const payload = {
             agentId: agent.id,
             prompt: message,
+            repositoryFullName: agent.workspace.githubRepositoryName,
+            toolSlugs,
             baseBranch: agent.workspace.currentBranch,
             isOrchestratorAgent: agent.isOrchestratorAgent,
         };

backend/src/providers/cursor.ts

@@ -27,6 +27,7 @@ export class CursorProvider implements CloudProvider {
         baseBranch: string;
         model?: string | null;
         images: MessageImage[];
+        environmentId?: number | null;
     }): Promise<Agent> {
         const agentRepository = AppDataSource.getRepository(Agent);
         const agent = agentRepository.create({

backend/src/providers/index.ts

@@ -26,6 +26,7 @@ export async function createAgentsFromProviders(params: {
     branchName: string;
     cloudProviders: CloudProviderConfig[];
     images: MessageImage[];
+    environmentId?: number | null;
 }): Promise<Agent> {
     let first: Agent | null = null;
     for (const config of params.cloudProviders) {
@@ -43,6 +44,7 @@ export async function createAgentsFromProviders(params: {
                 model: agentConfig.model,
                 isOrchestratorAgent: false,
                 images: params.images,
+                environmentId: params.environmentId,
             });
             if (!first) first = agent;
         }

backend/src/providers/jules.ts

@@ -28,6 +28,7 @@ export class JulesProvider implements CloudProvider {
         model?: string | null;
         isOrchestratorAgent: boolean;
         images: MessageImage[];
+        environmentId?: number | null;
     }): Promise<Agent> {
         const agentRepository = AppDataSource.getRepository(Agent);
         const agent = agentRepository.create({

backend/src/tools/kernel/index.ts

@@ -0,0 +1,49 @@
+import { browserSessions, consoleLogs, networkLogs } from './session';
+import { buildCreateSessionTool } from './tools/createSession';
+import { buildScreenshotTool } from './tools/screenshot';
+import { buildExecuteTool } from './tools/execute';
+import { buildNavigateTool } from './tools/navigate';
+import { buildClickTool } from './tools/click';
+import { buildTypeTool } from './tools/type';
+import { buildGetConsoleLogsTool } from './tools/getConsoleLogs';
+import { buildGetNetworkLogsTool } from './tools/getNetworkLogs';
+import { buildGetPageContentTool } from './tools/getPageContent';
+import { buildWaitForPageTool } from './tools/waitForPage';
+
+export type { BrowserToolResult } from './session';
+
+export interface SandboxUrl {
+    port: number;
+    url: string;
+}
+
+export function buildBrowserTools(params: { agentId: number; sandboxUrls?: SandboxUrl[] }) {
+    const { agentId, sandboxUrls } = params;
+
+    return {
+        browser_create_session: buildCreateSessionTool({ agentId, sandboxUrls }),
+        browser_screenshot: buildScreenshotTool({ agentId }),
+        browser_execute: buildExecuteTool({ agentId }),
+        browser_navigate: buildNavigateTool({ agentId }),
+        browser_click: buildClickTool({ agentId }),
+        browser_type: buildTypeTool({ agentId }),
+        browser_get_console_logs: buildGetConsoleLogsTool({ agentId }),
+        browser_get_network_logs: buildGetNetworkLogsTool({ agentId }),
+        browser_get_page_content: buildGetPageContentTool({ agentId }),
+        browser_wait_for_page: buildWaitForPageTool({ agentId }),
+    };
+}
+
+export async function cleanupBrowserSession(agentId: number): Promise<void> {
+    const session = browserSessions.get(agentId);
+    if (session) {
+        try {
+            await session.client.browsers.deleteByID(session.sessionId);
+        } catch {
+            // Ignore cleanup errors
+        }
+        browserSessions.delete(agentId);
+    }
+    consoleLogs.delete(agentId);
+    networkLogs.delete(agentId);
+}

backend/src/tools/kernel/session.ts

@@ -0,0 +1,24 @@
+import Kernel from '@onkernel/sdk';
+import type { ToolCallImage } from '../../db/entities/ToolCall';
+
+export interface BrowserSession {
+    sessionId: string;
+    client: Kernel;
+}
+
+export type BrowserToolResult = {
+    text: string;
+    images?: ToolCallImage[];
+};
+
+export const browserSessions: Map<number, BrowserSession> = new Map();
+export const consoleLogs: Map<number, Array<{ type: string; text: string; timestamp: string }>> = new Map();
+export const networkLogs: Map<number, Array<{ method: string; url: string; status?: number; timestamp: string }>> = new Map();
+
+export function getKernelClient(): Kernel {
+    const apiKey = process.env.KERNEL_API_KEY;
+    if (!apiKey) {
+        throw new Error('KERNEL_API_KEY environment variable is not set');
+    }
+    return new Kernel({ apiKey });
+}

backend/src/tools/kernel/tools/click.ts

@@ -0,0 +1,37 @@
+import { z } from 'zod';
+import { tool, zodSchema } from 'ai';
+import { emitStatus } from '../../../stream/events';
+import { browserSessions, type BrowserToolResult } from '../session';
+
+const clickSchema = z.object({
+    selector: z.string().describe('CSS selector for the element to click'),
+});
+
+export function buildClickTool(params: { agentId: number }) {
+    const { agentId } = params;
+
+    return tool({
+        description: 'Click on an element in the browser using a CSS selector.',
+        inputSchema: zodSchema(clickSchema),
+        execute: async (input): Promise<BrowserToolResult> => {
+            const { selector } = input;
+            await emitStatus(agentId, 'running', 'tool_browser_click', `Clicking ${selector}`, { arguments: input });
+
+            const session = browserSessions.get(agentId);
+            if (!session) {
+                return { text: 'Error: No browser session found. Call browser_create_session first.' };
+            }
+
+            const { sessionId, client } = session;
+
+            await client.browsers.playwright.execute(sessionId, {
+                code: `await page.click(${JSON.stringify(selector)});`,
+                timeout_sec: 30,
+            });
+
+            await emitStatus(agentId, 'running', 'tool_browser_click', `Clicked ${selector}`, { arguments: input });
+
+            return { text: `Successfully clicked on element: ${selector}` };
+        },
+    });
+}