Subscription (MCP / Claude Code) end-to-end + docs (v1.88 → v1.96)

ClaudeRevit/Services/BenchmarkRunner.cs

@@ -46,6 +46,8 @@ public static async Task RunAsync(
         bool resetBetweenTasks,
         int maxRoundsPerTask,
         int maxSecondsPerTask,
+        bool judgeViaClaudeCode,
+        bool runViaSubscription,
         Action<string> onStatus,
         Action<BenchmarkResult> onResult,
         CancellationToken ct)
@@ -55,15 +57,24 @@ public static async Task RunAsync(
         ToolDispatcher.ForceSuppress = true;
 
         // "claudecode" tests the MCP path end to end: the local `claude` CLI (on the subscription)
-        // drives the Revit tools through our MCP server. One impartial judge chat for the whole run.
-        var isClaudeCode = modelTag == "claudecode";
+        // drives the Revit tools through our MCP server. runViaSubscription does the same for any
+        // chosen model — the picked model is passed to the CLI via --model.
+        var isClaudeCode = modelTag == "claudecode" || runViaSubscription;
+        // The picked model → Claude Code --model alias (null = the CLI's default subscription model).
+        var ccModelAlias = modelTag == "claudecode" ? null : ClaudeCodeBackend.ModelAlias(modelTag);
+        // The judge can run on the subscription too (same claude.exe, no tools) so grading costs
+        // nothing on the API — needed since the account may sit at $0. It still grades from the
+        // objective probe only, so it stays impartial.
         string ccConfig = "", ccWorkDir = "", ccExe = "";
+        if (isClaudeCode || judgeViaClaudeCode)
+        {
+            ccExe = SettingsStore.ClaudeCodeExe;
+            ccWorkDir = McpServer.ClientWorkDir();
+        }
         if (isClaudeCode)
         {
             McpServer.Start();
             ccConfig = McpServer.WriteClientConfig();
-            ccWorkDir = McpServer.ClientWorkDir();
-            ccExe = SettingsStore.ClaudeCodeExe;
         }
         var judgeChat = new ChatService(ephemeral: true);
 
@@ -85,6 +96,7 @@ public static async Task RunAsync(
             // The model runs the task ONCE and recovers from any errors on its own — no judge hints.
             string finalText = "";
             string? error = null;
+            string? ccDiag = null;
             var budgetStopped = false;
             long inTok = 0, outTok = 0;
             var rounds = 0;
@@ -105,11 +117,19 @@ public static async Task RunAsync(
                             "mcp__clauderevit__*",
                             onText: t => sb.Append(t),
                             onTool: name => onStatus($"{task.Id} · {task.Title} · {name}"),
-                            taskCts.Token);
+                            taskCts.Token, model: ccModelAlias);
                         finalText = !string.IsNullOrEmpty(res.Text) ? res.Text : sb.ToString();
                         error = res.Error;
                         inTok = res.InputTokens; outTok = res.OutputTokens; rounds = res.NumTurns;
-                        modelUsed = "claude-code";
+                        modelUsed = ccModelAlias != null ? $"claude-code:{ccModelAlias}" : "claude-code";
+                        // Surface WHY a run did nothing: MCP connection status + result flag. A launch
+                        // with no tool calls and no tokens usually means MCP never connected.
+                        ccDiag = $"MCP: {res.McpStatus ?? "no init event"}" +
+                                 (res.IsError ? "; result=ERROR" : res.Subtype is { } st ? $"; result={st}" : "");
+                        if ((res.McpStatus == null ||
+                             res.McpStatus.IndexOf("connect", StringComparison.OrdinalIgnoreCase) < 0) &&
+                            outTok == 0)
+                            ccDiag += " — no Revit tools reached, nothing built";
                     }
                     catch (OperationCanceledException)
                     {
@@ -152,9 +172,12 @@ public static async Task RunAsync(
             onStatus($"{task.Id} · {task.Title} · grading…");
             var verdict = error != null
                 ? new Verdict(false, 0, "Run error: " + Truncate(error, 200), true)
-                : await JudgeAsync(judgeChat, judgeModel, task, before, after, finalText, ct);
+                : await JudgeAsync(judgeChat, judgeModel, task, before, after, finalText,
+                                   judgeViaClaudeCode, ccExe, ccWorkDir, ct);
             if (budgetStopped)
                 verdict = verdict with { Reason = "[stopped — task budget hit] " + verdict.Reason };
+            if (ccDiag != null)
+                verdict = verdict with { Reason = "[" + ccDiag + "] " + verdict.Reason };
 
             if (resetBetweenTasks)
                 await ResetAsync(baselineIds, ct);
@@ -208,7 +231,8 @@ private static async Task<string> StatsAsync(CancellationToken ct)
 
     private static async Task<Verdict> JudgeAsync(
         ChatService chat, string judgeModel, BenchmarkTask task,
-        string before, string after, string finalText, CancellationToken ct)
+        string before, string after, string finalText,
+        bool viaClaudeCode, string ccExe, string ccWorkDir, CancellationToken ct)
     {
         const string sys =
             "You are an impartial QA grader for a Revit modelling agent. Grade ONLY from the objective " +
@@ -226,7 +250,11 @@ private static async Task<Verdict> JudgeAsync(
             $"AGENT'S CLAIMED RESULT (unverified):\n{Truncate(finalText, 1000)}\n\nGrade now.";
         try
         {
-            var raw = await chat.RawCompleteAsync(judgeModel, sys, user, ct);
+            // On the subscription (no API cost) the CLI has no system-prompt flag in the same shape,
+            // so fold the grader instructions into the prompt; ParseVerdict tolerates surrounding prose.
+            var raw = viaClaudeCode
+                ? await ClaudeCodeBackend.CompleteAsync(ccExe, sys + "\n\n" + user, ccWorkDir, ct)
+                : await chat.RawCompleteAsync(judgeModel, sys, user, ct);
             return ParseVerdict(raw);
         }
         catch (Exception ex)

ClaudeRevit/Services/ChatService.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.ObjectModel;
+using System.IO;
 using System.Linq;
 using System.Text;
 using System.Text.Json;
@@ -181,7 +182,14 @@ public ChatService() : this(false) { }
     public ChatService(bool ephemeral)
     {
         _ephemeral = ephemeral;
-        if (!ephemeral) _history.AddRange(HistoryStore.LoadApiHistory());
+        if (!ephemeral)
+        {
+            _history.AddRange(HistoryStore.LoadApiHistory());
+            // Restore the Claude Code (subscription) session so the conversation continues after a
+            // Revit restart, matching how the API history persists.
+            try { if (File.Exists(ClaudeCodeSessionFile)) _claudeCodeSessionId = File.ReadAllText(ClaudeCodeSessionFile).Trim(); }
+            catch { /* non-fatal */ }
+        }
 
         // A restored long history must be eligible for compaction on the very FIRST send
         // after a restart — otherwise an oversized persisted conversation is replayed
@@ -206,6 +214,37 @@ public ChatService(bool ephemeral)
     // Set by the chat pane: progress ping each tool-call round (current, max) for the status line.
     public Action<int, int>? OnRound;
 
+    // Claude Code (subscription) mode keeps the CLI's session id so follow-up messages continue the
+    // same conversation via --resume. Reset on ClearHistory.
+    private string? _claudeCodeSessionId;
+
+    // Set by the chat pane: when true, the selected model runs through the Claude Code CLI on the
+    // subscription (via --model) instead of the pay-per-token API. The advisor/auto-escalation does
+    // NOT apply here — Claude Code runs its own loop with the one chosen model.
+    public bool SubscriptionMode;
+
+    private static string ClaudeCodeSessionFile => Path.Combine(
+        Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData),
+        "ClaudeRevit", "claudecode-session.txt");
+
+    private void PersistClaudeCodeSession()
+    {
+        if (_ephemeral) return;
+        try
+        {
+            if (string.IsNullOrEmpty(_claudeCodeSessionId))
+            {
+                if (File.Exists(ClaudeCodeSessionFile)) File.Delete(ClaudeCodeSessionFile);
+            }
+            else
+            {
+                Directory.CreateDirectory(Path.GetDirectoryName(ClaudeCodeSessionFile)!);
+                File.WriteAllText(ClaudeCodeSessionFile, _claudeCodeSessionId);
+            }
+        }
+        catch { /* non-fatal */ }
+    }
+
     public void RecreateClient() => _client = null;
 
     public void ClearHistory()
@@ -219,6 +258,8 @@ public void ClearHistory()
         _execCsharpOk = 0;
         _promoteNudged = false;
         _revealedCategories.Clear();
+        _claudeCodeSessionId = null;
+        PersistClaudeCodeSession();
         if (!_ephemeral) HistoryStore.Clear();
     }
 
@@ -260,6 +301,89 @@ public async Task<string> RawCompleteAsync(string modelTag, string systemPrompt,
         return string.Concat(msg.Content.Select(b => b.TryPickText(out var t) ? t.Text : ""));
     }
 
+    // Subscription mode: run the local Claude Code CLI headless, letting it drive the Revit tools
+    // through our in-process MCP server. Streams its narration into the pane and keeps the CLI's
+    // session id so follow-up messages continue the same conversation (--resume). Costs nothing on
+    // the API — the work runs on the user's Claude Pro/Max subscription.
+    private async Task SendViaClaudeCodeAsync(
+        ObservableCollection<ChatMessage> conversation, string prompt, Dispatcher ui,
+        string? modelAlias, CancellationToken ct)
+    {
+        if (!McpServer.IsRunning)
+        {
+            try { McpServer.Start(); }
+            catch (Exception ex)
+            {
+                throw new InvalidOperationException(
+                    "Couldn't start the MCP server that Claude Code needs — free the port in " +
+                    "Settings → MCP and try again. (" + ex.Message + ")");
+            }
+        }
+        var config = McpServer.WriteClientConfig();
+        var workDir = McpServer.ClientWorkDir();
+        var exe = SettingsStore.ClaudeCodeExe;
+
+        // Parity with the API path: give Claude Code the current document + selection so "this" /
+        // "the selected walls" resolve. The MCP session is long-lived, so we prepend this fresh each
+        // message (instructions, which carry memory/standards, are sent once at connect).
+        var contextedPrompt = prompt;
+        try
+        {
+            var contextJson = await ToolDispatcher.Instance.GetProjectContextAsync(ct);
+            var ctxHeader = "CURRENT DOCUMENT:\n" + contextJson;
+            var sel = SelectionService.Current;
+            if (sel.Ids.Count > 0)
+            {
+                var idList = sel.Ids.Count > 30
+                    ? string.Join(", ", sel.Ids.Take(30)) + $", … +{sel.Ids.Count - 30} more"
+                    : string.Join(", ", sel.Ids);
+                ctxHeader += $"\n\nCURRENT SELECTION: {sel.Description}. Element IDs: [{idList}]";
+            }
+            contextedPrompt = ctxHeader + "\n\n---\n\nUSER REQUEST:\n" + prompt;
+        }
+        catch { /* context is best-effort — fall back to the bare prompt */ }
+
+        ChatMessage? bubble = null;
+        void Append(string piece)
+        {
+            if (string.IsNullOrEmpty(piece)) return;
+            ui.InvokeAsync(() =>
+            {
+                if (bubble == null) { bubble = new ChatMessage { Role = "assistant", Text = "" }; conversation.Add(bubble); }
+                bubble.Text += piece;
+            });
+        }
+
+        var toolCount = 0;
+        var res = await ClaudeCodeBackend.RunAsync(
+            exe, contextedPrompt, workDir, config, resumeSessionId: _claudeCodeSessionId,
+            allowedToolsGlob: "mcp__clauderevit__*",
+            onText: Append,
+            onTool: _ => { toolCount++; OnRound?.Invoke(toolCount, toolCount); },
+            ct, model: modelAlias);
+
+        _claudeCodeSessionId = res.SessionId ?? _claudeCodeSessionId;
+        PersistClaudeCodeSession();
+
+        if (!string.IsNullOrEmpty(res.Error))
+        {
+            Append((bubble == null ? "" : "\n\n") + "⚠ " + res.Error);
+            return;
+        }
+        // Nothing streamed (e.g. a short answer delivered only in the final result event) — show it.
+        if (bubble == null && !string.IsNullOrEmpty(res.Text))
+            Append(res.Text);
+
+        if (SettingsStore.ShowTaskDiagnostics && bubble != null)
+        {
+            var tok = res.InputTokens + res.OutputTokens;
+            Append($"\n\n— claude-code (subscription — no API charge) · {res.NumTurns} turns · " +
+                   $"{res.DurationMs / 1000.0:0.0}s" +
+                   (tok > 0 ? $" · {tok:N0} tokens" : "") +
+                   (res.McpStatus != null ? $" · MCP {res.McpStatus}" : ""));
+        }
+    }
+
     public async Task SendAsync(
         ObservableCollection<ChatMessage> conversation,
         string model,
@@ -268,6 +392,20 @@ public async Task SendAsync(
         string? imageMime = null)
     {
         var ui = Dispatcher.CurrentDispatcher;
+
+        // Subscription path: the local Claude Code CLI drives the Revit tools through our MCP server.
+        // It runs its OWN agent loop, so bypass the whole Anthropic/alt pipeline (no API key, no tool
+        // schemas, no history compaction here) and just stream its output into the pane.
+        if (model == "claudecode" || SubscriptionMode)
+        {
+            var userText = conversation.LastOrDefault(m => m.Role == "user")?.Text ?? "";
+            if (string.IsNullOrWhiteSpace(userText)) return;
+            // "claudecode" = the CLI's default subscription model; any other pick maps to --model.
+            var alias = model == "claudecode" ? null : ClaudeCodeBackend.ModelAlias(model);
+            await SendViaClaudeCodeAsync(conversation, userText, ui, alias, ct);
+            return;
+        }
+
         bool alt = IsAlt(model);
         if (alt && !OpenAIBackend.IsConfigured)
             throw new InvalidOperationException(