diff --git a/devtools/grafana/dashboards/forge-issue-detail.json b/devtools/grafana/dashboards/forge-issue-detail.json index dd02972f..3bb09e9f 100644 --- a/devtools/grafana/dashboards/forge-issue-detail.json +++ b/devtools/grafana/dashboards/forge-issue-detail.json @@ -74,7 +74,7 @@ }, "spec": { "format": "table", - "rawSql": "SELECT t.metadata['workflow_step'] as step, round(sum(o.total_cost), 4) as cost FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE t.session_id = '${jira_issue}' AND t.metadata['workflow_step'] != '' GROUP BY step ORDER BY cost DESC" + "rawSql": "SELECT\n splitByChar(':', coalesce(nullIf(t.metadata['workflow_step'], ''), 'unknown'))[1] AS step,\n round(sum(o.total_cost), 4) AS cost\nFROM traces t FINAL\nJOIN observations o FINAL ON t.id = o.trace_id\nWHERE t.session_id = '${jira_issue}'\n AND coalesce(nullIf(t.metadata['workflow_step'], ''), '') != ''\nGROUP BY step\nORDER BY cost DESC" } }, "refId": "A", @@ -128,7 +128,7 @@ }, "spec": { "format": "table", - "rawSql": "SELECT t.metadata['workflow_step'] as step, sum(o.usage_details['input']) as input_tokens, sum(o.usage_details['output']) as output_tokens FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE t.session_id = '${jira_issue}' AND t.metadata['workflow_step'] != '' GROUP BY step ORDER BY input_tokens DESC" + "rawSql": "SELECT\n splitByChar(':', coalesce(nullIf(t.metadata['workflow_step'], ''), 'unknown'))[1] AS step,\n sum(coalesce(o.usage_details['input'], 0)) AS input_tokens,\n sum(coalesce(o.usage_details['output'], 0)) AS output_tokens\nFROM traces t FINAL\nJOIN observations o FINAL ON t.id = o.trace_id\nWHERE t.session_id = '${jira_issue}'\n AND coalesce(nullIf(t.metadata['workflow_step'], ''), '') != ''\nGROUP BY step\nORDER BY input_tokens DESC" } }, "refId": "A", @@ -201,7 +201,7 @@ }, "spec": { "format": "table", - "rawSql": "SELECT $__timeInterval(o.start_time) as time, avg(dateDiff('second', o.start_time, coalesce(o.end_time, o.start_time))) as avg_trace_latency_s FROM default.observations o FINAL WHERE o.trace_id IN (SELECT id FROM default.traces FINAL WHERE session_id = '${jira_issue}') GROUP BY time ORDER BY time" + "rawSql": "SELECT $__timeInterval(o.start_time) as time, avg(dateDiff('second', o.start_time, coalesce(o.end_time, o.start_time))) as avg_trace_latency_s FROM observations o FINAL WHERE o.trace_id IN (SELECT id FROM traces FINAL WHERE session_id = '${jira_issue}') GROUP BY time ORDER BY time" } }, "refId": "B", @@ -250,7 +250,7 @@ }, "spec": { "format": "table", - "rawSql": "SELECT t.id as id, o_agg.model, round(o_agg.latency_s, 1) as latency_s, round(o_agg.cost, 4) as cost, o_agg.total_tokens as tokens FROM default.traces t FINAL JOIN (SELECT trace_id, any(provided_model_name) as model, dateDiff('second', min(start_time), max(coalesce(end_time, start_time))) as latency_s, sum(total_cost) as cost, sum(usage_details['total']) as total_tokens FROM default.observations FINAL GROUP BY trace_id) o_agg ON t.id = o_agg.trace_id WHERE t.session_id = '${jira_issue}' AND t.metadata['workflow_step'] != '' ORDER BY t.timestamp" + "rawSql": "SELECT\n splitByChar(':', coalesce(nullIf(t.metadata['workflow_step'], ''), 'unknown'))[1] AS step,\n row_number() OVER (PARTITION BY splitByChar(':', coalesce(nullIf(t.metadata['workflow_step'], ''), 'unknown'))[1] ORDER BY t.timestamp ASC) AS iteration,\n t.id AS id,\n o_agg.model,\n round(o_agg.latency_s, 1) AS latency_s,\n round(o_agg.cost, 4) AS cost,\n o_agg.total_tokens AS tokens\nFROM traces t FINAL\nJOIN (\n SELECT\n trace_id,\n any(coalesce(nullIf(provided_model_name, ''), 'unknown')) AS model,\n dateDiff('second', min(start_time), max(coalesce(end_time, start_time))) AS latency_s,\n sum(coalesce(total_cost, 0)) AS cost,\n sum(coalesce(usage_details['total'], 0)) AS total_tokens\n FROM observations FINAL\n GROUP BY trace_id\n) o_agg ON t.id = o_agg.trace_id\nWHERE t.session_id = '${jira_issue}'\n AND coalesce(nullIf(t.metadata['workflow_step'], ''), '') != ''\nORDER BY t.timestamp" } }, "refId": "A", @@ -275,12 +275,62 @@ } }, "overrides": [ + { + "matcher": { + "id": "byName", + "options": "step" + }, + "properties": [ + { + "id": "displayName", + "value": "Step" + }, + { + "id": "custom", + "value": { + "align": "auto", + "displayMode": "auto" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "iteration" + }, + "properties": [ + { + "id": "displayName", + "value": "Iteration" + }, + { + "id": "unit", + "value": "none" + }, + { + "id": "decimals", + "value": 0 + }, + { + "id": "custom", + "value": { + "align": "auto", + "displayMode": "auto" + } + } + ] + }, { "matcher": { "id": "byName", "options": "cost" }, "properties": [ + { + "id": "displayName", + "value": "Cost" + }, { "id": "unit", "value": "currencyUSD" @@ -293,6 +343,10 @@ "options": "latency_s" }, "properties": [ + { + "id": "displayName", + "value": "Latency (s)" + }, { "id": "unit", "value": "s" @@ -305,6 +359,10 @@ "options": "id" }, "properties": [ + { + "id": "displayName", + "value": "Id" + }, { "id": "links", "value": [ @@ -316,6 +374,34 @@ ] } ] + }, + { + "matcher": { + "id": "byName", + "options": "tokens" + }, + "properties": [ + { + "id": "displayName", + "value": "Tokens" + }, + { + "id": "unit", + "value": "locale" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "model" + }, + "properties": [ + { + "id": "displayName", + "value": "Model" + } + ] } ] } @@ -378,7 +464,7 @@ }, "spec": { "format": "table", - "rawSql": "SELECT any(if(metadata['ticket_type'] = 'Feature', 1, 0)) as value FROM default.traces FINAL WHERE session_id = '${jira_issue}'" + "rawSql": "SELECT any(if(metadata['ticket_type'] = 'Feature', 1, 0)) as value FROM traces FINAL WHERE session_id = '${jira_issue}'" } }, "refId": "A", @@ -448,7 +534,7 @@ }, "spec": { "format": "table", - "rawSql": "SELECT uniqExact(id) as value FROM default.traces FINAL WHERE session_id = '${jira_issue}'" + "rawSql": "SELECT uniqExact(id) as value FROM traces FINAL WHERE session_id = '${jira_issue}'" } }, "refId": "A", @@ -503,7 +589,7 @@ }, "spec": { "format": "table", - "rawSql": "SELECT round(sum(total_cost), 4) as value FROM default.observations FINAL WHERE trace_id IN (SELECT id FROM default.traces FINAL WHERE session_id = '${jira_issue}')" + "rawSql": "SELECT round(sum(total_cost), 4) as value FROM observations FINAL WHERE trace_id IN (SELECT id FROM traces FINAL WHERE session_id = '${jira_issue}')" } }, "refId": "A", @@ -560,7 +646,7 @@ }, "spec": { "format": "table", - "rawSql": "SELECT round(avg(latency_s), 1) as value FROM (SELECT dateDiff('second', min(start_time), max(coalesce(end_time, start_time))) as latency_s FROM default.observations FINAL WHERE trace_id IN (SELECT id FROM default.traces FINAL WHERE session_id = '${jira_issue}') GROUP BY trace_id)" + "rawSql": "SELECT round(avg(latency_s), 1) as value FROM (SELECT dateDiff('second', min(start_time), max(coalesce(end_time, start_time))) as latency_s FROM observations FINAL WHERE trace_id IN (SELECT id FROM traces FINAL WHERE session_id = '${jira_issue}') GROUP BY trace_id) as sub" } }, "refId": "A", @@ -594,6 +680,201 @@ } } }, + "panel-39": { + "kind": "Panel", + "spec": { + "id": 39, + "title": "Cumulative Duration", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT sum(duration_s) as value FROM (SELECT dateDiff('second', min(o.start_time), max(coalesce(o.end_time, o.start_time))) as duration_s FROM traces t FINAL JOIN observations o FINAL ON t.id = o.trace_id WHERE t.session_id = '${jira_issue}' GROUP BY t.id) as sub" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 600, + "color": "red" + } + ] + } + }, + "overrides": [] + } + } + } + } + }, + "panel-40": { + "kind": "Panel", + "spec": { + "id": 40, + "title": "Total Tokens", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT sum(coalesce(usage_details['total'], 0)) as value FROM observations FINAL WHERE trace_id IN (SELECT id FROM traces FINAL WHERE session_id = '${jira_issue}')" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "fieldConfig": { + "defaults": { + "unit": "locale", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 1500000, + "color": "red" + } + ] + } + }, + "overrides": [] + } + } + } + } + }, + "panel-32": { + "kind": "Panel", + "spec": { + "id": 32, + "title": "CI Fix Attempts", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT count() as value FROM traces FINAL WHERE session_id = '${jira_issue}' AND splitByChar(':', coalesce(nullIf(metadata['workflow_step'], ''), 'unknown'))[1] = 'attempt_ci_fix'" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "", + "spec": { + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + } + } + }, "panel-8": { "kind": "Panel", "spec": { @@ -617,7 +898,7 @@ }, "spec": { "format": "table", - "rawSql": "SELECT t.metadata['workflow_step'] as step, dateDiff('second', min(o.start_time), max(coalesce(o.end_time, o.start_time))) as duration_s FROM default.traces t FINAL JOIN default.observations o FINAL ON t.id = o.trace_id WHERE t.session_id = '${jira_issue}' AND t.metadata['workflow_step'] != '' GROUP BY t.session_id, t.id, step ORDER BY min(o.start_time)" + "rawSql": "SELECT\n splitByChar(':', coalesce(nullIf(t.metadata['workflow_step'], ''), 'unknown'))[1] AS step,\n dateDiff('second', min(o.start_time), max(coalesce(o.end_time, o.start_time))) AS duration_s\nFROM traces t FINAL\nJOIN observations o FINAL ON t.id = o.trace_id\nWHERE t.session_id = '${jira_issue}'\n AND coalesce(nullIf(t.metadata['workflow_step'], ''), '') != ''\nGROUP BY t.session_id, t.id, step\nORDER BY min(o.start_time)" } }, "refId": "A", @@ -646,6 +927,145 @@ } } } + }, + "panel-28": { + "kind": "Panel", + "spec": { + "id": 28, + "title": "Iteration Count per Stage", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT /* NOTE: Keep this CASE WHEN block in sync with panel-35 */ CASE splitByChar(':', coalesce(nullIf(metadata['workflow_step'], ''), 'unknown'))[1]\n WHEN 'entry' THEN 'Entry'\n WHEN 'prd_approval_gate' THEN 'PRD'\n WHEN 'spec_approval_gate' THEN 'Spec'\n WHEN 'plan_approval_gate' THEN 'Epic Plan'\n WHEN 'task_approval_gate' THEN 'Tasks'\n WHEN 'create_pr' THEN 'PR Creation'\n WHEN 'human_review_gate' THEN 'Human Review'\n WHEN 'ci_evaluator' THEN 'CI'\n WHEN 'attempt_ci_fix' THEN 'CI Fix'\n ELSE splitByChar(':', coalesce(nullIf(metadata['workflow_step'], ''), 'unknown'))[1]\nEND AS workflow_step, count() AS iteration_count FROM traces FINAL WHERE session_id = '${jira_issue}' AND splitByChar(':', coalesce(nullIf(metadata['workflow_step'], ''), 'unknown'))[1] NOT IN ('', 'unknown') GROUP BY workflow_step ORDER BY iteration_count DESC" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": { + "orientation": "horizontal", + "xField": "workflow_step" + }, + "fieldConfig": { + "defaults": { + "noDataText": "No Data" + }, + "overrides": [] + } + } + } + } + }, + "panel-35": { + "kind": "Panel", + "spec": { + "id": 35, + "title": "Machine Time vs Idle Time per Stage", + "description": "", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana-clickhouse-datasource", + "version": "v0", + "datasource": { + "name": "langfuse-clickhouse" + }, + "spec": { + "format": "table", + "rawSql": "SELECT step, machine_s as machine_time, greatest(wall_s - machine_s, 0) as idle_time FROM (SELECT /* NOTE: Keep this CASE WHEN block in sync with panel-28 */ CASE splitByChar(':', coalesce(nullIf(t.metadata['workflow_step'], ''), 'unknown'))[1]\n WHEN 'entry' THEN 'Entry'\n WHEN 'prd_approval_gate' THEN 'PRD'\n WHEN 'spec_approval_gate' THEN 'Spec'\n WHEN 'plan_approval_gate' THEN 'Epic Plan'\n WHEN 'task_approval_gate' THEN 'Tasks'\n WHEN 'create_pr' THEN 'PR Creation'\n WHEN 'human_review_gate' THEN 'Human Review'\n WHEN 'ci_evaluator' THEN 'CI'\n WHEN 'attempt_ci_fix' THEN 'CI Fix'\n ELSE splitByChar(':', coalesce(nullIf(t.metadata['workflow_step'], ''), 'unknown'))[1]\nEND AS step, sum(dateDiff('second', o.start_time, coalesce(o.end_time, o.start_time))) as machine_s, dateDiff('second', min(o.start_time), max(coalesce(o.end_time, o.start_time))) as wall_s FROM traces t FINAL JOIN observations o FINAL ON t.id = o.trace_id WHERE t.session_id = '${jira_issue}' AND splitByChar(':', coalesce(nullIf(t.metadata['workflow_step'], ''), 'unknown'))[1] NOT IN ('', 'unknown') GROUP BY step) as sub ORDER BY machine_time + idle_time DESC" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "barchart", + "version": "", + "spec": { + "options": { + "orientation": "horizontal", + "stacking": "normal" + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "machine_time" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "idle_time" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + } + } + } + } } }, "layout": { @@ -666,7 +1086,7 @@ "spec": { "x": 0, "y": 0, - "width": 5, + "width": 4, "height": 4, "element": { "kind": "ElementReference", @@ -677,9 +1097,9 @@ { "kind": "GridLayoutItem", "spec": { - "x": 5, + "x": 4, "y": 0, - "width": 4, + "width": 3, "height": 4, "element": { "kind": "ElementReference", @@ -690,9 +1110,9 @@ { "kind": "GridLayoutItem", "spec": { - "x": 9, + "x": 7, "y": 0, - "width": 4, + "width": 3, "height": 4, "element": { "kind": "ElementReference", @@ -703,28 +1123,54 @@ { "kind": "GridLayoutItem", "spec": { - "x": 13, + "x": 10, "y": 0, - "width": 5, + "width": 4, "height": 4, "element": { "kind": "ElementReference", - "name": "panel-4" + "name": "panel-39" } } }, { "kind": "GridLayoutItem", "spec": { - "x": 18, + "x": 14, "y": 0, - "width": 6, + "width": 3, "height": 4, "element": { "kind": "ElementReference", "name": "panel-5" } } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 17, + "y": 0, + "width": 4, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-4" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 21, + "y": 0, + "width": 3, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-40" + } + } } ] } @@ -758,6 +1204,59 @@ } } }, + { + "kind": "RowsLayoutRow", + "spec": { + "title": "Iterations & Timing", + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 10, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-28" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 10, + "y": 0, + "width": 4, + "height": 4, + "element": { + "kind": "ElementReference", + "name": "panel-32" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 14, + "y": 0, + "width": 10, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-35" + } + } + } + ] + } + } + } + }, { "kind": "RowsLayoutRow", "spec": { @@ -933,12 +1432,12 @@ "name": "langfuse-clickhouse" }, "spec": { - "__legacyStringValue": "SELECT DISTINCT metadata['project_id'] as project FROM default.traces FINAL WHERE project != '' ORDER BY project" + "__legacyStringValue": "SELECT DISTINCT metadata['project_id'] as project FROM traces FINAL WHERE project != '' ORDER BY project" } }, "regex": "", "sort": "alphabeticalAsc", - "definition": "SELECT DISTINCT metadata['project_id'] as project FROM default.traces FINAL WHERE project != '' ORDER BY project", + "definition": "SELECT DISTINCT metadata['project_id'] as project FROM traces FINAL WHERE project != '' ORDER BY project", "options": [], "multi": true, "includeAll": true, @@ -965,12 +1464,12 @@ "name": "langfuse-clickhouse" }, "spec": { - "__legacyStringValue": "SELECT DISTINCT session_id FROM default.traces FINAL WHERE session_id IS NOT NULL AND session_id != '' AND metadata['project_id'] IN ($project_id) ORDER BY session_id" + "__legacyStringValue": "SELECT DISTINCT session_id FROM traces FINAL WHERE session_id IS NOT NULL AND session_id != '' AND metadata['project_id'] IN ($project_id) ORDER BY session_id" } }, "regex": "", "sort": "alphabeticalAsc", - "definition": "SELECT DISTINCT session_id FROM default.traces FINAL WHERE session_id IS NOT NULL AND session_id != '' AND metadata['project_id'] IN ($project_id) ORDER BY session_id", + "definition": "SELECT DISTINCT session_id FROM traces FINAL WHERE session_id IS NOT NULL AND session_id != '' AND metadata['project_id'] IN ($project_id) ORDER BY session_id", "options": [], "multi": false, "includeAll": false,