coder_eval/mkdocs.yml at main · UiPath/coder_eval · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
site_name: Coder Eval
site_description: >-
  Evaluate & benchmark AI coding agents and Claude Code skills — a sandboxed,
  reproducible framework with declarative YAML eval suites for Claude Code,
  Codex & Gemini, A/B experiments, weighted scoring, and CI gates.
site_url: https://coder-eval.com/docs/
repo_url: https://github.com/UiPath/coder_eval
repo_name: UiPath/coder_eval
edit_uri: edit/main/docs/
copyright: © 2026 UiPath — Apache License 2.0

theme:
  name: material
  features:
    - navigation.instant
    - navigation.tracking
    - navigation.sections
    - navigation.top
    - search.suggest
    - search.highlight
    - content.code.copy
    - content.action.edit
  palette:
    - media: "(prefers-color-scheme: light)"
      scheme: default
      primary: deep orange
      accent: deep orange
      toggle:
        icon: material/weather-night
        name: Switch to dark mode
    - media: "(prefers-color-scheme: dark)"
      scheme: slate
      primary: deep orange
      accent: deep orange
      toggle:
        icon: material/weather-sunny
        name: Switch to light mode
  icon:
    repo: fontawesome/brands/github

# The social plugin generates per-page Open Graph / Twitter card images (and
# the matching og:/twitter: meta tags) so shared links unfurl properly. It
# needs Cairo system libraries, so it is enabled only in CI (the Docs workflow
# installs the deps); local builds skip it.
plugins:
  - search
  - social:
      enabled: !ENV [CI, false]

markdown_extensions:
  - admonition
  - attr_list
  - md_in_html
  - tables
  - toc:
      permalink: true
  - pymdownx.highlight:
      anchor_linenums: true
  - pymdownx.superfences
  - pymdownx.inlinehilite
  - pymdownx.snippets
  - pymdownx.tabbed:
      alternate_style: true

extra:
  social:
    - icon: fontawesome/brands/github
      link: https://github.com/UiPath/coder_eval
    - icon: fontawesome/brands/python
      link: https://pypi.org/project/coder-eval/
  # docs_index: one-line blurb per nav page (docs-root-relative path -> blurb),
  # for every page EXCEPT tutorial leaves (tutorials/0*.md). This map plus `nav`
  # are the single source of truth for the flat index surfaces — README.md's
  # Documentation table, docs/index.md's "Where to go next" table, and the
  # ## Docs / ## Tutorials sections of docs/llms.txt. Regenerate them all with
  # `make docs-indexes`; CE028 fails the build if any of the three drifts, if a
  # nav page has no blurb (or vice-versa), or if a docs/ page is missing from nav.
  docs_index:
    index.md: "What Coder Eval is, its features, and a quick start"
    tutorials/README.md: "Step-by-step walkthroughs — start here"
    USER_GUIDE.md: "Full CLI, configuration, output, and environment-variable reference"
    TASK_DEFINITION_GUIDE.md: "The task-file schema — all criterion types, scoring, templates"
    agents/CLAUDE_CODE.md: "Configuring and running the default Claude Code agent"
    agents/CODEX.md: "Running the OpenAI Codex agent"
    agents/ANTIGRAVITY.md: "Running the Google Antigravity / Gemini agent"
    AB_EXPERIMENTS.md: "Compare models / tools / prompts across the same tasks"
    DATASETS.md: "Fan a single task out over a dataset"
    DIALOG_MODE.md: "Evaluate agents in multi-turn conversation via a simulated user"
    DOCKER_ISOLATION.md: "The container sandbox driver, with custom images"
    CI_GATE.md: "Run Coder Eval as a CI gate — the packaged Action, JUnit output, score floor"
    EXTENDING.md: "Author a custom agent, criterion, or model pricing via the plugin SPI"
    REPORT_SCHEMA.md: "Field-level reference for run.json / variant.json / task.json"
    comparison.md: "vs. SWE-bench, SkillsBench, Harbor, OpenAI Evals, hand-rolled scripts"

nav:
  - Home: index.md
  - Tutorials:
      - Overview: tutorials/README.md
      - 01 · Your first evaluation: tutorials/01-first-evaluation.md
      - 02 · Running Coder Eval in CI: tutorials/02-ci-pipeline.md
      - 03 · Browsing results with evalboard: tutorials/03-evalboard-local.md
      - 04 · Writing a task: tutorials/04-writing-a-task.md
      - 05 · Comparing two models: tutorials/05-comparing-models.md
      - 06 · Docker isolation: tutorials/06-use-docker-isolation.md
  - Guides:
      - User Guide: USER_GUIDE.md
      - Task Definition Guide: TASK_DEFINITION_GUIDE.md
  - Agents:
      - Claude Code: agents/CLAUDE_CODE.md
      - Codex: agents/CODEX.md
      - Antigravity (Gemini): agents/ANTIGRAVITY.md
  - Advanced:
      - A/B Experiments: AB_EXPERIMENTS.md
      - Bring Your Own Dataset: DATASETS.md
      - Dialog Mode: DIALOG_MODE.md
      - Docker Isolation: DOCKER_ISOLATION.md
      - CI Gate & GitHub Action: CI_GATE.md
      - Extending Coder Eval: EXTENDING.md
      - Report Schema: REPORT_SCHEMA.md
  - How It Compares: comparison.md