diff --git a/ScratchV-topic06-deliverable/.github/workflows/benchmark.yml b/ScratchV-topic06-deliverable/.github/workflows/benchmark.yml new file mode 100644 index 0000000..92c9878 --- /dev/null +++ b/ScratchV-topic06-deliverable/.github/workflows/benchmark.yml @@ -0,0 +1,41 @@ +name: ScratchV Course Benchmark Suite + +on: + push: + pull_request: + +jobs: + benchmark: + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install package and course benchmark dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e . + python -m pip install -r requirements-topic06.txt + + - name: Run unit tests + run: python -m pytest -q + + - name: Run course benchmark suite + run: | + python run_tests.py --benchmark 3 + + - name: Upload course benchmark reports + uses: actions/upload-artifact@v4 + with: + name: scratchv-course-benchmark-reports + path: | + reports/report.md + reports/report.json + reports/failures/ + reports/benchmark_baseline.json diff --git a/ScratchV-topic06-deliverable/LICENSE b/ScratchV-topic06-deliverable/LICENSE new file mode 100644 index 0000000..87324d7 --- /dev/null +++ b/ScratchV-topic06-deliverable/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 ScratchV + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/ScratchV-topic06-deliverable/README.md b/ScratchV-topic06-deliverable/README.md new file mode 100644 index 0000000..0a98a1c --- /dev/null +++ b/ScratchV-topic06-deliverable/README.md @@ -0,0 +1,227 @@ +# ScratchV 课题 06 交付说明 + +本测试套件不是独立项目，需要放在 ScratchV 仓库根目录下运行，并依赖 ScratchV 原项目环境。 +本目录是 ScratchV 课题 06 的编译正确性与性能回归测试套件，包含 23 个 DSL 测试用例、自动化测试脚本、依赖说明、CI 示例以及可重新生成的测试报告。 + +当前程序在原始自动编译、指令数统计、Benchmark、基线对比和报告功能上，进一步增加了真实 TinyFive 直通验证、元数据校验、编译与模拟超时、编译失败日志、统一 JSON schema、轻量/完整报告、可配置退化阈值和子集测试。测试套件不会在 TinyFive 不可用时回退到 stub，也不会使用测试套件内部的参考解释器代替真实编译结果。 + +真实验证路径为： + +```text +DSL -> ScratchV 编译器 -> RISC-V 汇编 -> TinyFive -> a0/x10 返回值 -> 期望值对比 +``` + +最近一次全量运行结果为 13 个通过、10 个失败。3 个 `branch` 用例因 TinyFive 模拟超时失败；另外 7 个 `dot`/`matmul` 用例涉及尚未完成的数组内存输入输出约定和编译器后端降低逻辑。测试套件负责真实暴露、隔离和记录这些问题，不负责实现 ScratchV 的分支、数组或矩阵代码生成。 + +## 运行依赖 + +该交付目录需要放在 ScratchV 项目环境中运行。`run_tests.py` 会导入 +`scratchv` 包，并通过 ScratchV 编译器把每个 DSL 用例编译为 RISC-V 汇编。 +调用真实 TinyFive 所需的 ScratchV 适配修改可参考： + +- [ScratchV PR #15](https://github.com/ScratchV-Compiler/ScratchV/pull/15) +- [ScratchV PR #17](https://github.com/ScratchV-Compiler/ScratchV/pull/17) + +安装基础测试依赖： + +```powershell +pip install -r requirements-topic06.txt +``` + +基础模式需要以下依赖： + +- `tinyfive`：用于模拟执行生成的 RISC-V 汇编 +- `pytest`：用于测试支持 + +如需生成 HTML 和 PNG 完整报告，再安装可选依赖： + +```powershell +pip install -r requirements-topic06-full.txt +``` + +其中 `jinja2` 用于生成 HTML，`matplotlib` 用于生成性能图表。 + +## 目录结构 + +```text +run_tests.py +requirements-topic06.txt +requirements-topic06-full.txt +tests_main/ + activation/ + branch/ + elementwise/ + loop/ + reduction/ + tensor/ +reports/ # 测试报告，可重新生成 + failures/ # 编译失败和超时日志 +.github/ + workflows/ + benchmark.yml +build/ # 编译输出的汇编文件，可重新生成 +``` + +`tests_main/` 下共有 23 个 DSL 用例。每个用例由 `.dsl` 文件和对应的 +`.meta.json` 文件组成，`.meta.json` 中定义输入、期望返回值和用例说明。 + +## 运行测试 + +在本目录下执行： + +```powershell +python run_tests.py +``` + +测试脚本会自动完成以下步骤： + +- 遍历 `tests_main/` 下的所有 `.dsl` 文件 +- 调用 ScratchV 编译器生成 RISC-V 汇编 +- 编译单个用例超过 30 秒时终止该编译进程，并继续运行后续用例 +- 编译失败或超时时将命令、返回码、stdout 和 stderr 保存到 `reports/failures/*.compile.log` +- 通过 ScratchV 的 TinyFive 适配层验证生成的汇编 +- 在独立子进程中执行 TinyFive；单次模拟超过 5 秒时终止该用例并继续后续测试 +- TinyFive 未安装或执行失败时明确判定为 FAIL，不回退到 stub +- 使用 TinyFive 返回寄存器 `a0/x10` 与 `.meta.json` 中的期望值进行结果对比 +- 使用 `time.perf_counter()` 记录编译、模拟和总耗时 +- 统计 PASS/FAIL 和指令数 +- 默认在 `reports/` 下生成 Markdown 和固定 schema 的 JSON 报告 + +## Benchmark 模式 + +重复运行每个用例并统计平均指令数： + +```powershell +python run_tests.py --benchmark 3 +``` + +如果首次模拟已经 timeout，该用例不会再执行 Benchmark 重复；如果重复过程中发生 timeout 或模拟失败，脚本会立即停止剩余次数。失败运行不会作为 `0` 加入平均值，报告会记录实际完成次数和 `benchmark_stopped_reason`。 + +更新性能基线： + +```powershell +python run_tests.py --benchmark 3 --update-baseline +``` + +性能退化阈值默认是 5%，可以按场景调整： + +```powershell +# 超过 2% 即判定为退化 +python run_tests.py --benchmark 3 --regression-threshold 2 + +# 超过 10% 才判定为退化 +python run_tests.py --benchmark 3 --regression-threshold 10 +``` + +阈值必须大于或等于 0，并会写入 Markdown、HTML 和 JSON 报告。 + +## 子集测试 + +按类别运行： + +```powershell +python run_tests.py --category activation +``` + +按用例名称进行大小写不敏感的包含匹配： + +```powershell +python run_tests.py --filter matmul +``` + +组合筛选： + +```powershell +python run_tests.py --category tensor --filter relu +``` + +没有匹配用例时脚本返回退出码 2，并保留已有报告。对子集使用 `--update-baseline` 时，只更新匹配用例的基线，不会删除其他用例的基线。 + +性能基线文件位于： + +```text +reports/benchmark_baseline.json +``` + +## 测试报告 + +默认运行后生成轻量报告： + +```text +reports/report.md +reports/report.json +``` + +需要 HTML 和 PNG 时运行： + +```powershell +python run_tests.py --full-report +``` + +完整模式额外生成： + +```text +reports/report.html +reports/course_report_instructions.png +``` + +`report.md` 适合提交或归档，`report.json` 供 CI 或其他程序稳定解析，`report.html` 适合在浏览器中查看。普通模式和 Benchmark 模式使用相同字段，Benchmark 专属字段在普通模式下为 `null`。JSON 顶层的 `selection` 会记录 `--category` 和 `--filter`，未筛选时两项均为 `null`。轻量模式不会加载 `jinja2` 或 `matplotlib`，也不会刷新已有的 HTML、PNG 文件。 + +编译失败时，报告中的 `compile_returncode`、`compile_timed_out`、`compile_error` 和 `compile_log` 会记录错误摘要及独立日志位置。成功用例的错误和日志字段为 `null`。 + +## 添加测试用例 + +新增用例时添加两个文件： + +```text +tests_main/{category}/{name}.dsl +tests_main/{category}/{name}.meta.json +``` + +DSL 示例： + +```text +result = add(a, b) +return result +``` + +元数据示例： + +```json +{ + "description": "Simple scalar add.", + "expected_output_type": "scalar", + "inputs": { + "a": 2, + "b": 3 + }, + "expected_return": 5 +} +``` + +矩阵输出需要声明类型、元素类型和形状： + +```json +{ + "expected_output_type": "tensor", + "output_dtype": "int32", + "output_shape": [2, 2], + "expected_return": [[19, 22], [43, 50]] +} +``` + +大型期望结果也可以使用 `expected_output_file` 引用同目录下的 JSON 文件。`expected_return` 与 `expected_output_file` 必须且只能填写一个。元数据中的类型、形状或元素值不合法时，仅当前用例标记为 FAIL，后续测试继续运行。 + +## CI 示例 + +GitHub Actions 示例位于： + +```text +.github/workflows/benchmark.yml +``` + +示例工作流会在 push 和 pull request 时安装基础依赖、运行测试并上传 Markdown、JSON、性能基线和编译失败日志。CI 默认使用轻量报告，不需要安装 `jinja2` 和 `matplotlib`。 + +## 详细设计 + +程序模块、报告字段、需求演进、Mentor Review 处理状态和当前失败原因见 `课题6设计文档.md`。 diff --git a/ScratchV-topic06-deliverable/reports/benchmark_baseline.json b/ScratchV-topic06-deliverable/reports/benchmark_baseline.json new file mode 100644 index 0000000..7a7596b --- /dev/null +++ b/ScratchV-topic06-deliverable/reports/benchmark_baseline.json @@ -0,0 +1,102 @@ +{ + "add_relu_relu": { + "category": "activation", + "avg_instr_count": 7.0, + "runs": 3 + }, + "relu_add": { + "category": "activation", + "avg_instr_count": 5.0, + "runs": 3 + }, + "relu_only": { + "category": "activation", + "avg_instr_count": 5.0, + "runs": 3 + }, + "relu_twice": { + "category": "activation", + "avg_instr_count": 6.0, + "runs": 3 + }, + "add_chain": { + "category": "elementwise", + "avg_instr_count": 4.0, + "runs": 3 + }, + "add_chain_3": { + "category": "elementwise", + "avg_instr_count": 5.0, + "runs": 3 + }, + "add_fan_in_4": { + "category": "elementwise", + "avg_instr_count": 5.0, + "runs": 3 + }, + "add_reuse": { + "category": "elementwise", + "avg_instr_count": 4.0, + "runs": 3 + }, + "vector_add": { + "category": "elementwise", + "avg_instr_count": 3.0, + "runs": 3 + }, + "loop_add_4": { + "category": "loop", + "avg_instr_count": 22.0, + "runs": 3 + }, + "loop_add_chain_4": { + "category": "loop", + "avg_instr_count": 26.0, + "runs": 3 + }, + "loop_relu_add_4": { + "category": "loop", + "avg_instr_count": 30.0, + "runs": 3 + }, + "dot_4": { + "category": "reduction", + "avg_instr_count": 3.0, + "runs": 3 + }, + "dot_8": { + "category": "reduction", + "avg_instr_count": 3.0, + "runs": 3 + }, + "dot_relu_4": { + "category": "reduction", + "avg_instr_count": 5.0, + "runs": 3 + }, + "dot_relu_8": { + "category": "reduction", + "avg_instr_count": 5.0, + "runs": 3 + }, + "matmul_2x2": { + "category": "tensor", + "avg_instr_count": 3.0, + "runs": 3 + }, + "matmul_4x4": { + "category": "tensor", + "avg_instr_count": 3.0, + "runs": 3 + }, + "matmul_add_2x2": { + "category": "tensor", + "avg_instr_count": 4.0, + "runs": 3 + }, + "matmul_relu_2x2": { + "category": "tensor", + "avg_instr_count": 5.0, + "runs": 3 + } +} \ No newline at end of file diff --git a/ScratchV-topic06-deliverable/reports/course_report_instructions.png b/ScratchV-topic06-deliverable/reports/course_report_instructions.png new file mode 100644 index 0000000..842bcde Binary files /dev/null and b/ScratchV-topic06-deliverable/reports/course_report_instructions.png differ diff --git a/ScratchV-topic06-deliverable/reports/report.html b/ScratchV-topic06-deliverable/reports/report.html new file mode 100644 index 0000000..5ab96da --- /dev/null +++ b/ScratchV-topic06-deliverable/reports/report.html @@ -0,0 +1,383 @@ + + + + + ScratchV 测试报告 + + + +

ScratchV DSL 编译器性能测试报告

+
+

用例总数：23，通过：20，失败：3，通过率：87.0%

+

测试目录：D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\tests_main，性能退化阈值：5.0%，单次模拟超时：5.0s

+
+ 课程版指令数图表 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
用例类别状态模拟后端平均指令数95% 置信区间编译耗时(s)模拟耗时(s)总耗时(s)变化率(%)是否退化描述
add_relu_reluactivationPASSnone7.00±0.000.05320.13230.48060.00FalseAdd input and bias, then apply ReLU twice.
relu_addactivationPASSnone5.00±0.000.05640.09810.44560.00FalseAdd input and bias, then apply one ReLU.
relu_onlyactivationPASSnone4.00±0.000.05160.10000.43590.00FalseApply ReLU directly to a single input value.
relu_twiceactivationPASSnone6.00±0.000.05330.09280.42650.00FalseApply ReLU twice to the same activation path.
if_elsebranchFAILtimeout0.00±0.000.05125.016820.09350.00Falseif/else branch returns subtraction result when flag is zero.
if_relubranchFAILtimeout0.00±0.000.05385.009020.10860.00Falseif/else branch combined with add and relu.
if_thenbranchFAILtimeout0.00±0.000.05495.009520.10930.00Falseif/else branch returns add result when flag is non-zero.
add_chainelementwisePASSnone4.00±0.000.05480.09690.43640.00FalseAdd a and b, then add c to the intermediate result.
add_chain_3elementwisePASSnone5.00±0.000.05250.09710.43570.00FalseChain three add operations across four symbolic inputs.
add_fan_in_4elementwisePASSnone5.00±0.000.05230.10050.43930.00FalseCompute two independent adds and then merge them with a final add.
add_reuseelementwisePASSnone4.00±0.000.05230.09350.43200.00FalseReuse the same intermediate add result on both operands of a second add.
vector_addelementwisePASSnone3.00±0.000.05350.09450.43630.00FalseSingle add over two symbolic vector inputs.
loop_add_4loopPASSnone22.00±0.000.05200.09410.43120.00FalseRun a four-iteration loop whose body computes one add; final returned value is the last loop-body result.
loop_add_chain_4loopPASSnone26.00±0.000.05340.09500.43750.00FalseRun a four-iteration loop whose body computes two chained adds; final returned value is the last loop-body result.
loop_relu_add_4loopPASSnone30.00±0.000.05110.09500.43020.00FalseRun a four-iteration loop whose body computes add followed by ReLU; final returned value is the last loop-body result.
dot_4reductionPASSnone3.00±0.000.05440.09390.43620.00FalseCompute the dot product of two symbolic vectors of length 4.
dot_8reductionPASSnone3.00±0.000.05370.10210.44740.00FalseCompute the dot product of two symbolic vectors of length 8.
dot_relu_4reductionPASSnone5.00±0.000.05230.09540.43420.00FalseCompute a length-4 dot product and pass it through ReLU.
dot_relu_8reductionPASSnone5.00±0.000.05240.09450.43500.00FalseCompute a length-8 dot product and pass it through ReLU.
matmul_2x2tensorPASSnone3.00±0.000.05180.09520.43280.00FalseCompute a symbolic 2x2 by 2x2 matrix multiplication.
matmul_4x4tensorPASSnone3.00±0.000.05290.09420.44000.00FalseCompute a symbolic 4x4 by 4x4 matrix multiplication.
matmul_add_2x2tensorPASSnone4.00±0.000.05220.09400.43160.00FalseCompute a 2x2 matmul and then add a symbolic bias term.
matmul_relu_2x2tensorPASSnone5.00±0.000.05140.09660.43420.00FalseCompute a 2x2 matmul and then apply ReLU to its result.
+ + \ No newline at end of file diff --git a/ScratchV-topic06-deliverable/reports/report.json b/ScratchV-topic06-deliverable/reports/report.json new file mode 100644 index 0000000..bda5373 --- /dev/null +++ b/ScratchV-topic06-deliverable/reports/report.json @@ -0,0 +1,956 @@ +{ + "schema_version": 1, + "mode": "normal", + "report_level": "light", + "regression_threshold_pct": 5.0, + "selection": { + "category": null, + "filter": null + }, + "generated_at": "2026-08-08T11:45:41", + "summary": { + "total": 23, + "passed": 13, + "failed": 10 + }, + "results": [ + { + "mode": "normal", + "name": "add_relu_relu", + "category": "activation", + "path": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\tests_main\\activation\\add_relu_relu.dsl", + "status": "PASS", + "description": "Add input and bias, then apply ReLU twice.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 7, + "actual": 7, + "matched": true, + "initial_registers": { + "t0": -3, + "t1": 10 + }, + "backend": "tinyfive", + "instr_count": 7, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_log": null, + "compile_time_sec": 0.07880300001124851, + "simulation_time_sec": 0.1846193000092171, + "total_time_sec": 0.2638619000208564, + "benchmark_runs": null, + "benchmark_stopped_reason": null, + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": null, + "regressed": null, + "asm": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\build\\add_relu_relu.s" + }, + { + "mode": "normal", + "name": "relu_add", + "category": "activation", + "path": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\tests_main\\activation\\relu_add.dsl", + "status": "PASS", + "description": "Add input and bias, then apply one ReLU.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 3, + "actual": 3, + "matched": true, + "initial_registers": { + "t0": -2, + "t1": 5 + }, + "backend": "tinyfive", + "instr_count": 5, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_log": null, + "compile_time_sec": 0.08887280002818443, + "simulation_time_sec": 0.14631609999923967, + "total_time_sec": 0.235515900014434, + "benchmark_runs": null, + "benchmark_stopped_reason": null, + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": null, + "regressed": null, + "asm": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\build\\relu_add.s" + }, + { + "mode": "normal", + "name": "relu_only", + "category": "activation", + "path": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\tests_main\\activation\\relu_only.dsl", + "status": "PASS", + "description": "Apply ReLU directly to a single input value.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 0, + "actual": 0, + "matched": true, + "initial_registers": { + "t0": -5 + }, + "backend": "tinyfive", + "instr_count": 5, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_log": null, + "compile_time_sec": 0.07508740000776015, + "simulation_time_sec": 0.1407900000049267, + "total_time_sec": 0.21604079997632653, + "benchmark_runs": null, + "benchmark_stopped_reason": null, + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": null, + "regressed": null, + "asm": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\build\\relu_only.s" + }, + { + "mode": "normal", + "name": "relu_twice", + "category": "activation", + "path": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\tests_main\\activation\\relu_twice.dsl", + "status": "PASS", + "description": "Apply ReLU twice to the same activation path.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 4, + "actual": 4, + "matched": true, + "initial_registers": { + "t0": 4 + }, + "backend": "tinyfive", + "instr_count": 6, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_log": null, + "compile_time_sec": 0.07831149999401532, + "simulation_time_sec": 0.14657159999478608, + "total_time_sec": 0.22510379998129793, + "benchmark_runs": null, + "benchmark_stopped_reason": null, + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": null, + "regressed": null, + "asm": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\build\\relu_twice.s" + }, + { + "mode": "normal", + "name": "if_else", + "category": "branch", + "path": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\tests_main\\branch\\if_else.dsl", + "status": "FAIL", + "description": "if/else branch returns subtraction result when flag is zero.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 5, + "actual": null, + "matched": false, + "initial_registers": { + "t1": 9, + "t2": 4 + }, + "backend": "timeout", + "instr_count": 0, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_log": null, + "compile_time_sec": 0.07771969999885187, + "simulation_time_sec": 5.020282899989979, + "total_time_sec": 5.098185700015165, + "benchmark_runs": null, + "benchmark_stopped_reason": null, + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": null, + "regressed": null, + "asm": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\build\\if_else.s" + }, + { + "mode": "normal", + "name": "if_relu", + "category": "branch", + "path": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\tests_main\\branch\\if_relu.dsl", + "status": "FAIL", + "description": "if/else branch combined with add and relu.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 0, + "actual": null, + "matched": false, + "initial_registers": {}, + "backend": "timeout", + "instr_count": 0, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_log": null, + "compile_time_sec": 0.07924190000630915, + "simulation_time_sec": 5.017901299986988, + "total_time_sec": 5.097439100005431, + "benchmark_runs": null, + "benchmark_stopped_reason": null, + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": null, + "regressed": null, + "asm": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\build\\if_relu.s" + }, + { + "mode": "normal", + "name": "if_then", + "category": "branch", + "path": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\tests_main\\branch\\if_then.dsl", + "status": "FAIL", + "description": "if/else branch returns add result when flag is non-zero.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 13, + "actual": null, + "matched": false, + "initial_registers": { + "t1": 9, + "t2": 4 + }, + "backend": "timeout", + "instr_count": 0, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_log": null, + "compile_time_sec": 0.07680909999180585, + "simulation_time_sec": 5.020291599998018, + "total_time_sec": 5.097542699979385, + "benchmark_runs": null, + "benchmark_stopped_reason": null, + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": null, + "regressed": null, + "asm": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\build\\if_then.s" + }, + { + "mode": "normal", + "name": "add_chain", + "category": "elementwise", + "path": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\tests_main\\elementwise\\add_chain.dsl", + "status": "PASS", + "description": "Add a and b, then add c to the intermediate result.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 9, + "actual": 9, + "matched": true, + "initial_registers": { + "t0": 2, + "t1": 3, + "t3": 4 + }, + "backend": "tinyfive", + "instr_count": 4, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_log": null, + "compile_time_sec": 0.07528699998511001, + "simulation_time_sec": 0.15915900000254624, + "total_time_sec": 0.23491749999811873, + "benchmark_runs": null, + "benchmark_stopped_reason": null, + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": null, + "regressed": null, + "asm": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\build\\add_chain.s" + }, + { + "mode": "normal", + "name": "add_chain_3", + "category": "elementwise", + "path": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\tests_main\\elementwise\\add_chain_3.dsl", + "status": "PASS", + "description": "Chain three add operations across four symbolic inputs.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 14, + "actual": 14, + "matched": true, + "initial_registers": { + "t0": 2, + "t1": 3, + "t3": 4, + "t5": 5 + }, + "backend": "tinyfive", + "instr_count": 5, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_log": null, + "compile_time_sec": 0.07411680000950582, + "simulation_time_sec": 0.14640270001837052, + "total_time_sec": 0.22091199998976663, + "benchmark_runs": null, + "benchmark_stopped_reason": null, + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": null, + "regressed": null, + "asm": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\build\\add_chain_3.s" + }, + { + "mode": "normal", + "name": "add_fan_in_4", + "category": "elementwise", + "path": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\tests_main\\elementwise\\add_fan_in_4.dsl", + "status": "PASS", + "description": "Compute two independent adds and then merge them with a final add.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 10, + "actual": 10, + "matched": true, + "initial_registers": { + "t0": 1, + "t1": 2, + "t3": 3, + "t4": 4 + }, + "backend": "tinyfive", + "instr_count": 5, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_log": null, + "compile_time_sec": 0.07103059999644756, + "simulation_time_sec": 0.13302649999968708, + "total_time_sec": 0.20422959999996237, + "benchmark_runs": null, + "benchmark_stopped_reason": null, + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": null, + "regressed": null, + "asm": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\build\\add_fan_in_4.s" + }, + { + "mode": "normal", + "name": "add_reuse", + "category": "elementwise", + "path": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\tests_main\\elementwise\\add_reuse.dsl", + "status": "PASS", + "description": "Reuse the same intermediate add result on both operands of a second add.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 10, + "actual": 10, + "matched": true, + "initial_registers": { + "t0": 2, + "t1": 3 + }, + "backend": "tinyfive", + "instr_count": 4, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_log": null, + "compile_time_sec": 0.06996160000562668, + "simulation_time_sec": 0.1360549999808427, + "total_time_sec": 0.20617819999461062, + "benchmark_runs": null, + "benchmark_stopped_reason": null, + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": null, + "regressed": null, + "asm": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\build\\add_reuse.s" + }, + { + "mode": "normal", + "name": "vector_add", + "category": "elementwise", + "path": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\tests_main\\elementwise\\vector_add.dsl", + "status": "PASS", + "description": "Single add over two symbolic vector inputs.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 5, + "actual": 5, + "matched": true, + "initial_registers": { + "t0": 2, + "t1": 3 + }, + "backend": "tinyfive", + "instr_count": 3, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_log": null, + "compile_time_sec": 0.07003299999632873, + "simulation_time_sec": 0.14579709997633472, + "total_time_sec": 0.21607930000755005, + "benchmark_runs": null, + "benchmark_stopped_reason": null, + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": null, + "regressed": null, + "asm": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\build\\vector_add.s" + }, + { + "mode": "normal", + "name": "loop_add_4", + "category": "loop", + "path": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\tests_main\\loop\\loop_add_4.dsl", + "status": "PASS", + "description": "Run a four-iteration loop whose body computes one add; final returned value is the last loop-body result.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 5, + "actual": 5, + "matched": true, + "initial_registers": { + "t0": 2, + "t1": 3 + }, + "backend": "tinyfive", + "instr_count": 22, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_log": null, + "compile_time_sec": 0.07936529998551123, + "simulation_time_sec": 0.14525349999894388, + "total_time_sec": 0.22499509999761358, + "benchmark_runs": null, + "benchmark_stopped_reason": null, + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": null, + "regressed": null, + "asm": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\build\\loop_add_4.s" + }, + { + "mode": "normal", + "name": "loop_add_chain_4", + "category": "loop", + "path": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\tests_main\\loop\\loop_add_chain_4.dsl", + "status": "PASS", + "description": "Run a four-iteration loop whose body computes two chained adds; final returned value is the last loop-body result.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 9, + "actual": 9, + "matched": true, + "initial_registers": { + "t0": 2, + "t1": 3, + "t4": 4 + }, + "backend": "tinyfive", + "instr_count": 26, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_log": null, + "compile_time_sec": 0.07717920001596212, + "simulation_time_sec": 0.15045019998797216, + "total_time_sec": 0.22800289999577217, + "benchmark_runs": null, + "benchmark_stopped_reason": null, + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": null, + "regressed": null, + "asm": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\build\\loop_add_chain_4.s" + }, + { + "mode": "normal", + "name": "loop_relu_add_4", + "category": "loop", + "path": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\tests_main\\loop\\loop_relu_add_4.dsl", + "status": "PASS", + "description": "Run a four-iteration loop whose body computes add followed by ReLU; final returned value is the last loop-body result.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 2, + "actual": 2, + "matched": true, + "initial_registers": { + "t0": -4, + "t1": 6 + }, + "backend": "tinyfive", + "instr_count": 30, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_log": null, + "compile_time_sec": 0.07636470001307316, + "simulation_time_sec": 0.13667730000452138, + "total_time_sec": 0.2132057000126224, + "benchmark_runs": null, + "benchmark_stopped_reason": null, + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": null, + "regressed": null, + "asm": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\build\\loop_relu_add_4.s" + }, + { + "mode": "normal", + "name": "dot_4", + "category": "reduction", + "path": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\tests_main\\reduction\\dot_4.dsl", + "status": "FAIL", + "description": "Compute the dot product of two symbolic vectors of length 4.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 70, + "actual": 0, + "matched": false, + "initial_registers": {}, + "backend": "tinyfive", + "instr_count": 3, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_log": null, + "compile_time_sec": 0.07195009998395108, + "simulation_time_sec": 0.13648230000399053, + "total_time_sec": 0.20867699998780154, + "benchmark_runs": null, + "benchmark_stopped_reason": null, + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": null, + "regressed": null, + "asm": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\build\\dot_4.s" + }, + { + "mode": "normal", + "name": "dot_8", + "category": "reduction", + "path": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\tests_main\\reduction\\dot_8.dsl", + "status": "FAIL", + "description": "Compute the dot product of two symbolic vectors of length 8.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 36, + "actual": 0, + "matched": false, + "initial_registers": {}, + "backend": "tinyfive", + "instr_count": 3, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_log": null, + "compile_time_sec": 0.07551269998657517, + "simulation_time_sec": 0.1408709999814164, + "total_time_sec": 0.21672679999028333, + "benchmark_runs": null, + "benchmark_stopped_reason": null, + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": null, + "regressed": null, + "asm": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\build\\dot_8.s" + }, + { + "mode": "normal", + "name": "dot_relu_4", + "category": "reduction", + "path": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\tests_main\\reduction\\dot_relu_4.dsl", + "status": "PASS", + "description": "Compute a length-4 dot product and pass it through ReLU.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 0, + "actual": 0, + "matched": true, + "initial_registers": {}, + "backend": "tinyfive", + "instr_count": 5, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_log": null, + "compile_time_sec": 0.07588799999211915, + "simulation_time_sec": 0.1373005000059493, + "total_time_sec": 0.21415960000013, + "benchmark_runs": null, + "benchmark_stopped_reason": null, + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": null, + "regressed": null, + "asm": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\build\\dot_relu_4.s" + }, + { + "mode": "normal", + "name": "dot_relu_8", + "category": "reduction", + "path": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\tests_main\\reduction\\dot_relu_8.dsl", + "status": "FAIL", + "description": "Compute a length-8 dot product and pass it through ReLU.", + "expected_type": "scalar", + "output_dtype": null, + "output_shape": null, + "expected": 8, + "actual": 0, + "matched": false, + "initial_registers": {}, + "backend": "tinyfive", + "instr_count": 5, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_log": null, + "compile_time_sec": 0.06948580002062954, + "simulation_time_sec": 0.13355569998384453, + "total_time_sec": 0.2032009999966249, + "benchmark_runs": null, + "benchmark_stopped_reason": null, + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": null, + "regressed": null, + "asm": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\build\\dot_relu_8.s" + }, + { + "mode": "normal", + "name": "matmul_2x2", + "category": "tensor", + "path": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\tests_main\\tensor\\matmul_2x2.dsl", + "status": "FAIL", + "description": "Compute a symbolic 2x2 by 2x2 matrix multiplication.", + "expected_type": "tensor", + "output_dtype": "int32", + "output_shape": [ + 2, + 2 + ], + "expected": [ + [ + 19, + 22 + ], + [ + 43, + 50 + ] + ], + "actual": 0, + "matched": false, + "initial_registers": {}, + "backend": "tinyfive", + "instr_count": 3, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_log": null, + "compile_time_sec": 0.07010060001630336, + "simulation_time_sec": 0.13320359998033382, + "total_time_sec": 0.20351309998659417, + "benchmark_runs": null, + "benchmark_stopped_reason": null, + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": null, + "regressed": null, + "asm": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\build\\matmul_2x2.s" + }, + { + "mode": "normal", + "name": "matmul_4x4", + "category": "tensor", + "path": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\tests_main\\tensor\\matmul_4x4.dsl", + "status": "FAIL", + "description": "Compute a symbolic 4x4 by 4x4 matrix multiplication.", + "expected_type": "tensor", + "output_dtype": "int32", + "output_shape": [ + 4, + 4 + ], + "expected": [ + [ + 1, + 2, + 3, + 4 + ], + [ + 5, + 6, + 7, + 8 + ], + [ + 9, + 10, + 11, + 12 + ], + [ + 13, + 14, + 15, + 16 + ] + ], + "actual": 0, + "matched": false, + "initial_registers": {}, + "backend": "tinyfive", + "instr_count": 3, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_log": null, + "compile_time_sec": 0.07288160000462085, + "simulation_time_sec": 0.1335090999782551, + "total_time_sec": 0.2066616999800317, + "benchmark_runs": null, + "benchmark_stopped_reason": null, + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": null, + "regressed": null, + "asm": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\build\\matmul_4x4.s" + }, + { + "mode": "normal", + "name": "matmul_add_2x2", + "category": "tensor", + "path": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\tests_main\\tensor\\matmul_add_2x2.dsl", + "status": "FAIL", + "description": "Compute a 2x2 matmul and then add a symbolic bias term.", + "expected_type": "tensor", + "output_dtype": "int32", + "output_shape": [ + 2, + 2 + ], + "expected": [ + [ + 20, + 23 + ], + [ + 44, + 51 + ] + ], + "actual": 0, + "matched": false, + "initial_registers": {}, + "backend": "tinyfive", + "instr_count": 4, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_log": null, + "compile_time_sec": 0.07536990000517108, + "simulation_time_sec": 0.15975869999965653, + "total_time_sec": 0.23648309998679906, + "benchmark_runs": null, + "benchmark_stopped_reason": null, + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": null, + "regressed": null, + "asm": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\build\\matmul_add_2x2.s" + }, + { + "mode": "normal", + "name": "matmul_relu_2x2", + "category": "tensor", + "path": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\tests_main\\tensor\\matmul_relu_2x2.dsl", + "status": "FAIL", + "description": "Compute a 2x2 matmul and then apply ReLU to its result.", + "expected_type": "tensor", + "output_dtype": "int32", + "output_shape": [ + 2, + 2 + ], + "expected": [ + [ + 0, + 2 + ], + [ + 0, + 4 + ] + ], + "actual": 0, + "matched": false, + "initial_registers": {}, + "backend": "tinyfive", + "instr_count": 5, + "compile_returncode": 0, + "compile_timed_out": false, + "compile_error": null, + "compile_log": null, + "compile_time_sec": 0.0864644999965094, + "simulation_time_sec": 0.17334539999137633, + "total_time_sec": 0.2609300999902189, + "benchmark_runs": null, + "benchmark_stopped_reason": null, + "avg_instr_count": null, + "min_instr_count": null, + "max_instr_count": null, + "ci95_instr_count": null, + "baseline_instr_count": null, + "delta": null, + "delta_pct": null, + "threshold_pct": null, + "regressed": null, + "asm": "D:\\PycharmProjects\\ScratchV\\ScratchV\\ScratchV-topic06-deliverable\\build\\matmul_relu_2x2.s" + } + ] +} \ No newline at end of file diff --git a/ScratchV-topic06-deliverable/reports/report.md b/ScratchV-topic06-deliverable/reports/report.md new file mode 100644 index 0000000..181f81b --- /dev/null +++ b/ScratchV-topic06-deliverable/reports/report.md @@ -0,0 +1,740 @@ +# ScratchV DSL 编译器性能测试报告 + +## 测试概览 + +- Schema 版本: 1 +- 运行模式: normal +- 类别筛选: null +- 名称筛选: null +- 生成时间: 2026-08-08 11:45:41 +- 用例总数: 23 +- 通过数量: 13 +- 失败数量: 10 +- 通过率: 56.5% +- 测试目录: `D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\tests_main` +- 汇编输出目录: `D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build` +- 性能基线文件: `D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\reports\benchmark_baseline.json` +- 性能退化阈值: 5.00% +- 单次编译超时: 30s +- 单次模拟超时: 5s + +## 测试结果 + +| 用例 | 类别 | 状态 | 编译返回码 | 编译日志 | 模拟后端 | 指令数 | Benchmark 次数 | Benchmark 停止原因 | 平均指令数 | 95% 置信区间 | 最小 | 最大 | 编译耗时(s) | 模拟耗时(s) | 总耗时(s) | 基线 | 变化量 | 变化率(%) | 退化阈值(%) | 是否退化 | 预期输出 | TinyFive 输出 | 输出匹配 | 汇编文件 | +|---|---|---|---:|---|---|---:|---:|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|---|---|---|---| +| add_relu_relu | activation | PASS | 0 | null | tinyfive | 7 | null | null | null | null | null | null | 0.0788 | 0.1846 | 0.2639 | null | null | null | null | null | 7 | 7 | True | D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\add_relu_relu.s | +| relu_add | activation | PASS | 0 | null | tinyfive | 5 | null | null | null | null | null | null | 0.0889 | 0.1463 | 0.2355 | null | null | null | null | null | 3 | 3 | True | D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\relu_add.s | +| relu_only | activation | PASS | 0 | null | tinyfive | 5 | null | null | null | null | null | null | 0.0751 | 0.1408 | 0.2160 | null | null | null | null | null | 0 | 0 | True | D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\relu_only.s | +| relu_twice | activation | PASS | 0 | null | tinyfive | 6 | null | null | null | null | null | null | 0.0783 | 0.1466 | 0.2251 | null | null | null | null | null | 4 | 4 | True | D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\relu_twice.s | +| if_else | branch | FAIL | 0 | null | timeout | 0 | null | null | null | null | null | null | 0.0777 | 5.0203 | 5.0982 | null | null | null | null | null | 5 | None | False | D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\if_else.s | +| if_relu | branch | FAIL | 0 | null | timeout | 0 | null | null | null | null | null | null | 0.0792 | 5.0179 | 5.0974 | null | null | null | null | null | 0 | None | False | D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\if_relu.s | +| if_then | branch | FAIL | 0 | null | timeout | 0 | null | null | null | null | null | null | 0.0768 | 5.0203 | 5.0975 | null | null | null | null | null | 13 | None | False | D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\if_then.s | +| add_chain | elementwise | PASS | 0 | null | tinyfive | 4 | null | null | null | null | null | null | 0.0753 | 0.1592 | 0.2349 | null | null | null | null | null | 9 | 9 | True | D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\add_chain.s | +| add_chain_3 | elementwise | PASS | 0 | null | tinyfive | 5 | null | null | null | null | null | null | 0.0741 | 0.1464 | 0.2209 | null | null | null | null | null | 14 | 14 | True | D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\add_chain_3.s | +| add_fan_in_4 | elementwise | PASS | 0 | null | tinyfive | 5 | null | null | null | null | null | null | 0.0710 | 0.1330 | 0.2042 | null | null | null | null | null | 10 | 10 | True | D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\add_fan_in_4.s | +| add_reuse | elementwise | PASS | 0 | null | tinyfive | 4 | null | null | null | null | null | null | 0.0700 | 0.1361 | 0.2062 | null | null | null | null | null | 10 | 10 | True | D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\add_reuse.s | +| vector_add | elementwise | PASS | 0 | null | tinyfive | 3 | null | null | null | null | null | null | 0.0700 | 0.1458 | 0.2161 | null | null | null | null | null | 5 | 5 | True | D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\vector_add.s | +| loop_add_4 | loop | PASS | 0 | null | tinyfive | 22 | null | null | null | null | null | null | 0.0794 | 0.1453 | 0.2250 | null | null | null | null | null | 5 | 5 | True | D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\loop_add_4.s | +| loop_add_chain_4 | loop | PASS | 0 | null | tinyfive | 26 | null | null | null | null | null | null | 0.0772 | 0.1505 | 0.2280 | null | null | null | null | null | 9 | 9 | True | D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\loop_add_chain_4.s | +| loop_relu_add_4 | loop | PASS | 0 | null | tinyfive | 30 | null | null | null | null | null | null | 0.0764 | 0.1367 | 0.2132 | null | null | null | null | null | 2 | 2 | True | D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\loop_relu_add_4.s | +| dot_4 | reduction | FAIL | 0 | null | tinyfive | 3 | null | null | null | null | null | null | 0.0720 | 0.1365 | 0.2087 | null | null | null | null | null | 70 | 0 | False | D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\dot_4.s | +| dot_8 | reduction | FAIL | 0 | null | tinyfive | 3 | null | null | null | null | null | null | 0.0755 | 0.1409 | 0.2167 | null | null | null | null | null | 36 | 0 | False | D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\dot_8.s | +| dot_relu_4 | reduction | PASS | 0 | null | tinyfive | 5 | null | null | null | null | null | null | 0.0759 | 0.1373 | 0.2142 | null | null | null | null | null | 0 | 0 | True | D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\dot_relu_4.s | +| dot_relu_8 | reduction | FAIL | 0 | null | tinyfive | 5 | null | null | null | null | null | null | 0.0695 | 0.1336 | 0.2032 | null | null | null | null | null | 8 | 0 | False | D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\dot_relu_8.s | +| matmul_2x2 | tensor | FAIL | 0 | null | tinyfive | 3 | null | null | null | null | null | null | 0.0701 | 0.1332 | 0.2035 | null | null | null | null | null | [[19, 22], [43, 50]] | 0 | False | D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\matmul_2x2.s | +| matmul_4x4 | tensor | FAIL | 0 | null | tinyfive | 3 | null | null | null | null | null | null | 0.0729 | 0.1335 | 0.2067 | null | null | null | null | null | [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]] | 0 | False | D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\matmul_4x4.s | +| matmul_add_2x2 | tensor | FAIL | 0 | null | tinyfive | 4 | null | null | null | null | null | null | 0.0754 | 0.1598 | 0.2365 | null | null | null | null | null | [[20, 23], [44, 51]] | 0 | False | D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\matmul_add_2x2.s | +| matmul_relu_2x2 | tensor | FAIL | 0 | null | tinyfive | 5 | null | null | null | null | null | null | 0.0865 | 0.1733 | 0.2609 | null | null | null | null | null | [[0, 2], [0, 4]] | 0 | False | D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\matmul_relu_2x2.s | + +## 用例详情 + +### add_relu_relu + +- 类别: activation +- 描述: Add input and bias, then apply ReLU twice. +- 预期输出 (scalar): 7 +- TinyFive 输出: 7 +- 输出是否匹配: True +- TinyFive 初始寄存器: {'t0': -3, 't1': 10} +- 模拟后端: tinyfive +- 指令数: 7 +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: null +- Benchmark 停止原因: null +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0788 +- 模拟耗时(s): 0.1846 +- 总耗时(s): 0.2639 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): null +- 是否性能退化: null +- 汇编文件: D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\add_relu_relu.s + +### relu_add + +- 类别: activation +- 描述: Add input and bias, then apply one ReLU. +- 预期输出 (scalar): 3 +- TinyFive 输出: 3 +- 输出是否匹配: True +- TinyFive 初始寄存器: {'t0': -2, 't1': 5} +- 模拟后端: tinyfive +- 指令数: 5 +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: null +- Benchmark 停止原因: null +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0889 +- 模拟耗时(s): 0.1463 +- 总耗时(s): 0.2355 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): null +- 是否性能退化: null +- 汇编文件: D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\relu_add.s + +### relu_only + +- 类别: activation +- 描述: Apply ReLU directly to a single input value. +- 预期输出 (scalar): 0 +- TinyFive 输出: 0 +- 输出是否匹配: True +- TinyFive 初始寄存器: {'t0': -5} +- 模拟后端: tinyfive +- 指令数: 5 +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: null +- Benchmark 停止原因: null +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0751 +- 模拟耗时(s): 0.1408 +- 总耗时(s): 0.2160 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): null +- 是否性能退化: null +- 汇编文件: D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\relu_only.s + +### relu_twice + +- 类别: activation +- 描述: Apply ReLU twice to the same activation path. +- 预期输出 (scalar): 4 +- TinyFive 输出: 4 +- 输出是否匹配: True +- TinyFive 初始寄存器: {'t0': 4} +- 模拟后端: tinyfive +- 指令数: 6 +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: null +- Benchmark 停止原因: null +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0783 +- 模拟耗时(s): 0.1466 +- 总耗时(s): 0.2251 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): null +- 是否性能退化: null +- 汇编文件: D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\relu_twice.s + +### if_else + +- 类别: branch +- 描述: if/else branch returns subtraction result when flag is zero. +- 预期输出 (scalar): 5 +- TinyFive 输出: None +- 输出是否匹配: False +- TinyFive 初始寄存器: {'t1': 9, 't2': 4} +- 模拟后端: timeout +- 指令数: 0 +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: null +- Benchmark 停止原因: null +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0777 +- 模拟耗时(s): 5.0203 +- 总耗时(s): 5.0982 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): null +- 是否性能退化: null +- 汇编文件: D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\if_else.s + +### if_relu + +- 类别: branch +- 描述: if/else branch combined with add and relu. +- 预期输出 (scalar): 0 +- TinyFive 输出: None +- 输出是否匹配: False +- TinyFive 初始寄存器: {} +- 模拟后端: timeout +- 指令数: 0 +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: null +- Benchmark 停止原因: null +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0792 +- 模拟耗时(s): 5.0179 +- 总耗时(s): 5.0974 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): null +- 是否性能退化: null +- 汇编文件: D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\if_relu.s + +### if_then + +- 类别: branch +- 描述: if/else branch returns add result when flag is non-zero. +- 预期输出 (scalar): 13 +- TinyFive 输出: None +- 输出是否匹配: False +- TinyFive 初始寄存器: {'t1': 9, 't2': 4} +- 模拟后端: timeout +- 指令数: 0 +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: null +- Benchmark 停止原因: null +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0768 +- 模拟耗时(s): 5.0203 +- 总耗时(s): 5.0975 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): null +- 是否性能退化: null +- 汇编文件: D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\if_then.s + +### add_chain + +- 类别: elementwise +- 描述: Add a and b, then add c to the intermediate result. +- 预期输出 (scalar): 9 +- TinyFive 输出: 9 +- 输出是否匹配: True +- TinyFive 初始寄存器: {'t0': 2, 't1': 3, 't3': 4} +- 模拟后端: tinyfive +- 指令数: 4 +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: null +- Benchmark 停止原因: null +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0753 +- 模拟耗时(s): 0.1592 +- 总耗时(s): 0.2349 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): null +- 是否性能退化: null +- 汇编文件: D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\add_chain.s + +### add_chain_3 + +- 类别: elementwise +- 描述: Chain three add operations across four symbolic inputs. +- 预期输出 (scalar): 14 +- TinyFive 输出: 14 +- 输出是否匹配: True +- TinyFive 初始寄存器: {'t0': 2, 't1': 3, 't3': 4, 't5': 5} +- 模拟后端: tinyfive +- 指令数: 5 +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: null +- Benchmark 停止原因: null +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0741 +- 模拟耗时(s): 0.1464 +- 总耗时(s): 0.2209 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): null +- 是否性能退化: null +- 汇编文件: D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\add_chain_3.s + +### add_fan_in_4 + +- 类别: elementwise +- 描述: Compute two independent adds and then merge them with a final add. +- 预期输出 (scalar): 10 +- TinyFive 输出: 10 +- 输出是否匹配: True +- TinyFive 初始寄存器: {'t0': 1, 't1': 2, 't3': 3, 't4': 4} +- 模拟后端: tinyfive +- 指令数: 5 +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: null +- Benchmark 停止原因: null +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0710 +- 模拟耗时(s): 0.1330 +- 总耗时(s): 0.2042 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): null +- 是否性能退化: null +- 汇编文件: D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\add_fan_in_4.s + +### add_reuse + +- 类别: elementwise +- 描述: Reuse the same intermediate add result on both operands of a second add. +- 预期输出 (scalar): 10 +- TinyFive 输出: 10 +- 输出是否匹配: True +- TinyFive 初始寄存器: {'t0': 2, 't1': 3} +- 模拟后端: tinyfive +- 指令数: 4 +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: null +- Benchmark 停止原因: null +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0700 +- 模拟耗时(s): 0.1361 +- 总耗时(s): 0.2062 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): null +- 是否性能退化: null +- 汇编文件: D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\add_reuse.s + +### vector_add + +- 类别: elementwise +- 描述: Single add over two symbolic vector inputs. +- 预期输出 (scalar): 5 +- TinyFive 输出: 5 +- 输出是否匹配: True +- TinyFive 初始寄存器: {'t0': 2, 't1': 3} +- 模拟后端: tinyfive +- 指令数: 3 +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: null +- Benchmark 停止原因: null +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0700 +- 模拟耗时(s): 0.1458 +- 总耗时(s): 0.2161 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): null +- 是否性能退化: null +- 汇编文件: D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\vector_add.s + +### loop_add_4 + +- 类别: loop +- 描述: Run a four-iteration loop whose body computes one add; final returned value is the last loop-body result. +- 预期输出 (scalar): 5 +- TinyFive 输出: 5 +- 输出是否匹配: True +- TinyFive 初始寄存器: {'t0': 2, 't1': 3} +- 模拟后端: tinyfive +- 指令数: 22 +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: null +- Benchmark 停止原因: null +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0794 +- 模拟耗时(s): 0.1453 +- 总耗时(s): 0.2250 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): null +- 是否性能退化: null +- 汇编文件: D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\loop_add_4.s + +### loop_add_chain_4 + +- 类别: loop +- 描述: Run a four-iteration loop whose body computes two chained adds; final returned value is the last loop-body result. +- 预期输出 (scalar): 9 +- TinyFive 输出: 9 +- 输出是否匹配: True +- TinyFive 初始寄存器: {'t0': 2, 't1': 3, 't4': 4} +- 模拟后端: tinyfive +- 指令数: 26 +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: null +- Benchmark 停止原因: null +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0772 +- 模拟耗时(s): 0.1505 +- 总耗时(s): 0.2280 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): null +- 是否性能退化: null +- 汇编文件: D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\loop_add_chain_4.s + +### loop_relu_add_4 + +- 类别: loop +- 描述: Run a four-iteration loop whose body computes add followed by ReLU; final returned value is the last loop-body result. +- 预期输出 (scalar): 2 +- TinyFive 输出: 2 +- 输出是否匹配: True +- TinyFive 初始寄存器: {'t0': -4, 't1': 6} +- 模拟后端: tinyfive +- 指令数: 30 +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: null +- Benchmark 停止原因: null +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0764 +- 模拟耗时(s): 0.1367 +- 总耗时(s): 0.2132 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): null +- 是否性能退化: null +- 汇编文件: D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\loop_relu_add_4.s + +### dot_4 + +- 类别: reduction +- 描述: Compute the dot product of two symbolic vectors of length 4. +- 预期输出 (scalar): 70 +- TinyFive 输出: 0 +- 输出是否匹配: False +- TinyFive 初始寄存器: {} +- 模拟后端: tinyfive +- 指令数: 3 +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: null +- Benchmark 停止原因: null +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0720 +- 模拟耗时(s): 0.1365 +- 总耗时(s): 0.2087 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): null +- 是否性能退化: null +- 汇编文件: D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\dot_4.s + +### dot_8 + +- 类别: reduction +- 描述: Compute the dot product of two symbolic vectors of length 8. +- 预期输出 (scalar): 36 +- TinyFive 输出: 0 +- 输出是否匹配: False +- TinyFive 初始寄存器: {} +- 模拟后端: tinyfive +- 指令数: 3 +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: null +- Benchmark 停止原因: null +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0755 +- 模拟耗时(s): 0.1409 +- 总耗时(s): 0.2167 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): null +- 是否性能退化: null +- 汇编文件: D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\dot_8.s + +### dot_relu_4 + +- 类别: reduction +- 描述: Compute a length-4 dot product and pass it through ReLU. +- 预期输出 (scalar): 0 +- TinyFive 输出: 0 +- 输出是否匹配: True +- TinyFive 初始寄存器: {} +- 模拟后端: tinyfive +- 指令数: 5 +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: null +- Benchmark 停止原因: null +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0759 +- 模拟耗时(s): 0.1373 +- 总耗时(s): 0.2142 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): null +- 是否性能退化: null +- 汇编文件: D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\dot_relu_4.s + +### dot_relu_8 + +- 类别: reduction +- 描述: Compute a length-8 dot product and pass it through ReLU. +- 预期输出 (scalar): 8 +- TinyFive 输出: 0 +- 输出是否匹配: False +- TinyFive 初始寄存器: {} +- 模拟后端: tinyfive +- 指令数: 5 +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: null +- Benchmark 停止原因: null +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0695 +- 模拟耗时(s): 0.1336 +- 总耗时(s): 0.2032 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): null +- 是否性能退化: null +- 汇编文件: D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\dot_relu_8.s + +### matmul_2x2 + +- 类别: tensor +- 描述: Compute a symbolic 2x2 by 2x2 matrix multiplication. +- 预期输出 (tensor): [[19, 22], [43, 50]] +- TinyFive 输出: 0 +- 输出是否匹配: False +- TinyFive 初始寄存器: {} +- 模拟后端: tinyfive +- 指令数: 3 +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: null +- Benchmark 停止原因: null +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0701 +- 模拟耗时(s): 0.1332 +- 总耗时(s): 0.2035 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): null +- 是否性能退化: null +- 汇编文件: D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\matmul_2x2.s + +### matmul_4x4 + +- 类别: tensor +- 描述: Compute a symbolic 4x4 by 4x4 matrix multiplication. +- 预期输出 (tensor): [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]] +- TinyFive 输出: 0 +- 输出是否匹配: False +- TinyFive 初始寄存器: {} +- 模拟后端: tinyfive +- 指令数: 3 +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: null +- Benchmark 停止原因: null +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0729 +- 模拟耗时(s): 0.1335 +- 总耗时(s): 0.2067 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): null +- 是否性能退化: null +- 汇编文件: D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\matmul_4x4.s + +### matmul_add_2x2 + +- 类别: tensor +- 描述: Compute a 2x2 matmul and then add a symbolic bias term. +- 预期输出 (tensor): [[20, 23], [44, 51]] +- TinyFive 输出: 0 +- 输出是否匹配: False +- TinyFive 初始寄存器: {} +- 模拟后端: tinyfive +- 指令数: 4 +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: null +- Benchmark 停止原因: null +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0754 +- 模拟耗时(s): 0.1598 +- 总耗时(s): 0.2365 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): null +- 是否性能退化: null +- 汇编文件: D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\matmul_add_2x2.s + +### matmul_relu_2x2 + +- 类别: tensor +- 描述: Compute a 2x2 matmul and then apply ReLU to its result. +- 预期输出 (tensor): [[0, 2], [0, 4]] +- TinyFive 输出: 0 +- 输出是否匹配: False +- TinyFive 初始寄存器: {} +- 模拟后端: tinyfive +- 指令数: 5 +- 编译返回码: 0 +- 编译是否超时: False +- 编译错误摘要: null +- 编译失败日志: null +- Benchmark 重复次数: null +- Benchmark 停止原因: null +- 平均指令数: null +- 95% 置信区间: null +- 最小指令数: null +- 最大指令数: null +- 编译耗时(s): 0.0865 +- 模拟耗时(s): 0.1733 +- 总耗时(s): 0.2609 +- 基线指令数: null +- 性能变化量: null +- 性能变化率(%): null +- 性能退化阈值(%): null +- 是否性能退化: null +- 汇编文件: D:\PycharmProjects\ScratchV\ScratchV\ScratchV-topic06-deliverable\build\matmul_relu_2x2.s + diff --git a/ScratchV-topic06-deliverable/requirements-topic06-full.txt b/ScratchV-topic06-deliverable/requirements-topic06-full.txt new file mode 100644 index 0000000..b8738ed --- /dev/null +++ b/ScratchV-topic06-deliverable/requirements-topic06-full.txt @@ -0,0 +1,3 @@ +-r requirements-topic06.txt +jinja2 +matplotlib diff --git a/ScratchV-topic06-deliverable/requirements-topic06.txt b/ScratchV-topic06-deliverable/requirements-topic06.txt new file mode 100644 index 0000000..0e8d4ba --- /dev/null +++ b/ScratchV-topic06-deliverable/requirements-topic06.txt @@ -0,0 +1,2 @@ +tinyfive +pytest diff --git a/ScratchV-topic06-deliverable/run_tests.py b/ScratchV-topic06-deliverable/run_tests.py new file mode 100644 index 0000000..aa0cc97 --- /dev/null +++ b/ScratchV-topic06-deliverable/run_tests.py @@ -0,0 +1,1854 @@ +import argparse +import json +import math +import os +import re +import shlex +import subprocess +import sys +import tempfile +import time +from datetime import datetime +from pathlib import Path + +from scratchv.simulator.tinyfive import verify_assembly + +SUITE_DIR = Path(__file__).resolve().parent +PROJECT_ROOT = SUITE_DIR.parent +TEST_DIR = SUITE_DIR / "tests_main" +BUILD_DIR = SUITE_DIR / "build" +REPORT_DIR = SUITE_DIR / "reports" +REPORT_FILE = REPORT_DIR / "report.md" +HTML_REPORT_FILE = REPORT_DIR / "report.html" +JSON_REPORT_FILE = REPORT_DIR / "report.json" +CHART_FILE = REPORT_DIR / "course_report_instructions.png" +BASELINE_FILE = REPORT_DIR / "benchmark_baseline.json" +FAILURE_DIR = REPORT_DIR / "failures" +REGRESSION_THRESHOLD_PCT = 5.0 +COMPILE_TIMEOUT_SEC = 30.0 +SIMULATION_TIMEOUT_SEC = 5.0 +INPUT_REGISTERS = [ + "t0", "t1", "t2", "t3", "t4", "t5", "t6", + "s0", "s1", "s2", "s3", "s4", "s5", "s6", "s7", + "s8", "s9", "s10", "s11", +] + + +def run_compile(dsl_file: Path, timeout: float = COMPILE_TIMEOUT_SEC): + output_file = BUILD_DIR / (dsl_file.stem + ".s") + + cmd = [ + sys.executable, + "-m", + "scratchv.main", + str(dsl_file), + "-o", + str(output_file), + "--optimize", + "all", + "--dump-ir", + ] + + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + encoding="utf-8", + errors="ignore", + cwd=PROJECT_ROOT, + timeout=timeout, + ) + except subprocess.TimeoutExpired as exc: + stdout = exc.stdout.decode("utf-8", errors="ignore") if isinstance(exc.stdout, bytes) else (exc.stdout or "") + stderr = exc.stderr.decode("utf-8", errors="ignore") if isinstance(exc.stderr, bytes) else (exc.stderr or "") + timeout_message = f"compile timeout after {timeout:.0f}s" + stderr = f"{stderr.rstrip()}\n{timeout_message}" if stderr else timeout_message + result = subprocess.CompletedProcess( + args=cmd, + returncode=124, + stdout=stdout, + stderr=stderr, + ) + + return result, output_file + + +def _last_nonempty_line(text): + lines = [line.strip() for line in (text or "").splitlines() if line.strip()] + return lines[-1] if lines else None + + +def write_compile_failure_log(dsl_file, result, output_file, compile_time_sec): + FAILURE_DIR.mkdir(parents=True, exist_ok=True) + safe_name = re.sub( + r"[^A-Za-z0-9_.-]+", + "_", + f"{dsl_file.parent.name}-{dsl_file.stem}", + ) + log_file = FAILURE_DIR / f"{safe_name}.compile.log" + args = result.args + if isinstance(args, (list, tuple)): + command = shlex.join(str(arg) for arg in args) + else: + command = str(args or "(compiler was not started)") + stdout = result.stdout or "" + stderr = result.stderr or "" + log_file.write_text( + "\n".join([ + f"Timestamp: {datetime.now().isoformat(timespec='seconds')}", + f"Case: {dsl_file}", + f"Command: {command}", + f"Return code: {result.returncode}", + f"Timed out: {result.returncode == 124}", + f"Compile time (s): {compile_time_sec:.4f}", + f"Assembly output: {output_file}", + f"Assembly exists: {output_file.exists()}", + "", + "--- stdout ---", + stdout, + "", + "--- stderr ---", + stderr, + "", + ]), + encoding="utf-8", + ) + return log_file + + +SUPPORTED_OUTPUT_DTYPES = {"bool", "int32", "int64", "float32", "float64"} + + +def _value_shape(value): + if not isinstance(value, list): + return () + if not value: + return (0,) + child_shapes = [_value_shape(item) for item in value] + if any(shape != child_shapes[0] for shape in child_shapes[1:]): + raise ValueError("expected output must be a rectangular tensor") + return (len(value),) + child_shapes[0] + + +def _validate_output_dtype(value, dtype): + values = value if isinstance(value, list) else [value] + for item in values: + if isinstance(item, list): + _validate_output_dtype(item, dtype) + elif dtype == "bool" and not isinstance(item, bool): + raise ValueError(f"expected output contains non-bool value: {item!r}") + elif dtype.startswith("int") and (isinstance(item, bool) or not isinstance(item, int)): + raise ValueError(f"expected output contains non-integer value: {item!r}") + elif dtype.startswith("float") and (isinstance(item, bool) or not isinstance(item, (int, float))): + raise ValueError(f"expected output contains non-numeric value: {item!r}") + + +def _load_expected_output(metadata, meta_file): + has_inline = "expected_return" in metadata + has_file = "expected_output_file" in metadata + if has_inline == has_file: + raise ValueError("define exactly one of expected_return or expected_output_file") + + if has_inline: + return metadata["expected_return"] + + expected_file = meta_file.parent / metadata["expected_output_file"] + payload = json.loads(expected_file.read_text(encoding="utf-8")) + if isinstance(payload, dict) and "expected_return" in payload: + return payload["expected_return"] + return payload + + +def _validate_metadata(metadata, meta_file): + expected = _load_expected_output(metadata, meta_file) + output_type = metadata.get("expected_output_type", "scalar") + if output_type == "return_value": + output_type = "scalar" + if output_type not in {"scalar", "tensor"}: + raise ValueError(f"unsupported expected_output_type: {output_type}") + + actual_shape = _value_shape(expected) + if output_type == "scalar" and actual_shape: + raise ValueError("scalar expected output cannot contain a list") + if output_type == "tensor" and not actual_shape: + raise ValueError("tensor expected output must contain a nested JSON array") + + declared_shape = metadata.get("output_shape") + if output_type == "tensor" and declared_shape is None: + raise ValueError("tensor expected output requires output_shape") + if declared_shape is not None: + if not isinstance(declared_shape, list) or any( + isinstance(size, bool) or not isinstance(size, int) or size < 0 + for size in declared_shape + ): + raise ValueError("output_shape must be a list of non-negative integers") + if tuple(declared_shape) != actual_shape: + raise ValueError( + f"output_shape {declared_shape} does not match expected output shape {list(actual_shape)}" + ) + + dtype = metadata.get("output_dtype") + if output_type == "tensor" and dtype is None: + raise ValueError("tensor expected output requires output_dtype") + if dtype is not None: + if dtype not in SUPPORTED_OUTPUT_DTYPES: + raise ValueError(f"unsupported output_dtype: {dtype}") + _validate_output_dtype(expected, dtype) + + metadata["expected_output_type"] = output_type + metadata["expected_return"] = expected + return metadata + + +def load_metadata(dsl_file: Path): + meta_file = dsl_file.with_suffix(".meta.json") + if not meta_file.exists(): + return { + "description": "", + "expected_output_type": "scalar", + "expected_return": None, + "_metadata_error": f"metadata file not found: {meta_file}", + } + try: + metadata = json.loads(meta_file.read_text(encoding="utf-8")) + return _validate_metadata(metadata, meta_file) + except (OSError, ValueError, TypeError, json.JSONDecodeError) as exc: + return { + "description": "", + "expected_output_type": "invalid", + "expected_return": None, + "_metadata_error": str(exc), + } + + +def _optimized_ir_text(compiler_output: str) -> str: + marker = "; --- IR Dump (after" + marker_pos = compiler_output.find(marker) + if marker_pos == -1: + return compiler_output + return compiler_output[marker_pos:] + + +def infer_initial_registers(compiler_output: str, inputs: dict) -> dict[str, int]: + """Infer scalar input registers from optimized IR first-use order.""" + ir_text = _optimized_ir_text(compiler_output) + allocation_order: list[str] = [] + seen: set[str] = set() + + def remember(name: str): + if name not in seen: + seen.add(name) + allocation_order.append(name) + + for raw_line in ir_text.splitlines(): + line = raw_line.strip() + if not line or line.startswith(";") or line.startswith("fun") or line.startswith("."): + continue + if line.startswith("return "): + continue + + match = re.match(r"\$(?P[A-Za-z_][A-Za-z0-9_]*)\s*=\s*(?P.*)", line) + if not match: + continue + + rhs_names = re.findall(r"\$([A-Za-z_][A-Za-z0-9_]*)", match.group("rhs")) + for name in rhs_names: + remember(name) + remember(match.group("dst")) + + result: dict[str, int] = {} + for idx, name in enumerate(allocation_order): + if idx >= len(INPUT_REGISTERS): + break + value = inputs.get(name) + if isinstance(value, bool): + result[INPUT_REGISTERS[idx]] = int(value) + elif isinstance(value, int): + result[INPUT_REGISTERS[idx]] = value + elif isinstance(value, float) and value.is_integer(): + result[INPUT_REGISTERS[idx]] = int(value) + return result + + +def run_simulation( + asm_file: Path, + initial_registers: dict[str, int] | None = None, + timeout: float = SIMULATION_TIMEOUT_SEC, +): + if not asm_file.exists(): + return { + "success": False, + "instr_count": 0, + "return_value": None, + "backend": "none", + "error": "assembly file not found", + } + + code = "\n".join([ + "import json, sys", + "from pathlib import Path", + "from scratchv.simulator.tinyfive import verify_assembly", + "asm = Path(sys.argv[1]).read_text(encoding='utf-8')", + "initial_registers = json.loads(sys.argv[2])", + "try:", + " result = verify_assembly(asm, initial_registers=initial_registers)", + "except Exception as exc:", + " result = {'success': False, 'instr_count': 0, " + "'return_value': None, 'backend': 'tinyfive', 'error': str(exc)}", + "print(json.dumps(result, ensure_ascii=False))", + ]) + + try: + completed = subprocess.run( + [ + sys.executable, + "-c", + code, + str(asm_file), + json.dumps(initial_registers or {}), + ], + capture_output=True, + text=True, + encoding="utf-8", + errors="ignore", + cwd=PROJECT_ROOT, + timeout=timeout, + ) + except subprocess.TimeoutExpired: + return { + "success": False, + "instr_count": 0, + "return_value": None, + "backend": "timeout", + "error": f"simulation timeout after {timeout:.0f}s", + } + + if completed.returncode != 0: + return { + "success": False, + "instr_count": 0, + "return_value": None, + "backend": "tinyfive", + "error": (completed.stderr or completed.stdout or "simulation failed").strip(), + } + + try: + return json.loads(completed.stdout.strip().splitlines()[-1]) + except (IndexError, json.JSONDecodeError) as exc: + return { + "success": False, + "instr_count": 0, + "return_value": None, + "backend": "tinyfive", + "error": f"invalid simulation output: {exc}", + } + + +def values_equal(lhs, rhs): + if isinstance(lhs, list) and isinstance(rhs, list): + if len(lhs) != len(rhs): + return False + return all(values_equal(a, b) for a, b in zip(lhs, rhs)) + if isinstance(lhs, float) or isinstance(rhs, float): + return math.isclose(lhs, rhs, rel_tol=1e-7, abs_tol=1e-7) + return lhs == rhs + + +def summarize_benchmark_runs(instr_counts): + if not instr_counts: + return { + "runs": 0, + "avg_instr_count": None, + "min_instr_count": None, + "max_instr_count": None, + "ci95_instr_count": None, + } + avg = sum(instr_counts) / len(instr_counts) + if len(instr_counts) > 1: + variance = sum((value - avg) ** 2 for value in instr_counts) / (len(instr_counts) - 1) + ci95 = 1.96 * math.sqrt(variance) / math.sqrt(len(instr_counts)) + else: + ci95 = 0.0 + return { + "runs": len(instr_counts), + "avg_instr_count": avg, + "min_instr_count": min(instr_counts), + "max_instr_count": max(instr_counts), + "ci95_instr_count": ci95, + } + + +def detect_regression( + avg_instr_count, + baseline_instr_count, + threshold_pct=REGRESSION_THRESHOLD_PCT, +): + delta = avg_instr_count - baseline_instr_count + delta_pct = 0.0 if baseline_instr_count == 0 else (delta / baseline_instr_count) * 100.0 + return { + "baseline_instr_count": baseline_instr_count, + "delta": round(delta, 4), + "delta_pct": round(delta_pct, 4), + "threshold_pct": threshold_pct, + "regressed": delta_pct > threshold_pct, + } + + +def load_baseline(): + if not BASELINE_FILE.exists(): + return {} + return json.loads(BASELINE_FILE.read_text(encoding="utf-8")) + + +def save_baseline(results, preserve_existing=False): + REPORT_DIR.mkdir(exist_ok=True) + payload = load_baseline() if preserve_existing else {} + for r in results: + if r["avg_instr_count"] is None: + continue + payload[r["name"]] = { + "category": r["category"], + "avg_instr_count": r["avg_instr_count"], + "runs": r["benchmark_runs"], + } + BASELINE_FILE.write_text(json.dumps(payload, indent=2), encoding="utf-8") + + +def generate_report_text(results, passed, failed): + lines = [] + lines.append("# ScratchV DSL 编译器性能测试报告\n\n") + + lines.append("## 测试概览\n\n") + lines.append(f"- 用例总数: {len(results)}\n") + lines.append(f"- 通过数量: {passed}\n") + lines.append(f"- 失败数量: {failed}\n\n") + + benchmark_mode = any(r.get("benchmark_runs", 1) > 1 for r in results) + if benchmark_mode: + lines.append("## 性能基准概览\n\n") + lines.append("- 运行模式: benchmark\n") + lines.append(f"- 性能基线文件: `{BASELINE_FILE}`\n\n") + + lines.append("## 测试结果\n\n") + if benchmark_mode: + lines.append("| 测试用例 | 类别 | 状态 | 模拟后端 | 平均指令数 | 最小值 | 最大值 | 基线 | 变化率 | 是否退化 | 预期输出 | 实际输出 | 是否匹配 | 汇编文件 |\n") + lines.append("|---|---|---|---|---:|---:|---:|---:|---:|---|---|---|---|---|\n") + else: + lines.append("| 测试用例 | 类别 | 状态 | 模拟后端 | 指令数 | 预期输出 | 实际输出 | 是否匹配 | 汇编文件 |\n") + lines.append("|---|---|---|---|---:|---|---|---|---|\n") + + for r in results: + expected = str(r["expected"]).replace("\n", " ").replace("|", "\\|") + actual = str(r["actual"]).replace("\n", " ").replace("|", "\\|") + if benchmark_mode: + lines.append( + f"| {r['name']} | {r['category']} | {r['status']} | {r['backend']} | " + f"{r['avg_instr_count']:.2f} | {r['min_instr_count']} | {r['max_instr_count']} | " + f"{r['baseline_instr_count']:.2f} | {r['delta_pct']:.2f} | {r['regressed']} | " + f"{expected} | {actual} | {r['matched']} | {r['asm']} |\n" + ) + else: + lines.append( + f"| {r['name']} | {r['category']} | {r['status']} | {r['backend']} | " + f"{r['instr_count']} | {expected} | {actual} | {r['matched']} | {r['asm']} |\n" + ) + + lines.append("\n## 性能图表\n\n") + chart_cases = [r["name"] for r in results] + chart_instr = [str(round(r.get("avg_instr_count", r["instr_count"]), 2)) for r in results] + lines.append("### 各测试用例指令数\n\n") + lines.append("```mermaid\n") + lines.append("xychart-beta\n") + lines.append(' title "各测试用例指令数"\n') + lines.append(" x-axis [" + ", ".join(f'"{name}"' for name in chart_cases) + "]\n") + max_instr = max((r.get("avg_instr_count", r["instr_count"]) for r in results), default=0) + lines.append(f' y-axis "指令数" 0 --> {max_instr + 2}\n') + lines.append(" bar [" + ", ".join(chart_instr) + "]\n") + lines.append("```\n\n") + + category_totals = {} + for r in results: + category_totals[r["category"]] = category_totals.get(r["category"], 0) + r.get("avg_instr_count", r["instr_count"]) + lines.append("### 各类别指令数占比\n\n") + lines.append("```mermaid\n") + lines.append("pie showData\n") + lines.append(' title 各类别指令数占比\n') + for category, total in sorted(category_totals.items()): + lines.append(f' "{category}" : {total}\n') + lines.append("```\n") + + lines.append("\n## 用例详情\n\n") + for r in results: + lines.append(f"### {r['name']}\n\n") + lines.append(f"- 类别: {r['category']}\n") + lines.append(f"- 描述: {r['description']}\n") + lines.append(f"- 预期输出 ({r['expected_type']}): {r['expected']}\n") + lines.append(f"- 实际输出: {r['actual']}\n") + lines.append(f"- 是否匹配: {r['matched']}\n") + lines.append(f"- 模拟后端: {r['backend']}\n") + if benchmark_mode: + lines.append(f"- Benchmark 重复次数: {r['benchmark_runs']}\n") + lines.append(f"- 平均指令数: {r['avg_instr_count']:.2f}\n") + lines.append(f"- 最小指令数: {r['min_instr_count']}\n") + lines.append(f"- 最大指令数: {r['max_instr_count']}\n") + lines.append(f"- 基线指令数: {r['baseline_instr_count']:.2f}\n") + lines.append(f"- 性能变化率 (%): {r['delta_pct']:.2f}\n") + lines.append(f"- 是否性能退化: {r['regressed']}\n") + else: + lines.append(f"- 指令数: {r['instr_count']}\n") + lines.append(f"- 汇编文件: {r['asm']}\n\n") + + return "".join(lines) + + +def write_html_report(results, passed, failed): + try: + from jinja2 import Template + except ImportError: + return None + + pass_rate = 0.0 if not results else passed / len(results) * 100.0 + template = Template(""" + + + + ScratchV 课程版测试报告 + + + +

ScratchV DSL 编译器性能测试报告

+
+

用例总数：{{ total }}，通过：{{ passed }}，失败：{{ failed }}，通过率：{{ "%.1f"|format(pass_rate) }}%

+

测试目录：{{ test_dir }}，性能退化阈值：{{ threshold }}%

+
+ 课程版指令数图表 + + + + + + {% for r in results %} + + + + + + + + + + + {% endfor %} + +
用例类别状态平均指令数95%置信区间变化率(%)是否退化描述
{{ r.name }}{{ r.category }}{{ r.status }}{{ "%.2f"|format(r.avg_instr_count) }}±{{ "%.2f"|format(r.ci95_instr_count) }}{{ "%.2f"|format(r.delta_pct) }}{{ r.regressed }}{{ r.description }}
+ + +""") + HTML_REPORT_FILE.write_text( + template.render( + total=len(results), + passed=passed, + failed=failed, + pass_rate=pass_rate, + test_dir=str(TEST_DIR), + threshold=REGRESSION_THRESHOLD_PCT, + chart_name=CHART_FILE.name, + results=results, + ), + encoding="utf-8", + ) + return HTML_REPORT_FILE + + +def generate_report_text(results, passed, failed): + benchmark_mode = any(r.get("benchmark_runs", 1) > 1 for r in results) + pass_rate = 0.0 if not results else passed / len(results) * 100.0 + lines = [ + "# ScratchV DSL 编译器性能测试报告\n\n", + "## 测试概览\n\n", + f"- 生成时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n", + f"- 用例总数: {len(results)}\n", + f"- 通过数量: {passed}\n", + f"- 失败数量: {failed}\n", + f"- 通过率: {pass_rate:.1f}%\n", + f"- 测试目录: `{TEST_DIR}`\n", + f"- 汇编输出目录: `{BUILD_DIR}`\n", + f"- 性能基线文件: `{BASELINE_FILE}`\n", + f"- 性能退化阈值: {REGRESSION_THRESHOLD_PCT:.1f}%\n\n", + "## 测试结果\n\n", + ] + + if benchmark_mode: + lines.extend([ + "| 用例 | 类别 | 状态 | 模拟后端 | 平均指令数 | 95%置信区间 | 最小 | 最大 | 基线 | 变化率(%) | 是否退化 | 预期输出 | 实际输出 | 输出匹配 | 汇编文件 |\n", + "|---|---|---|---|---:|---:|---:|---:|---:|---:|---|---|---|---|---|\n", + ]) + else: + lines.extend([ + "| 用例 | 类别 | 状态 | 模拟后端 | 指令数 | 预期输出 | 实际输出 | 输出匹配 | 汇编文件 |\n", + "|---|---|---|---|---:|---|---|---|---|\n", + ]) + + for r in results: + expected = _markdown_cell(r["expected"]) + actual = _markdown_cell(r["actual"]) + if benchmark_mode: + lines.append( + f"| {r['name']} | {r['category']} | {r['status']} | {r['backend']} | " + f"{r['avg_instr_count']:.2f} | ±{r['ci95_instr_count']:.2f} | " + f"{r['min_instr_count']} | {r['max_instr_count']} | " + f"{r['baseline_instr_count']:.2f} | {r['delta_pct']:.2f} | " + f"{r['regressed']} | {expected} | {actual} | {r['matched']} | {r['asm']} |\n" + ) + else: + lines.append( + f"| {r['name']} | {r['category']} | {r['status']} | {r['backend']} | " + f"{r['instr_count']} | {expected} | {actual} | {r['matched']} | {r['asm']} |\n" + ) + + lines.extend([ + "\n## 性能图表\n\n", + f"![课程版指令数图表]({CHART_FILE.name})\n\n", + "### Mermaid 图表\n\n", + "```mermaid\n", + "xychart-beta\n", + ' title "各测试用例指令数"\n', + " x-axis [" + ", ".join(f'"{r["name"]}"' for r in results) + "]\n", + ]) + max_instr = max((r.get("avg_instr_count", r["instr_count"]) for r in results), default=0) + chart_values = [str(round(r.get("avg_instr_count", r["instr_count"]), 2)) for r in results] + lines.extend([ + f' y-axis "指令数" 0 --> {max_instr + 2}\n', + " bar [" + ", ".join(chart_values) + "]\n", + "```\n\n", + "## 用例详情\n\n", + ]) + + for r in results: + lines.extend([ + f"### {r['name']}\n\n", + f"- 类别: {r['category']}\n", + f"- 描述: {r['description']}\n", + f"- 预期输出 ({r['expected_type']}): {r['expected']}\n", + f"- 实际输出: {r['actual']}\n", + f"- 输出是否匹配: {r['matched']}\n", + f"- 模拟后端: {r['backend']}\n", + f"- 汇编文件: {r['asm']}\n", + ]) + if benchmark_mode: + lines.extend([ + f"- Benchmark 重复次数: {r['benchmark_runs']}\n", + f"- 平均指令数: {r['avg_instr_count']:.2f}\n", + f"- 95% 置信区间: ±{r['ci95_instr_count']:.2f}\n", + f"- 最小指令数: {r['min_instr_count']}\n", + f"- 最大指令数: {r['max_instr_count']}\n", + f"- 基线指令数: {r['baseline_instr_count']:.2f}\n", + f"- 性能变化率: {r['delta_pct']:.2f}%\n", + f"- 性能退化阈值: {r['threshold_pct']:.2f}%\n", + f"- 是否性能退化: {r['regressed']}\n", + ]) + else: + lines.append(f"- 指令数: {r['instr_count']}\n") + lines.append("\n") + + return "".join(lines) + + +def write_report(results, passed, failed): + REPORT_DIR.mkdir(exist_ok=True) + + REPORT_FILE.write_text(generate_report_text(results, passed, failed), encoding="utf-8") + print(f"\nReport written to {REPORT_FILE}") + + +def _markdown_cell(value): + return str(value).replace("\n", " ").replace("|", "\\|") + + +def generate_report_text(results, passed, failed): + benchmark_mode = any(r.get("benchmark_runs", 1) > 1 for r in results) + pass_rate = 0.0 if not results else passed / len(results) * 100.0 + lines = [ + "# ScratchV DSL 编译器性能测试报告\n\n", + "## 测试概览\n\n", + f"- 生成时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n", + f"- 用例总数: {len(results)}\n", + f"- 通过数量: {passed}\n", + f"- 失败数量: {failed}\n", + f"- 通过率: {pass_rate:.1f}%\n", + f"- 测试目录: `{TEST_DIR}`\n", + f"- 汇编输出目录: `{BUILD_DIR}`\n", + f"- 性能基线文件: `{BASELINE_FILE}`\n", + f"- 性能退化阈值: {REGRESSION_THRESHOLD_PCT:.1f}%\n\n", + "## 测试结果\n\n", + ] + + if benchmark_mode: + lines.extend([ + "| 用例 | 类别 | 状态 | 模拟后端 | 平均指令数 | 95%置信区间 | 最小 | 最大 | 基线 | 变化率(%) | 是否退化 | 预期输出 | 实际输出 | 输出匹配 | 汇编文件 |\n", + "|---|---|---|---|---:|---:|---:|---:|---:|---:|---|---|---|---|---|\n", + ]) + else: + lines.extend([ + "| 用例 | 类别 | 状态 | 模拟后端 | 指令数 | 预期输出 | 实际输出 | 输出匹配 | 汇编文件 |\n", + "|---|---|---|---|---:|---|---|---|---|\n", + ]) + + for r in results: + expected = _markdown_cell(r["expected"]) + actual = _markdown_cell(r["actual"]) + if benchmark_mode: + lines.append( + f"| {r['name']} | {r['category']} | {r['status']} | {r['backend']} | " + f"{r['avg_instr_count']:.2f} | ±{r['ci95_instr_count']:.2f} | " + f"{r['min_instr_count']} | {r['max_instr_count']} | " + f"{r['baseline_instr_count']:.2f} | {r['delta_pct']:.2f} | " + f"{r['regressed']} | {expected} | {actual} | {r['matched']} | {r['asm']} |\n" + ) + else: + lines.append( + f"| {r['name']} | {r['category']} | {r['status']} | {r['backend']} | " + f"{r['instr_count']} | {expected} | {actual} | {r['matched']} | {r['asm']} |\n" + ) + + lines.extend([ + "\n## 性能图表\n\n", + f"![课程版指令数图表]({CHART_FILE.name})\n\n", + "### Mermaid 图表\n\n", + "```mermaid\n", + "xychart-beta\n", + ' title "各测试用例指令数"\n', + " x-axis [" + ", ".join(f'"{r["name"]}"' for r in results) + "]\n", + ]) + max_instr = max((r.get("avg_instr_count", r["instr_count"]) for r in results), default=0) + chart_values = [str(round(r.get("avg_instr_count", r["instr_count"]), 2)) for r in results] + lines.extend([ + f' y-axis "指令数" 0 --> {max_instr + 2}\n', + " bar [" + ", ".join(chart_values) + "]\n", + "```\n\n", + "## 用例详情\n\n", + ]) + + for r in results: + lines.extend([ + f"### {r['name']}\n\n", + f"- 类别: {r['category']}\n", + f"- 描述: {r['description']}\n", + f"- 预期输出 ({r['expected_type']}): {r['expected']}\n", + f"- 实际输出: {r['actual']}\n", + f"- 输出是否匹配: {r['matched']}\n", + f"- 模拟后端: {r['backend']}\n", + f"- 汇编文件: {r['asm']}\n", + ]) + if benchmark_mode: + lines.extend([ + f"- Benchmark 重复次数: {r['benchmark_runs']}\n", + f"- 平均指令数: {r['avg_instr_count']:.2f}\n", + f"- 95% 置信区间: ±{r['ci95_instr_count']:.2f}\n", + f"- 最小指令数: {r['min_instr_count']}\n", + f"- 最大指令数: {r['max_instr_count']}\n", + f"- 基线指令数: {r['baseline_instr_count']:.2f}\n", + f"- 性能变化率: {r['delta_pct']:.2f}%\n", + f"- 性能退化阈值: {r['threshold_pct']:.2f}%\n", + f"- 是否性能退化: {r['regressed']}\n", + ]) + else: + lines.append(f"- 指令数: {r['instr_count']}\n") + lines.append("\n") + + return "".join(lines) + + +def generate_report_text(results, passed, failed): + benchmark_mode = any(r.get("benchmark_runs", 1) > 1 for r in results) + pass_rate = 0.0 if not results else passed / len(results) * 100.0 + lines = [ + "# ScratchV DSL 编译器性能测试报告\n\n", + "## 测试概览\n\n", + f"- 生成时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n", + f"- 用例总数: {len(results)}\n", + f"- 通过数量: {passed}\n", + f"- 失败数量: {failed}\n", + f"- 通过率: {pass_rate:.1f}%\n", + f"- 测试目录: `{TEST_DIR}`\n", + f"- 汇编输出目录: `{BUILD_DIR}`\n", + f"- 性能基线文件: `{BASELINE_FILE}`\n", + f"- 性能退化阈值: {REGRESSION_THRESHOLD_PCT:.1f}%\n\n", + "## 测试结果\n\n", + ] + + if benchmark_mode: + lines.extend([ + "| 用例 | 类别 | 状态 | 模拟后端 | 平均指令数 | 95%置信区间 | 最小 | 最大 | 基线 | 变化率(%) | 是否退化 | 预期输出 | 实际输出 | 输出匹配 | 汇编文件 |\n", + "|---|---|---|---|---:|---:|---:|---:|---:|---:|---|---|---|---|---|\n", + ]) + else: + lines.extend([ + "| 用例 | 类别 | 状态 | 模拟后端 | 指令数 | 预期输出 | 实际输出 | 输出匹配 | 汇编文件 |\n", + "|---|---|---|---|---:|---|---|---|---|\n", + ]) + + for r in results: + expected = _markdown_cell(r["expected"]) + actual = _markdown_cell(r["actual"]) + if benchmark_mode: + lines.append( + f"| {r['name']} | {r['category']} | {r['status']} | {r['backend']} | " + f"{r['avg_instr_count']:.2f} | ±{r['ci95_instr_count']:.2f} | " + f"{r['min_instr_count']} | {r['max_instr_count']} | " + f"{r['baseline_instr_count']:.2f} | {r['delta_pct']:.2f} | " + f"{r['regressed']} | {expected} | {actual} | {r['matched']} | {r['asm']} |\n" + ) + else: + lines.append( + f"| {r['name']} | {r['category']} | {r['status']} | {r['backend']} | " + f"{r['instr_count']} | {expected} | {actual} | {r['matched']} | {r['asm']} |\n" + ) + + lines.extend([ + "\n## 性能图表\n\n", + f"![课程版指令数图表]({CHART_FILE.name})\n\n", + "### Mermaid 图表\n\n", + "```mermaid\n", + "xychart-beta\n", + ' title "各测试用例指令数"\n', + " x-axis [" + ", ".join(f'"{r["name"]}"' for r in results) + "]\n", + ]) + max_instr = max((r.get("avg_instr_count", r["instr_count"]) for r in results), default=0) + chart_values = [str(round(r.get("avg_instr_count", r["instr_count"]), 2)) for r in results] + lines.extend([ + f' y-axis "指令数" 0 --> {max_instr + 2}\n', + " bar [" + ", ".join(chart_values) + "]\n", + "```\n\n", + "## 用例详情\n\n", + ]) + + for r in results: + lines.extend([ + f"### {r['name']}\n\n", + f"- 类别: {r['category']}\n", + f"- 描述: {r['description']}\n", + f"- 预期输出 ({r['expected_type']}): {r['expected']}\n", + f"- 实际输出: {r['actual']}\n", + f"- 输出是否匹配: {r['matched']}\n", + f"- 模拟后端: {r['backend']}\n", + f"- 汇编文件: {r['asm']}\n", + ]) + if benchmark_mode: + lines.extend([ + f"- Benchmark 重复次数: {r['benchmark_runs']}\n", + f"- 平均指令数: {r['avg_instr_count']:.2f}\n", + f"- 95% 置信区间: ±{r['ci95_instr_count']:.2f}\n", + f"- 最小指令数: {r['min_instr_count']}\n", + f"- 最大指令数: {r['max_instr_count']}\n", + f"- 基线指令数: {r['baseline_instr_count']:.2f}\n", + f"- 性能变化率: {r['delta_pct']:.2f}%\n", + f"- 性能退化阈值: {r['threshold_pct']:.2f}%\n", + f"- 是否性能退化: {r['regressed']}\n", + ]) + else: + lines.append(f"- 指令数: {r['instr_count']}\n") + lines.append("\n") + + return "".join(lines) + + +def write_chart(results): + try: + mpl_config_dir = Path(tempfile.gettempdir()) / "scratchv-matplotlib" + mpl_config_dir.mkdir(parents=True, exist_ok=True) + os.environ.setdefault("MPLCONFIGDIR", str(mpl_config_dir)) + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + except ImportError: + return None + + names = [r["name"] for r in results] + values = [_reported_instr_count(r) for r in results] + width = max(10, len(names) * 0.45) + fig, ax = plt.subplots(figsize=(width, 5)) + ax.bar(range(len(names)), values, color="#2563eb") + ax.set_title("ScratchV Course Benchmark Instruction Counts") + ax.set_ylabel("Instructions") + ax.set_xticks(range(len(names))) + ax.set_xticklabels(names, rotation=60, ha="right", fontsize=8) + ax.grid(axis="y", linestyle="--", alpha=0.35) + fig.tight_layout() + fig.savefig(CHART_FILE, dpi=160) + plt.close(fig) + return CHART_FILE + + +def _reported_instr_count(result): + average = result.get("avg_instr_count") + return result["instr_count"] if average is None else average + + +def _report_value(value, precision=None, prefix=""): + if value is None: + return "null" + if precision is not None: + return f"{prefix}{value:.{precision}f}" + return f"{prefix}{value}" + + +def write_html_report(results, passed, failed): + try: + from jinja2 import Template + except ImportError: + return None + + pass_rate = 0.0 if not results else passed / len(results) * 100.0 + template = Template(""" + + + + ScratchV 课程版测试报告 + + + +

ScratchV DSL 编译器性能测试报告

+
+

用例总数：{{ total }}，通过：{{ passed }}，失败：{{ failed }}，通过率：{{ "%.1f"|format(pass_rate) }}%

+

测试目录：{{ test_dir }}，性能退化阈值：{{ threshold }}%

+
+ 课程版指令数图表 + + + + + + {% for r in results %} + + + + + + + + + + + {% endfor %} + +
用例类别状态平均指令数95%置信区间变化率(%)是否退化描述
{{ r.name }}{{ r.category }}{{ r.status }}{{ "%.2f"|format(r.avg_instr_count) }}±{{ "%.2f"|format(r.ci95_instr_count) }}{{ "%.2f"|format(r.delta_pct) }}{{ r.regressed }}{{ r.description }}
+ + +""") + HTML_REPORT_FILE.write_text( + template.render( + total=len(results), + passed=passed, + failed=failed, + pass_rate=pass_rate, + test_dir=str(TEST_DIR), + threshold=REGRESSION_THRESHOLD_PCT, + chart_name=CHART_FILE.name, + results=results, + ), + encoding="utf-8", + ) + return HTML_REPORT_FILE + + +def generate_report_text_cn(results, passed, failed): + benchmark_mode = any(r.get("benchmark_runs", 1) > 1 for r in results) + pass_rate = 0.0 if not results else passed / len(results) * 100.0 + lines = [ + "# ScratchV DSL 编译器性能测试报告\n\n", + "## 测试概览\n\n", + f"- 生成时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n", + f"- 用例总数: {len(results)}\n", + f"- 通过数量: {passed}\n", + f"- 失败数量: {failed}\n", + f"- 通过率: {pass_rate:.1f}%\n", + f"- 测试目录: `{TEST_DIR}`\n", + f"- 汇编输出目录: `{BUILD_DIR}`\n", + f"- 性能基线文件: `{BASELINE_FILE}`\n", + f"- 性能退化阈值: {REGRESSION_THRESHOLD_PCT:.1f}%\n\n", + "## 测试结果\n\n", + ] + + if benchmark_mode: + lines.extend([ + "| 用例 | 类别 | 状态 | 模拟后端 | 平均指令数 | 95%置信区间 | 最小 | 最大 | 基线 | 变化率(%) | 是否退化 | 预期输出 | 实际输出 | 输出匹配 | 汇编文件 |\n", + "|---|---|---|---|---:|---:|---:|---:|---:|---:|---|---|---|---|---|\n", + ]) + else: + lines.extend([ + "| 用例 | 类别 | 状态 | 模拟后端 | 指令数 | 预期输出 | 实际输出 | 输出匹配 | 汇编文件 |\n", + "|---|---|---|---|---:|---|---|---|---|\n", + ]) + + for r in results: + expected = _markdown_cell(r["expected"]) + actual = _markdown_cell(r["actual"]) + if benchmark_mode: + lines.append( + f"| {r['name']} | {r['category']} | {r['status']} | {r['backend']} | " + f"{r['avg_instr_count']:.2f} | ±{r['ci95_instr_count']:.2f} | " + f"{r['min_instr_count']} | {r['max_instr_count']} | " + f"{r['baseline_instr_count']:.2f} | {r['delta_pct']:.2f} | " + f"{r['regressed']} | {expected} | {actual} | {r['matched']} | {r['asm']} |\n" + ) + else: + lines.append( + f"| {r['name']} | {r['category']} | {r['status']} | {r['backend']} | " + f"{r['instr_count']} | {expected} | {actual} | {r['matched']} | {r['asm']} |\n" + ) + + lines.extend([ + "\n## 性能图表\n\n", + f"![课程版指令数图表]({CHART_FILE.name})\n\n", + "### Mermaid 图表\n\n", + "```mermaid\n", + "xychart-beta\n", + ' title "各测试用例指令数"\n', + " x-axis [" + ", ".join(f'"{r["name"]}"' for r in results) + "]\n", + ]) + max_instr = max((r.get("avg_instr_count", r["instr_count"]) for r in results), default=0) + chart_values = [str(round(r.get("avg_instr_count", r["instr_count"]), 2)) for r in results] + lines.extend([ + f' y-axis "指令数" 0 --> {max_instr + 2}\n', + " bar [" + ", ".join(chart_values) + "]\n", + "```\n\n", + "## 用例详情\n\n", + ]) + + for r in results: + lines.extend([ + f"### {r['name']}\n\n", + f"- 类别: {r['category']}\n", + f"- 描述: {r['description']}\n", + f"- 预期输出 ({r['expected_type']}): {r['expected']}\n", + f"- 实际输出: {r['actual']}\n", + f"- 输出是否匹配: {r['matched']}\n", + f"- 模拟后端: {r['backend']}\n", + f"- 汇编文件: {r['asm']}\n", + ]) + if benchmark_mode: + lines.extend([ + f"- Benchmark 重复次数: {r['benchmark_runs']}\n", + f"- 平均指令数: {r['avg_instr_count']:.2f}\n", + f"- 95% 置信区间: ±{r['ci95_instr_count']:.2f}\n", + f"- 最小指令数: {r['min_instr_count']}\n", + f"- 最大指令数: {r['max_instr_count']}\n", + f"- 基线指令数: {r['baseline_instr_count']:.2f}\n", + f"- 性能变化率: {r['delta_pct']:.2f}%\n", + f"- 性能退化阈值: {r['threshold_pct']:.2f}%\n", + f"- 是否性能退化: {r['regressed']}\n", + ]) + else: + lines.append(f"- 指令数: {r['instr_count']}\n") + lines.append("\n") + + return "".join(lines) + + +def write_html_report_cn(results, passed, failed): + try: + from jinja2 import Template + except ImportError: + return None + + pass_rate = 0.0 if not results else passed / len(results) * 100.0 + template = Template(""" + + + + ScratchV 课程版测试报告 + + + +

ScratchV DSL 编译器性能测试报告

+
+

用例总数：{{ total }}，通过：{{ passed }}，失败：{{ failed }}，通过率：{{ "%.1f"|format(pass_rate) }}%

+

测试目录：{{ test_dir }}，性能退化阈值：{{ threshold }}%

+
+ 课程版指令数图表 + + + + + + {% for r in results %} + + + + + + + + + + + {% endfor %} + +
用例类别状态平均指令数95%置信区间变化率(%)是否退化描述
{{ r.name }}{{ r.category }}{{ r.status }}{{ "%.2f"|format(r.avg_instr_count) }}±{{ "%.2f"|format(r.ci95_instr_count) }}{{ "%.2f"|format(r.delta_pct) }}{{ r.regressed }}{{ r.description }}
+ + +""") + HTML_REPORT_FILE.write_text( + template.render( + total=len(results), + passed=passed, + failed=failed, + pass_rate=pass_rate, + test_dir=str(TEST_DIR), + threshold=REGRESSION_THRESHOLD_PCT, + chart_name=CHART_FILE.name, + results=results, + ), + encoding="utf-8", + ) + return HTML_REPORT_FILE + + +def generate_report_text_cn(results, passed, failed): + benchmark_mode = any(r.get("benchmark_runs", 1) > 1 for r in results) + pass_rate = 0.0 if not results else passed / len(results) * 100.0 + lines = [ + "# ScratchV DSL 编译器性能测试报告\n\n", + "## 测试概览\n\n", + f"- 生成时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n", + f"- 用例总数: {len(results)}\n", + f"- 通过数量: {passed}\n", + f"- 失败数量: {failed}\n", + f"- 通过率: {pass_rate:.1f}%\n", + f"- 测试目录: `{TEST_DIR}`\n", + f"- 汇编输出目录: `{BUILD_DIR}`\n", + f"- 性能基线文件: `{BASELINE_FILE}`\n", + f"- 性能退化阈值: {REGRESSION_THRESHOLD_PCT:.1f}%\n", + f"- 单次模拟超时: {SIMULATION_TIMEOUT_SEC:.0f}s\n\n", + "## 测试结果\n\n", + ] + + if benchmark_mode: + lines.extend([ + "| 用例 | 类别 | 状态 | 模拟后端 | 平均指令数 | 95% 置信区间 | 最小 | 最大 | 编译耗时(s) | 模拟耗时(s) | 总耗时(s) | 基线 | 变化率(%) | 是否退化 | 预期输出 | TinyFive 输出 | 输出匹配 | 汇编文件 |\n", + "|---|---|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|---|---|---|---|\n", + ]) + else: + lines.extend([ + "| 用例 | 类别 | 状态 | 模拟后端 | 指令数 | 编译耗时(s) | 模拟耗时(s) | 总耗时(s) | 预期输出 | TinyFive 输出 | 输出匹配 | 汇编文件 |\n", + "|---|---|---|---|---:|---:|---:|---:|---|---|---|---|\n", + ]) + + for r in results: + expected = _markdown_cell(r["expected"]) + actual = _markdown_cell(r["actual"]) + if benchmark_mode: + lines.append( + f"| {r['name']} | {r['category']} | {r['status']} | {r['backend']} | " + f"{r['avg_instr_count']:.2f} | ±{r['ci95_instr_count']:.2f} | " + f"{r['min_instr_count']} | {r['max_instr_count']} | " + f"{r['compile_time_sec']:.4f} | {r['simulation_time_sec']:.4f} | " + f"{r['total_time_sec']:.4f} | {r['baseline_instr_count']:.2f} | " + f"{r['delta_pct']:.2f} | {r['regressed']} | {expected} | " + f"{actual} | {r['matched']} | {r['asm']} |\n" + ) + else: + lines.append( + f"| {r['name']} | {r['category']} | {r['status']} | {r['backend']} | " + f"{r['instr_count']} | {r['compile_time_sec']:.4f} | " + f"{r['simulation_time_sec']:.4f} | {r['total_time_sec']:.4f} | " + f"{expected} | {actual} | {r['matched']} | {r['asm']} |\n" + ) + + lines.extend([ + "\n## 性能图表\n\n", + f"![课程版指令数图表]({CHART_FILE.name})\n\n", + "### Mermaid 图表\n\n", + "```mermaid\n", + "xychart-beta\n", + ' title "各测试用例指令数"\n', + " x-axis [" + ", ".join(f'"{r["name"]}"' for r in results) + "]\n", + ]) + max_instr = max((r.get("avg_instr_count", r["instr_count"]) for r in results), default=0) + chart_values = [str(round(r.get("avg_instr_count", r["instr_count"]), 2)) for r in results] + lines.extend([ + f' y-axis "指令数" 0 --> {max_instr + 2}\n', + " bar [" + ", ".join(chart_values) + "]\n", + "```\n\n", + "## 用例详情\n\n", + ]) + + for r in results: + lines.extend([ + f"### {r['name']}\n\n", + f"- 类别: {r['category']}\n", + f"- 描述: {r['description']}\n", + f"- 预期输出 ({r['expected_type']}): {r['expected']}\n", + f"- TinyFive 输出: {r['actual']}\n", + f"- 输出是否匹配: {r['matched']}\n", + f"- TinyFive 初始寄存器: {r['initial_registers']}\n", + f"- 模拟后端: {r['backend']}\n", + f"- 编译耗时(s): {r['compile_time_sec']:.4f}\n", + f"- 模拟耗时(s): {r['simulation_time_sec']:.4f}\n", + f"- 总耗时(s): {r['total_time_sec']:.4f}\n", + f"- 汇编文件: {r['asm']}\n", + ]) + if benchmark_mode: + lines.extend([ + f"- Benchmark 重复次数: {r['benchmark_runs']}\n", + f"- 平均指令数: {r['avg_instr_count']:.2f}\n", + f"- 95% 置信区间: ±{r['ci95_instr_count']:.2f}\n", + f"- 最小指令数: {r['min_instr_count']}\n", + f"- 最大指令数: {r['max_instr_count']}\n", + f"- 基线指令数: {r['baseline_instr_count']:.2f}\n", + f"- 性能变化率: {r['delta_pct']:.2f}%\n", + f"- 性能退化阈值: {r['threshold_pct']:.2f}%\n", + f"- 是否性能退化: {r['regressed']}\n", + ]) + else: + lines.append(f"- 指令数: {r['instr_count']}\n") + lines.append("\n") + + return "".join(lines) + + +def write_html_report_cn(results, passed, failed): + try: + from jinja2 import Template + except ImportError: + return None + + pass_rate = 0.0 if not results else passed / len(results) * 100.0 + template = Template(""" + + + + ScratchV 测试报告 + + + +

ScratchV DSL 编译器性能测试报告

+
+

用例总数：{{ total }}，通过：{{ passed }}，失败：{{ failed }}，通过率：{{ "%.1f"|format(pass_rate) }}%

+

测试目录：{{ test_dir }}，性能退化阈值：{{ threshold }}%，单次模拟超时：{{ timeout }}s

+
+ 课程版指令数图表 + + + + + + + + + + + {% for r in results %} + + + + + + + + + + + + + + + {% endfor %} + +
用例类别状态模拟后端平均指令数95% 置信区间编译耗时(s)模拟耗时(s)总耗时(s)变化率(%)是否退化描述
{{ r.name }}{{ r.category }}{{ r.status }}{{ r.backend }}{{ "%.2f"|format(r.avg_instr_count) }}±{{ "%.2f"|format(r.ci95_instr_count) }}{{ "%.4f"|format(r.compile_time_sec) }}{{ "%.4f"|format(r.simulation_time_sec) }}{{ "%.4f"|format(r.total_time_sec) }}{{ "%.2f"|format(r.delta_pct) }}{{ r.regressed }}{{ r.description }}
+ + +""") + HTML_REPORT_FILE.write_text( + template.render( + total=len(results), + passed=passed, + failed=failed, + pass_rate=pass_rate, + test_dir=str(TEST_DIR), + threshold=REGRESSION_THRESHOLD_PCT, + timeout=SIMULATION_TIMEOUT_SEC, + chart_name=CHART_FILE.name, + results=results, + ), + encoding="utf-8", + ) + return HTML_REPORT_FILE + + +def generate_unified_report_text_cn( + results, + passed, + failed, + regression_threshold_pct=REGRESSION_THRESHOLD_PCT, + selection_category=None, + selection_filter=None, + include_chart=False, +): + mode = results[0]["mode"] if results else "normal" + pass_rate = 0.0 if not results else passed / len(results) * 100.0 + lines = [ + "# ScratchV DSL 编译器性能测试报告\n\n", + "## 测试概览\n\n", + f"- Schema 版本: 1\n", + f"- 运行模式: {mode}\n", + f"- 类别筛选: {_report_value(selection_category)}\n", + f"- 名称筛选: {_report_value(selection_filter)}\n", + f"- 生成时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n", + f"- 用例总数: {len(results)}\n", + f"- 通过数量: {passed}\n", + f"- 失败数量: {failed}\n", + f"- 通过率: {pass_rate:.1f}%\n", + f"- 测试目录: `{TEST_DIR}`\n", + f"- 汇编输出目录: `{BUILD_DIR}`\n", + f"- 性能基线文件: `{BASELINE_FILE}`\n", + f"- 性能退化阈值: {regression_threshold_pct:.2f}%\n", + f"- 单次编译超时: {COMPILE_TIMEOUT_SEC:.0f}s\n", + f"- 单次模拟超时: {SIMULATION_TIMEOUT_SEC:.0f}s\n\n", + "## 测试结果\n\n", + "| 用例 | 类别 | 状态 | 编译返回码 | 编译日志 | 模拟后端 | 指令数 | Benchmark 次数 | Benchmark 停止原因 | 平均指令数 | 95% 置信区间 | 最小 | 最大 | 编译耗时(s) | 模拟耗时(s) | 总耗时(s) | 基线 | 变化量 | 变化率(%) | 退化阈值(%) | 是否退化 | 预期输出 | TinyFive 输出 | 输出匹配 | 汇编文件 |\n", + "|---|---|---|---:|---|---|---:|---:|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|---|---|---|---|\n", + ] + + for result in results: + lines.append( + f"| {result['name']} | {result['category']} | {result['status']} | " + f"{result['compile_returncode']} | {_report_value(result['compile_log'])} | " + f"{result['backend']} | {result['instr_count']} | " + f"{_report_value(result['benchmark_runs'])} | " + f"{_report_value(result['benchmark_stopped_reason'])} | " + f"{_report_value(result['avg_instr_count'], 2)} | " + f"{_report_value(result['ci95_instr_count'], 2, '±')} | " + f"{_report_value(result['min_instr_count'])} | " + f"{_report_value(result['max_instr_count'])} | " + f"{result['compile_time_sec']:.4f} | {result['simulation_time_sec']:.4f} | " + f"{result['total_time_sec']:.4f} | " + f"{_report_value(result['baseline_instr_count'], 2)} | " + f"{_report_value(result['delta'], 2)} | " + f"{_report_value(result['delta_pct'], 2)} | " + f"{_report_value(result['threshold_pct'], 2)} | " + f"{_report_value(result['regressed'])} | {_markdown_cell(result['expected'])} | " + f"{_markdown_cell(result['actual'])} | {result['matched']} | {result['asm']} |\n" + ) + + if include_chart: + lines.extend([ + "\n## 性能图表\n\n", + f"![课程版指令数图表]({CHART_FILE.name})\n\n", + ]) + lines.append("\n## 用例详情\n\n") + for result in results: + lines.extend([ + f"### {result['name']}\n\n", + f"- 类别: {result['category']}\n", + f"- 描述: {result['description']}\n", + f"- 预期输出 ({result['expected_type']}): {result['expected']}\n", + f"- TinyFive 输出: {result['actual']}\n", + f"- 输出是否匹配: {result['matched']}\n", + f"- TinyFive 初始寄存器: {result['initial_registers']}\n", + f"- 模拟后端: {result['backend']}\n", + f"- 指令数: {result['instr_count']}\n", + f"- 编译返回码: {result['compile_returncode']}\n", + f"- 编译是否超时: {result['compile_timed_out']}\n", + f"- 编译错误摘要: {_report_value(result['compile_error'])}\n", + f"- 编译失败日志: {_report_value(result['compile_log'])}\n", + f"- Benchmark 重复次数: {_report_value(result['benchmark_runs'])}\n", + f"- Benchmark 停止原因: {_report_value(result['benchmark_stopped_reason'])}\n", + f"- 平均指令数: {_report_value(result['avg_instr_count'], 2)}\n", + f"- 95% 置信区间: {_report_value(result['ci95_instr_count'], 2, '±')}\n", + f"- 最小指令数: {_report_value(result['min_instr_count'])}\n", + f"- 最大指令数: {_report_value(result['max_instr_count'])}\n", + f"- 编译耗时(s): {result['compile_time_sec']:.4f}\n", + f"- 模拟耗时(s): {result['simulation_time_sec']:.4f}\n", + f"- 总耗时(s): {result['total_time_sec']:.4f}\n", + f"- 基线指令数: {_report_value(result['baseline_instr_count'], 2)}\n", + f"- 性能变化量: {_report_value(result['delta'], 2)}\n", + f"- 性能变化率(%): {_report_value(result['delta_pct'], 2)}\n", + f"- 性能退化阈值(%): {_report_value(result['threshold_pct'], 2)}\n", + f"- 是否性能退化: {_report_value(result['regressed'])}\n", + f"- 汇编文件: {result['asm']}\n\n", + ]) + return "".join(lines) + + +def write_unified_html_report_cn( + results, + passed, + failed, + regression_threshold_pct=REGRESSION_THRESHOLD_PCT, + selection_category=None, + selection_filter=None, +): + try: + from jinja2 import Template + except ImportError: + return None + + template = Template(""" + + + + ScratchV 测试报告 + + + +

ScratchV DSL 编译器性能测试报告

+
+

Schema 版本：1，运行模式：{{ mode }}

+

类别筛选：{{ fmt(selection_category) }}，名称筛选：{{ fmt(selection_filter) }}

+

用例总数：{{ total }}，通过：{{ passed }}，失败：{{ failed }}

+

性能退化阈值：{{ fmt(regression_threshold_pct, 2) }}%

+
+ 课程版指令数图表 +
+ + + + + + + + {% for r in results %} + + + + + + + + + + + {% endfor %} +
用例类别状态编译返回码编译日志模拟后端指令数Benchmark 次数Benchmark 停止原因平均指令数95% 置信区间最小最大编译耗时(s)模拟耗时(s)总耗时(s)基线变化量变化率(%)退化阈值(%)是否退化预期输出TinyFive 输出输出匹配汇编文件
{{ r.name }}{{ r.category }}{{ r.status }}{{ r.compile_returncode }}{{ fmt(r.compile_log) }}{{ r.backend }}{{ r.instr_count }}{{ fmt(r.benchmark_runs) }}{{ fmt(r.benchmark_stopped_reason) }}{{ fmt(r.avg_instr_count, 2) }}{{ fmt(r.ci95_instr_count, 2, '±') }}{{ fmt(r.min_instr_count) }}{{ fmt(r.max_instr_count) }}{{ fmt(r.compile_time_sec, 4) }}{{ fmt(r.simulation_time_sec, 4) }}{{ fmt(r.total_time_sec, 4) }}{{ fmt(r.baseline_instr_count, 2) }}{{ fmt(r.delta, 2) }}{{ fmt(r.delta_pct, 2) }}{{ fmt(r.threshold_pct, 2) }}{{ fmt(r.regressed) }}{{ r.expected }}{{ r.actual }}{{ r.matched }}{{ r.asm }}
+ + +""") + mode = results[0]["mode"] if results else "normal" + HTML_REPORT_FILE.write_text( + template.render( + mode=mode, + total=len(results), + passed=passed, + failed=failed, + regression_threshold_pct=regression_threshold_pct, + selection_category=selection_category, + selection_filter=selection_filter, + chart_name=CHART_FILE.name, + results=results, + fmt=_report_value, + ), + encoding="utf-8", + ) + return HTML_REPORT_FILE + + +def write_json_report( + results, + passed, + failed, + regression_threshold_pct=REGRESSION_THRESHOLD_PCT, + selection_category=None, + selection_filter=None, + full_report=False, +): + mode = results[0]["mode"] if results else "normal" + payload = { + "schema_version": 1, + "mode": mode, + "report_level": "full" if full_report else "light", + "regression_threshold_pct": regression_threshold_pct, + "selection": { + "category": selection_category, + "filter": selection_filter, + }, + "generated_at": datetime.now().isoformat(timespec="seconds"), + "summary": { + "total": len(results), + "passed": passed, + "failed": failed, + }, + "results": results, + } + JSON_REPORT_FILE.write_text( + json.dumps(payload, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + return JSON_REPORT_FILE + + +def write_report( + results, + passed, + failed, + regression_threshold_pct=REGRESSION_THRESHOLD_PCT, + selection_category=None, + selection_filter=None, + full_report=False, +): + REPORT_DIR.mkdir(exist_ok=True) + chart_path = write_chart(results) if full_report else None + REPORT_FILE.write_text( + generate_unified_report_text_cn( + results, + passed, + failed, + regression_threshold_pct=regression_threshold_pct, + selection_category=selection_category, + selection_filter=selection_filter, + include_chart=chart_path is not None, + ), + encoding="utf-8", + ) + html_path = ( + write_unified_html_report_cn( + results, + passed, + failed, + regression_threshold_pct=regression_threshold_pct, + selection_category=selection_category, + selection_filter=selection_filter, + ) + if full_report else None + ) + json_path = write_json_report( + results, + passed, + failed, + regression_threshold_pct=regression_threshold_pct, + selection_category=selection_category, + selection_filter=selection_filter, + full_report=full_report, + ) + print(f"\nMarkdown report written to {REPORT_FILE}") + print(f"JSON report written to {json_path}") + if full_report: + if html_path: + print(f"HTML report written to {html_path}") + else: + print("HTML report skipped: jinja2 is not installed") + if chart_path: + print(f"Chart written to {chart_path}") + else: + print("Chart skipped: matplotlib is not installed") + + +def non_negative_float(value): + parsed = float(value) + if parsed < 0: + raise argparse.ArgumentTypeError("must be greater than or equal to 0") + return parsed + + +def select_dsl_files(test_dir, category=None, name_filter=None): + dsl_files = sorted(test_dir.rglob("*.dsl"), key=lambda path: str(path).lower()) + if category: + expected_category = category.lower() + dsl_files = [ + path for path in dsl_files + if path.parent.name.lower() == expected_category + ] + if name_filter: + expected_name = name_filter.lower() + dsl_files = [ + path for path in dsl_files + if expected_name in path.stem.lower() + ] + return dsl_files + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description="Run ScratchV DSL benchmark suite.") + parser.add_argument("--benchmark", type=int, default=0, metavar="N", + help="Run each case N times and report average instruction count.") + parser.add_argument("--update-baseline", action="store_true", + help="Write current benchmark averages to the baseline file.") + parser.add_argument("--full-report", action="store_true", + help="Also generate the optional HTML report and PNG chart.") + parser.add_argument( + "--regression-threshold", + type=non_negative_float, + default=REGRESSION_THRESHOLD_PCT, + metavar="PERCENT", + help="Mark instruction-count increases above this percentage as regressions (default: 5).", + ) + parser.add_argument( + "--category", + help="Only run cases in this test category (for example: activation or tensor).", + ) + parser.add_argument( + "--filter", + dest="name_filter", + help="Only run cases whose file name contains this text.", + ) + return parser.parse_args(argv) + + +def main(argv=None): + args = parse_args(argv) + BUILD_DIR.mkdir(exist_ok=True) + baseline = load_baseline() if args.benchmark else {} + + all_dsl_files = list(TEST_DIR.rglob("*.dsl")) + + if not all_dsl_files: + print("No DSL test files found.") + return 2 + + dsl_files = select_dsl_files( + TEST_DIR, + category=args.category, + name_filter=args.name_filter, + ) + if not dsl_files: + print("No DSL test cases matched the selected filters.") + return 2 + + passed = 0 + failed = 0 + results = [] + + print("Running DSL compiler tests...") + print("=" * 50) + + for dsl_file in dsl_files: + print(f"\n[TEST] {dsl_file}") + + case_start = time.perf_counter() + meta = load_metadata(dsl_file) + compile_start = time.perf_counter() + metadata_error = meta.get("_metadata_error") + if metadata_error: + output_file = BUILD_DIR / (dsl_file.stem + ".s") + result = subprocess.CompletedProcess( + args=[], + returncode=2, + stdout="", + stderr=f"metadata error: {metadata_error}", + ) + else: + result, output_file = run_compile(dsl_file) + compile_time_sec = time.perf_counter() - compile_start + compile_log = None + compile_error = None + if result.returncode != 0: + compile_log = write_compile_failure_log( + dsl_file, + result, + output_file, + compile_time_sec, + ) + compile_error = _last_nonempty_line(result.stderr or result.stdout) + initial_registers = infer_initial_registers( + (result.stdout or "") + "\n" + (result.stderr or ""), + meta.get("inputs", {}), + ) + simulation_start = time.perf_counter() + sim_result = run_simulation(output_file, initial_registers) if result.returncode == 0 else { + "success": False, + "instr_count": 0, + "return_value": None, + "backend": "none", + "error": (result.stderr or result.stdout or "compile failed").strip(), + } + simulation_time_sec = time.perf_counter() - simulation_start + expected_value = meta.get("expected_return") + actual_value = sim_result.get("return_value") + matched = bool(sim_result.get("success")) and values_equal(actual_value, expected_value) + benchmark_counts = [] + benchmark_summary = { + "runs": None, + "avg_instr_count": None, + "min_instr_count": None, + "max_instr_count": None, + "ci95_instr_count": None, + } + regression = { + "baseline_instr_count": None, + "delta": None, + "delta_pct": None, + "threshold_pct": None, + "regressed": None, + } + benchmark_stopped_reason = None + + if args.benchmark > 0: + regression["threshold_pct"] = args.regression_threshold + if result.returncode != 0 or not output_file.exists(): + benchmark_stopped_reason = "benchmark skipped: compile failed" + benchmark_summary = summarize_benchmark_runs([]) + elif sim_result.get("backend") == "timeout": + benchmark_stopped_reason = "benchmark skipped: initial simulation timeout" + benchmark_summary = summarize_benchmark_runs([]) + elif not sim_result.get("success"): + benchmark_stopped_reason = "benchmark skipped: initial simulation failed" + benchmark_summary = summarize_benchmark_runs([]) + else: + for run_index in range(1, args.benchmark + 1): + benchmark_result = run_simulation(output_file, initial_registers) + if benchmark_result.get("backend") == "timeout": + benchmark_stopped_reason = ( + f"benchmark stopped: timeout on run {run_index}" + ) + break + if not benchmark_result.get("success"): + benchmark_stopped_reason = ( + f"benchmark stopped: simulation failed on run {run_index}" + ) + break + benchmark_counts.append(benchmark_result.get("instr_count", 0)) + benchmark_summary = summarize_benchmark_runs(benchmark_counts) + + if benchmark_summary["avg_instr_count"] is not None and benchmark_stopped_reason is None: + regression["regressed"] = False + + baseline_entry = baseline.get(dsl_file.stem) + if ( + baseline_entry + and benchmark_summary["avg_instr_count"] is not None + and benchmark_stopped_reason is None + ): + regression = detect_regression( + avg_instr_count=benchmark_summary["avg_instr_count"], + baseline_instr_count=baseline_entry.get("avg_instr_count", 0.0), + threshold_pct=args.regression_threshold, + ) + total_time_sec = time.perf_counter() - case_start + + ok = ( + result.returncode == 0 + and output_file.exists() + and sim_result["success"] + and matched + and regression["regressed"] is not True + and benchmark_stopped_reason is None + ) + + if ok: + print("PASS") + passed += 1 + status = "PASS" + else: + print("FAIL") + failed += 1 + status = "FAIL" + if sim_result.get("error"): + print(sim_result["error"]) + else: + print( + "output mismatch: " + f"expected={expected_value}, " + f"tinyfive={actual_value}" + ) + + results.append({ + "mode": "benchmark" if args.benchmark > 0 else "normal", + "name": dsl_file.stem, + "category": dsl_file.parent.name, + "path": str(dsl_file), + "status": status, + "description": meta.get("description", ""), + "expected_type": meta.get("expected_output_type", "scalar"), + "output_dtype": meta.get("output_dtype"), + "output_shape": meta.get("output_shape"), + "expected": expected_value, + "actual": actual_value, + "matched": matched, + "initial_registers": initial_registers, + "backend": sim_result.get("backend", "none"), + "instr_count": sim_result.get("instr_count", 0), + "compile_returncode": result.returncode, + "compile_timed_out": result.returncode == 124, + "compile_error": compile_error, + "compile_log": str(compile_log) if compile_log else None, + "compile_time_sec": compile_time_sec, + "simulation_time_sec": simulation_time_sec, + "total_time_sec": total_time_sec, + "benchmark_runs": benchmark_summary["runs"], + "benchmark_stopped_reason": benchmark_stopped_reason, + "avg_instr_count": benchmark_summary["avg_instr_count"], + "min_instr_count": benchmark_summary["min_instr_count"], + "max_instr_count": benchmark_summary["max_instr_count"], + "ci95_instr_count": benchmark_summary["ci95_instr_count"], + "baseline_instr_count": regression["baseline_instr_count"], + "delta": regression["delta"], + "delta_pct": regression["delta_pct"], + "threshold_pct": regression["threshold_pct"], + "regressed": regression["regressed"], + "asm": str(output_file), + }) + + print("\n" + "=" * 50) + print(f"Total: {len(dsl_files)}") + print(f"Passed: {passed}") + print(f"Failed: {failed}") + + if args.benchmark and args.update_baseline: + save_baseline( + results, + preserve_existing=bool(args.category or args.name_filter), + ) + print(f"Baseline written to {BASELINE_FILE}") + + write_report( + results, + passed, + failed, + regression_threshold_pct=args.regression_threshold, + selection_category=args.category, + selection_filter=args.name_filter, + full_report=args.full_report, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ScratchV-topic06-deliverable/setup.py b/ScratchV-topic06-deliverable/setup.py new file mode 100644 index 0000000..6068493 --- /dev/null +++ b/ScratchV-topic06-deliverable/setup.py @@ -0,0 +1,3 @@ +from setuptools import setup + +setup() diff --git a/ScratchV-topic06-deliverable/tests_main/activation/add_relu_relu.dsl b/ScratchV-topic06-deliverable/tests_main/activation/add_relu_relu.dsl new file mode 100644 index 0000000..52c1325 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/activation/add_relu_relu.dsl @@ -0,0 +1,5 @@ +# Add followed by two ReLU stages +x = add(input, bias) +y = relu(x) +result = relu(y) +return result diff --git a/ScratchV-topic06-deliverable/tests_main/activation/add_relu_relu.meta.json b/ScratchV-topic06-deliverable/tests_main/activation/add_relu_relu.meta.json new file mode 100644 index 0000000..e587d8a --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/activation/add_relu_relu.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Add input and bias, then apply ReLU twice.", + "expected_output_type": "return_value", + "inputs": { + "input": -3, + "bias": 10 + }, + "expected_return": 7 +} diff --git a/ScratchV-topic06-deliverable/tests_main/activation/relu_add.dsl b/ScratchV-topic06-deliverable/tests_main/activation/relu_add.dsl new file mode 100644 index 0000000..7190772 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/activation/relu_add.dsl @@ -0,0 +1,4 @@ +# Add + ReLU activation +x = add(input, bias) +y = relu(x) +return y \ No newline at end of file diff --git a/ScratchV-topic06-deliverable/tests_main/activation/relu_add.meta.json b/ScratchV-topic06-deliverable/tests_main/activation/relu_add.meta.json new file mode 100644 index 0000000..f1b7f84 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/activation/relu_add.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Add input and bias, then apply one ReLU.", + "expected_output_type": "return_value", + "inputs": { + "input": -2, + "bias": 5 + }, + "expected_return": 3 +} diff --git a/ScratchV-topic06-deliverable/tests_main/activation/relu_only.dsl b/ScratchV-topic06-deliverable/tests_main/activation/relu_only.dsl new file mode 100644 index 0000000..404a911 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/activation/relu_only.dsl @@ -0,0 +1,3 @@ +# Single ReLU activation +result = relu(x) +return result diff --git a/ScratchV-topic06-deliverable/tests_main/activation/relu_only.meta.json b/ScratchV-topic06-deliverable/tests_main/activation/relu_only.meta.json new file mode 100644 index 0000000..b08af80 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/activation/relu_only.meta.json @@ -0,0 +1,8 @@ +{ + "description": "Apply ReLU directly to a single input value.", + "expected_output_type": "return_value", + "inputs": { + "x": -5 + }, + "expected_return": 0 +} diff --git a/ScratchV-topic06-deliverable/tests_main/activation/relu_twice.dsl b/ScratchV-topic06-deliverable/tests_main/activation/relu_twice.dsl new file mode 100644 index 0000000..7d2defe --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/activation/relu_twice.dsl @@ -0,0 +1,4 @@ +# Two-stage ReLU activation +x = relu(input) +result = relu(x) +return result diff --git a/ScratchV-topic06-deliverable/tests_main/activation/relu_twice.meta.json b/ScratchV-topic06-deliverable/tests_main/activation/relu_twice.meta.json new file mode 100644 index 0000000..fabbb38 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/activation/relu_twice.meta.json @@ -0,0 +1,8 @@ +{ + "description": "Apply ReLU twice to the same activation path.", + "expected_output_type": "return_value", + "inputs": { + "input": 4 + }, + "expected_return": 4 +} diff --git a/ScratchV-topic06-deliverable/tests_main/branch/if_else.dsl b/ScratchV-topic06-deliverable/tests_main/branch/if_else.dsl new file mode 100644 index 0000000..6541768 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/branch/if_else.dsl @@ -0,0 +1,8 @@ +# Branch takes the else path when flag is zero. +if (flag != 0): +result = add(a, b) +return result +else +result = sub(a, b) +return result +endif diff --git a/ScratchV-topic06-deliverable/tests_main/branch/if_else.meta.json b/ScratchV-topic06-deliverable/tests_main/branch/if_else.meta.json new file mode 100644 index 0000000..fdacf6a --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/branch/if_else.meta.json @@ -0,0 +1,10 @@ +{ + "description": "if/else branch returns subtraction result when flag is zero.", + "expected_output_type": "return_value", + "inputs": { + "flag": 0, + "a": 9, + "b": 4 + }, + "expected_return": 5 +} diff --git a/ScratchV-topic06-deliverable/tests_main/branch/if_relu.dsl b/ScratchV-topic06-deliverable/tests_main/branch/if_relu.dsl new file mode 100644 index 0000000..1346060 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/branch/if_relu.dsl @@ -0,0 +1,8 @@ +# Branch selects whether to apply ReLU after an add. +sum = add(a, b) +if (use_relu != 0): +result = relu(sum) +return result +else +return sum +endif diff --git a/ScratchV-topic06-deliverable/tests_main/branch/if_relu.meta.json b/ScratchV-topic06-deliverable/tests_main/branch/if_relu.meta.json new file mode 100644 index 0000000..1925fb5 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/branch/if_relu.meta.json @@ -0,0 +1,10 @@ +{ + "description": "if/else branch combined with add and relu.", + "expected_output_type": "return_value", + "inputs": { + "use_relu": 1, + "a": -8, + "b": 3 + }, + "expected_return": 0 +} diff --git a/ScratchV-topic06-deliverable/tests_main/branch/if_then.dsl b/ScratchV-topic06-deliverable/tests_main/branch/if_then.dsl new file mode 100644 index 0000000..c305c3d --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/branch/if_then.dsl @@ -0,0 +1,8 @@ +# Branch takes the then path when flag is non-zero. +if (flag != 0): +result = add(a, b) +return result +else +result = sub(a, b) +return result +endif diff --git a/ScratchV-topic06-deliverable/tests_main/branch/if_then.meta.json b/ScratchV-topic06-deliverable/tests_main/branch/if_then.meta.json new file mode 100644 index 0000000..04748d1 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/branch/if_then.meta.json @@ -0,0 +1,10 @@ +{ + "description": "if/else branch returns add result when flag is non-zero.", + "expected_output_type": "return_value", + "inputs": { + "flag": 1, + "a": 9, + "b": 4 + }, + "expected_return": 13 +} diff --git a/ScratchV-topic06-deliverable/tests_main/elementwise/add_chain.dsl b/ScratchV-topic06-deliverable/tests_main/elementwise/add_chain.dsl new file mode 100644 index 0000000..d3917c9 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/elementwise/add_chain.dsl @@ -0,0 +1,4 @@ +# Chain of two vector adds +x = add(a, b) +y = add(x, c) +return y \ No newline at end of file diff --git a/ScratchV-topic06-deliverable/tests_main/elementwise/add_chain.meta.json b/ScratchV-topic06-deliverable/tests_main/elementwise/add_chain.meta.json new file mode 100644 index 0000000..b07adaf --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/elementwise/add_chain.meta.json @@ -0,0 +1,10 @@ +{ + "description": "Add a and b, then add c to the intermediate result.", + "expected_output_type": "return_value", + "inputs": { + "a": 2, + "b": 3, + "c": 4 + }, + "expected_return": 9 +} diff --git a/ScratchV-topic06-deliverable/tests_main/elementwise/add_chain_3.dsl b/ScratchV-topic06-deliverable/tests_main/elementwise/add_chain_3.dsl new file mode 100644 index 0000000..aeca4df --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/elementwise/add_chain_3.dsl @@ -0,0 +1,5 @@ +# Chain of three add operations +x = add(a, b) +y = add(x, c) +z = add(y, d) +return z diff --git a/ScratchV-topic06-deliverable/tests_main/elementwise/add_chain_3.meta.json b/ScratchV-topic06-deliverable/tests_main/elementwise/add_chain_3.meta.json new file mode 100644 index 0000000..368b3aa --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/elementwise/add_chain_3.meta.json @@ -0,0 +1,11 @@ +{ + "description": "Chain three add operations across four symbolic inputs.", + "expected_output_type": "return_value", + "inputs": { + "a": 2, + "b": 3, + "c": 4, + "d": 5 + }, + "expected_return": 14 +} diff --git a/ScratchV-topic06-deliverable/tests_main/elementwise/add_fan_in_4.dsl b/ScratchV-topic06-deliverable/tests_main/elementwise/add_fan_in_4.dsl new file mode 100644 index 0000000..0e4449e --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/elementwise/add_fan_in_4.dsl @@ -0,0 +1,5 @@ +# Fan-in add over four inputs +x = add(a, b) +y = add(c, d) +result = add(x, y) +return result diff --git a/ScratchV-topic06-deliverable/tests_main/elementwise/add_fan_in_4.meta.json b/ScratchV-topic06-deliverable/tests_main/elementwise/add_fan_in_4.meta.json new file mode 100644 index 0000000..3eb1f4a --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/elementwise/add_fan_in_4.meta.json @@ -0,0 +1,11 @@ +{ + "description": "Compute two independent adds and then merge them with a final add.", + "expected_output_type": "return_value", + "inputs": { + "a": 1, + "b": 2, + "c": 3, + "d": 4 + }, + "expected_return": 10 +} diff --git a/ScratchV-topic06-deliverable/tests_main/elementwise/add_reuse.dsl b/ScratchV-topic06-deliverable/tests_main/elementwise/add_reuse.dsl new file mode 100644 index 0000000..4618948 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/elementwise/add_reuse.dsl @@ -0,0 +1,4 @@ +# Reuse intermediate add result +x = add(a, b) +result = add(x, x) +return result diff --git a/ScratchV-topic06-deliverable/tests_main/elementwise/add_reuse.meta.json b/ScratchV-topic06-deliverable/tests_main/elementwise/add_reuse.meta.json new file mode 100644 index 0000000..2d2c396 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/elementwise/add_reuse.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Reuse the same intermediate add result on both operands of a second add.", + "expected_output_type": "return_value", + "inputs": { + "a": 2, + "b": 3 + }, + "expected_return": 10 +} diff --git a/ScratchV-topic06-deliverable/tests_main/elementwise/vector_add.dsl b/ScratchV-topic06-deliverable/tests_main/elementwise/vector_add.dsl new file mode 100644 index 0000000..36ac648 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/elementwise/vector_add.dsl @@ -0,0 +1,3 @@ +# Vector add +result = add(a, b) +return result \ No newline at end of file diff --git a/ScratchV-topic06-deliverable/tests_main/elementwise/vector_add.meta.json b/ScratchV-topic06-deliverable/tests_main/elementwise/vector_add.meta.json new file mode 100644 index 0000000..7347fff --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/elementwise/vector_add.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Single add over two symbolic vector inputs.", + "expected_output_type": "return_value", + "inputs": { + "a": 2, + "b": 3 + }, + "expected_return": 5 +} diff --git a/ScratchV-topic06-deliverable/tests_main/loop/loop_add_4.dsl b/ScratchV-topic06-deliverable/tests_main/loop/loop_add_4.dsl new file mode 100644 index 0000000..381f1c0 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/loop/loop_add_4.dsl @@ -0,0 +1,5 @@ +# Loop with repeated add body over 4 iterations +for i = 0, 4 +x = add(a, b) +endfor +return x diff --git a/ScratchV-topic06-deliverable/tests_main/loop/loop_add_4.meta.json b/ScratchV-topic06-deliverable/tests_main/loop/loop_add_4.meta.json new file mode 100644 index 0000000..fc31fae --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/loop/loop_add_4.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Run a four-iteration loop whose body computes one add; final returned value is the last loop-body result.", + "expected_output_type": "return_value", + "inputs": { + "a": 2, + "b": 3 + }, + "expected_return": 5 +} diff --git a/ScratchV-topic06-deliverable/tests_main/loop/loop_add_chain_4.dsl b/ScratchV-topic06-deliverable/tests_main/loop/loop_add_chain_4.dsl new file mode 100644 index 0000000..eca595b --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/loop/loop_add_chain_4.dsl @@ -0,0 +1,6 @@ +# Loop with chained add inside the body +for i = 0, 4 +x = add(a, b) +y = add(x, c) +endfor +return y diff --git a/ScratchV-topic06-deliverable/tests_main/loop/loop_add_chain_4.meta.json b/ScratchV-topic06-deliverable/tests_main/loop/loop_add_chain_4.meta.json new file mode 100644 index 0000000..154d91d --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/loop/loop_add_chain_4.meta.json @@ -0,0 +1,10 @@ +{ + "description": "Run a four-iteration loop whose body computes two chained adds; final returned value is the last loop-body result.", + "expected_output_type": "return_value", + "inputs": { + "a": 2, + "b": 3, + "c": 4 + }, + "expected_return": 9 +} diff --git a/ScratchV-topic06-deliverable/tests_main/loop/loop_relu_add_4.dsl b/ScratchV-topic06-deliverable/tests_main/loop/loop_relu_add_4.dsl new file mode 100644 index 0000000..44f3080 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/loop/loop_relu_add_4.dsl @@ -0,0 +1,6 @@ +# Loop with add followed by ReLU in the body +for i = 0, 4 +x = add(input, bias) +y = relu(x) +endfor +return y diff --git a/ScratchV-topic06-deliverable/tests_main/loop/loop_relu_add_4.meta.json b/ScratchV-topic06-deliverable/tests_main/loop/loop_relu_add_4.meta.json new file mode 100644 index 0000000..8baca28 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/loop/loop_relu_add_4.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Run a four-iteration loop whose body computes add followed by ReLU; final returned value is the last loop-body result.", + "expected_output_type": "return_value", + "inputs": { + "input": -4, + "bias": 6 + }, + "expected_return": 2 +} diff --git a/ScratchV-topic06-deliverable/tests_main/reduction/dot_4.dsl b/ScratchV-topic06-deliverable/tests_main/reduction/dot_4.dsl new file mode 100644 index 0000000..2118c72 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/reduction/dot_4.dsl @@ -0,0 +1,3 @@ +# Dot product of two 4-element vectors +result = dot(a, b, len:4) +return result \ No newline at end of file diff --git a/ScratchV-topic06-deliverable/tests_main/reduction/dot_4.meta.json b/ScratchV-topic06-deliverable/tests_main/reduction/dot_4.meta.json new file mode 100644 index 0000000..748f260 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/reduction/dot_4.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Compute the dot product of two symbolic vectors of length 4.", + "expected_output_type": "return_value", + "inputs": { + "a": [1, 2, 3, 4], + "b": [5, 6, 7, 8] + }, + "expected_return": 70 +} diff --git a/ScratchV-topic06-deliverable/tests_main/reduction/dot_8.dsl b/ScratchV-topic06-deliverable/tests_main/reduction/dot_8.dsl new file mode 100644 index 0000000..e1b71fa --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/reduction/dot_8.dsl @@ -0,0 +1,3 @@ +# Dot product of two 8-element vectors +result = dot(a, b, len:8) +return result diff --git a/ScratchV-topic06-deliverable/tests_main/reduction/dot_8.meta.json b/ScratchV-topic06-deliverable/tests_main/reduction/dot_8.meta.json new file mode 100644 index 0000000..d3172a2 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/reduction/dot_8.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Compute the dot product of two symbolic vectors of length 8.", + "expected_output_type": "return_value", + "inputs": { + "a": [1, 2, 3, 4, 5, 6, 7, 8], + "b": [1, 1, 1, 1, 1, 1, 1, 1] + }, + "expected_return": 36 +} diff --git a/ScratchV-topic06-deliverable/tests_main/reduction/dot_relu_4.dsl b/ScratchV-topic06-deliverable/tests_main/reduction/dot_relu_4.dsl new file mode 100644 index 0000000..279fc19 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/reduction/dot_relu_4.dsl @@ -0,0 +1,4 @@ +# Dot product followed by ReLU +x = dot(a, b, len:4) +result = relu(x) +return result diff --git a/ScratchV-topic06-deliverable/tests_main/reduction/dot_relu_4.meta.json b/ScratchV-topic06-deliverable/tests_main/reduction/dot_relu_4.meta.json new file mode 100644 index 0000000..24910bc --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/reduction/dot_relu_4.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Compute a length-4 dot product and pass it through ReLU.", + "expected_output_type": "return_value", + "inputs": { + "a": [1, -2, 3, -4], + "b": [2, 3, 4, 5] + }, + "expected_return": 0 +} diff --git a/ScratchV-topic06-deliverable/tests_main/reduction/dot_relu_8.dsl b/ScratchV-topic06-deliverable/tests_main/reduction/dot_relu_8.dsl new file mode 100644 index 0000000..d2b906d --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/reduction/dot_relu_8.dsl @@ -0,0 +1,4 @@ +# Dot product of length 8 followed by ReLU +x = dot(a, b, len:8) +result = relu(x) +return result diff --git a/ScratchV-topic06-deliverable/tests_main/reduction/dot_relu_8.meta.json b/ScratchV-topic06-deliverable/tests_main/reduction/dot_relu_8.meta.json new file mode 100644 index 0000000..776fab5 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/reduction/dot_relu_8.meta.json @@ -0,0 +1,9 @@ +{ + "description": "Compute a length-8 dot product and pass it through ReLU.", + "expected_output_type": "return_value", + "inputs": { + "a": [1, 0, 1, 0, 1, 0, 1, 0], + "b": [2, 2, 2, 2, 2, 2, 2, 2] + }, + "expected_return": 8 +} diff --git a/ScratchV-topic06-deliverable/tests_main/tensor/matmul_2x2.dsl b/ScratchV-topic06-deliverable/tests_main/tensor/matmul_2x2.dsl new file mode 100644 index 0000000..f0a80d9 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/tensor/matmul_2x2.dsl @@ -0,0 +1,3 @@ +# Matrix multiplication: 2x2 * 2x2 +result = matmul(A, B, m:2, n:2, k:2) +return result \ No newline at end of file diff --git a/ScratchV-topic06-deliverable/tests_main/tensor/matmul_2x2.meta.json b/ScratchV-topic06-deliverable/tests_main/tensor/matmul_2x2.meta.json new file mode 100644 index 0000000..31a0848 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/tensor/matmul_2x2.meta.json @@ -0,0 +1,11 @@ +{ + "description": "Compute a symbolic 2x2 by 2x2 matrix multiplication.", + "expected_output_type": "tensor", + "output_dtype": "int32", + "output_shape": [2, 2], + "inputs": { + "A": [[1, 2], [3, 4]], + "B": [[5, 6], [7, 8]] + }, + "expected_return": [[19, 22], [43, 50]] +} diff --git a/ScratchV-topic06-deliverable/tests_main/tensor/matmul_4x4.dsl b/ScratchV-topic06-deliverable/tests_main/tensor/matmul_4x4.dsl new file mode 100644 index 0000000..ab5a865 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/tensor/matmul_4x4.dsl @@ -0,0 +1,3 @@ +# Matrix multiplication: 4x4 * 4x4 +result = matmul(A, B, m:4, n:4, k:4) +return result diff --git a/ScratchV-topic06-deliverable/tests_main/tensor/matmul_4x4.meta.json b/ScratchV-topic06-deliverable/tests_main/tensor/matmul_4x4.meta.json new file mode 100644 index 0000000..2adb159 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/tensor/matmul_4x4.meta.json @@ -0,0 +1,11 @@ +{ + "description": "Compute a symbolic 4x4 by 4x4 matrix multiplication.", + "expected_output_type": "tensor", + "output_dtype": "int32", + "output_shape": [4, 4], + "inputs": { + "A": [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]], + "B": [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]] + }, + "expected_return": [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]] +} diff --git a/ScratchV-topic06-deliverable/tests_main/tensor/matmul_add_2x2.dsl b/ScratchV-topic06-deliverable/tests_main/tensor/matmul_add_2x2.dsl new file mode 100644 index 0000000..d41a2cb --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/tensor/matmul_add_2x2.dsl @@ -0,0 +1,4 @@ +# Matrix multiplication followed by add +x = matmul(A, B, m:2, n:2, k:2) +result = add(x, bias) +return result diff --git a/ScratchV-topic06-deliverable/tests_main/tensor/matmul_add_2x2.meta.json b/ScratchV-topic06-deliverable/tests_main/tensor/matmul_add_2x2.meta.json new file mode 100644 index 0000000..b336c80 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/tensor/matmul_add_2x2.meta.json @@ -0,0 +1,12 @@ +{ + "description": "Compute a 2x2 matmul and then add a symbolic bias term.", + "expected_output_type": "tensor", + "output_dtype": "int32", + "output_shape": [2, 2], + "inputs": { + "A": [[1, 2], [3, 4]], + "B": [[5, 6], [7, 8]], + "bias": [[1, 1], [1, 1]] + }, + "expected_return": [[20, 23], [44, 51]] +} diff --git a/ScratchV-topic06-deliverable/tests_main/tensor/matmul_relu_2x2.dsl b/ScratchV-topic06-deliverable/tests_main/tensor/matmul_relu_2x2.dsl new file mode 100644 index 0000000..28c7c61 --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/tensor/matmul_relu_2x2.dsl @@ -0,0 +1,4 @@ +# Matrix multiplication followed by ReLU +x = matmul(A, B, m:2, n:2, k:2) +result = relu(x) +return result diff --git a/ScratchV-topic06-deliverable/tests_main/tensor/matmul_relu_2x2.meta.json b/ScratchV-topic06-deliverable/tests_main/tensor/matmul_relu_2x2.meta.json new file mode 100644 index 0000000..e2efb5e --- /dev/null +++ b/ScratchV-topic06-deliverable/tests_main/tensor/matmul_relu_2x2.meta.json @@ -0,0 +1,11 @@ +{ + "description": "Compute a 2x2 matmul and then apply ReLU to its result.", + "expected_output_type": "tensor", + "output_dtype": "int32", + "output_shape": [2, 2], + "inputs": { + "A": [[-1, 2], [-3, 4]], + "B": [[1, 0], [0, 1]] + }, + "expected_return": [[0, 2], [0, 4]] +} diff --git "a/ScratchV-topic06-deliverable/\350\257\276\351\242\2306\350\256\276\350\256\241\346\226\207\346\241\243.md" "b/ScratchV-topic06-deliverable/\350\257\276\351\242\2306\350\256\276\350\256\241\346\226\207\346\241\243.md" new file mode 100644 index 0000000..058c128 --- /dev/null +++ "b/ScratchV-topic06-deliverable/\350\257\276\351\242\2306\350\256\276\350\256\241\346\226\207\346\241\243.md" @@ -0,0 +1,631 @@ +# ScratchV 课题 06 性能测试套件设计文档 + +> 文档版本：v1.2 +> 编写日期：2026-07-22 +> 更新日期：2026-08-08 +> 交付目录：`ScratchV-topic06-deliverable` +> 核心文件：`run_tests.py`、`tests_main/`、`reports/`、`README.md` +> 功能范围：DSL 用例管理、真实 TinyFive 验证、元数据校验、编译与模拟超时保护、失败现场保存、指令数与耗时统计、Benchmark、性能基线对比、子集测试、统一 Markdown/JSON/HTML 报告、CI 示例 + +--- + +## 一、功能介绍 + +### 1.1 功能概述 + +本测试套件用于验证 ScratchV 编译器在多个 DSL 程序上的编译正确性、模拟结果和性能表现。测试脚本会自动遍历或按条件筛选 `tests_main/` 下的 DSL 用例，调用 ScratchV 编译器生成 RISC-V 汇编，再通过真实 TinyFive 适配层执行汇编，并将返回值与用例元数据中的期望值比较。测试结束后会生成适合人工查看和 CI 解析的报告。 + +当前套件包含 23 个 DSL 测试用例，覆盖以下类别： + +| 类别 | 说明 | +|---|---| +| `activation` | ReLU 等激活函数组合 | +| `elementwise` | 标量/向量加法、链式加法、复用输入 | +| `loop` | `for/endfor` 循环类用例 | +| `branch` | `if/else/endif` 分支类用例 | +| `reduction` | `dot` 等归约类用例 | +| `tensor` | `matmul` 等张量计算类用例 | + +测试输出不仅包含 PASS/FAIL，还包含编译状态、TinyFive 后端状态、期望值与实际值、指令数、编译耗时、模拟耗时、总耗时、Benchmark 平均值、95% 置信区间、性能基线对比和失败日志路径等信息。 + +### 1.2 设计目标 + +- **自动化**：一条命令完成所有用例的编译、模拟、结果比对和报告生成。 +- **真实性**：只使用 `DSL -> ScratchV -> RISC-V -> TinyFive` 路径判断结果，不使用 stub 或测试套件内置解释器替代真实后端。 +- **可扩展**：新增测试只需要添加 `.dsl` 和 `.meta.json` 文件，不需要修改主测试逻辑。 +- **可解释**：报告中展示每个用例的状态、预期输出、实际输出、模拟后端、指令数和耗时。 +- **可对比**：支持保存 Benchmark 基线，并在后续运行中检查性能变化率。 +- **可集成**：普通模式和 Benchmark 模式采用统一 JSON schema，便于 CI 稳定解析。 +- **可选择**：支持按类别和名称运行子集，缩短定位单个模块问题时的等待时间。 +- **避免卡死**：编译阶段设置 30 秒超时，TinyFive 模拟阶段设置 5 秒超时，单个异常用例不会阻塞整套测试。 +- **便于诊断**：编译失败时保存命令、返回码、stdout、stderr 和耗时等现场信息。 +- **轻量运行**：默认报告不依赖绘图库和模板库，需要可视化时再通过 `--full-report` 启用。 + +### 1.3 需求演进与当前范围 + +项目最初按照 W1-W12 实现自动编译、模拟、结果对比、性能统计、报告和 CI。经过实际接入 TinyFive 以及 Mentor Review 后，程序增加了可靠性和工程化能力。新增内容不是另一套测试目标，而是保证原有目标能够在真实编译链路中稳定执行。 + +| 来源 | 当前实现内容 | 解决的问题 | +|---|---|---| +| 原始 W1-W4 | DSL 用例格式、自动调用编译器和模拟器、PASS/FAIL 对比 | 完成基本自动化正确性测试 | +| 原始 W5-W10 | 指令数、耗时、Benchmark、置信区间、基线和退化检测 | 完成性能测量与回归比较 | +| 原始 W11-W12 | GitHub Actions 示例和使用文档 | 支持 CI 和交付使用 | +| S1、A2 | 真实 TinyFive 直通验证，禁用 stub 回退，模拟子进程超时 | 防止模拟结果被替代，并避免模拟卡死 | +| S2、T1 | 编译超时和独立失败日志 | 防止编译卡死并保留崩溃现场 | +| A1 | 标量/张量期望值、dtype/shape 和外部期望文件校验 | 提高测试元数据的表达能力和错误可读性 | +| S3 | Markdown、HTML、JSON 统一字段，未执行数据使用 `null` | 避免普通模式和 Benchmark 模式的报告解析冲突 | +| I1 | 默认轻量报告与 `--full-report` 可选完整报告 | 降低日常测试和 CI 的依赖成本 | +| I2、I3 | 可配置退化阈值、Benchmark 失败短路 | 适应不同检查标准并避免重复等待失败用例 | +| T2 | `--category`、`--filter` 和子集基线合并 | 支持快速定位和增量测试 | + +因此，当前程序的交付边界不仅是“运行 23 个用例并生成性能报告”，还包括真实后端验证、异常隔离、稳定报告接口、失败诊断和子集执行。编译器后端本身的分支、数组及矩阵指令生成不属于测试套件的实现范围；测试套件负责真实暴露并记录这些问题。 + +--- + +## 二、目录结构 + +交付目录当前结构如下： + +```text +ScratchV-topic06-deliverable/ + README.md + 课题6设计文档.md + LICENSE + requirements-topic06.txt + requirements-topic06-full.txt + run_tests.py + setup.py + tests_main/ + activation/ + branch/ + elementwise/ + loop/ + reduction/ + tensor/ + reports/ + failures/ + report.md + report.json + report.html + course_report_instructions.png + benchmark_baseline.json + .github/ + workflows/ + benchmark.yml +``` + +其中： + +- `run_tests.py` 是测试主程序。 +- `tests_main/` 保存 DSL 测试用例。 +- `reports/` 保存测试运行后生成的报告和基线文件。 +- `reports/failures/` 保存编译失败或超时用例的独立诊断日志。 +- `requirements-topic06.txt` 保存基础测试依赖。 +- `requirements-topic06-full.txt` 保存 HTML 和 PNG 报告所需的可选依赖。 +- `.github/workflows/benchmark.yml` 是 GitHub Actions 示例配置。 + +`build/` 目录如果存在，是运行测试时生成的汇编输出目录，不是必须提交的源码内容。 + +--- + +## 三、测试用例格式 + +每个测试用例由两个文件组成： + +```text +tests_main/{category}/{name}.dsl +tests_main/{category}/{name}.meta.json +``` + +示例 DSL： + +```text +result = add(a, b) +return result +``` + +示例元数据： + +```json +{ + "description": "Simple scalar add.", + "expected_output_type": "scalar", + "inputs": { + "a": 2, + "b": 3 + }, + "expected_return": 5 +} +``` + +这里没有采用单独的 `.expected` 和 `.desc` 文件，而是使用 `.meta.json` 统一保存输入、描述和期望输出。这样结构更集中，也方便 Python 脚本直接解析。 + +`expected_output_type` 支持两种值： + +- `scalar`：标量结果，使用数字或布尔值；旧值 `return_value` 会兼容转换为 `scalar`。 +- `tensor`：向量或多维数组，必须同时声明 `output_dtype` 和 `output_shape`。 + +张量示例： + +```json +{ + "description": "2x2 matrix multiplication", + "expected_output_type": "tensor", + "output_dtype": "int32", + "output_shape": [2, 2], + "inputs": { + "A": [[1, 2], [3, 4]], + "B": [[5, 6], [7, 8]] + }, + "expected_return": [[19, 22], [43, 50]] +} +``` + +支持的 `output_dtype` 为 `bool`、`int32`、`int64`、`float32` 和 `float64`。加载元数据时，测试脚本会递归检查数组是否规则、实际形状是否与 `output_shape` 一致，以及每个元素是否符合声明的类型。 + +期望结果较大时，可以不使用内联的 `expected_return`，改为引用当前用例目录下的 JSON 文件： + +```json +{ + "expected_output_type": "tensor", + "output_dtype": "int32", + "output_shape": [2, 2], + "expected_output_file": "matmul_2x2.expected.json" +} +``` + +`expected_return` 和 `expected_output_file` 必须且只能存在一个。外部 JSON 可以直接保存标量或数组，也可以保存包含 `expected_return` 字段的对象。格式、shape 或 dtype 校验失败时，该用例报告 `metadata error` 并判定为 FAIL，不影响后续用例。 + +这一协议解决的是期望值的表达和校验。张量用例要在真实 TinyFive 路径下通过，还需要编译器后端约定输出内存地址并生成完整的数组计算及写回指令；这属于后端输入输出约定，不由元数据格式代替。 + +--- + +## 四、核心实现设计 + +### 4.1 编译阶段 + +函数：`run_compile(dsl_file: Path, timeout: float = 30.0)` + +作用： + +1. 接收一个 `.dsl` 文件路径。 +2. 调用 ScratchV 编译器： + +```powershell +python -m scratchv.main -o --optimize all --dump-ir +``` + +3. 将生成的 RISC-V 汇编写入 `build/{case_name}.s`。 +4. 返回编译进程结果和汇编输出路径。 + +这一阶段使用 `subprocess.run()` 调用编译器，能够捕获 `stdout`、`stderr` 和返回码。调用设置了默认 30 秒的 `timeout`：如果编译器在某个 DSL 上卡死，脚本会终止该编译子进程，返回错误 `compile timeout after 30s`，将当前用例标记为 FAIL，并继续运行后续用例。编译超时使用返回码 `124` 表示，不会进入 TinyFive 模拟阶段。 + +编译返回码非 0 时，`write_compile_failure_log()` 会生成： + +```text +reports/failures/{category}-{case}.compile.log +``` + +日志保存以下现场信息： + +- 失败时间和 DSL 用例路径。 +- 完整编译命令。 +- 返回码和是否触发 30 秒 timeout。 +- 编译耗时、目标汇编路径以及汇编文件是否存在。 +- 完整 stdout 和 stderr，包括 Python traceback。 + +报告中的每条结果固定包含 `compile_returncode`、`compile_timed_out`、`compile_error` 和 `compile_log`。`compile_error` 保存最后一条非空错误信息，方便 CI 快速显示；`compile_log` 指向完整日志。成功用例的错误摘要和日志路径为 `null`。 + +### 4.2 模拟验证阶段 + +函数：`run_simulation(asm_file: Path, timeout: float = 5.0)` + +作用： + +1. 读取编译生成的 `.s` 汇编文件。 +2. 在独立 Python 子进程中调用： + +```python +from scratchv.simulator.tinyfive import verify_assembly +``` + +3. 获取 TinyFive 适配层返回的模拟结果。 +4. 如果模拟超过 5 秒，则终止该子进程，并返回： + +```python +{ + "success": False, + "backend": "timeout", + "error": "simulation timeout after 5s" +} +``` + +TinyFive 执行采用两层保护： + +1. `verify_assembly()` 内部调用 `m.run(instructions=100_000_000)`，最多执行一亿条指令，防止生成的汇编无限运行。 +2. `run_simulation()` 在外部使用独立子进程和 5 秒 timeout。即使 TinyFive 的指令上限执行过慢，或者模拟器内部其他步骤没有及时返回，主测试进程仍能终止该子进程。 + +如果真实 TinyFive 没有安装，`verify_assembly()` 会立即返回： + +```python +{ + "success": False, + "instr_count": 0, + "return_value": None, + "backend": "tinyfive", + "error": "tinyfive not installed" +} +``` + +测试套件不会在 TinyFive 不可用时自动回退到 stub，因此报告中的 `backend: tinyfive` 表示实际调用了 TinyFive。TinyFive 未安装、执行异常或超过 5 秒都会使当前用例判定为 FAIL，但后续用例仍会继续执行。 + +这个设计解决了 branch 分支用例在真实 TinyFive 路径下可能长时间不返回的问题。当前 3 个 branch 用例触发 timeout，说明保护机制生效；它们的汇编跳转问题仍属于编译器后端问题，不属于 A2 保护机制本身。 + +### 4.3 TinyFive 直通验证 + +当前测试的输出正确性已经改为由 TinyFive 直通路径判断。流程是： + +```text +DSL + -> ScratchV 编译器 + -> RISC-V 汇编 + -> TinyFive 模拟执行 + -> 读取 a0/x10 作为 return_value + -> 与 .meta.json 中的 expected_return 对比 +``` + +`run_tests.py` 会根据编译器输出的优化后 IR 推断标量输入对应的初始寄存器，并传给 TinyFive 适配层。TinyFive 执行结束后，`scratchv.simulator.tinyfive.verify_assembly()` 会返回 `return_value`，该值来自 RISC-V 返回寄存器 `a0/x10`。 + +报告中的“TinyFive 输出”和“输出是否匹配”均来自真实编译产物的 TinyFive 执行结果。 + +### 4.4 PASS/FAIL 判断 + +一个用例通过需要同时满足： + +```python +ok = ( + result.returncode == 0 + and output_file.exists() + and sim_result["success"] + and matched + and not regression["regressed"] +) +``` + +也就是说，必须同时满足： + +- 编译成功。 +- 汇编文件生成成功。 +- TinyFive 模拟成功。 +- TinyFive 返回值与期望输出匹配。 +- Benchmark 模式下没有超过性能退化阈值。 +- Benchmark 重复没有因 timeout 或模拟失败提前停止。 + +如果 TinyFive 超时或执行失败，`sim_result["success"]` 为 `False`，该用例直接判定为 FAIL，不会使用其他执行结果替代。 + +### 4.5 指令数统计 + +模拟结果中包含 `instr_count` 字段。测试脚本会把该字段写入报告。 + +普通模式和 Benchmark 模式使用完全相同的报告字段。`instr_count` 始终记录首次 TinyFive 模拟的指令数；Benchmark 模式另外填写重复运行次数、平均值、最小值、最大值和 95% 置信区间。普通模式中的 Benchmark 专属字段统一写为 `null`，不再用 `0` 冒充未执行的统计结果。 + +相关函数： + +- `summarize_benchmark_runs(instr_counts)` +- `detect_regression(avg_instr_count, baseline_instr_count)` + +固定字段包括： + +| 字段 | 普通模式 | Benchmark 模式 | +|---|---|---| +| `mode` | `"normal"` | `"benchmark"` | +| `instr_count` | 单次模拟指令数 | 首次模拟指令数 | +| `compile_returncode` | 编译器返回码 | 编译器返回码 | +| `compile_timed_out` | 是否触发编译 timeout | 是否触发编译 timeout | +| `compile_error` / `compile_log` | 成功时为 `null`，失败时为摘要和日志路径 | 与普通模式相同 | +| `benchmark_runs` | `null` | 实际重复次数 | +| `benchmark_stopped_reason` | `null` | 正常完成时为 `null`，提前停止时记录原因 | +| `avg_instr_count` | `null` | 平均指令数 | +| `min_instr_count` / `max_instr_count` | `null` | 最小值/最大值 | +| `ci95_instr_count` | `null` | 95% 置信区间 | +| `baseline_instr_count` | `null` | 有基线时为基线值，否则为 `null` | +| `delta` / `delta_pct` | `null` | 有基线时为变化量和变化率，否则为 `null` | +| `threshold_pct` / `regressed` | `null` | 退化阈值和判断结果 | + +Markdown 和 HTML 报告固定展示相同列，不再根据模式增删列。CI 推荐读取 `reports/report.json`，其顶层固定包含 `schema_version`、`mode`、`generated_at`、`summary` 和 `results`，当前 schema 版本为 `1`。 + +### 4.6 耗时统计 + +测试脚本使用 `time.perf_counter()` 记录每个用例的耗时。 + +当前记录的字段包括： + +| 字段 | 说明 | +|---|---| +| `compile_time_sec` | 编译 DSL 到 RISC-V 汇编的耗时 | +| `simulation_time_sec` | TinyFive 模拟验证耗时 | +| `total_time_sec` | 单个用例完整处理耗时 | + +这些字段会输出到 Markdown 和 HTML 报告中，便于观察哪些用例编译慢、模拟慢或因为 timeout 导致耗时较长。 + +### 4.7 Benchmark 与性能基线 + +运行： + +```powershell +python run_tests.py --benchmark 3 +``` + +表示每个用例重复模拟 3 次，统计平均指令数和 95% 置信区间。 + +运行： + +```powershell +python run_tests.py --benchmark 3 --update-baseline +``` + +表示将当前 Benchmark 结果保存为性能基线： + +```text +reports/benchmark_baseline.json +``` + +后续再次运行： + +```powershell +python run_tests.py --benchmark 3 +``` + +脚本会读取基线文件，并计算变化率： + +```text +变化率 = (当前平均指令数 - 基线指令数) / 基线指令数 * 100% +``` + +如果变化率超过配置阈值，则认为发生性能退化。默认阈值仍为 5%，可以使用命令行参数覆盖： + +```powershell +python run_tests.py --benchmark 3 --regression-threshold 2 +``` + +上面的命令表示指令数相对基线增加超过 2% 就判定为退化。参数必须大于或等于 0；未指定时使用 `REGRESSION_THRESHOLD_PCT = 5.0`。实际阈值会传给 `detect_regression()`，并写入 Markdown、HTML 和 JSON 报告。这样日常开发、严格发布检查和宽松趋势监控可以使用不同标准。 + +Benchmark 模式会自动处理 timeout： + +1. 首次 TinyFive 模拟已经 timeout 时，直接跳过该用例的全部 Benchmark 重复。 +2. Benchmark 重复过程中一旦发生 timeout 或模拟失败，立即停止剩余次数。 +3. 只有成功运行的 `instr_count` 才进入平均值、最小值、最大值和置信区间计算，失败结果不会作为 `0` 混入统计。 +4. 报告中的 `benchmark_runs` 记录实际成功次数，`benchmark_stopped_reason` 记录跳过或提前停止原因。 +5. 没有成功 Benchmark 结果的用例不会写入性能基线，也不会进行变化率比较。 + +例如 branch 用例首次模拟等待 5 秒后 timeout，`--benchmark 3` 不会再额外等待 15 秒。 + +--- + +## 五、报告生成设计 + +### 5.1 Markdown 报告 + +输出文件： + +```text +reports/report.md +``` + +内容包括： + +- 测试概览。 +- 用例总数、通过数量、失败数量、通过率。 +- 每个用例的状态、模拟后端、指令数、耗时、预期输出、实际输出。 +- Benchmark 模式下的平均指令数、置信区间、基线、变化率和是否退化。 +- Mermaid 指令数图表。 +- 每个用例的详细说明。 + +普通模式和 Benchmark 模式使用相同表头，未产生的 Benchmark 数据显示为 `null`。 + +Markdown 是默认轻量报告，不需要 `jinja2` 或 `matplotlib`。只有使用 `--full-report` 且图表生成成功时，Markdown 才会加入 PNG 图表链接，避免引用未生成或未刷新的图片。 + +### 5.2 JSON 报告 + +输出文件： + +```text +reports/report.json +``` + +JSON 报告是提供给 CI 和其他程序的稳定解析接口。两种运行模式下字段集合保持一致，CI 只需要检查 `mode` 和字段值，不需要维护两套解析器。 + +顶层 `selection` 字段记录本次使用的 `category` 和 `filter`；未筛选时两个值均为 `null`，便于 CI 判断报告覆盖的是全量还是子集测试。 + +### 5.3 HTML 报告 + +输出文件： + +```text +reports/report.html +``` + +HTML 报告主要用于浏览器查看，报告中会使用不同颜色标识 PASS 和 FAIL。它只在指定 `--full-report` 时生成，需要安装可选依赖 `jinja2`。 + +### 5.4 图表 + +输出文件： + +```text +reports/course_report_instructions.png +``` + +图表由 `matplotlib` 生成，展示不同测试用例的指令数对比。它只在指定 `--full-report` 时生成；默认轻量模式不会导入 `matplotlib`，因此 CI 无需安装图表依赖。 + +--- + +## 六、运行方式 + +### 6.1 安装依赖 + +基础测试： + +```powershell +pip install -r requirements-topic06.txt +``` + +完整报告： + +```powershell +pip install -r requirements-topic06-full.txt +``` + +### 6.2 普通测试 + +```powershell +python run_tests.py +``` + +默认只生成 `report.md` 和 `report.json`。生成 HTML 和 PNG 完整报告： + +```powershell +python run_tests.py --full-report +``` + +### 6.3 子集测试 + +按测试目录类别筛选： + +```powershell +python run_tests.py --category activation +``` + +按 DSL 文件名进行大小写不敏感的包含匹配： + +```powershell +python run_tests.py --filter matmul +``` + +两个条件可以组合，组合时必须同时满足： + +```powershell +python run_tests.py --category tensor --filter relu +``` + +筛选结果按路径排序，保证本地和 CI 的执行顺序稳定。没有匹配用例时返回退出码 2，不生成或覆盖报告。子集模式结合 `--update-baseline` 时会加载现有基线并只更新本次成功完成 Benchmark 的用例；全量模式仍会重建整份基线。 + +适合快速查看所有用例的 PASS/FAIL 和基本报告。 + +### 6.4 Benchmark 测试 + +```powershell +python run_tests.py --benchmark 3 +``` + +适合统计平均指令数和置信区间。 + +### 6.5 更新性能基线 + +```powershell +python run_tests.py --benchmark 3 --update-baseline +``` + +适合在修改编译器前或稳定版本上保存基线，供后续性能回归对比使用。 + +--- + +## 七、Mentor Review 对应状态 + +mentor review 中提出的问题分为架构、API、实现和测试覆盖四类。当前文档按真实代码状态整理如下： + +| 编号 | 问题 | 当前处理状态 | 说明 | +|---|---|---|---| +| S1 | 参考解释器与 ScratchV 编译器语义脱节 | 已处理 | 原方案由测试套件单独实现 DSL 求值逻辑，可能绕过真实编译路径，并形成两套需要同步维护的算子语义。现已删除参考解释器及其报告字段；测试结果只使用 `DSL -> ScratchV 编译器 -> RISC-V 汇编 -> TinyFive` 真实路径，并将 TinyFive 的 `return_value` 与 `expected_return` 比较。 | +| S2 | 编译阶段缺 timeout | 已处理 | `run_compile()` 默认设置 30 秒 timeout；超时后返回编译失败结果并继续执行后续用例。 | +| S3 | Benchmark/普通模式报告格式冲突 | 已处理 | Markdown、HTML 和 JSON 使用固定字段；普通模式的 Benchmark 专属字段为 `null`，CI 使用 `report.json` 统一解析。 | +| A1 | `.meta.json` 期望值表达能力有限 | 已处理 | 支持 `scalar`/`tensor`、嵌套数组、dtype/shape 校验和 `expected_output_file`；非法元数据只使当前用例失败。 | +| A2 | `verify_assembly` 调用风险 | 已处理 | TinyFive 未安装时明确失败且不回退 stub；内部设置一亿条指令上限，外层使用独立子进程和 5 秒 timeout，异常只影响当前用例。 | +| I1 | 报告依赖过重 | 已处理 | 默认只生成 Markdown 和 JSON，且不加载 `jinja2`/`matplotlib`；`--full-report` 才生成 HTML 和 PNG，可选依赖单独放在 `requirements-topic06-full.txt`。 | +| I2 | 5% 退化阈值过于刚性 | 已处理 | 新增 `--regression-threshold`，默认 5%，支持非负自定义值；实际阈值参与退化判断并写入 Markdown、HTML 和 JSON 报告。 | +| I3 | Benchmark 重复运行 timeout 用例 | 已处理 | 首次模拟 timeout 时跳过全部重复；重复过程中 timeout/失败则立即停止，失败结果不计入统计，报告记录实际次数和停止原因。 | +| T1 | 编译器 crash 无上下文保存 | 已处理 | 编译失败或超时时保存命令、返回码、耗时、stdout、stderr 和汇编状态到独立日志，并在 Markdown、HTML、JSON 和 CI artifact 中关联日志路径。 | +| T2 | 不支持增量/子集测试 | 已处理 | 新增大小写不敏感的 `--category` 和 `--filter`，支持组合筛选、稳定排序和无匹配退出码 2；子集更新基线采用合并策略。 | + +--- + +## 八、当前测试现状 + +最近一次运行结果： + +```text +Total: 23 +Passed: 13 +Failed: 10 +``` + +失败用例分为两类。 + +第一类是 `branch/` 目录下 3 个 if 分支用例： + +```text +branch/if_else.dsl +branch/if_relu.dsl +branch/if_then.dsl +``` + +失败原因是 TinyFive 模拟阶段 5 秒超时： + +```text +simulation timeout after 5s +``` + +第二类是以下 7 个数组或矩阵用例： + +```text +reduction/dot_4.dsl +reduction/dot_8.dsl +reduction/dot_relu_8.dsl +tensor/matmul_2x2.dsl +tensor/matmul_4x4.dsl +tensor/matmul_add_2x2.dsl +tensor/matmul_relu_2x2.dsl +``` + +这些用例失败有两层原因： + +1. 测试套件目前只根据优化后 IR 初始化标量寄存器，会跳过 `.meta.json` 中的数组和矩阵输入，也没有把数组写入 TinyFive 内存、传递首地址或从结果内存读取矩阵。 +2. 更根本的原因是 RISC-V 指令选择器当前把 `dot` 和 `matmul` 都降低成单条 `mul`。生成汇编没有数组元素读取指令、循环累加或矩阵结果写回，也没有使用 IR 中保存的 `length`、`m`、`n`、`k` 属性。因此只完善测试套件的内存初始化仍不能得到正确结果，后续需要先明确数组调用约定并实现后端降低逻辑。 + +`reduction/dot_relu_4.dsl` 当前显示 PASS，但该用例期望值恰好为 0，而未初始化的数组输入也使 TinyFive 返回 0，属于偶然匹配，不能证明 `dot` 已正确实现。 + +当前报告中的另外 12 个通过用例可以完成编译、TinyFive 模拟和真实返回值对比。 + +--- + +## 九、CI 集成 + +目录中包含 GitHub Actions 示例： + +```text +.github/workflows/benchmark.yml +``` + +CI 的目标是在 push 或 pull request 时自动运行测试套件，并上传测试报告，方便查看每次修改后的正确性和性能变化。 + +CI 示例使用 `requirements-topic06.txt` 安装基础依赖，默认只上传 `report.md`、`report.json` 和性能基线，不安装 `jinja2`、`matplotlib`，也不生成 HTML、PNG。需要人工查看完整可视化报告时，可以在本地安装 `requirements-topic06-full.txt` 并使用 `--full-report`。 + +--- + +## 十、已完成工作对照 + +| 阶段 | 完成情况 | +|---|---| +| W1 学习编译命令和 TinyFive 用法 | 已完成 | +| W2 使用 subprocess 自动调用编译器和模拟器 | 已完成 | +| W3 设计测试用例格式 | 已完成，采用 `.dsl + .meta.json` | +| W4 遍历用例并输出 PASS/FAIL | 已完成 | +| W5 提取指令数并加入报告 | 已完成 | +| W6 matplotlib 图表和 HTML 报告 | 已完成 | +| W7 扩充到 15 个以上用例并覆盖分支循环 | 已完成，共 23 个 | +| W8 使用 `time.perf_counter()` 统计耗时 | 已完成 | +| W9 回归测试模式和 5% 退化阈值 | 已完成 | +| W10 `--benchmark` 重复运行和置信区间 | 已完成 | +| W11 GitHub Actions CI 示例 | 已完成 | +| W12 完整文档 | 已完成 | + +--- diff --git a/scratchv/backend/regalloc_linear.py b/scratchv/backend/regalloc_linear.py index a326131..37baaf4 100644 --- a/scratchv/backend/regalloc_linear.py +++ b/scratchv/backend/regalloc_linear.py @@ -192,10 +192,12 @@ def __init__(self, phys_regs: Optional[list[str]] = None): ) self.stack_slot: int = 0 self.alloc_map: dict[str, str] = {} - self.spill_code: list[tuple[int, str, str]] = ( - [] # (position, op, operand) - ) + self.spill_code: dict[int, list[str]] = {} # pos -> [sw asm lines] self._spill_slots: dict[str, int] = {} # vreg -> slot offset + self._reloads: dict[int, list[tuple[str, int]]] = ( + {} # pos -> [(vreg, slot), ...] + ) + self._spilled: set[str] = set() # ------------------------------------------------------------------ # Live interval computation @@ -273,6 +275,8 @@ def allocate(self, intervals: list[LiveInterval]) -> dict[str, str]: self.alloc_map.clear() self.spill_code.clear() self._spill_slots.clear() + self._reloads.clear() + self._spilled.clear() # Active list: (interval, phys_reg) sorted by increasing end active: list[tuple[LiveInterval, str]] = [] @@ -320,19 +324,12 @@ def spill(self, current: LiveInterval, """Select a register to spill and emit spill code. Chooses the active interval with the farthest end position to spill. - - Parameters - ---------- - current: - The live interval that needs a register. - active: - Currently active intervals. - free_regs: - List of free registers (will be appended to if a spill succeeds). + Records reload positions for the spilled interval so that + ``get_allocated_code`` can insert ``lw`` before each future use. Returns ------- - The physical register freed by spilling, or None if no spill possible. + The physical register freed by spilling, or None if current is spilled. """ if not active: return None @@ -350,23 +347,37 @@ def spill(self, current: LiveInterval, # Only spill if the current interval ends earlier if current.end <= spill_interval.end: - # Spill the farthest interval + # Spill the farthest active interval (victim) slot = self._get_spill_slot(spill_interval.vreg) active.pop(spill_idx) - # Emit store after the definition point - self.spill_code.append( - (spill_interval.start, "sw", - f"{spill_reg}, {slot}(sp) # spill {spill_interval.vreg}") + self.spill_code.setdefault(spill_interval.start, []).append( + f" sw {spill_reg}, {slot}(sp) # spill {spill_interval.vreg}" ) + # Remove stale mapping so codegen won't use the freed register + if spill_interval.vreg in self.alloc_map: + del self.alloc_map[spill_interval.vreg] + self._spilled.add(spill_interval.vreg) + # Record reload at every future use of the spilled vreg + for use_pos in spill_interval.uses: + if use_pos > current.start: + self._reloads.setdefault(use_pos, []).append( + (spill_interval.vreg, slot)) free_regs.append(spill_reg) return spill_reg # Otherwise, spill the current interval slot = self._get_spill_slot(current.vreg) - self.spill_code.append( - (current.start, "sw", - f"{self.phys_regs[0]}, {slot}(sp) # spill {current.vreg}") + # Assign a temporary register for the definition instruction + temp_reg = self.phys_regs[0] + self.alloc_map[current.vreg] = temp_reg + self._spilled.add(current.vreg) + self.spill_code.setdefault(current.start, []).append( + f" sw {temp_reg}, {slot}(sp) # spill {current.vreg}" ) + for use_pos in current.uses: + if use_pos > current.start: + self._reloads.setdefault(use_pos, []).append( + (current.vreg, slot)) return None def _get_spill_slot(self, vreg: str) -> int: @@ -380,41 +391,60 @@ def _get_spill_slot(self, vreg: str) -> int: # Code generation # ------------------------------------------------------------------ - def get_allocated_code(self, block: list[LsInstruction]) -> str: - """Generate allocated assembly code for the block. + def emit(self, block: list[LsInstruction]) -> str: + """Main entry point: allocate registers and emit assembly. - Parameters - ---------- - block: - The original block of LsInstruction objects. + Computes live intervals, runs linear-scan allocation, then + generates assembly with spill stores and reloads interleaved. + """ + intervals = self.compute_live_intervals(block) + self.allocate(intervals) + return self.get_allocated_code(block) - Returns - ------- - RISC-V assembly text with physical registers and spill code. + def get_allocated_code(self, block: list[LsInstruction]) -> str: + """Generate allocated assembly with spill stores and reloads. + + Walks the instruction block in order. Before each instruction + that uses a spilled vreg, a reload ``lw`` is inserted. After + each instruction that defines a spilled vreg, a spill ``sw`` + is inserted. """ lines: list[str] = [] - rename = self.alloc_map - - # Build a position -> spill load map - spill_loads: dict[int, list[str]] = {} - for pos, op, operand in self.spill_code: - if "sw" in op: - lines.append(f" {op} {operand}") - else: - if pos not in spill_loads: - spill_loads[pos] = [] - spill_loads[pos].append(f" {op} {operand}") + rename: dict[str, str] = dict(self.alloc_map) for inst in block: - # Insert spill loads before instruction - if inst.id in spill_loads: - for load_line in spill_loads[inst.id]: - lines.append(load_line) + # Insert reloads before the instruction + if inst.id in self._reloads: + for vreg, slot in self._reloads[inst.id]: + reload_reg = self._pick_reload_reg(rename) + lines.append( + f" lw {reload_reg}, {slot}(sp)" + f" # reload {vreg}" + ) + rename[vreg] = reload_reg lines.append(inst.to_asm(rename)) + # Insert spill stores after the instruction + if inst.id in self.spill_code: + lines.extend(self.spill_code[inst.id]) + return "\n".join(lines) + def _pick_reload_reg(self, rename: dict[str, str]) -> str: + """Pick a free physical register for a reload ``lw``. + + Chooses any allocatable register not currently mapped in + *rename*. Falls back to the first physical register if all + are occupied (should not happen for the current workload where + simultaneously-live vregs never exceed the register pool). + """ + used: set[str] = set(rename.values()) + for reg in self.phys_regs: + if reg not in used: + return reg + return self.phys_regs[0] + # ------------------------------------------------------------------ # Report # ------------------------------------------------------------------ diff --git a/scratchv/backend/register_alloc.py b/scratchv/backend/register_alloc.py index 9e304c8..15d3e1b 100644 --- a/scratchv/backend/register_alloc.py +++ b/scratchv/backend/register_alloc.py @@ -105,7 +105,6 @@ def _allocate_greedy(self) -> list[MachineInstr]: for instr in self.instructions: if instr.op == MachineOp.LABEL: - self._flush_regs() self._emit(instr) continue diff --git a/scratchv/backend/riscv_encoder.py b/scratchv/backend/riscv_encoder.py index efe4713..e0bea27 100644 --- a/scratchv/backend/riscv_encoder.py +++ b/scratchv/backend/riscv_encoder.py @@ -6,6 +6,7 @@ from __future__ import annotations +import re import struct from enum import IntEnum @@ -175,32 +176,116 @@ class RISCVAEncoder: def __init__(self): self.labels: dict[str, int] = {} # label -> instruction index self.pending_fixups: list[tuple[int, str, str]] = [] + self._max_counter = 0 + self._temp_reg = 0 + + # ── Pseudo-instruction expansion ────────────────────────────────── + + def _find_free_temp(self, asm_text: str) -> int: + """Scan assembly text for used registers; return first free temp. + + Preference order: t6, t5, t4, t3, t2, t1, t0 (x31 down to x5). + """ + used = set() + for match in re.finditer( + r'\b(zero|ra|sp|gp|tp|t[0-6]|s\d+|a\d+|fp|x\d+)\b', + asm_text, + ): + name = match.group(1) + if name in REG_MAP: + used.add(REG_MAP[name]) + for r in [31, 30, 29, 28, 7, 6, 5]: + if r not in used: + return r + return 31 # fallback + + def _expand_pseudo(self, line: str) -> list[str]: + """Expand one possibly-pseudo line into standard RISC-V lines. + + Returns a list of lines (may be empty for skipped directives). + """ + if not line or line.startswith(".") or line.endswith(":"): + return [line] + + tokens = line.replace(",", " ").split() + if not tokens: + return [line] + + op = tokens[0].lower() + + # max rd, rs1, rs2 → 4-instruction sequence + if op == "max": + rd = tokens[1] if len(tokens) > 1 else "x0" + rs1 = tokens[2] if len(tokens) > 2 else "x0" + rs2 = tokens[3] if len(tokens) > 3 else "x0" + n = self._max_counter + self._max_counter += 1 + return [ + f"bge {rs1}, {rs2}, .__max_then_{n}", + f"addi {rd}, x0, 0", + f"j .__max_end_{n}", + f".__max_then_{n}:", + f"addi {rd}, {rs1}, 0", + f".__max_end_{n}:", + ] + + # Branch-with-immediate: beq/bne/blt/bge rs1, imm, label + # → li xTEMP, imm; beq/bne/blt/bge rs1, xTEMP, label + if op in ("beq", "bne", "blt", "bge"): + if len(tokens) >= 4: + op2 = tokens[2].rstrip(",") + if op2 not in REG_MAP and not op2.startswith("x") and not op2.startswith("%"): + try: + imm = int(op2) + except ValueError: + pass + else: + temp = f"x{self._temp_reg}" + label = tokens[3] + return [ + f"li {temp}, {imm}", + f"{op} {tokens[1]}, {temp}, {label}", + ] + + return [line] + + # ── Assembly pass ───────────────────────────────────────────────── def assemble(self, asm_text: str) -> bytearray: """Assemble RISC-V assembly text to flat binary.""" + # Pre-scan: find a free temp register for pseudo expansion + clean_text = "\n".join( + line.split("#")[0] for line in asm_text.split("\n") + ) + self._temp_reg = self._find_free_temp(clean_text) + lines = asm_text.strip().split("\n") - instructions: list[tuple] = [] # (encoded_word, comment) + instructions: list[tuple] = [] # (encoded_word, fixup_or_None) - # Pass 1: collect labels and encode + # Pass 1: expand pseudos, collect labels, encode for line in lines: line = line.split("#")[0].strip() if not line: continue - # Skip directives (but not local labels like .Lxxx) - if line.startswith(".") and not line.startswith(".L"): - continue - - # Label detection (including .L local labels) - if line.endswith(":"): - name = line[:-1].strip() - self.labels[name] = len(instructions) - continue - - # Parse instruction - encoded = self._encode_line(line, len(instructions)) - if encoded is not None: - instructions.append(encoded) + # Expand pseudo-instructions (one line → possibly many) + expanded = self._expand_pseudo(line) + for exp_line in expanded: + exp_line = exp_line.split("#")[0].strip() + if not exp_line: + continue + + # Skip directives, but keep all labels (including .L and .__) + if exp_line.startswith(".") and not exp_line.endswith(":"): + continue + if exp_line.endswith(":"): + name = exp_line[:-1].strip() + self.labels[name] = len(instructions) + continue + + encoded = self._encode_line(exp_line, len(instructions)) + if encoded is not None: + instructions.append(encoded) # Pass 2: apply label fixups result = bytearray() @@ -214,8 +299,8 @@ def assemble(self, asm_text: str) -> bytearray: def _encode_line( self, line: str, idx: int, ) -> tuple[int, tuple[str, str] | None] | None: - """Encode a single assembly line.""" - # Tokenize + """Encode a single assembly line (standard RISC-V only — pseudos + should already be expanded by ``_expand_pseudo``).""" tokens = line.replace(",", " ").split() if not tokens: return None @@ -230,11 +315,35 @@ def _encode_line( rs1 = _reg_num(operands[1]) rs2 = _reg_num(operands[2]) word = _r_type(rd, rs1, rs2, F3_ADD_SUB, F7_ADD) + elif op == "rem": + rd = _reg_num(operands[0]) + rs1 = _reg_num(operands[1]) + rs2 = _reg_num(operands[2]) + word = _r_type(rd, rs1, rs2, 0b110, F7_MULDIV) elif op == "sub": rd = _reg_num(operands[0]) rs1 = _reg_num(operands[1]) rs2 = _reg_num(operands[2]) word = _r_type(rd, rs1, rs2, F3_ADD_SUB, F7_SUB) + elif op == "srai": + rd = _reg_num(operands[0]) + rs1 = _reg_num(operands[1]) + shamt = self._parse_imm(operands[2]) & 0x1F + word = _i_type(rd, rs1, shamt | (0b0100000 << 5), F3_SRL_SRA) + # Shamt is encoded in lower 5 bits of the 12-bit immediate; + # the upper 7 bits are 0100000 for SRAI. + imm12 = shamt | (0b0100000 << 5) + word = _i_type(rd, rs1, imm12, F3_SRL_SRA) + elif op == "xor": + rd = _reg_num(operands[0]) + rs1 = _reg_num(operands[1]) + rs2 = _reg_num(operands[2]) + word = _r_type(rd, rs1, rs2, F3_XOR, 0b0000000) + elif op == "and": + rd = _reg_num(operands[0]) + rs1 = _reg_num(operands[1]) + rs2 = _reg_num(operands[2]) + word = _r_type(rd, rs1, rs2, F3_AND, 0b0000000) elif op == "mul": rd = _reg_num(operands[0]) rs1 = _reg_num(operands[1]) @@ -255,8 +364,20 @@ def _encode_line( offset, rs1 = self._parse_mem(operands[1]) word = _i_type(rd, rs1, offset, F3_LW, RVOpcode.LOAD) elif op == "sw": - rs2 = _reg_num(operands[0]) - offset, rs1 = self._parse_mem(operands[1]) + # Handle both standard (sw rs2, offset(rs1)) and compiler + # (sw rs1(offset), rs2) syntax. + if "(" in operands[0]: + # Compiler syntax: sw rs1(offset), rs2 + mem = operands[0].strip() + base = mem[:mem.index("(")] + off_str = mem[mem.index("(") + 1:mem.index(")")] + offset = self._parse_imm(off_str) if off_str else 0 + rs1 = _reg_num(base) + rs2 = _reg_num(operands[1]) + else: + # Standard syntax: sw rs2, offset(rs1) + rs2 = _reg_num(operands[0]) + offset, rs1 = self._parse_mem(operands[1]) word = _s_type(rs1, rs2, offset, F3_SW) elif op == "beq": rs1 = _reg_num(operands[0]) @@ -302,10 +423,8 @@ def _encode_line( if -2048 <= imm <= 2047: word = _i_type(rd, 0, imm, F3_ADD_SUB) else: - # lui + addi sequence — will be handled later upper = (imm + 0x800) >> 12 word = _u_type(rd, upper) - # Store second instruction self._pending_li = (rd, imm & 0xFFF) elif op == "mv": rd = _reg_num(operands[0]) @@ -317,10 +436,7 @@ def _encode_line( fixup = ("call", label) word = _u_type(1, 0) else: - # call without label (runtime call, target in comment) - # Encode as auipc ra, 0 + jalr (nop-like, handled by emulator) word = _i_type(1, 1, 0, 0, RVOpcode.JALR) - # Store runtime call info for later fixup fixup = ("runtime_call", "") elif op == "ret": word = _i_type(0, 1, 0, 0, RVOpcode.JALR) @@ -337,19 +453,6 @@ def _encode_line( else: imm = self._parse_imm(operands[2]) word = _i_type(rd, rs1, imm, F3_SLT) - elif op == "max": - # Expand pseudo: blt rs1, rs2, +8; mv rd, rs2; j +8; mv rd, rs1 - # For single instruction encoding, emit as add (simplified) - rd = _reg_num(operands[0]) if len(operands) > 0 else 0 - rs1 = _reg_num(operands[1]) if len(operands) > 1 else 0 - if len(operands) > 2 and (operands[2].startswith("%") or operands[2] in REG_MAP): - rs2 = _reg_num(operands[2]) - else: - rs2 = 0 - word = _r_type(rd, rs1, rs2, F3_ADD_SUB, 0b0000000) - # Store as multi-instruction expansion - self._pending_max = (rd, rs1, rs2) - fixup = ("max_expand", "") elif op == "nop": word = _i_type(0, 0, 0, F3_ADD_SUB) else: @@ -362,8 +465,6 @@ def _apply_fixup(self, word: int, fixup: tuple, current_idx: int) -> int: kind, label = fixup if kind == "runtime_call": return word - if kind == "max_expand": - return word # already encoded target_idx = self.labels.get(label, current_idx) offset = target_idx - current_idx diff --git a/scratchv/compiler.py b/scratchv/compiler.py index 7e61036..4141308 100644 --- a/scratchv/compiler.py +++ b/scratchv/compiler.py @@ -403,21 +403,17 @@ def _generate_riscv_linear(self, program) -> str: selector = InstructionSelector(program) machine_instrs = selector.run() - alloc = RegisterAllocator(machine_instrs, mode=self.config.reg_alloc) - allocated = alloc.run() - - # Optional: use linear-scan instead + # Linear-scan: skip greedy allocator, use liveness-driven allocator if self.config.reg_alloc == "linear": from scratchv.backend.regalloc_linear import ( LinearScanAllocator, block_from_machine_instrs, ) - ls_insts = block_from_machine_instrs(allocated) + ls_insts = block_from_machine_instrs(machine_instrs) lsa = LinearScanAllocator() - intervals = lsa.compute_live_intervals(ls_insts) - lsa.allocate(intervals) - # Use linear-scan allocated code as assembly directly - return lsa.get_allocated_code(ls_insts) + return lsa.emit(ls_insts) + alloc = RegisterAllocator(machine_instrs, mode=self.config.reg_alloc) + allocated = alloc.run() emitter = AsmEmitter(allocated) return emitter.emit() diff --git a/scratchv/simulator/tinyfive.py b/scratchv/simulator/tinyfive.py index 203a045..8f7e87d 100644 --- a/scratchv/simulator/tinyfive.py +++ b/scratchv/simulator/tinyfive.py @@ -227,16 +227,6 @@ def load_binary(self, words: list[int], origin: int = 0): self._pc = origin self._code_words = words - def load_asm(self, asm_lines: list[str], origin: int = 0x200): - self._pc = origin - self._code_words = [] - - for line in asm_lines: - line = line.split("#")[0].strip() - if not line or line.endswith(":"): - continue - self._code_words.append(line) - def load_data(self, data: bytes, addr: int): for i, b in enumerate(data): self.memory[addr + i] = b @@ -270,26 +260,98 @@ def pc(self, val: int): self._pc = val -def verify_assembly(asm_code: str, verbose: bool = False) -> dict: +_REG_NUMS: dict[str, int] = { + "zero": 0, "x0": 0, + "ra": 1, "x1": 1, + "sp": 2, "x2": 2, + "gp": 3, "x3": 3, + "tp": 4, "x4": 4, + "t0": 5, "x5": 5, + "t1": 6, "x6": 6, + "t2": 7, "x7": 7, + "s0": 8, "fp": 8, "x8": 8, + "s1": 9, "x9": 9, + "a0": 10, "x10": 10, + "a1": 11, "x11": 11, + "a2": 12, "x12": 12, + "a3": 13, "x13": 13, + "a4": 14, "x14": 14, + "a5": 15, "x15": 15, + "a6": 16, "x16": 16, + "a7": 17, "x17": 17, + "s2": 18, "x18": 18, + "s3": 19, "x19": 19, + "s4": 20, "x20": 20, + "s5": 21, "x21": 21, + "s6": 22, "x22": 22, + "s7": 23, "x23": 23, + "s8": 24, "x24": 24, + "s9": 25, "x25": 25, + "s10": 26, "x26": 26, + "s11": 27, "x27": 27, + "t3": 28, "x28": 28, + "t4": 29, "x29": 29, + "t5": 30, "x30": 30, + "t6": 31, "x31": 31, +} + + +def verify_assembly( + asm_code: str, + verbose: bool = False, + initial_registers: Optional[dict[str, int]] = None, +) -> dict: """Verify generated assembly by running it in TinyFive. + Uses ``RISCVAEncoder`` to assemble text → binary, then loads via + ``load_binary()`` for reliable execution. Falls back to + ``load_asm()`` if encoding fails. + Args: asm_code: RISC-V assembly text. verbose: Print performance info. Returns: - dict with keys: success, instr_count, error + dict with keys: success, instr_count, return_value, error """ m = ProfiledMachine(mem_size=128 * 1024 * 1024) if not m.available: return { "success": False, "instr_count": 0, + "return_value": None, + "backend": "tinyfive", "error": "tinyfive not installed", } - lines = asm_code.strip().split("\n") - load_asm(lines, origin=0) + # Primary path: assemble to binary via our encoder, then load. + try: + from scratchv.backend.riscv_encoder import assemble_to_binary + + binary = assemble_to_binary(asm_code) + if len(binary) > 0: + words = [ + int.from_bytes(binary[i:i + 4], "little") + for i in range(0, len(binary), 4) + ] + m.load_binary(words, origin=0) + # Point ra past the end of valid memory. The compiler emits + # ``jalr zero, ra`` for ``ret``, but ra is 0 on startup. + # By setting ra to an out-of-bounds address, the instruction + # fetch after the jump triggers an IndexError that the + # ``except Exception`` in ``run()`` catches cleanly. + m.set_reg(1, m.mem_size) # ra = x1 = out-of-bounds + else: + raise ValueError("assembler produced empty binary") + except Exception as enc_err: + # Fallback: try TinyFive's limited asm() parser. + lines = asm_code.strip().split("\n") + m.load_asm(lines, origin=0) + + for reg_name, value in (initial_registers or {}).items(): + reg_num = _REG_NUMS.get(reg_name) + if reg_num is not None: + m.set_reg(reg_num, int(value)) try: m.run(instructions=100_000_000) @@ -297,7 +359,15 @@ def verify_assembly(asm_code: str, verbose: bool = False) -> dict: return { "success": False, "instr_count": m.instr_count, + "return_value": None, + "backend": "tinyfive", "error": str(e), } - return {"success": True, "instr_count": m.instr_count, "error": None} + return { + "success": True, + "instr_count": m.instr_count, + "return_value": m.get_reg(10), + "backend": "tinyfive", + "error": None, + }