From b92f987a0c9f6a87977652d4ac774a4d1a472122 Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Fri, 10 Jul 2026 17:58:57 +0900 Subject: [PATCH 1/8] [TOGSim] Order dependency sets by instruction id, not by heap address Instruction::_deps is a std::set>, whose default comparator orders by pointer value. That only looked deterministic because the whole TileGraph was allocated up front in one monotonically growing run, so addresses happened to track creation order and fire() released dependents in creation order. It is not a property of the code. Running the same binary on the same trace under a different allocator changes the reported cycle count: kernel glibc malloc tcmalloc 677alforuj6 7852 7846 6lxncgaxtnh 3490 3491 7ascx6t7uu7 2888 2889 Order the sets by _global_inst_id instead. The id is intrinsic to the instruction, so the release order -- and the issue order and the cycle count -- no longer depend on the heap layout. All three allocators above now agree. Five of twelve real kernels shift by 1..6 cycles; a two-tenant model shifts by 416 of 1.4M (0.03%). Those numbers were allocator artefacts, not the model. This is also a prerequisite for building the TileGraph on demand, which frees and reuses tiles and would otherwise perturb the release order. (cherry picked from commit 1d067ef6f81c2330897104f9461070636b092ed0) --- TOGSim/include/Instruction.h | 16 +++++++++++++--- TOGSim/src/Instruction.cc | 5 +++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/TOGSim/include/Instruction.h b/TOGSim/include/Instruction.h index 74c5e339..122be8a4 100644 --- a/TOGSim/include/Instruction.h +++ b/TOGSim/include/Instruction.h @@ -32,6 +32,15 @@ typedef uint64_t cycle_type; std::string opcode_to_string(Opcode opcode); std::string format_tag_key_list_hex(const std::vector& tag_keys); +class Instruction; +// Order dependents by creation order, NOT by heap address: std::set's +// default comparator sorts by pointer value, which makes fire()'s release order -- +// and with it the issue order and the reported cycle count -- allocator-dependent. +struct DepLess { + bool operator()(const std::shared_ptr& a, + const std::shared_ptr& b) const; +}; + class Instruction : public std::enable_shared_from_this { public: Instruction(Opcode opcode, cycle_type compute_cycle, size_t num_parents, addr_type dram_addr, @@ -51,7 +60,7 @@ class Instruction : public std::enable_shared_from_this { for (auto& c : _deps[static_cast(e)]) c->dec_ready_counter(); _deps[static_cast(e)].clear(); } - const std::set>& get_deps(DepEvent e) { + const std::set, DepLess>& get_deps(DepEvent e) { return _deps[static_cast(e)]; } void set_assigned_sa(int s) { _assigned_sa = s; } @@ -159,8 +168,9 @@ class Instruction : public std::enable_shared_from_this { size_t ready_counter = 0; // parents not yet finished; the minimal Instruction(Opcode) // ctor (barriers) relies on this default + inc_ready_counter // Per-event subscriber sets: _deps[ISSUE] released at issue (occupancy), - // _deps[DONE] at finish (latency). std::set dedups + fixes the release order. - std::array>, + // _deps[DONE] at finish (latency). std::set dedups and, via DepLess, iterates in + // instruction-id order, so the release order does not depend on the allocator. + std::array, DepLess>, static_cast(DepEvent::COUNT)> _deps; std::vector tile_size; std::vector tile_stride; diff --git a/TOGSim/src/Instruction.cc b/TOGSim/src/Instruction.cc index ee184a1a..718e2f93 100644 --- a/TOGSim/src/Instruction.cc +++ b/TOGSim/src/Instruction.cc @@ -49,6 +49,11 @@ Instruction::Instruction(Opcode opcode) _tile_numel = 1; } +bool DepLess::operator()(const std::shared_ptr& a, + const std::shared_ptr& b) const { + return a->get_global_inst_id() < b->get_global_inst_id(); +} + void Instruction::finish_instruction() { fire(DepEvent::DONE); // latency consumers finished = true; From 3e0f4c9f00f85233738e082fbd5df204e11ac5ce Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Fri, 10 Jul 2026 17:59:25 +0900 Subject: [PATCH 2/8] [TOGSim] Keep an instruction's sram_release version ids ascending sram_finalize() tags each buffer version's last reader while walking the version map, which is ordered by version id, so an instruction that frees several versions receives them ascending. Make that an invariant of add_sram_release instead of an accident of the caller's iteration order. No behaviour change today. It lets the builder tag a version the moment it closes -- in buffer order -- and still produce a byte-identical graph, which the on-demand construction that follows relies on. (cherry picked from commit 7a52bc9cba9be59830af7890ac936508b9cd20a4) --- TOGSim/include/Instruction.h | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/TOGSim/include/Instruction.h b/TOGSim/include/Instruction.h index 122be8a4..0ce9b268 100644 --- a/TOGSim/include/Instruction.h +++ b/TOGSim/include/Instruction.h @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -139,7 +140,13 @@ class Instruction : public std::enable_shared_from_this { // last reader frees it on issue via `_sram_release_allocs`. void set_sram_alloc(int64_t id) { _sram_alloc_id = id; } int64_t get_sram_alloc() const { return _sram_alloc_id; } - void add_sram_release(int64_t id) { _sram_release_allocs.push_back(id); } + // Keep the list ascending by version id. The Core frees each version regardless + // of order, but the invariant lets a builder that tags versions as they close be + // byte-comparable with one that walks the version map at the end. + void add_sram_release(int64_t id) { + auto& v = _sram_release_allocs; + v.insert(std::upper_bound(v.begin(), v.end(), id), id); + } const std::vector& get_sram_release() const { return _sram_release_allocs; } // bytes this instruction's buffer occupies in the spad. A DMA derives it from // the tile it moves; a compute output gets it set explicitly by the bridge (the From 021a3fb5874053cc86808ee59af12bc25dea99f8 Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Thu, 9 Jul 2026 15:07:06 +0900 Subject: [PATCH 3/8] [TOGSim] Track a buffer's writers in one struct so the UNION link is O(K) link()'s is_mm_accum UNION branch rescanned all of writers(b) on every one of the K matmuls that accumulate into b, to add a dep edge to the non-MATMUL producer (the init/bias that seeded the accumulator). But a UNION only ever appends MATMULs -- that producer is fixed until the next REPLACE -- so the rescan found the same one instruction every time and skipped everything else. Measured with a counter on an 8x8 conv whose K_tiles is 784 (50176 accumulating matmuls): the scan ran 1,258,840,576 times = N(N+1)/2, and only 50176 of those iterations produced an add_dep. 99.996 percent of the work was wasted. The real CI case (test_conv2d batch2 128->512 14x14 k7) has N=2458624, i.e. about 3e12 iterations, which is why it never finished. Replace the bare writers(b) vector with a BufferWriters struct that keeps the initializer alongside the producer set, so the UNION reads it in O(1) instead of rescanning. Both mutations (REPLACE, JOIN) are now methods, which is what keeps the two consistent -- the old vector let any caller assign writers[b] directly and silently desync a parallel map. The scan drops to 50176 on that case (25088x fewer iterations) and the set of add_dep edges is unchanged. Total execution cycles are identical on all three cases checked (33000, 129461, 202082), before and after. (cherry picked from commit 83f9f336b5d9be0007104f3af58a61f2eebae45f) --- TOGSim/src/togsim_trace_bridge.cc | 73 +++++++++++++++++++++---------- 1 file changed, 51 insertions(+), 22 deletions(-) diff --git a/TOGSim/src/togsim_trace_bridge.cc b/TOGSim/src/togsim_trace_bridge.cc index 46e6f7b3..74da75b9 100644 --- a/TOGSim/src/togsim_trace_bridge.cc +++ b/TOGSim/src/togsim_trace_bridge.cc @@ -58,6 +58,28 @@ std::shared_ptr make_compute(const togsim::TraceRec& t) { return inst; } +// Core's compute-unit enum, as the producer encodes it in TraceRec::compute_type. +constexpr int MATMUL_CT = 1, PRELOAD_CT = 2; + +// The current producers of one SRAM buffer. Normally a write REPLACEs them, but +// the K matmuls of Y += X@W commute: each only waits on whoever INITIALIZED the +// accumulator, so it JOINs the set and reads initializer() in O(1), not O(K). +class BufferWriters { + public: + void replace(const std::shared_ptr& w) { // normal write: sole producer + _all.assign(1, w); + _initializer = (w->get_compute_type() != MATMUL_CT) ? w : nullptr; + } + void accumulate(const std::shared_ptr& mm) { _all.push_back(mm); } + + const std::vector>& all() const { return _all; } + const std::shared_ptr& initializer() const { return _initializer; } + + private: + std::vector> _all; + std::shared_ptr _initializer; // the non-MATMUL member of _all, if any +}; + } // namespace std::unique_ptr trace_to_tilegraph(const togsim::RunResult& run, @@ -74,10 +96,16 @@ std::unique_ptr trace_to_tilegraph(const togsim::RunResult& run, // The explicit dependency DAG (sec 10): per SRAM buffer, writers(b) is the SET of // current producers' DONE-handles and readers(b) its consumers. Scoped per // work-item, so distinct work-items are independent (-> parallel). - std::map>> writers; // buffer id -> current producers (DONE-handles) - // 1 load : N barriers, so track the CURRENT load per (tag_id, tag_slot), not a - // FIFO. Each load takes a fresh `uniq` and its iteration's barriers reuse it. - // Correct only because a load nest and its consumers run in order. Per work-item. + std::map writers; + // An async dma is paired with its explicit memory_barrier(s) by the runtime tag + // (tag_id, tag_slot). It is 1 load : N barriers (the load happens once per + // reduction iteration; each consumer in that iteration is preceded by a wait on + // the same tag), so we track the CURRENT (most recent) load per (tag_id, + // tag_slot) -- not a FIFO. Each load gets a fresh `uniq` Core key, so successive + // reduction iterations (multi-tile-K, conv) never collide in the tag table; the + // iteration's barriers reuse that load's uniq. Correct because the load nest and + // its consumer nest run in order within the reduction body (no cross-iteration + // prefetch). Scoped per work-item. std::map, std::pair>> current_dma; // Dedup barriers on the CURRENT load of a (tag_id, tag_slot): a conv reads one @@ -109,8 +137,8 @@ std::unique_ptr trace_to_tilegraph(const togsim::RunResult& run, // The single dataflow rule (sec 10.3). READ b: depend on all writers(b), ISSUE // when both are SA ops else DONE. WRITE b: replace writers(b), except a commutative - // matmul accumulator, which waits only the seed and unions. WAR: resource models. - const int MATMUL_CT = 1, PRELOAD_CT = 2; + // matmul accumulator, which joins them and waits only the initializer. WAR: + // resource models. auto is_mm_accum = [&](const std::shared_ptr& inst, int64_t b, const std::vector& writes) { if (inst->get_compute_type() != MATMUL_CT) return false; @@ -124,7 +152,7 @@ std::unique_ptr trace_to_tilegraph(const togsim::RunResult& run, if (is_mm_accum(inst, b, writes)) continue; // accumulator read -> handled in WRITE (UNION) auto it = writers.find(b); if (it != writers.end()) - for (auto& w : it->second) { + for (auto& w : it->second.all()) { int pct = w->get_compute_type(); // both SA ops -> occupancy (overlap on the SA pipeline); else latency. DepEvent on = (inst->get_compute_type() == MATMUL_CT && @@ -134,15 +162,12 @@ std::unique_ptr trace_to_tilegraph(const togsim::RunResult& run, } } for (int64_t b : writes) { - if (is_mm_accum(inst, b, writes)) { // UNION (commutative accumulate) - auto it = writers.find(b); - if (it != writers.end()) - for (auto& s : it->second) - if (s->get_compute_type() != MATMUL_CT) - s->add_dep(inst, DepEvent::DONE); // wait the init/bias seed only - writers[b].push_back(inst); // join; no reset, no co-matmul edge - } else { // REPLACE (normal output; resets the producer set) - writers[b] = { inst }; + auto& w = writers[b]; + if (is_mm_accum(inst, b, writes)) { // JOIN: commutative, waits only the init + if (const auto& init = w.initializer()) init->add_dep(inst, DepEvent::DONE); + w.accumulate(inst); + } else { // REPLACE: a normal output resets the set + w.replace(inst); } } tile->append_instuction(inst); @@ -253,10 +278,14 @@ std::unique_ptr trace_to_tilegraph(const togsim::RunResult& run, // so consumers gate on arrival. A sync load blocks to arrival itself. if (t.is_async) current_dma[{t.tag_id, t.tag_slot}] = {uniq, inst}; for (int64_t b : t.write_bufs) { - // No hard WAR edge: load-buffer reuse is modeled by the SRAM version/ - // capacity machinery (sec 10.4); an edge would force single-buffering. The - // accumulator is not a load buffer -- link()'s REPLACE branch handles it. - writers[b] = { inst }; + // No hard WAR edge here: load-buffer reuse (double-buffering, X_spad/ + // W_spad reloaded each reduction iter) is modeled by the SRAM + // version/capacity machinery (sram_on_load), which sizes how many + // versions physically coexist. A latency WAR edge would force + // single-buffering and kill the overlap the spad permits. (The + // accumulator Y is NOT a load buffer -> its cross-tile WAR is handled by + // the REPLACE branch of link() when the next tile's init overwrites it.) + writers[b].replace(inst); sram_on_load(b, inst); // occupy spad } } @@ -272,7 +301,7 @@ std::unique_ptr trace_to_tilegraph(const togsim::RunResult& run, // so the buffer's consumers gate on it, instead of emitting a redundant barrier. auto bf = bar_for_load.find({t.tag_id, t.tag_slot}); if (bf != bar_for_load.end() && bf->second.first == uniq) { - for (int64_t b : t.write_bufs) writers[b] = { bf->second.second }; + for (int64_t b : t.write_bufs) writers[b].replace(bf->second.second); continue; } auto bar = make_mem_bar(t, uniq); @@ -281,7 +310,7 @@ std::unique_ptr trace_to_tilegraph(const togsim::RunResult& run, tile->append_instuction(bar); // the bar is the load's DONE-handle: REPLACE writers(b) with it (no WAR -- the // load already WAR'd the prior readers when it wrote). - for (int64_t b : t.write_bufs) writers[b] = { bar }; + for (int64_t b : t.write_bufs) writers[b].replace(bar); bar_for_load[{t.tag_id, t.tag_slot}] = {uniq, bar}; } else if (t.kind == TraceRec::COMPUTE) { auto inst = make_compute(t); From 97298366cd20b032ce62645219d3d3194b89d81f Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Fri, 10 Jul 2026 21:26:06 +0900 Subject: [PATCH 4/8] [TOGSim] Do not build instruction trace strings when the trace is off core_trace_log::trace_instruction_line() takes formatted strings, so its arguments were built at every call site whether or not spdlog would keep them: each issued instruction paid a TraceLogTag::pad15() and a fmt::format over its tile dims and strides, which spdlog::trace() then dropped. The same holds for two spdlog::trace() calls in DMA::cycle that format the instruction's addr_name. Guard them with core_trace_log::trace_enabled(). Output is unchanged when the trace is on; when it is off (the default) an 8x8 conv2d simulation drops from 77.59s to 76.84s of CPU. Modest -- the string work was never the bottleneck -- but it is pure waste. (cherry picked from commit a220fb48b16781cb72c0cf826a7d0180c2bc7c1a) --- TOGSim/include/CoreTraceLog.h | 10 ++++++++++ TOGSim/src/Core.cc | 18 +++++++++--------- TOGSim/src/DMA.cc | 3 ++- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/TOGSim/include/CoreTraceLog.h b/TOGSim/include/CoreTraceLog.h index 2ce51012..47a3e6f8 100644 --- a/TOGSim/include/CoreTraceLog.h +++ b/TOGSim/include/CoreTraceLog.h @@ -3,6 +3,8 @@ #include #include +#include + #include "Instruction.h" #include "TraceLogTags.h" @@ -12,6 +14,14 @@ */ namespace core_trace_log { +/** Whether the instruction trace is actually being emitted. + * + * The trace_* helpers below take formatted strings, so their arguments are + * built whether or not spdlog will keep them: every issued instruction paid two + * std::string constructions (fmt::format over its tile dims/strides) that + * spdlog::trace then dropped. Guard the call sites with this. */ +inline bool trace_enabled() { return spdlog::should_log(spdlog::level::trace); } + std::string format_dma_inst_issued_detail(Instruction& inst); /** Opcode + (detail...) for DMA issue / skip traces. */ std::string format_dma_inst_issued_trace_line(Instruction& inst); diff --git a/TOGSim/src/Core.cc b/TOGSim/src/Core.cc index 6097f287..48f264ce 100644 --- a/TOGSim/src/Core.cc +++ b/TOGSim/src/Core.cc @@ -228,7 +228,7 @@ void Core::dma_cycle() { core_trace_log::log_error_dma_instruction_invalid(_core_cycle, _id); exit(EXIT_FAILURE); } else if (finished_inst->get_opcode() == Opcode::MEMORY_BAR) { - core_trace_log::trace_instruction_line(_core_cycle, + if (core_trace_log::trace_enabled()) core_trace_log::trace_instruction_line(_core_cycle, _id, TraceLogTag::pad15(TraceLogTag::kInstructionFinished), finished_inst->get_global_inst_id(), @@ -307,7 +307,7 @@ void Core::cycle() { finish_instruction(inst); else _dma.register_tag_waiter(inst->subgraph_id, key, inst); - core_trace_log::trace_instruction_line(_core_cycle, + if (core_trace_log::trace_enabled()) core_trace_log::trace_instruction_line(_core_cycle, _id, TraceLogTag::pad15( TraceLogTag::kInstructionSkipped), @@ -320,7 +320,7 @@ void Core::cycle() { } else { // load occupies its spad bytes on issue; stall (retry next cycle) if full. if (!try_occupy_sram(inst)) break; - core_trace_log::trace_instruction_line(_core_cycle, + if (core_trace_log::trace_enabled()) core_trace_log::trace_instruction_line(_core_cycle, _id, TraceLogTag::pad15( TraceLogTag::kInstructionIssued), @@ -335,7 +335,7 @@ void Core::cycle() { } case Opcode::MOVOUT: release_sram(inst); // store issued -> free the tiles it drained - core_trace_log::trace_instruction_line(_core_cycle, + if (core_trace_log::trace_enabled()) core_trace_log::trace_instruction_line(_core_cycle, _id, TraceLogTag::pad15(TraceLogTag::kInstructionIssued), inst->get_global_inst_id(), @@ -411,7 +411,7 @@ void Core::cycle() { continue; // old code fell through to it++ on the // erased (invalidated) iterator -> UB } else { - core_trace_log::trace_instruction_line(_core_cycle, + if (core_trace_log::trace_enabled()) core_trace_log::trace_instruction_line(_core_cycle, _id, TraceLogTag::pad15( TraceLogTag::kInstructionIssued), @@ -443,7 +443,7 @@ void Core::cycle() { } else { _dma.register_tag_waiter(inst->subgraph_id, key, inst); } - core_trace_log::trace_instruction_line(_core_cycle, + if (core_trace_log::trace_enabled()) core_trace_log::trace_instruction_line(_core_cycle, _id, TraceLogTag::pad15( TraceLogTag::kInstructionIssued), @@ -498,7 +498,7 @@ void Core::finish_instruction(std::shared_ptr& inst, InstFinishTrac core_trace_log::log_error_dram_responses_trace_not_finished(_core_cycle, _id); exit(EXIT_FAILURE); } - core_trace_log::trace_instruction_line(_core_cycle, + if (core_trace_log::trace_enabled()) core_trace_log::trace_instruction_line(_core_cycle, _id, TraceLogTag::pad15(TraceLogTag::kAllDramResponsesReceived), inst->get_global_inst_id(), @@ -515,7 +515,7 @@ void Core::finish_instruction(std::shared_ptr& inst, InstFinishTrac const char* trace_tag = (tag == InstFinishTraceTag::DmaIssueComplete) ? TraceLogTag::kAsyncDmaAllRequestsIssued : TraceLogTag::kInstructionFinished; - core_trace_log::trace_instruction_line(_core_cycle, + if (core_trace_log::trace_enabled()) core_trace_log::trace_instruction_line(_core_cycle, _id, TraceLogTag::pad15(trace_tag), inst->get_global_inst_id(), @@ -562,7 +562,7 @@ void Core::push_memory_response(mem_fetch* response) { if (!owner_inst->got_first_response()) { // first data of this load arrived owner_inst->mark_first_response(); - core_trace_log::trace_instruction_line(_core_cycle, _id, + if (core_trace_log::trace_enabled()) core_trace_log::trace_instruction_line(_core_cycle, _id, TraceLogTag::pad15(TraceLogTag::kFirstDramResponse), owner_inst->get_global_inst_id(), core_trace_log::format_instruction_detail_line(*owner_inst)); diff --git a/TOGSim/src/DMA.cc b/TOGSim/src/DMA.cc index 5d509953..ab8e05f1 100644 --- a/TOGSim/src/DMA.cc +++ b/TOGSim/src/DMA.cc @@ -33,7 +33,7 @@ std::shared_ptr> DMA::get_memory_access(cycle_type core_ bool is_cacheable = owner_subgraph->is_cacheable(base_daddr, base_daddr + _dram_req_size); - if (_l2_datacache_enabled) { + if (_l2_datacache_enabled && spdlog::should_log(spdlog::level::trace)) { spdlog::trace( "[{}][Core {}][{}][INST_ID={}] dram=0x{:016x} cacheable={}", core_cycle, @@ -43,6 +43,7 @@ std::shared_ptr> DMA::get_memory_access(cycle_type core_ base_daddr, is_cacheable); } + if (spdlog::should_log(spdlog::level::trace)) spdlog::trace( "[{}][Core {}][{}][INST_ID={}] core_id={} subgraph_id={} numa_id={} addr_name={} is_write={}", core_cycle, From b962934ca2af75c9cd43692e995439f26d9ec756 Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Fri, 10 Jul 2026 21:26:38 +0900 Subject: [PATCH 5/8] [TOGSim] Stop recounting a preload's matmul consumers on every issue scan Core's issue scan walks a tile's ready list every cycle and issues at most one instruction. For a PRELOAD it counted the matmuls subscribed to its ISSUE event by iterating the dependency set -- to size the weight-slot refcount -- before asking whether a weight slot was even free. A preload that stalls waiting for a slot is revisited on the next cycle, and the next, and recounted every time. One preload's ISSUE set can hold hundreds of thousands of matmuls. Measured on an 8x8 conv2d (CM16): that count was 83% of the whole simulation (Core.cc:358-359 plus the std::set iteration inlined into it), and matmul_consumers() was entered 762,064,838 times to perform 25,088 counts. Two fixes: * cache the count on the Instruction -- it is a property of the graph and cannot change once the tile is built; * ask pick_free_weight_sa() first, so a preload that cannot issue costs a compare instead of a walk. Safe to reorder: try_occupy_sram() is a no-op for a preload (the virtual SA-weights buffer is untracked), so nothing has been reserved that would need giving back, and the call count -- hence the round-robin cursor it advances -- is unchanged. CM16 (conv, 803k cycles) 78.5s -> 29.3s (2.80x) MM16 (gemm, 803k cycles) 55.6s -> 38.9s (1.43x) MM36 (gemm, 9.8M cycles) ~1h56m -> 5m51s Cycle counts unchanged: verified on 297 kernels. The scan itself is now the top cost -- a stalled preload is still visited once per cycle. Making the ready list resource-aware is the next step. (cherry picked from commit e3707ffcb1e8703e843defd15236b69e65d9983d) --- TOGSim/include/Instruction.h | 5 +++++ TOGSim/src/Core.cc | 11 ++++++----- TOGSim/src/Instruction.cc | 11 +++++++++++ 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/TOGSim/include/Instruction.h b/TOGSim/include/Instruction.h index 0ce9b268..19ebf34c 100644 --- a/TOGSim/include/Instruction.h +++ b/TOGSim/include/Instruction.h @@ -73,6 +73,10 @@ class Instruction : public std::enable_shared_from_this { void set_tile_group(int g) { _tile_group = g; } int get_tile_group() const { return _tile_group; } bool check_ready() { return ready_counter == 0; } + // This preload's weight-slot refcount: the MATMULs subscribed to its ISSUE event. + // Fixed once the tile is built, but Core's issue scan asks on every stalled cycle + // and one preload can have 300k subscribers -- so count once and cache. + int matmul_consumers(); const Opcode get_opcode() { return opcode; } bool is_dma_read() { return opcode == Opcode::MOVIN; } bool is_dma_write() { return opcode == Opcode::MOVOUT; } @@ -208,4 +212,5 @@ class Instruction : public std::enable_shared_from_this { int _assigned_sa = -1; std::shared_ptr _weight_token; int _tile_group = -1; // trace-only work-item id (see set_tile_group) + int _n_matmul_consumers = -1; // lazily counted; see matmul_consumers() }; \ No newline at end of file diff --git a/TOGSim/src/Core.cc b/TOGSim/src/Core.cc index 48f264ce..8588fede 100644 --- a/TOGSim/src/Core.cc +++ b/TOGSim/src/Core.cc @@ -354,15 +354,16 @@ void Core::cycle() { int sa_idx = -1; if (ct == MATMUL || ct == PRELOAD) { if (ct == PRELOAD) { - int n_consumers = 0; // matmuls reusing this weight - for (auto& c : inst->get_deps(DepEvent::ISSUE)) - if (c->get_compute_type() == MATMUL) n_consumers++; + // Ask for the slot FIRST: with none free the preload cannot issue, + // so nothing else about it is worth computing. Safe to reorder -- + // try_occupy_sram above is a no-op for a preload (weights untracked). + sa_idx = pick_free_weight_sa(); + if (sa_idx < 0) break; // all weight slots full -> stall (retry) + const int n_consumers = inst->matmul_consumers(); // cached if (n_consumers == 0) { // weight-slot model needs >=1 consumer spdlog::error("preload has no matmul consumer (weight-slot model invariant)"); exit(EXIT_FAILURE); } - sa_idx = pick_free_weight_sa(); - if (sa_idx < 0) break; // all weight slots full -> stall (retry) _weight_slots_used[sa_idx]++; auto tok = std::make_shared(WeightToken{sa_idx, n_consumers}); for (auto& c : inst->get_deps(DepEvent::ISSUE)) diff --git a/TOGSim/src/Instruction.cc b/TOGSim/src/Instruction.cc index 718e2f93..0f866de3 100644 --- a/TOGSim/src/Instruction.cc +++ b/TOGSim/src/Instruction.cc @@ -54,6 +54,17 @@ bool DepLess::operator()(const std::shared_ptr& a, return a->get_global_inst_id() < b->get_global_inst_id(); } +int Instruction::matmul_consumers() { + if (_n_matmul_consumers < 0) { + constexpr int kMatmul = 1; // Core's MATMUL compute-unit enum + int n = 0; + for (auto& c : _deps[static_cast(DepEvent::ISSUE)]) + if (c->get_compute_type() == kMatmul) n++; + _n_matmul_consumers = n; + } + return _n_matmul_consumers; +} + void Instruction::finish_instruction() { fire(DepEvent::DONE); // latency consumers finished = true; From 97a9dba985c045d1ab77d4933866debd4eb4db72 Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Mon, 13 Jul 2026 14:20:52 +0900 Subject: [PATCH 6/8] [TOGSim] Add LazyProducer: record dispatches, replay one work-item on demand togsim_dispatch(ctx, fn, iv, n) hands the runtime the work-item's function pointer and its induction variables, and the tile body reads nothing but ctx and iv. So running togsim_kernel records (fn, iv, core) per dispatch WITHOUT running any body, and any single work-item can be replayed later on its own. That is now all togsim_dispatch does: register the work-item and return. There is no mode to select -- LazyProducer::open() runs the kernel once (which only walks its outer loop nest), and run_item(i) replays work-item i, returning its record stream in a reused buffer. No whole-kernel stream is ever materialized. run_producer(), the eager whole-kernel run, becomes every work-item's records concatenated. That is an identical stream: a producer emits nothing outside a dispatch (checked: 0 records). It stays only so its one caller -- the TileGraph builder -- keeps compiling; the next commit switches that caller over and deletes it. --- TOGSim/include/togsim_loader.h | 46 +++++++++++-- TOGSim/src/togsim_runtime.cc | 117 +++++++++++++++++++++++++-------- 2 files changed, 130 insertions(+), 33 deletions(-) diff --git a/TOGSim/include/togsim_loader.h b/TOGSim/include/togsim_loader.h index fef63e54..634fd3e2 100644 --- a/TOGSim/include/togsim_loader.h +++ b/TOGSim/include/togsim_loader.h @@ -1,7 +1,7 @@ #pragma once // togsim_loader.h -- the TOGSim half (not the producer ABI): `dlopen` a producer -// `.so`, run its `togsim_kernel`, record the emitted instructions. The -// "materializing sink" of sec 5.3 / 9.7; the stream goes to togsim_trace_bridge.h. +// `.so`, run its `togsim_kernel`, and record the emitted instructions as +// TraceRecs (sec 5.3 / 9.7). togsim_trace_bridge.h turns them into a TileGraph. #include #include @@ -34,18 +34,52 @@ struct TraceRec { int64_t overlapping; // looked up from the cycle table }; +// Eager whole-kernel run. Transitional: the only caller is the TileGraph builder, +// which switches to LazyProducer in the next commit -- this goes away with it. struct RunResult { bool ok = false; std::vector trace; }; - -// Load `so_path`, run its `togsim_kernel`, and return the recorded trace. -// `tensor_base` gives each tensor argument's DRAM base, `cyc`/`ovl` the cycle table. -// Work-items round-robin only over `partition_cores` (empty/null -> core 0). RunResult run_producer(const char* so_path, const int64_t* shape_args, int32_t n_shape, const uint64_t* tensor_base, int32_t n_tensors, const int64_t* cyc, const int64_t* ovl, int32_t n_tiles, const int32_t* partition_cores, int32_t n_partition_cores); +// The producer is run ON DEMAND, one work-item at a time (materializing the whole +// record stream cost 12.7 GiB on a large 8x8 conv). A tile body reads only ctx and +// iv, so a work-item is (fn, iv, core) -- replayable on its own, whenever needed. +struct WorkItem { + void* fn = nullptr; // togsim_tile_fn + std::vector iv; // the enclosing parallel loop indices + int32_t core = 0; // round-robin binding, fixed when it is registered +}; + +class LazyProducer { + public: + LazyProducer() = default; + ~LazyProducer(); + LazyProducer(const LazyProducer&) = delete; + LazyProducer& operator=(const LazyProducer&) = delete; + + // dlopen the .so and run togsim_kernel once: every togsim_dispatch registers + // its work-item and returns without running the tile body, so num_items() is + // known but no record is emitted yet. + bool open(const char* so_path, const int64_t* shape_args, int32_t n_shape, + const uint64_t* tensor_base, int32_t n_tensors, + const int64_t* cyc, const int64_t* ovl, int32_t n_tiles, + const int32_t* partition_cores, int32_t n_partition_cores); + + size_t num_items() const; + // Replay work-item `i` and return its record stream (TILE_BEGIN, body, + // TILE_END). The returned vector is a buffer reused by the next call. + const std::vector& run_item(size_t i); + + private: + struct EmitCtx* _ctx = nullptr; // opaque; owns the work-item list + void* _lib = nullptr; + std::vector _bases; + std::vector _cyc, _ovl; +}; + } // namespace togsim diff --git a/TOGSim/src/togsim_runtime.cc b/TOGSim/src/togsim_runtime.cc index 1a76231b..c4736ba8 100644 --- a/TOGSim/src/togsim_runtime.cc +++ b/TOGSim/src/togsim_runtime.cc @@ -22,17 +22,22 @@ struct EmitCtx { std::vector cores{0}; // the partition's core ids; dispatch round-robins over these // mutable run state int32_t rr = 0; // round-robin cursor into `cores` - int32_t cur_core = -1; // current work-item's core - std::vector trace; + int32_t cur_core = -1; // core of the work-item being replayed + std::vector items; // every dispatch, recorded by togsim_kernel + std::vector trace; // records of the work-item being replayed }; namespace { +// A fresh TraceRec of kind `k` for core `core` (fields zeroed). inline togsim::TraceRec blank(togsim::TraceRec::Kind k, int32_t core) { togsim::TraceRec r{}; r.kind = k; r.core = core; return r; } +inline void emit_rec(EmitCtx* ctx, const togsim::TraceRec& r) { + ctx->trace.push_back(r); +} } // namespace extern "C" { @@ -40,14 +45,14 @@ extern "C" { int32_t togsim_abi_version(void) { return TOGSIM_ABI_VERSION; } void togsim_dispatch(EmitCtx* ctx, togsim_tile_fn fn, int64_t* iv, int32_t n_iv) { - // Work-item wrapper (sec 9.3): round-robin over THIS partition's cores only -- - // a work-item on another partition's core would sit in this partition's scheduler - // forever. TILE_BEGIN/TILE_END bracket the ops `fn` emits under ctx->cur_core. - ctx->cur_core = ctx->cores.empty() ? 0 - : ctx->cores[ctx->rr++ % (int32_t)ctx->cores.size()]; - ctx->trace.push_back(blank(togsim::TraceRec::TILE_BEGIN, ctx->cur_core)); - fn(ctx, iv, n_iv); - ctx->trace.push_back(blank(togsim::TraceRec::TILE_END, ctx->cur_core)); + // Register the work-item; LazyProducer::run_item runs its body later, on demand. + // Round-robin over THIS partition's cores only -- a work-item on another + // partition's core would sit in this partition's scheduler forever. + togsim::WorkItem w; + w.fn = (void*)fn; + w.core = ctx->cores.empty() ? 0 : ctx->cores[ctx->rr++ % (int32_t)ctx->cores.size()]; + if (iv && n_iv > 0) w.iv.assign(iv, iv + n_iv); + ctx->items.push_back(std::move(w)); } void togsim_dma(EmitCtx* ctx, int32_t dir, int32_t arg_id, @@ -62,13 +67,17 @@ void togsim_dma(EmitCtx* ctx, int32_t dir, int32_t arg_id, togsim::TraceRec r = blank(togsim::TraceRec::DMA, ctx->cur_core); r.dir = dir; r.arg_id = arg_id; r.elem_bits = elem_bits; r.is_async = is_async; r.addr = addr; r.tag_id = tag_id; r.tag_slot = tag_slot; + if (dims) r.dims.reserve(ndim); + if (strides) r.strides.reserve(ndim); + r.read_bufs.reserve(n_read); + r.write_bufs.reserve(n_write); for (int32_t i = 0; i < ndim; ++i) { if (dims) r.dims.push_back(dims[i]); if (strides) r.strides.push_back(strides[i]); } for (int32_t i = 0; i < n_read; ++i) r.read_bufs.push_back(read_bufs[i]); for (int32_t i = 0; i < n_write; ++i) r.write_bufs.push_back(write_bufs[i]); - ctx->trace.push_back(r); + emit_rec(ctx, r); } void togsim_compute(EmitCtx* ctx, uint64_t tile_id, int32_t compute_type, @@ -79,46 +88,100 @@ void togsim_compute(EmitCtx* ctx, uint64_t tile_id, int32_t compute_type, togsim::TraceRec r = blank(togsim::TraceRec::COMPUTE, ctx->cur_core); r.tile_id = tile_id; r.compute_type = compute_type; + r.read_bufs.reserve(n_read); + r.write_bufs.reserve(n_write); for (int32_t i = 0; i < n_read; ++i) r.read_bufs.push_back(read_bufs[i]); for (int32_t i = 0; i < n_write; ++i) r.write_bufs.push_back(write_bufs[i]); if (ctx->cyc && (int32_t)tile_id < ctx->n_tiles) r.cycle = ctx->cyc[tile_id]; if (ctx->ovl && (int32_t)tile_id < ctx->n_tiles) r.overlapping = ctx->ovl[tile_id]; - ctx->trace.push_back(r); + emit_rec(ctx, r); } void togsim_memory_barrier(EmitCtx* ctx, int32_t tag_id, uint64_t tag_slot, const int64_t* write_bufs, int32_t n_write) { togsim::TraceRec r = blank(togsim::TraceRec::MEMORY_BAR, ctx->cur_core); r.tag_id = tag_id; r.tag_slot = tag_slot; + r.write_bufs.reserve(n_write); for (int32_t i = 0; i < n_write; ++i) r.write_bufs.push_back(write_bufs[i]); - ctx->trace.push_back(r); + emit_rec(ctx, r); } } // extern "C" namespace togsim { +namespace { +void init_ctx(EmitCtx& ctx, const uint64_t* tensor_base, int32_t n_tensors, + const int64_t* cyc, const int64_t* ovl, int32_t n_tiles, + const int32_t* partition_cores, int32_t n_partition_cores) { + ctx.tensor_base = tensor_base; ctx.n_tensors = n_tensors; + ctx.cyc = cyc; ctx.ovl = ovl; ctx.n_tiles = n_tiles; + ctx.cores.assign(partition_cores, partition_cores + (n_partition_cores > 0 ? n_partition_cores : 0)); + if (ctx.cores.empty()) ctx.cores.push_back(0); +} +} // namespace + +LazyProducer::~LazyProducer() { + delete _ctx; // _lib intentionally left open: the tile fns must stay callable +} + +bool LazyProducer::open(const char* so_path, const int64_t* shape_args, int32_t n_shape, + const uint64_t* tensor_base, int32_t n_tensors, + const int64_t* cyc, const int64_t* ovl, int32_t n_tiles, + const int32_t* partition_cores, int32_t n_partition_cores) { + // own the tables: the caller's buffers are its locals, and the tile fns are + // invoked long after it returns. + if (tensor_base && n_tensors > 0) _bases.assign(tensor_base, tensor_base + n_tensors); + if (cyc && n_tiles > 0) _cyc.assign(cyc, cyc + n_tiles); + if (ovl && n_tiles > 0) _ovl.assign(ovl, ovl + n_tiles); + + _ctx = new EmitCtx(); + init_ctx(*_ctx, _bases.empty() ? nullptr : _bases.data(), (int32_t)_bases.size(), + _cyc.empty() ? nullptr : _cyc.data(), _ovl.empty() ? nullptr : _ovl.data(), + (int32_t)_cyc.size(), partition_cores, n_partition_cores); + + _lib = dlopen(so_path, RTLD_NOW | RTLD_GLOBAL); + if (!_lib) { fprintf(stderr, "togsim: dlopen failed: %s\n", dlerror()); return false; } + auto kernel = (void (*)(EmitCtx*, int64_t*, int32_t))dlsym(_lib, "togsim_kernel"); + if (!kernel) { fprintf(stderr, "togsim: dlsym togsim_kernel failed: %s\n", dlerror()); return false; } + + // Running the kernel only walks its outer loop nest: every togsim_dispatch + // registers a work-item and returns, so no tile body runs and no record is + // emitted yet. run_item() replays an individual work-item later. + kernel(_ctx, (int64_t*)shape_args, n_shape); + return true; +} + +size_t LazyProducer::num_items() const { return _ctx->items.size(); } + +const std::vector& LazyProducer::run_item(size_t i) { + _ctx->trace.clear(); + if (i >= _ctx->items.size()) return _ctx->trace; + WorkItem& w = _ctx->items[i]; + _ctx->cur_core = w.core; // the binding fixed when registered + emit_rec(_ctx, blank(TraceRec::TILE_BEGIN, w.core)); + ((togsim_tile_fn)w.fn)(_ctx, w.iv.empty() ? nullptr : w.iv.data(), (int32_t)w.iv.size()); + emit_rec(_ctx, blank(TraceRec::TILE_END, w.core)); + return _ctx->trace; // one work-item's records; overwritten by the next call +} + +// The eager whole-kernel trace, now just every work-item's records concatenated. +// Transitional (togsim_loader.h): its one caller switches to LazyProducer next. RunResult run_producer(const char* so_path, const int64_t* shape_args, int32_t n_shape, const uint64_t* tensor_base, int32_t n_tensors, const int64_t* cyc, const int64_t* ovl, int32_t n_tiles, const int32_t* partition_cores, int32_t n_partition_cores) { RunResult res; - void* lib = dlopen(so_path, RTLD_NOW | RTLD_GLOBAL); - if (!lib) { fprintf(stderr, "togsim: dlopen failed: %s\n", dlerror()); return res; } - auto emit = (void (*)(EmitCtx*, int64_t*, int32_t))dlsym(lib, "togsim_kernel"); - if (!emit) { fprintf(stderr, "togsim: dlsym togsim_kernel failed: %s\n", dlerror()); return res; } - - EmitCtx ctx; - ctx.tensor_base = tensor_base; ctx.n_tensors = n_tensors; - ctx.cyc = cyc; ctx.ovl = ovl; ctx.n_tiles = n_tiles; - ctx.cores.assign(partition_cores, partition_cores + (n_partition_cores > 0 ? n_partition_cores : 0)); - if (ctx.cores.empty()) ctx.cores.push_back(0); - emit(&ctx, (int64_t*)shape_args, n_shape); - + LazyProducer p; + if (!p.open(so_path, shape_args, n_shape, tensor_base, n_tensors, + cyc, ovl, n_tiles, partition_cores, n_partition_cores)) + return res; + for (size_t i = 0; i < p.num_items(); i++) { + const std::vector& recs = p.run_item(i); + res.trace.insert(res.trace.end(), recs.begin(), recs.end()); + } res.ok = true; - res.trace = std::move(ctx.trace); return res; } - } // namespace togsim From 700d9147f6eeccbc2db584037e84d4a998a61850 Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Mon, 13 Jul 2026 14:22:19 +0900 Subject: [PATCH 7/8] [TOGSim] Build the TileGraph one dispatch tile at a time The builder materialized every dispatch tile up front and handed the finished graph to the Simulator, even though a Core only ever holds one or two tiles (Core::max_concurrent). Peak memory scaled with the dispatch count: an 8x8 conv2d(x[2,128,14,14], w[512,128,7,7], pad=3) -- 49 dispatch tiles, 17.7M instructions, 37.0M dependency edges -- needed 12.7 GiB and was SIGKILLed in CI, while the identical torch.mm needed 1.8 GiB. Build a tile only when a Core asks for work: * trace_to_tilegraph now takes the producer .so directly. BuildState::index() records each dispatch as a WorkItem and collects buffer footprints; it makes no Instruction. * BuildState::build_one_tile() materializes the next work-item's tile on demand, wired to the graph via TileGraph::set_tile_source. A finished subgraph is dropped instead of parked in a keep-everything vector, which is what releases a tile's Instructions once the Core is done. * sram_schedule() precomputes each buffer version's last reader in a pass that allocates nothing, so the builder no longer retains every reader of every version to end-of-stream (7.5M shared_ptrs on the conv) just to free them. Sound because a dispatch tile is dependency-closed: flush() resets writers/seeds/tag maps at every tile boundary, so no dependency edge crosses tiles (measured: 0 of 37.0M). Buffer versions do cross tiles and their bookkeeping is kept; only where it lives moved. Peak graph-build memory: MM16 232->71 MiB, MM36 1.83 GiB->477 MiB (3.9x), CI36 12.7 GiB->537 MiB (24.3x), and it no longer grows with the dispatch count. Cycle counts unchanged (verified on the 2-core 2-partition config and others). Deletes run_producer()/RunResult with its last caller: nothing materializes a whole-kernel record stream any more. --- TOGSim/include/TileGraph.h | 14 +- TOGSim/include/togsim_loader.h | 12 - TOGSim/include/togsim_runtime.h | 4 +- TOGSim/include/togsim_trace_bridge.h | 15 +- TOGSim/src/TileGraph.cc | 37 ++- TOGSim/src/main.cc | 11 +- TOGSim/src/togsim_runtime.cc | 20 -- TOGSim/src/togsim_trace_bridge.cc | 355 +++++++++++++++++---------- 8 files changed, 280 insertions(+), 188 deletions(-) diff --git a/TOGSim/include/TileGraph.h b/TOGSim/include/TileGraph.h index 4cad9355..1df9460e 100644 --- a/TOGSim/include/TileGraph.h +++ b/TOGSim/include/TileGraph.h @@ -4,6 +4,7 @@ #include #include #include +#include #include "Tile.h" #include "IntervalTree.h" @@ -39,7 +40,16 @@ class TileGraph { public: TileGraph(std::string path, std::string name) : _path(path), _name(name), _subgraph_vec(), _cpu_graph_map() {} void append_subgraph(std::shared_ptr subgraph); + + // On-demand construction: `_pull` materializes the NEXT dispatch tile, or returns + // nullptr once the producer is exhausted. allocate_subgraph calls it only when a + // core needs work, so peak memory is O(tiles in flight), not O(dispatches). + void set_tile_source(std::function()> pull) { + _pull = std::move(pull); + } + bool empty(int core_id) { + if (_pull && !_exhausted) return false; // more tiles can still be made if (_vec_index != _subgraph_vec.size()) { return false; } @@ -132,13 +142,15 @@ class TileGraph { private: int _vec_index=0; + std::function()> _pull; // empty -> legacy + bool _exhausted = false; + bool refill(); std::string _path; std::string _name = "?"; unsigned int _kernel_id = 0; std::vector _loop_index_list; std::vector> _ranges; std::vector> _subgraph_vec; - std::vector> _finished_subgraph_vec; std::map>> _cpu_graph_map; std::shared_ptr> _cache_plan; cycle_type _arrival_time; diff --git a/TOGSim/include/togsim_loader.h b/TOGSim/include/togsim_loader.h index 634fd3e2..4600926b 100644 --- a/TOGSim/include/togsim_loader.h +++ b/TOGSim/include/togsim_loader.h @@ -34,18 +34,6 @@ struct TraceRec { int64_t overlapping; // looked up from the cycle table }; -// Eager whole-kernel run. Transitional: the only caller is the TileGraph builder, -// which switches to LazyProducer in the next commit -- this goes away with it. -struct RunResult { - bool ok = false; - std::vector trace; -}; -RunResult run_producer(const char* so_path, - const int64_t* shape_args, int32_t n_shape, - const uint64_t* tensor_base, int32_t n_tensors, - const int64_t* cyc, const int64_t* ovl, int32_t n_tiles, - const int32_t* partition_cores, int32_t n_partition_cores); - // The producer is run ON DEMAND, one work-item at a time (materializing the whole // record stream cost 12.7 GiB on a large 8x8 conv). A tile body reads only ctx and // iv, so a work-item is (fn, iv, core) -- replayable on its own, whenever needed. diff --git a/TOGSim/include/togsim_runtime.h b/TOGSim/include/togsim_runtime.h index c3183926..cac17b17 100644 --- a/TOGSim/include/togsim_runtime.h +++ b/TOGSim/include/togsim_runtime.h @@ -14,8 +14,8 @@ extern "C" { #define TOGSIM_ABI_VERSION 12 int32_t togsim_abi_version(void); -// Opaque per-invocation context owned by TOGSim. Holds the record sink and the -// tile_id->cycle lookup. Never dereferenced by the producer. +// Opaque per-invocation context owned by TOGSim. Holds the recorded trace and +// the tile_id->cycle lookup. Never dereferenced by the producer. typedef struct EmitCtx EmitCtx; // Direction for togsim_dma. diff --git a/TOGSim/include/togsim_trace_bridge.h b/TOGSim/include/togsim_trace_bridge.h index 212cd4d6..c105fcd0 100644 --- a/TOGSim/include/togsim_trace_bridge.h +++ b/TOGSim/include/togsim_trace_bridge.h @@ -7,6 +7,15 @@ #include "TileGraph.h" #include "togsim_loader.h" -// Build a TileGraph from a recorded trace. `name` labels the graph. -std::unique_ptr trace_to_tilegraph(const togsim::RunResult& run, - const std::string& name); +// Build a TileGraph straight from the trace producer `so_path`. The graph is built +// ON DEMAND, one togsim_dispatch work-item at a time (see togsim_loader.h): peak +// memory is O(tiles in flight), not O(dispatches). Sound because a tile is +// dependency-closed; SRAM buffer VERSIONS do cross tiles though. +// +// `name` labels the graph. Returns nullptr if the producer fails to load or run. +std::unique_ptr trace_to_tilegraph( + const char* so_path, const int64_t* shape_args, int32_t n_shape, + const uint64_t* tensor_base, int32_t n_tensors, + const int64_t* cyc, const int64_t* ovl, int32_t n_tiles, + const int32_t* partition_cores, int32_t n_partition_cores, + const std::string& name); diff --git a/TOGSim/src/TileGraph.cc b/TOGSim/src/TileGraph.cc index 120d49e2..08f8330f 100644 --- a/TOGSim/src/TileGraph.cc +++ b/TOGSim/src/TileGraph.cc @@ -51,8 +51,20 @@ void TileGraph::append_subgraph(std::shared_ptr subgraph) { _subgraph_vec.push_back(std::move(subgraph)); } +// Materialize the next dispatch tile, on demand. Returns false once the producer +// is exhausted. Called only when a core has no work left, so the builder never +// runs ahead of the simulation. +bool TileGraph::refill() { + if (!_pull || _exhausted) return false; + auto sg = _pull(); + if (!sg) { _exhausted = true; return false; } + sg->init_cache_plan(_cache_plan); + _subgraph_vec.push_back(std::move(sg)); + return true; +} + bool TileGraph::is_finished() { - bool finished = _subgraph_vec.empty(); + bool finished = _subgraph_vec.empty() && (!_pull || _exhausted); /* Check all outer loop is allocated */ if (!finished) return finished; @@ -104,18 +116,21 @@ std::shared_ptr TileGraph::get_tile(int core_id, int slot_id) { } void TileGraph::allocate_subgraph(int core_id, int slot_id) { - if (_cpu_graph_map[core_id][slot_id] != nullptr) { - _finished_subgraph_vec.push_back(_cpu_graph_map[core_id][slot_id]); + // Drop the finished subgraph: this releases its Instructions. + if (_cpu_graph_map[core_id][slot_id] != nullptr) _cpu_graph_map[core_id][slot_id] = nullptr; - } - for (auto it = _subgraph_vec.begin(); it != _subgraph_vec.end(); ++it) { - if ((*it)->get_core_id() == -1 || (*it)->get_core_id() == core_id) { - std::shared_ptr subgraph = *it; - _cpu_graph_map[core_id][slot_id] = subgraph; - _subgraph_vec.erase(it); - return; + // Work-items round-robin over the partition's cores, so the next tile for THIS + // core may be a few dispatches ahead: keep pulling until one turns up. + for (;;) { + for (auto it = _subgraph_vec.begin(); it != _subgraph_vec.end(); ++it) { + if ((*it)->get_core_id() == -1 || (*it)->get_core_id() == core_id) { + std::shared_ptr subgraph = *it; + _cpu_graph_map[core_id][slot_id] = subgraph; + _subgraph_vec.erase(it); + return; + } } + if (!refill()) return; // exhausted: leave the slot empty } - return; } \ No newline at end of file diff --git a/TOGSim/src/main.cc b/TOGSim/src/main.cc index 4ccaa4ae..0ef98eff 100644 --- a/TOGSim/src/main.cc +++ b/TOGSim/src/main.cc @@ -40,12 +40,11 @@ std::unique_ptr build_trace_tilegraph(Simulator* simulator, while (ct >> c >> o) { cyc.push_back(c); ovl.push_back(o); } } if (cyc.empty()) { cyc.assign(256, 128); ovl.assign(256, 0); } - auto run = togsim::run_producer(trace_so_path.c_str(), nullptr, 0, - bases.data(), (int)bases.size(), - cyc.data(), ovl.data(), (int)cyc.size(), - partition_cores.data(), (int32_t)partition_cores.size()); - if (!run.ok) return nullptr; - return trace_to_tilegraph(run, "trace_kernel"); + return trace_to_tilegraph(trace_so_path.c_str(), nullptr, 0, + bases.data(), (int)bases.size(), + cyc.data(), ovl.data(), (int)cyc.size(), + partition_cores.data(), (int32_t)partition_cores.size(), + "trace_kernel"); } void launchKernel(Simulator* simulator, unsigned int kernel_id, std::string onnx_path, std::string attribute_path, const YAML::Node& config_yaml, cycle_type request_time=0, int partition_id=0, int device_id=0) { diff --git a/TOGSim/src/togsim_runtime.cc b/TOGSim/src/togsim_runtime.cc index c4736ba8..2c91b212 100644 --- a/TOGSim/src/togsim_runtime.cc +++ b/TOGSim/src/togsim_runtime.cc @@ -164,24 +164,4 @@ const std::vector& LazyProducer::run_item(size_t i) { emit_rec(_ctx, blank(TraceRec::TILE_END, w.core)); return _ctx->trace; // one work-item's records; overwritten by the next call } - -// The eager whole-kernel trace, now just every work-item's records concatenated. -// Transitional (togsim_loader.h): its one caller switches to LazyProducer next. -RunResult run_producer(const char* so_path, - const int64_t* shape_args, int32_t n_shape, - const uint64_t* tensor_base, int32_t n_tensors, - const int64_t* cyc, const int64_t* ovl, int32_t n_tiles, - const int32_t* partition_cores, int32_t n_partition_cores) { - RunResult res; - LazyProducer p; - if (!p.open(so_path, shape_args, n_shape, tensor_base, n_tensors, - cyc, ovl, n_tiles, partition_cores, n_partition_cores)) - return res; - for (size_t i = 0; i < p.num_items(); i++) { - const std::vector& recs = p.run_item(i); - res.trace.insert(res.trace.end(), recs.begin(), recs.end()); - } - res.ok = true; - return res; -} } // namespace togsim diff --git a/TOGSim/src/togsim_trace_bridge.cc b/TOGSim/src/togsim_trace_bridge.cc index 74da75b9..9c25355f 100644 --- a/TOGSim/src/togsim_trace_bridge.cc +++ b/TOGSim/src/togsim_trace_bridge.cc @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -80,50 +81,130 @@ class BufferWriters { std::shared_ptr _initializer; // the non-MATMUL member of _all, if any }; -} // namespace - -std::unique_ptr trace_to_tilegraph(const togsim::RunResult& run, - const std::string& name) { - using togsim::TraceRec; - auto tg = std::make_unique(name, name); - // Empty cache plan (no L2/CMEM persistence) -- append_subgraph propagates it - // to each subgraph, and DMA::is_cacheable dereferences it, so it must be a - // valid (if empty) IntervalTree rather than null. - tg->init_cache_plan({}); +// All builder state for one kernel, driven one dispatch tile at a time. Owned by +// the TileGraph's tile source, so it survives between materializations. link() is +// the dependency DAG, sram_effects() the buffer-version rule, feed() the dispatch. +struct BuildState { + togsim::LazyProducer prod; + size_t next = 0; // next work-item to materialize + std::map buf_bytes; + std::shared_ptr sg_out; // the tile just built std::shared_ptr sg; std::shared_ptr tile; - // The explicit dependency DAG (sec 10): per SRAM buffer, writers(b) is the SET of - // current producers' DONE-handles and readers(b) its consumers. Scoped per - // work-item, so distinct work-items are independent (-> parallel). std::map writers; - // An async dma is paired with its explicit memory_barrier(s) by the runtime tag - // (tag_id, tag_slot). It is 1 load : N barriers (the load happens once per - // reduction iteration; each consumer in that iteration is preceded by a wait on - // the same tag), so we track the CURRENT (most recent) load per (tag_id, - // tag_slot) -- not a FIFO. Each load gets a fresh `uniq` Core key, so successive - // reduction iterations (multi-tile-K, conv) never collide in the tag table; the - // iteration's barriers reuse that load's uniq. Correct because the load nest and - // its consumer nest run in order within the reduction body (no cross-iteration - // prefetch). Scoped per work-item. std::map, std::pair>> current_dma; - // Dedup barriers on the CURRENT load of a (tag_id, tag_slot): a conv reads one - // loaded subtile from many matmuls, so its per-consumer waits collapse to one bar. - // A new load bumps uniq, so a genuine new wait still gets its own bar. std::map, std::pair>> bar_for_load; - int64_t next_tag = 0; // mints a unique Core tag key per dma record - int cur_tile_group = -1; // work-item index, bumped per TILE_BEGIN (trace grouping) - std::set cur_tile_bufs; // distinct spad buffers this tile touches - size_t cur_tile_footprint = 0; // their footprint sum = the tile's resident set + int64_t next_tag = 0; + int cur_tile_group = -1; + std::set cur_tile_bufs; + size_t cur_tile_footprint = 0; + + // SRAM buffer versions, NOT scoped to a work-item: the spad is one physical + // resource, so a buffer reused by the next tile is a new version that waits for + // the old one to free. sram_schedule() precomputes the schedule; see below. + int64_t next_alloc = 0; + std::map cur_alloc; // buf -> current version id + std::map open_ver; // buf -> version still accepting writes + std::vector has_readers; // version -> ever read? + std::vector> last_reader; // version -> (work-item, record) + size_t item = 0, rec = 0; // position of the record being fed + + static size_t rec_bytes(const togsim::TraceRec& t) { // single source of the tile footprint + size_t numel = 1; + for (auto d : t.dims) numel *= (size_t)d; + return numel * (t.elem_bits / 8); + } + + // ---- index + footprint -------------------------------------------------- + // Open the producer, then replay every work-item once to size each buffer: + // buf_bytes must be known before sram_schedule() and before the build. + bool index(const char* so_path, const int64_t* shape_args, int32_t n_shape, + const uint64_t* tensor_base, int32_t n_tensors, + const int64_t* cyc, const int64_t* ovl, int32_t n_tiles, + const int32_t* partition_cores, int32_t n_partition_cores) { + using togsim::TraceRec; + if (!prod.open(so_path, shape_args, n_shape, tensor_base, n_tensors, + cyc, ovl, n_tiles, partition_cores, n_partition_cores)) + return false; + for (size_t i = 0; i < prod.num_items(); i++) + for (const TraceRec& t : prod.run_item(i)) { + if (t.kind != TraceRec::DMA) continue; + const auto& bs = (t.dir == 1) ? t.read_bufs : t.write_bufs; // store reads spad, load writes spad + for (int64_t b : bs) buf_bytes[b] = rec_bytes(t); + } + return true; + } + + // ---- the SRAM buffer-version rule (THE single source of truth) ----------- + // What `t` does to the spad: `reads` consume (and close) a buffer's current + // version, `opens` may start a new one. sram_schedule() and feed() must agree on + // this or version ids drift, so both call it. Apply `reads` before `opens`. + void sram_effects(const togsim::TraceRec& t, + std::vector& reads, + std::vector& opens) const { + using togsim::TraceRec; + reads.clear(); + opens.clear(); + auto has = [](const std::vector& v, int64_t b) { + return std::find(v.begin(), v.end(), b) != v.end(); + }; + if (t.kind == TraceRec::DMA) { + if (t.dir == 1) reads = t.read_bufs; // store drains the spad + else opens = t.write_bufs; // load fills it + } else if (t.kind == TraceRec::COMPUTE) { + // An in-place buffer (read AND written: the accumulator, an in-place + // vector op) is version-transparent -- it neither closes nor opens one. + for (int64_t b : t.read_bufs) + if (!has(t.write_bufs, b)) reads.push_back(b); // consuming reads + for (int64_t b : t.write_bufs) + if (!has(t.read_bufs, b) && buf_bytes.count(b)) // fresh outputs; a + opens.push_back(b); // never-DMA'd buf has + } // no size -> untracked + } - auto flush = [&]() { + // ---- version lifetimes, precomputed (allocates no Instruction) ----------- + // Per buffer version, record whether anything reads it and where its LAST reader + // sits, so feed() can tag that reader as it goes rather than retain every reader. + void sram_schedule() { + using togsim::TraceRec; + int64_t alloc = 0; + std::map cur; // buf -> current version id + std::map open; // buf -> version still accepting writes + std::vector reads, opens; + + for (size_t wi = 0; wi < prod.num_items(); wi++) { + size_t pos = 0; + for (const TraceRec& t : prod.run_item(wi)) { + sram_effects(t, reads, opens); + for (int64_t b : reads) { + auto f = cur.find(b); + if (f == cur.end()) continue; // untracked buffer + has_readers[f->second] = 1; + last_reader[f->second] = {wi, pos}; // keep the LAST one + open[b] = false; // next write starts a new version + } + for (int64_t b : opens) + if (!cur.count(b) || !open[b]) { // a read closed it -> new version + cur[b] = alloc++; + open[b] = true; + has_readers.push_back(0); + last_reader.emplace_back((size_t)-1, (size_t)-1); + } + pos++; + } + } + } + + // ---- per-tile close ----------------------------------------------------- + void flush() { if (sg && tile) { tile->set_spad_footprint(cur_tile_footprint); // distinct-buffer resident set (1- vs 2-dispatch) sg->add_tile(tile); tile->set_owner(sg); - tg->append_subgraph(sg); + sg_out = sg; // hand this tile to the consumer } sg.reset(); tile.reset(); @@ -133,21 +214,21 @@ std::unique_ptr trace_to_tilegraph(const togsim::RunResult& run, cur_tile_bufs.clear(); cur_tile_footprint = 0; next_tag = 0; - }; - - // The single dataflow rule (sec 10.3). READ b: depend on all writers(b), ISSUE - // when both are SA ops else DONE. WRITE b: replace writers(b), except a commutative - // matmul accumulator, which joins them and waits only the initializer. WAR: - // resource models. - auto is_mm_accum = [&](const std::shared_ptr& inst, int64_t b, - const std::vector& writes) { + } + + // ---- dependency DAG (sec 10.3) ------------------------------------------ + // READ b: depend on all writers(b), ISSUE when both are SA ops else DONE. + // WRITE b: replace writers(b), except a commutative matmul accumulator, which + // joins them and waits only the initializer. WAR: resource models. + bool is_mm_accum(const std::shared_ptr& inst, int64_t b, + const std::vector& writes) { if (inst->get_compute_type() != MATMUL_CT) return false; for (int64_t w : writes) if (w == b) return true; return false; - }; - auto link = [&](std::shared_ptr inst, - const std::vector& reads, - const std::vector& writes) { + } + void link(std::shared_ptr inst, + const std::vector& reads, + const std::vector& writes) { for (int64_t b : reads) { if (is_mm_accum(inst, b, writes)) continue; // accumulator read -> handled in WRITE (UNION) auto it = writers.find(b); @@ -171,79 +252,64 @@ std::unique_ptr trace_to_tilegraph(const togsim::RunResult& run, } } tile->append_instuction(inst); - }; - - // SRAM-capacity tracking (sec 10.4): a coarse tile is one version of its buffer, - // freed once all its consumers have issued. NOT reset in flush() -- the spad is a - // physical per-core resource. v1 is single-core; multi-core would key by (core, buf). - int64_t next_alloc = 0; - std::map cur_alloc; // buf -> current version id - std::map open_ver; // buf -> version still accepting writes - struct Ver { std::vector> loads, readers; }; - std::map vers; - // Spad bytes per buffer id, from the DMA records that touch it (a compute output - // takes its footprint from its store). Pre-pass, so it is known before the compute. - auto rec_bytes = [](const TraceRec& t) { // single source of the tile footprint - size_t numel = 1; - for (auto d : t.dims) numel *= (size_t)d; - return numel * (t.elem_bits / 8); - }; - std::map buf_bytes; - for (const auto& t : run.trace) { - if (t.kind != TraceRec::DMA) continue; - const auto& bs = (t.dir == 1) ? t.read_bufs : t.write_bufs; // store reads spad, load writes spad - for (int64_t b : bs) buf_bytes[b] = rec_bytes(t); } - // Add each buffer once to the current tile's footprint (reloads in a K-loop reuse the same id). - auto note_bufs = [&](const std::vector& bufs) { - for (int64_t b : bufs) + + // ---- SRAM-capacity tracking (buffer-version allocations, sec 10.4) ------- + // A coarse tile = one version of its buffer; the fine DMAs filling it share one + // allocation, freed once every consumer has issued. Tracks DMA-loaded buffers and + // compute outputs, but not the virtual SA-weights (weight slots model those). + void note_bufs(const std::vector& bufs) { + for (int64_t b : bufs) // once per buffer: a K-loop reload reuses the same id if (cur_tile_bufs.insert(b).second) { auto it = buf_bytes.find(b); if (it != buf_bytes.end()) cur_tile_footprint += it->second; } - }; - auto sram_on_load = [&](int64_t b, const std::shared_ptr& ld) { - if (!cur_alloc.count(b) || !open_ver[b]) { // a read closed it -> new version - cur_alloc[b] = next_alloc++; - open_ver[b] = true; - vers[cur_alloc[b]] = {}; - } - ld->set_sram_alloc(cur_alloc[b]); - vers[cur_alloc[b]].loads.push_back(ld); - }; - // A compute that freshly produces b opens a version like a load, carrying b's - // footprint; its last reader frees it -- identical lifecycle to a load. - auto sram_on_write = [&](int64_t b, const std::shared_ptr& w) { - auto bb = buf_bytes.find(b); - if (bb == buf_bytes.end()) return; // size unknown (never DMA'd) -> untracked - if (!cur_alloc.count(b) || !open_ver[b]) { // a consuming read closed it -> new version - cur_alloc[b] = next_alloc++; - open_ver[b] = true; - vers[cur_alloc[b]] = {}; - w->set_sram_alloc(cur_alloc[b]); - w->set_sram_footprint(bb->second); - vers[cur_alloc[b]].loads.push_back(w); + } + // a version nothing ever reads is never freed -> leave it untracked + int64_t tracked(int64_t a) { return has_readers[a] ? a : (int64_t)-1; } + + // Run record `t`'s spad effects against the live version state and tag `inst` + // with what it allocates and frees. Same rule as sram_schedule() (sram_effects), + // so the version ids agree; this pass just also touches the Instruction. + void sram_apply(const togsim::TraceRec& t, const std::shared_ptr& inst) { + using togsim::TraceRec; + std::vector reads, opens; + sram_effects(t, reads, opens); + + for (int64_t b : reads) { // consume the current version + auto it = cur_alloc.find(b); + if (it == cur_alloc.end()) continue; // untracked buffer + // Its LAST reader frees it. sram_schedule() already found where that reader + // sits, so tag it here rather than retaining every reader to end-of-stream. + if (last_reader[it->second] == std::make_pair(item, rec)) + inst->add_sram_release(it->second); + open_ver[b] = false; // next write starts a new version } - // already-open version (further producing writes): same physical bytes, no re-add. - }; - auto sram_on_read = [&](int64_t b, const std::shared_ptr& rd) { - auto it = cur_alloc.find(b); - if (it == cur_alloc.end()) return; // not a load buffer -> untracked - vers[it->second].readers.push_back(rd); - open_ver[b] = false; // next write starts a new version - }; - auto sram_finalize = [&]() { // tag only each version's LAST reader - for (auto& kv : vers) { - auto& v = kv.second; - if (v.readers.empty()) { // no consumer -> never freed: untrack - for (auto& ld : v.loads) ld->set_sram_alloc(-1); - continue; + for (int64_t b : opens) { + const bool fresh = !cur_alloc.count(b) || !open_ver[b]; // a read closed it + if (fresh) { + cur_alloc[b] = next_alloc++; + open_ver[b] = true; + } + if (t.kind == TraceRec::DMA) { + // Every fine DMA that fills the buffer carries the version id, so the + // reloads of a reduction's K-loop share one allocation. Its footprint is + // derived from the tile it moves, so nothing to set here. + inst->set_sram_alloc(tracked(cur_alloc[b])); + } else if (fresh) { + // A compute that freshly produces b opens a version like a load and must + // carry b's size explicitly (it was never DMA'd in). Further producing + // writes into an already-open version are the same physical bytes. + inst->set_sram_alloc(tracked(cur_alloc[b])); + inst->set_sram_footprint(buf_bytes.at(b)); } - v.readers.back()->add_sram_release(kv.first); // it frees the whole version on issue } - }; + } - for (const auto& t : run.trace) { + // ---- the record consumer (one record at a time) ------------------------- + void feed(const togsim::TraceRec& t) { + using togsim::TraceRec; + struct RecTick { size_t& r; ~RecTick() { ++r; } } tick{rec}; if (t.kind == TraceRec::TILE_BEGIN) { // togsim_dispatch opened a work-item -> new subgraph (bound to its core) + // tile. The scope runs until the matching TILE_END (the dispatch wrapper @@ -253,13 +319,13 @@ std::unique_ptr trace_to_tilegraph(const togsim::RunResult& run, sg->set_core_id(t.core); tile = std::make_shared(Tile::Status::INITIALIZED); cur_tile_group++; - continue; + return; } if (t.kind == TraceRec::TILE_END) { flush(); // close the work-item explicitly (scope = the tile fn call) - continue; + return; } - if (!tile) continue; // defensive: ops before the first TILE_BEGIN + if (!tile) return; // defensive: ops before the first TILE_BEGIN if (t.kind == TraceRec::DMA) { int64_t uniq = next_tag++; // fresh Core tag key per dma record @@ -270,25 +336,18 @@ std::unique_ptr trace_to_tilegraph(const togsim::RunResult& run, if (t.dir == 1) { // STORE // store reads the result buffer(s) -> link() JOINs all their writers. link(inst, t.read_bufs, t.write_bufs); - for (int64_t b : t.read_bufs) sram_on_read(b, inst); // store frees what it drains } else { // LOAD tile->append_instuction(inst); // async load: the CURRENT load for this (tag_id, tag_slot), with a fresh // uniq its barriers reuse. writers = the dma until its barrier overwrites it, // so consumers gate on arrival. A sync load blocks to arrival itself. if (t.is_async) current_dma[{t.tag_id, t.tag_slot}] = {uniq, inst}; - for (int64_t b : t.write_bufs) { - // No hard WAR edge here: load-buffer reuse (double-buffering, X_spad/ - // W_spad reloaded each reduction iter) is modeled by the SRAM - // version/capacity machinery (sram_on_load), which sizes how many - // versions physically coexist. A latency WAR edge would force - // single-buffering and kill the overlap the spad permits. (The - // accumulator Y is NOT a load buffer -> its cross-tile WAR is handled by - // the REPLACE branch of link() when the next tile's init overwrites it.) - writers[b].replace(inst); - sram_on_load(b, inst); // occupy spad - } + // No hard WAR edge: load-buffer reuse is modeled by the SRAM version / + // capacity machinery (sram_apply), which caps how many versions coexist. A + // latency WAR edge would force single-buffering and kill the spad overlap. + for (int64_t b : t.write_bufs) writers[b].replace(inst); } + sram_apply(t, inst); // a store frees what it drains; a load occupies the spad } else if (t.kind == TraceRec::MEMORY_BAR) { // The explicit async-DMA sync. Pair with the CURRENT load for this (tag_id, // tag_slot), reusing its uniq: the dma releases the bar at issue, the bar parks @@ -302,7 +361,7 @@ std::unique_ptr trace_to_tilegraph(const togsim::RunResult& run, auto bf = bar_for_load.find({t.tag_id, t.tag_slot}); if (bf != bar_for_load.end() && bf->second.first == uniq) { for (int64_t b : t.write_bufs) writers[b].replace(bf->second.second); - continue; + return; } auto bar = make_mem_bar(t, uniq); bar->set_tile_group(cur_tile_group); @@ -317,17 +376,47 @@ std::unique_ptr trace_to_tilegraph(const togsim::RunResult& run, inst->set_tile_group(cur_tile_group); link(inst, t.read_bufs, t.write_bufs); note_bufs(t.read_bufs); note_bufs(t.write_bufs); // distinct-buffer footprint for 1- vs 2-dispatch - // in-place buffers (read AND written) are version-transparent (accumulator, - // in-place vector): skip the self-read and the self-write so footprint is not - // double-counted. read_bufs/write_bufs are tiny, so a linear scan beats a set. - auto in = [](const std::vector& v, int64_t b) { - return std::find(v.begin(), v.end(), b) != v.end(); - }; - for (int64_t b : t.read_bufs) if (!in(t.write_bufs, b)) sram_on_read(b, inst); // consuming reads - for (int64_t b : t.write_bufs) if (!in(t.read_bufs, b)) sram_on_write(b, inst); // fresh outputs + sram_apply(t, inst); // consuming reads free their version; fresh outputs open one } } - flush(); - sram_finalize(); // readers per version are now final -> set each version's refcount + + // ---- materialize exactly one dispatch tile ------------------------------ + // Returns the work-item `next`'s subgraph; nullptr once the producer is + // exhausted. Called from the TileGraph's tile source, on demand. + std::shared_ptr build_one_tile() { + if (next >= prod.num_items()) return nullptr; + item = next; rec = 0; + for (const togsim::TraceRec& t : prod.run_item(next++)) feed(t); + auto out = std::move(sg_out); + sg_out.reset(); + return out; + } +}; + +} // namespace + +std::unique_ptr trace_to_tilegraph( + const char* so_path, const int64_t* shape_args, int32_t n_shape, + const uint64_t* tensor_base, int32_t n_tensors, + const int64_t* cyc, const int64_t* ovl, int32_t n_tiles, + const int32_t* partition_cores, int32_t n_partition_cores, + const std::string& name) { + using togsim::TraceRec; + auto S = std::make_shared(); + + // Index the dispatches (records each work-item's fn/iv/core) and collect each + // buffer's spad size. Builds no Instruction. + if (!S->index(so_path, shape_args, n_shape, tensor_base, n_tensors, + cyc, ovl, n_tiles, partition_cores, n_partition_cores)) + return nullptr; + + S->sram_schedule(); // buffer-version lifetimes, materializing no Instruction + + auto tg = std::make_unique(name, name); + // Empty cache plan (no L2/CMEM persistence) -- the tile source propagates it + // to each subgraph, and DMA::is_cacheable dereferences it, so it must be a + // valid (if empty) IntervalTree rather than null. + tg->init_cache_plan({}); + tg->set_tile_source([S]() { return S->build_one_tile(); }); return tg; } From bb2080a677ada55d6ab122b0f466585e90a1f7f2 Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Mon, 13 Jul 2026 14:22:36 +0900 Subject: [PATCH 8/8] [TOGSim] Stop reallocating the vectors behind every emitted instruction Instruction's constructor took its five vectors by value and copied them into the members, allocating a second buffer per vector for every DMA and barrier. And prepare_tag_key() pushed two elements onto an empty vector, allocating at capacity 1 and immediately reallocating at 2. Move instead of copy, and reserve the exact tag-key size. These are millions of allocations on the graph-build path that never had to happen. No behaviour change. --- TOGSim/src/Instruction.cc | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/TOGSim/src/Instruction.cc b/TOGSim/src/Instruction.cc index 0f866de3..60375763 100644 --- a/TOGSim/src/Instruction.cc +++ b/TOGSim/src/Instruction.cc @@ -1,5 +1,7 @@ #include "Instruction.h" +#include + #include uint64_t Instruction::_next_global_inst_id = 0; @@ -32,14 +34,16 @@ Instruction::Instruction(Opcode opcode, cycle_type compute_cycle, size_t num_par addr_type dram_addr, std::vector tile_size, std::vector tile_stride, size_t elem_bits, std::vector tag_idx_list, std::vector tag_stride_list, std::vector accum_tag_idx_list) + // The vectors are taken by value, so move them into the members: copying them + // allocated a second buffer per vector for every DMA and barrier instruction. : opcode(opcode), compute_cycle(compute_cycle), ready_counter(num_parents), dram_addr(dram_addr), - tile_size(tile_size), tile_stride(tile_stride), _elem_bits(elem_bits), - _tag_idx_list(tag_idx_list), _tag_stride_list(tag_stride_list), - _accum_tag_idx_list(accum_tag_idx_list) { + tile_size(std::move(tile_size)), tile_stride(std::move(tile_stride)), _elem_bits(elem_bits), + _tag_idx_list(std::move(tag_idx_list)), _tag_stride_list(std::move(tag_stride_list)), + _accum_tag_idx_list(std::move(accum_tag_idx_list)) { _global_inst_id = _next_global_inst_id++; assert(_tag_idx_list.size()==_tag_stride_list.size()); _tile_numel = 1; - for (auto dim : tile_size) + for (auto dim : this->tile_size) // the parameter was moved from _tile_numel *= dim; } @@ -82,6 +86,9 @@ void Instruction::dec_waiting_request() { void Instruction::prepare_tag_key() { /* Calculate tag key */ int64_t key_offset = 0; + // exact size: the two unconditional pushes otherwise grow 0 -> 1 -> 2, i.e. an + // allocate + reallocate + free for every DMA and barrier. + _tag_key.reserve(2 + _accum_tag_idx_list.size()); _tag_key.push_back(_addr_id); for (size_t i = 0; i < _tag_idx_list.size(); i++) key_offset += _tag_idx_list.at(i) * _tag_stride_list.at(i);