diff --git a/TOGSim/include/CoreTraceLog.h b/TOGSim/include/CoreTraceLog.h index 2ce51012..47a3e6f8 100644 --- a/TOGSim/include/CoreTraceLog.h +++ b/TOGSim/include/CoreTraceLog.h @@ -3,6 +3,8 @@ #include #include +#include + #include "Instruction.h" #include "TraceLogTags.h" @@ -12,6 +14,14 @@ */ namespace core_trace_log { +/** Whether the instruction trace is actually being emitted. + * + * The trace_* helpers below take formatted strings, so their arguments are + * built whether or not spdlog will keep them: every issued instruction paid two + * std::string constructions (fmt::format over its tile dims/strides) that + * spdlog::trace then dropped. Guard the call sites with this. */ +inline bool trace_enabled() { return spdlog::should_log(spdlog::level::trace); } + std::string format_dma_inst_issued_detail(Instruction& inst); /** Opcode + (detail...) for DMA issue / skip traces. */ std::string format_dma_inst_issued_trace_line(Instruction& inst); diff --git a/TOGSim/include/Instruction.h b/TOGSim/include/Instruction.h index 74c5e339..19ebf34c 100644 --- a/TOGSim/include/Instruction.h +++ b/TOGSim/include/Instruction.h @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -32,6 +33,15 @@ typedef uint64_t cycle_type; std::string opcode_to_string(Opcode opcode); std::string format_tag_key_list_hex(const std::vector& tag_keys); +class Instruction; +// Order dependents by creation order, NOT by heap address: std::set's +// default comparator sorts by pointer value, which makes fire()'s release order -- +// and with it the issue order and the reported cycle count -- allocator-dependent. +struct DepLess { + bool operator()(const std::shared_ptr& a, + const std::shared_ptr& b) const; +}; + class Instruction : public std::enable_shared_from_this { public: Instruction(Opcode opcode, cycle_type compute_cycle, size_t num_parents, addr_type dram_addr, @@ -51,7 +61,7 @@ class Instruction : public std::enable_shared_from_this { for (auto& c : _deps[static_cast(e)]) c->dec_ready_counter(); _deps[static_cast(e)].clear(); } - const std::set>& get_deps(DepEvent e) { + const std::set, DepLess>& get_deps(DepEvent e) { return _deps[static_cast(e)]; } void set_assigned_sa(int s) { _assigned_sa = s; } @@ -63,6 +73,10 @@ class Instruction : public std::enable_shared_from_this { void set_tile_group(int g) { _tile_group = g; } int get_tile_group() const { return _tile_group; } bool check_ready() { return ready_counter == 0; } + // This preload's weight-slot refcount: the MATMULs subscribed to its ISSUE event. + // Fixed once the tile is built, but Core's issue scan asks on every stalled cycle + // and one preload can have 300k subscribers -- so count once and cache. + int matmul_consumers(); const Opcode get_opcode() { return opcode; } bool is_dma_read() { return opcode == Opcode::MOVIN; } bool is_dma_write() { return opcode == Opcode::MOVOUT; } @@ -130,7 +144,13 @@ class Instruction : public std::enable_shared_from_this { // last reader frees it on issue via `_sram_release_allocs`. void set_sram_alloc(int64_t id) { _sram_alloc_id = id; } int64_t get_sram_alloc() const { return _sram_alloc_id; } - void add_sram_release(int64_t id) { _sram_release_allocs.push_back(id); } + // Keep the list ascending by version id. The Core frees each version regardless + // of order, but the invariant lets a builder that tags versions as they close be + // byte-comparable with one that walks the version map at the end. + void add_sram_release(int64_t id) { + auto& v = _sram_release_allocs; + v.insert(std::upper_bound(v.begin(), v.end(), id), id); + } const std::vector& get_sram_release() const { return _sram_release_allocs; } // bytes this instruction's buffer occupies in the spad. A DMA derives it from // the tile it moves; a compute output gets it set explicitly by the bridge (the @@ -159,8 +179,9 @@ class Instruction : public std::enable_shared_from_this { size_t ready_counter = 0; // parents not yet finished; the minimal Instruction(Opcode) // ctor (barriers) relies on this default + inc_ready_counter // Per-event subscriber sets: _deps[ISSUE] released at issue (occupancy), - // _deps[DONE] at finish (latency). std::set dedups + fixes the release order. - std::array>, + // _deps[DONE] at finish (latency). std::set dedups and, via DepLess, iterates in + // instruction-id order, so the release order does not depend on the allocator. + std::array, DepLess>, static_cast(DepEvent::COUNT)> _deps; std::vector tile_size; std::vector tile_stride; @@ -191,4 +212,5 @@ class Instruction : public std::enable_shared_from_this { int _assigned_sa = -1; std::shared_ptr _weight_token; int _tile_group = -1; // trace-only work-item id (see set_tile_group) + int _n_matmul_consumers = -1; // lazily counted; see matmul_consumers() }; \ No newline at end of file diff --git a/TOGSim/include/TileGraph.h b/TOGSim/include/TileGraph.h index 4cad9355..1df9460e 100644 --- a/TOGSim/include/TileGraph.h +++ b/TOGSim/include/TileGraph.h @@ -4,6 +4,7 @@ #include #include #include +#include #include "Tile.h" #include "IntervalTree.h" @@ -39,7 +40,16 @@ class TileGraph { public: TileGraph(std::string path, std::string name) : _path(path), _name(name), _subgraph_vec(), _cpu_graph_map() {} void append_subgraph(std::shared_ptr subgraph); + + // On-demand construction: `_pull` materializes the NEXT dispatch tile, or returns + // nullptr once the producer is exhausted. allocate_subgraph calls it only when a + // core needs work, so peak memory is O(tiles in flight), not O(dispatches). + void set_tile_source(std::function()> pull) { + _pull = std::move(pull); + } + bool empty(int core_id) { + if (_pull && !_exhausted) return false; // more tiles can still be made if (_vec_index != _subgraph_vec.size()) { return false; } @@ -132,13 +142,15 @@ class TileGraph { private: int _vec_index=0; + std::function()> _pull; // empty -> legacy + bool _exhausted = false; + bool refill(); std::string _path; std::string _name = "?"; unsigned int _kernel_id = 0; std::vector _loop_index_list; std::vector> _ranges; std::vector> _subgraph_vec; - std::vector> _finished_subgraph_vec; std::map>> _cpu_graph_map; std::shared_ptr> _cache_plan; cycle_type _arrival_time; diff --git a/TOGSim/include/togsim_loader.h b/TOGSim/include/togsim_loader.h index fef63e54..4600926b 100644 --- a/TOGSim/include/togsim_loader.h +++ b/TOGSim/include/togsim_loader.h @@ -1,7 +1,7 @@ #pragma once // togsim_loader.h -- the TOGSim half (not the producer ABI): `dlopen` a producer -// `.so`, run its `togsim_kernel`, record the emitted instructions. The -// "materializing sink" of sec 5.3 / 9.7; the stream goes to togsim_trace_bridge.h. +// `.so`, run its `togsim_kernel`, and record the emitted instructions as +// TraceRecs (sec 5.3 / 9.7). togsim_trace_bridge.h turns them into a TileGraph. #include #include @@ -34,18 +34,40 @@ struct TraceRec { int64_t overlapping; // looked up from the cycle table }; -struct RunResult { - bool ok = false; - std::vector trace; +// The producer is run ON DEMAND, one work-item at a time (materializing the whole +// record stream cost 12.7 GiB on a large 8x8 conv). A tile body reads only ctx and +// iv, so a work-item is (fn, iv, core) -- replayable on its own, whenever needed. +struct WorkItem { + void* fn = nullptr; // togsim_tile_fn + std::vector iv; // the enclosing parallel loop indices + int32_t core = 0; // round-robin binding, fixed when it is registered }; -// Load `so_path`, run its `togsim_kernel`, and return the recorded trace. -// `tensor_base` gives each tensor argument's DRAM base, `cyc`/`ovl` the cycle table. -// Work-items round-robin only over `partition_cores` (empty/null -> core 0). -RunResult run_producer(const char* so_path, - const int64_t* shape_args, int32_t n_shape, - const uint64_t* tensor_base, int32_t n_tensors, - const int64_t* cyc, const int64_t* ovl, int32_t n_tiles, - const int32_t* partition_cores, int32_t n_partition_cores); +class LazyProducer { + public: + LazyProducer() = default; + ~LazyProducer(); + LazyProducer(const LazyProducer&) = delete; + LazyProducer& operator=(const LazyProducer&) = delete; + + // dlopen the .so and run togsim_kernel once: every togsim_dispatch registers + // its work-item and returns without running the tile body, so num_items() is + // known but no record is emitted yet. + bool open(const char* so_path, const int64_t* shape_args, int32_t n_shape, + const uint64_t* tensor_base, int32_t n_tensors, + const int64_t* cyc, const int64_t* ovl, int32_t n_tiles, + const int32_t* partition_cores, int32_t n_partition_cores); + + size_t num_items() const; + // Replay work-item `i` and return its record stream (TILE_BEGIN, body, + // TILE_END). The returned vector is a buffer reused by the next call. + const std::vector& run_item(size_t i); + + private: + struct EmitCtx* _ctx = nullptr; // opaque; owns the work-item list + void* _lib = nullptr; + std::vector _bases; + std::vector _cyc, _ovl; +}; } // namespace togsim diff --git a/TOGSim/include/togsim_runtime.h b/TOGSim/include/togsim_runtime.h index c3183926..cac17b17 100644 --- a/TOGSim/include/togsim_runtime.h +++ b/TOGSim/include/togsim_runtime.h @@ -14,8 +14,8 @@ extern "C" { #define TOGSIM_ABI_VERSION 12 int32_t togsim_abi_version(void); -// Opaque per-invocation context owned by TOGSim. Holds the record sink and the -// tile_id->cycle lookup. Never dereferenced by the producer. +// Opaque per-invocation context owned by TOGSim. Holds the recorded trace and +// the tile_id->cycle lookup. Never dereferenced by the producer. typedef struct EmitCtx EmitCtx; // Direction for togsim_dma. diff --git a/TOGSim/include/togsim_trace_bridge.h b/TOGSim/include/togsim_trace_bridge.h index 212cd4d6..c105fcd0 100644 --- a/TOGSim/include/togsim_trace_bridge.h +++ b/TOGSim/include/togsim_trace_bridge.h @@ -7,6 +7,15 @@ #include "TileGraph.h" #include "togsim_loader.h" -// Build a TileGraph from a recorded trace. `name` labels the graph. -std::unique_ptr trace_to_tilegraph(const togsim::RunResult& run, - const std::string& name); +// Build a TileGraph straight from the trace producer `so_path`. The graph is built +// ON DEMAND, one togsim_dispatch work-item at a time (see togsim_loader.h): peak +// memory is O(tiles in flight), not O(dispatches). Sound because a tile is +// dependency-closed; SRAM buffer VERSIONS do cross tiles though. +// +// `name` labels the graph. Returns nullptr if the producer fails to load or run. +std::unique_ptr trace_to_tilegraph( + const char* so_path, const int64_t* shape_args, int32_t n_shape, + const uint64_t* tensor_base, int32_t n_tensors, + const int64_t* cyc, const int64_t* ovl, int32_t n_tiles, + const int32_t* partition_cores, int32_t n_partition_cores, + const std::string& name); diff --git a/TOGSim/src/Core.cc b/TOGSim/src/Core.cc index 6097f287..8588fede 100644 --- a/TOGSim/src/Core.cc +++ b/TOGSim/src/Core.cc @@ -228,7 +228,7 @@ void Core::dma_cycle() { core_trace_log::log_error_dma_instruction_invalid(_core_cycle, _id); exit(EXIT_FAILURE); } else if (finished_inst->get_opcode() == Opcode::MEMORY_BAR) { - core_trace_log::trace_instruction_line(_core_cycle, + if (core_trace_log::trace_enabled()) core_trace_log::trace_instruction_line(_core_cycle, _id, TraceLogTag::pad15(TraceLogTag::kInstructionFinished), finished_inst->get_global_inst_id(), @@ -307,7 +307,7 @@ void Core::cycle() { finish_instruction(inst); else _dma.register_tag_waiter(inst->subgraph_id, key, inst); - core_trace_log::trace_instruction_line(_core_cycle, + if (core_trace_log::trace_enabled()) core_trace_log::trace_instruction_line(_core_cycle, _id, TraceLogTag::pad15( TraceLogTag::kInstructionSkipped), @@ -320,7 +320,7 @@ void Core::cycle() { } else { // load occupies its spad bytes on issue; stall (retry next cycle) if full. if (!try_occupy_sram(inst)) break; - core_trace_log::trace_instruction_line(_core_cycle, + if (core_trace_log::trace_enabled()) core_trace_log::trace_instruction_line(_core_cycle, _id, TraceLogTag::pad15( TraceLogTag::kInstructionIssued), @@ -335,7 +335,7 @@ void Core::cycle() { } case Opcode::MOVOUT: release_sram(inst); // store issued -> free the tiles it drained - core_trace_log::trace_instruction_line(_core_cycle, + if (core_trace_log::trace_enabled()) core_trace_log::trace_instruction_line(_core_cycle, _id, TraceLogTag::pad15(TraceLogTag::kInstructionIssued), inst->get_global_inst_id(), @@ -354,15 +354,16 @@ void Core::cycle() { int sa_idx = -1; if (ct == MATMUL || ct == PRELOAD) { if (ct == PRELOAD) { - int n_consumers = 0; // matmuls reusing this weight - for (auto& c : inst->get_deps(DepEvent::ISSUE)) - if (c->get_compute_type() == MATMUL) n_consumers++; + // Ask for the slot FIRST: with none free the preload cannot issue, + // so nothing else about it is worth computing. Safe to reorder -- + // try_occupy_sram above is a no-op for a preload (weights untracked). + sa_idx = pick_free_weight_sa(); + if (sa_idx < 0) break; // all weight slots full -> stall (retry) + const int n_consumers = inst->matmul_consumers(); // cached if (n_consumers == 0) { // weight-slot model needs >=1 consumer spdlog::error("preload has no matmul consumer (weight-slot model invariant)"); exit(EXIT_FAILURE); } - sa_idx = pick_free_weight_sa(); - if (sa_idx < 0) break; // all weight slots full -> stall (retry) _weight_slots_used[sa_idx]++; auto tok = std::make_shared(WeightToken{sa_idx, n_consumers}); for (auto& c : inst->get_deps(DepEvent::ISSUE)) @@ -411,7 +412,7 @@ void Core::cycle() { continue; // old code fell through to it++ on the // erased (invalidated) iterator -> UB } else { - core_trace_log::trace_instruction_line(_core_cycle, + if (core_trace_log::trace_enabled()) core_trace_log::trace_instruction_line(_core_cycle, _id, TraceLogTag::pad15( TraceLogTag::kInstructionIssued), @@ -443,7 +444,7 @@ void Core::cycle() { } else { _dma.register_tag_waiter(inst->subgraph_id, key, inst); } - core_trace_log::trace_instruction_line(_core_cycle, + if (core_trace_log::trace_enabled()) core_trace_log::trace_instruction_line(_core_cycle, _id, TraceLogTag::pad15( TraceLogTag::kInstructionIssued), @@ -498,7 +499,7 @@ void Core::finish_instruction(std::shared_ptr& inst, InstFinishTrac core_trace_log::log_error_dram_responses_trace_not_finished(_core_cycle, _id); exit(EXIT_FAILURE); } - core_trace_log::trace_instruction_line(_core_cycle, + if (core_trace_log::trace_enabled()) core_trace_log::trace_instruction_line(_core_cycle, _id, TraceLogTag::pad15(TraceLogTag::kAllDramResponsesReceived), inst->get_global_inst_id(), @@ -515,7 +516,7 @@ void Core::finish_instruction(std::shared_ptr& inst, InstFinishTrac const char* trace_tag = (tag == InstFinishTraceTag::DmaIssueComplete) ? TraceLogTag::kAsyncDmaAllRequestsIssued : TraceLogTag::kInstructionFinished; - core_trace_log::trace_instruction_line(_core_cycle, + if (core_trace_log::trace_enabled()) core_trace_log::trace_instruction_line(_core_cycle, _id, TraceLogTag::pad15(trace_tag), inst->get_global_inst_id(), @@ -562,7 +563,7 @@ void Core::push_memory_response(mem_fetch* response) { if (!owner_inst->got_first_response()) { // first data of this load arrived owner_inst->mark_first_response(); - core_trace_log::trace_instruction_line(_core_cycle, _id, + if (core_trace_log::trace_enabled()) core_trace_log::trace_instruction_line(_core_cycle, _id, TraceLogTag::pad15(TraceLogTag::kFirstDramResponse), owner_inst->get_global_inst_id(), core_trace_log::format_instruction_detail_line(*owner_inst)); diff --git a/TOGSim/src/DMA.cc b/TOGSim/src/DMA.cc index 5d509953..ab8e05f1 100644 --- a/TOGSim/src/DMA.cc +++ b/TOGSim/src/DMA.cc @@ -33,7 +33,7 @@ std::shared_ptr> DMA::get_memory_access(cycle_type core_ bool is_cacheable = owner_subgraph->is_cacheable(base_daddr, base_daddr + _dram_req_size); - if (_l2_datacache_enabled) { + if (_l2_datacache_enabled && spdlog::should_log(spdlog::level::trace)) { spdlog::trace( "[{}][Core {}][{}][INST_ID={}] dram=0x{:016x} cacheable={}", core_cycle, @@ -43,6 +43,7 @@ std::shared_ptr> DMA::get_memory_access(cycle_type core_ base_daddr, is_cacheable); } + if (spdlog::should_log(spdlog::level::trace)) spdlog::trace( "[{}][Core {}][{}][INST_ID={}] core_id={} subgraph_id={} numa_id={} addr_name={} is_write={}", core_cycle, diff --git a/TOGSim/src/Instruction.cc b/TOGSim/src/Instruction.cc index ee184a1a..60375763 100644 --- a/TOGSim/src/Instruction.cc +++ b/TOGSim/src/Instruction.cc @@ -1,5 +1,7 @@ #include "Instruction.h" +#include + #include uint64_t Instruction::_next_global_inst_id = 0; @@ -32,14 +34,16 @@ Instruction::Instruction(Opcode opcode, cycle_type compute_cycle, size_t num_par addr_type dram_addr, std::vector tile_size, std::vector tile_stride, size_t elem_bits, std::vector tag_idx_list, std::vector tag_stride_list, std::vector accum_tag_idx_list) + // The vectors are taken by value, so move them into the members: copying them + // allocated a second buffer per vector for every DMA and barrier instruction. : opcode(opcode), compute_cycle(compute_cycle), ready_counter(num_parents), dram_addr(dram_addr), - tile_size(tile_size), tile_stride(tile_stride), _elem_bits(elem_bits), - _tag_idx_list(tag_idx_list), _tag_stride_list(tag_stride_list), - _accum_tag_idx_list(accum_tag_idx_list) { + tile_size(std::move(tile_size)), tile_stride(std::move(tile_stride)), _elem_bits(elem_bits), + _tag_idx_list(std::move(tag_idx_list)), _tag_stride_list(std::move(tag_stride_list)), + _accum_tag_idx_list(std::move(accum_tag_idx_list)) { _global_inst_id = _next_global_inst_id++; assert(_tag_idx_list.size()==_tag_stride_list.size()); _tile_numel = 1; - for (auto dim : tile_size) + for (auto dim : this->tile_size) // the parameter was moved from _tile_numel *= dim; } @@ -49,6 +53,22 @@ Instruction::Instruction(Opcode opcode) _tile_numel = 1; } +bool DepLess::operator()(const std::shared_ptr& a, + const std::shared_ptr& b) const { + return a->get_global_inst_id() < b->get_global_inst_id(); +} + +int Instruction::matmul_consumers() { + if (_n_matmul_consumers < 0) { + constexpr int kMatmul = 1; // Core's MATMUL compute-unit enum + int n = 0; + for (auto& c : _deps[static_cast(DepEvent::ISSUE)]) + if (c->get_compute_type() == kMatmul) n++; + _n_matmul_consumers = n; + } + return _n_matmul_consumers; +} + void Instruction::finish_instruction() { fire(DepEvent::DONE); // latency consumers finished = true; @@ -66,6 +86,9 @@ void Instruction::dec_waiting_request() { void Instruction::prepare_tag_key() { /* Calculate tag key */ int64_t key_offset = 0; + // exact size: the two unconditional pushes otherwise grow 0 -> 1 -> 2, i.e. an + // allocate + reallocate + free for every DMA and barrier. + _tag_key.reserve(2 + _accum_tag_idx_list.size()); _tag_key.push_back(_addr_id); for (size_t i = 0; i < _tag_idx_list.size(); i++) key_offset += _tag_idx_list.at(i) * _tag_stride_list.at(i); diff --git a/TOGSim/src/TileGraph.cc b/TOGSim/src/TileGraph.cc index 120d49e2..08f8330f 100644 --- a/TOGSim/src/TileGraph.cc +++ b/TOGSim/src/TileGraph.cc @@ -51,8 +51,20 @@ void TileGraph::append_subgraph(std::shared_ptr subgraph) { _subgraph_vec.push_back(std::move(subgraph)); } +// Materialize the next dispatch tile, on demand. Returns false once the producer +// is exhausted. Called only when a core has no work left, so the builder never +// runs ahead of the simulation. +bool TileGraph::refill() { + if (!_pull || _exhausted) return false; + auto sg = _pull(); + if (!sg) { _exhausted = true; return false; } + sg->init_cache_plan(_cache_plan); + _subgraph_vec.push_back(std::move(sg)); + return true; +} + bool TileGraph::is_finished() { - bool finished = _subgraph_vec.empty(); + bool finished = _subgraph_vec.empty() && (!_pull || _exhausted); /* Check all outer loop is allocated */ if (!finished) return finished; @@ -104,18 +116,21 @@ std::shared_ptr TileGraph::get_tile(int core_id, int slot_id) { } void TileGraph::allocate_subgraph(int core_id, int slot_id) { - if (_cpu_graph_map[core_id][slot_id] != nullptr) { - _finished_subgraph_vec.push_back(_cpu_graph_map[core_id][slot_id]); + // Drop the finished subgraph: this releases its Instructions. + if (_cpu_graph_map[core_id][slot_id] != nullptr) _cpu_graph_map[core_id][slot_id] = nullptr; - } - for (auto it = _subgraph_vec.begin(); it != _subgraph_vec.end(); ++it) { - if ((*it)->get_core_id() == -1 || (*it)->get_core_id() == core_id) { - std::shared_ptr subgraph = *it; - _cpu_graph_map[core_id][slot_id] = subgraph; - _subgraph_vec.erase(it); - return; + // Work-items round-robin over the partition's cores, so the next tile for THIS + // core may be a few dispatches ahead: keep pulling until one turns up. + for (;;) { + for (auto it = _subgraph_vec.begin(); it != _subgraph_vec.end(); ++it) { + if ((*it)->get_core_id() == -1 || (*it)->get_core_id() == core_id) { + std::shared_ptr subgraph = *it; + _cpu_graph_map[core_id][slot_id] = subgraph; + _subgraph_vec.erase(it); + return; + } } + if (!refill()) return; // exhausted: leave the slot empty } - return; } \ No newline at end of file diff --git a/TOGSim/src/main.cc b/TOGSim/src/main.cc index 4ccaa4ae..0ef98eff 100644 --- a/TOGSim/src/main.cc +++ b/TOGSim/src/main.cc @@ -40,12 +40,11 @@ std::unique_ptr build_trace_tilegraph(Simulator* simulator, while (ct >> c >> o) { cyc.push_back(c); ovl.push_back(o); } } if (cyc.empty()) { cyc.assign(256, 128); ovl.assign(256, 0); } - auto run = togsim::run_producer(trace_so_path.c_str(), nullptr, 0, - bases.data(), (int)bases.size(), - cyc.data(), ovl.data(), (int)cyc.size(), - partition_cores.data(), (int32_t)partition_cores.size()); - if (!run.ok) return nullptr; - return trace_to_tilegraph(run, "trace_kernel"); + return trace_to_tilegraph(trace_so_path.c_str(), nullptr, 0, + bases.data(), (int)bases.size(), + cyc.data(), ovl.data(), (int)cyc.size(), + partition_cores.data(), (int32_t)partition_cores.size(), + "trace_kernel"); } void launchKernel(Simulator* simulator, unsigned int kernel_id, std::string onnx_path, std::string attribute_path, const YAML::Node& config_yaml, cycle_type request_time=0, int partition_id=0, int device_id=0) { diff --git a/TOGSim/src/togsim_runtime.cc b/TOGSim/src/togsim_runtime.cc index 1a76231b..2c91b212 100644 --- a/TOGSim/src/togsim_runtime.cc +++ b/TOGSim/src/togsim_runtime.cc @@ -22,17 +22,22 @@ struct EmitCtx { std::vector cores{0}; // the partition's core ids; dispatch round-robins over these // mutable run state int32_t rr = 0; // round-robin cursor into `cores` - int32_t cur_core = -1; // current work-item's core - std::vector trace; + int32_t cur_core = -1; // core of the work-item being replayed + std::vector items; // every dispatch, recorded by togsim_kernel + std::vector trace; // records of the work-item being replayed }; namespace { +// A fresh TraceRec of kind `k` for core `core` (fields zeroed). inline togsim::TraceRec blank(togsim::TraceRec::Kind k, int32_t core) { togsim::TraceRec r{}; r.kind = k; r.core = core; return r; } +inline void emit_rec(EmitCtx* ctx, const togsim::TraceRec& r) { + ctx->trace.push_back(r); +} } // namespace extern "C" { @@ -40,14 +45,14 @@ extern "C" { int32_t togsim_abi_version(void) { return TOGSIM_ABI_VERSION; } void togsim_dispatch(EmitCtx* ctx, togsim_tile_fn fn, int64_t* iv, int32_t n_iv) { - // Work-item wrapper (sec 9.3): round-robin over THIS partition's cores only -- - // a work-item on another partition's core would sit in this partition's scheduler - // forever. TILE_BEGIN/TILE_END bracket the ops `fn` emits under ctx->cur_core. - ctx->cur_core = ctx->cores.empty() ? 0 - : ctx->cores[ctx->rr++ % (int32_t)ctx->cores.size()]; - ctx->trace.push_back(blank(togsim::TraceRec::TILE_BEGIN, ctx->cur_core)); - fn(ctx, iv, n_iv); - ctx->trace.push_back(blank(togsim::TraceRec::TILE_END, ctx->cur_core)); + // Register the work-item; LazyProducer::run_item runs its body later, on demand. + // Round-robin over THIS partition's cores only -- a work-item on another + // partition's core would sit in this partition's scheduler forever. + togsim::WorkItem w; + w.fn = (void*)fn; + w.core = ctx->cores.empty() ? 0 : ctx->cores[ctx->rr++ % (int32_t)ctx->cores.size()]; + if (iv && n_iv > 0) w.iv.assign(iv, iv + n_iv); + ctx->items.push_back(std::move(w)); } void togsim_dma(EmitCtx* ctx, int32_t dir, int32_t arg_id, @@ -62,13 +67,17 @@ void togsim_dma(EmitCtx* ctx, int32_t dir, int32_t arg_id, togsim::TraceRec r = blank(togsim::TraceRec::DMA, ctx->cur_core); r.dir = dir; r.arg_id = arg_id; r.elem_bits = elem_bits; r.is_async = is_async; r.addr = addr; r.tag_id = tag_id; r.tag_slot = tag_slot; + if (dims) r.dims.reserve(ndim); + if (strides) r.strides.reserve(ndim); + r.read_bufs.reserve(n_read); + r.write_bufs.reserve(n_write); for (int32_t i = 0; i < ndim; ++i) { if (dims) r.dims.push_back(dims[i]); if (strides) r.strides.push_back(strides[i]); } for (int32_t i = 0; i < n_read; ++i) r.read_bufs.push_back(read_bufs[i]); for (int32_t i = 0; i < n_write; ++i) r.write_bufs.push_back(write_bufs[i]); - ctx->trace.push_back(r); + emit_rec(ctx, r); } void togsim_compute(EmitCtx* ctx, uint64_t tile_id, int32_t compute_type, @@ -79,46 +88,80 @@ void togsim_compute(EmitCtx* ctx, uint64_t tile_id, int32_t compute_type, togsim::TraceRec r = blank(togsim::TraceRec::COMPUTE, ctx->cur_core); r.tile_id = tile_id; r.compute_type = compute_type; + r.read_bufs.reserve(n_read); + r.write_bufs.reserve(n_write); for (int32_t i = 0; i < n_read; ++i) r.read_bufs.push_back(read_bufs[i]); for (int32_t i = 0; i < n_write; ++i) r.write_bufs.push_back(write_bufs[i]); if (ctx->cyc && (int32_t)tile_id < ctx->n_tiles) r.cycle = ctx->cyc[tile_id]; if (ctx->ovl && (int32_t)tile_id < ctx->n_tiles) r.overlapping = ctx->ovl[tile_id]; - ctx->trace.push_back(r); + emit_rec(ctx, r); } void togsim_memory_barrier(EmitCtx* ctx, int32_t tag_id, uint64_t tag_slot, const int64_t* write_bufs, int32_t n_write) { togsim::TraceRec r = blank(togsim::TraceRec::MEMORY_BAR, ctx->cur_core); r.tag_id = tag_id; r.tag_slot = tag_slot; + r.write_bufs.reserve(n_write); for (int32_t i = 0; i < n_write; ++i) r.write_bufs.push_back(write_bufs[i]); - ctx->trace.push_back(r); + emit_rec(ctx, r); } } // extern "C" namespace togsim { -RunResult run_producer(const char* so_path, - const int64_t* shape_args, int32_t n_shape, - const uint64_t* tensor_base, int32_t n_tensors, - const int64_t* cyc, const int64_t* ovl, int32_t n_tiles, - const int32_t* partition_cores, int32_t n_partition_cores) { - RunResult res; - void* lib = dlopen(so_path, RTLD_NOW | RTLD_GLOBAL); - if (!lib) { fprintf(stderr, "togsim: dlopen failed: %s\n", dlerror()); return res; } - auto emit = (void (*)(EmitCtx*, int64_t*, int32_t))dlsym(lib, "togsim_kernel"); - if (!emit) { fprintf(stderr, "togsim: dlsym togsim_kernel failed: %s\n", dlerror()); return res; } - - EmitCtx ctx; +namespace { +void init_ctx(EmitCtx& ctx, const uint64_t* tensor_base, int32_t n_tensors, + const int64_t* cyc, const int64_t* ovl, int32_t n_tiles, + const int32_t* partition_cores, int32_t n_partition_cores) { ctx.tensor_base = tensor_base; ctx.n_tensors = n_tensors; ctx.cyc = cyc; ctx.ovl = ovl; ctx.n_tiles = n_tiles; ctx.cores.assign(partition_cores, partition_cores + (n_partition_cores > 0 ? n_partition_cores : 0)); if (ctx.cores.empty()) ctx.cores.push_back(0); - emit(&ctx, (int64_t*)shape_args, n_shape); +} +} // namespace - res.ok = true; - res.trace = std::move(ctx.trace); - return res; +LazyProducer::~LazyProducer() { + delete _ctx; // _lib intentionally left open: the tile fns must stay callable } +bool LazyProducer::open(const char* so_path, const int64_t* shape_args, int32_t n_shape, + const uint64_t* tensor_base, int32_t n_tensors, + const int64_t* cyc, const int64_t* ovl, int32_t n_tiles, + const int32_t* partition_cores, int32_t n_partition_cores) { + // own the tables: the caller's buffers are its locals, and the tile fns are + // invoked long after it returns. + if (tensor_base && n_tensors > 0) _bases.assign(tensor_base, tensor_base + n_tensors); + if (cyc && n_tiles > 0) _cyc.assign(cyc, cyc + n_tiles); + if (ovl && n_tiles > 0) _ovl.assign(ovl, ovl + n_tiles); + + _ctx = new EmitCtx(); + init_ctx(*_ctx, _bases.empty() ? nullptr : _bases.data(), (int32_t)_bases.size(), + _cyc.empty() ? nullptr : _cyc.data(), _ovl.empty() ? nullptr : _ovl.data(), + (int32_t)_cyc.size(), partition_cores, n_partition_cores); + + _lib = dlopen(so_path, RTLD_NOW | RTLD_GLOBAL); + if (!_lib) { fprintf(stderr, "togsim: dlopen failed: %s\n", dlerror()); return false; } + auto kernel = (void (*)(EmitCtx*, int64_t*, int32_t))dlsym(_lib, "togsim_kernel"); + if (!kernel) { fprintf(stderr, "togsim: dlsym togsim_kernel failed: %s\n", dlerror()); return false; } + + // Running the kernel only walks its outer loop nest: every togsim_dispatch + // registers a work-item and returns, so no tile body runs and no record is + // emitted yet. run_item() replays an individual work-item later. + kernel(_ctx, (int64_t*)shape_args, n_shape); + return true; +} + +size_t LazyProducer::num_items() const { return _ctx->items.size(); } + +const std::vector& LazyProducer::run_item(size_t i) { + _ctx->trace.clear(); + if (i >= _ctx->items.size()) return _ctx->trace; + WorkItem& w = _ctx->items[i]; + _ctx->cur_core = w.core; // the binding fixed when registered + emit_rec(_ctx, blank(TraceRec::TILE_BEGIN, w.core)); + ((togsim_tile_fn)w.fn)(_ctx, w.iv.empty() ? nullptr : w.iv.data(), (int32_t)w.iv.size()); + emit_rec(_ctx, blank(TraceRec::TILE_END, w.core)); + return _ctx->trace; // one work-item's records; overwritten by the next call +} } // namespace togsim diff --git a/TOGSim/src/togsim_trace_bridge.cc b/TOGSim/src/togsim_trace_bridge.cc index 46e6f7b3..9c25355f 100644 --- a/TOGSim/src/togsim_trace_bridge.cc +++ b/TOGSim/src/togsim_trace_bridge.cc @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -58,44 +59,152 @@ std::shared_ptr make_compute(const togsim::TraceRec& t) { return inst; } -} // namespace +// Core's compute-unit enum, as the producer encodes it in TraceRec::compute_type. +constexpr int MATMUL_CT = 1, PRELOAD_CT = 2; -std::unique_ptr trace_to_tilegraph(const togsim::RunResult& run, - const std::string& name) { - using togsim::TraceRec; - auto tg = std::make_unique(name, name); - // Empty cache plan (no L2/CMEM persistence) -- append_subgraph propagates it - // to each subgraph, and DMA::is_cacheable dereferences it, so it must be a - // valid (if empty) IntervalTree rather than null. - tg->init_cache_plan({}); +// The current producers of one SRAM buffer. Normally a write REPLACEs them, but +// the K matmuls of Y += X@W commute: each only waits on whoever INITIALIZED the +// accumulator, so it JOINs the set and reads initializer() in O(1), not O(K). +class BufferWriters { + public: + void replace(const std::shared_ptr& w) { // normal write: sole producer + _all.assign(1, w); + _initializer = (w->get_compute_type() != MATMUL_CT) ? w : nullptr; + } + void accumulate(const std::shared_ptr& mm) { _all.push_back(mm); } + + const std::vector>& all() const { return _all; } + const std::shared_ptr& initializer() const { return _initializer; } + + private: + std::vector> _all; + std::shared_ptr _initializer; // the non-MATMUL member of _all, if any +}; + +// All builder state for one kernel, driven one dispatch tile at a time. Owned by +// the TileGraph's tile source, so it survives between materializations. link() is +// the dependency DAG, sram_effects() the buffer-version rule, feed() the dispatch. +struct BuildState { + togsim::LazyProducer prod; + size_t next = 0; // next work-item to materialize + std::map buf_bytes; + std::shared_ptr sg_out; // the tile just built std::shared_ptr sg; std::shared_ptr tile; - // The explicit dependency DAG (sec 10): per SRAM buffer, writers(b) is the SET of - // current producers' DONE-handles and readers(b) its consumers. Scoped per - // work-item, so distinct work-items are independent (-> parallel). - std::map>> writers; // buffer id -> current producers (DONE-handles) - // 1 load : N barriers, so track the CURRENT load per (tag_id, tag_slot), not a - // FIFO. Each load takes a fresh `uniq` and its iteration's barriers reuse it. - // Correct only because a load nest and its consumers run in order. Per work-item. + std::map writers; std::map, std::pair>> current_dma; - // Dedup barriers on the CURRENT load of a (tag_id, tag_slot): a conv reads one - // loaded subtile from many matmuls, so its per-consumer waits collapse to one bar. - // A new load bumps uniq, so a genuine new wait still gets its own bar. std::map, std::pair>> bar_for_load; - int64_t next_tag = 0; // mints a unique Core tag key per dma record - int cur_tile_group = -1; // work-item index, bumped per TILE_BEGIN (trace grouping) - std::set cur_tile_bufs; // distinct spad buffers this tile touches - size_t cur_tile_footprint = 0; // their footprint sum = the tile's resident set + int64_t next_tag = 0; + int cur_tile_group = -1; + std::set cur_tile_bufs; + size_t cur_tile_footprint = 0; + + // SRAM buffer versions, NOT scoped to a work-item: the spad is one physical + // resource, so a buffer reused by the next tile is a new version that waits for + // the old one to free. sram_schedule() precomputes the schedule; see below. + int64_t next_alloc = 0; + std::map cur_alloc; // buf -> current version id + std::map open_ver; // buf -> version still accepting writes + std::vector has_readers; // version -> ever read? + std::vector> last_reader; // version -> (work-item, record) + size_t item = 0, rec = 0; // position of the record being fed + + static size_t rec_bytes(const togsim::TraceRec& t) { // single source of the tile footprint + size_t numel = 1; + for (auto d : t.dims) numel *= (size_t)d; + return numel * (t.elem_bits / 8); + } + + // ---- index + footprint -------------------------------------------------- + // Open the producer, then replay every work-item once to size each buffer: + // buf_bytes must be known before sram_schedule() and before the build. + bool index(const char* so_path, const int64_t* shape_args, int32_t n_shape, + const uint64_t* tensor_base, int32_t n_tensors, + const int64_t* cyc, const int64_t* ovl, int32_t n_tiles, + const int32_t* partition_cores, int32_t n_partition_cores) { + using togsim::TraceRec; + if (!prod.open(so_path, shape_args, n_shape, tensor_base, n_tensors, + cyc, ovl, n_tiles, partition_cores, n_partition_cores)) + return false; + for (size_t i = 0; i < prod.num_items(); i++) + for (const TraceRec& t : prod.run_item(i)) { + if (t.kind != TraceRec::DMA) continue; + const auto& bs = (t.dir == 1) ? t.read_bufs : t.write_bufs; // store reads spad, load writes spad + for (int64_t b : bs) buf_bytes[b] = rec_bytes(t); + } + return true; + } + + // ---- the SRAM buffer-version rule (THE single source of truth) ----------- + // What `t` does to the spad: `reads` consume (and close) a buffer's current + // version, `opens` may start a new one. sram_schedule() and feed() must agree on + // this or version ids drift, so both call it. Apply `reads` before `opens`. + void sram_effects(const togsim::TraceRec& t, + std::vector& reads, + std::vector& opens) const { + using togsim::TraceRec; + reads.clear(); + opens.clear(); + auto has = [](const std::vector& v, int64_t b) { + return std::find(v.begin(), v.end(), b) != v.end(); + }; + if (t.kind == TraceRec::DMA) { + if (t.dir == 1) reads = t.read_bufs; // store drains the spad + else opens = t.write_bufs; // load fills it + } else if (t.kind == TraceRec::COMPUTE) { + // An in-place buffer (read AND written: the accumulator, an in-place + // vector op) is version-transparent -- it neither closes nor opens one. + for (int64_t b : t.read_bufs) + if (!has(t.write_bufs, b)) reads.push_back(b); // consuming reads + for (int64_t b : t.write_bufs) + if (!has(t.read_bufs, b) && buf_bytes.count(b)) // fresh outputs; a + opens.push_back(b); // never-DMA'd buf has + } // no size -> untracked + } + + // ---- version lifetimes, precomputed (allocates no Instruction) ----------- + // Per buffer version, record whether anything reads it and where its LAST reader + // sits, so feed() can tag that reader as it goes rather than retain every reader. + void sram_schedule() { + using togsim::TraceRec; + int64_t alloc = 0; + std::map cur; // buf -> current version id + std::map open; // buf -> version still accepting writes + std::vector reads, opens; + + for (size_t wi = 0; wi < prod.num_items(); wi++) { + size_t pos = 0; + for (const TraceRec& t : prod.run_item(wi)) { + sram_effects(t, reads, opens); + for (int64_t b : reads) { + auto f = cur.find(b); + if (f == cur.end()) continue; // untracked buffer + has_readers[f->second] = 1; + last_reader[f->second] = {wi, pos}; // keep the LAST one + open[b] = false; // next write starts a new version + } + for (int64_t b : opens) + if (!cur.count(b) || !open[b]) { // a read closed it -> new version + cur[b] = alloc++; + open[b] = true; + has_readers.push_back(0); + last_reader.emplace_back((size_t)-1, (size_t)-1); + } + pos++; + } + } + } - auto flush = [&]() { + // ---- per-tile close ----------------------------------------------------- + void flush() { if (sg && tile) { tile->set_spad_footprint(cur_tile_footprint); // distinct-buffer resident set (1- vs 2-dispatch) sg->add_tile(tile); tile->set_owner(sg); - tg->append_subgraph(sg); + sg_out = sg; // hand this tile to the consumer } sg.reset(); tile.reset(); @@ -105,26 +214,26 @@ std::unique_ptr trace_to_tilegraph(const togsim::RunResult& run, cur_tile_bufs.clear(); cur_tile_footprint = 0; next_tag = 0; - }; - - // The single dataflow rule (sec 10.3). READ b: depend on all writers(b), ISSUE - // when both are SA ops else DONE. WRITE b: replace writers(b), except a commutative - // matmul accumulator, which waits only the seed and unions. WAR: resource models. - const int MATMUL_CT = 1, PRELOAD_CT = 2; - auto is_mm_accum = [&](const std::shared_ptr& inst, int64_t b, - const std::vector& writes) { + } + + // ---- dependency DAG (sec 10.3) ------------------------------------------ + // READ b: depend on all writers(b), ISSUE when both are SA ops else DONE. + // WRITE b: replace writers(b), except a commutative matmul accumulator, which + // joins them and waits only the initializer. WAR: resource models. + bool is_mm_accum(const std::shared_ptr& inst, int64_t b, + const std::vector& writes) { if (inst->get_compute_type() != MATMUL_CT) return false; for (int64_t w : writes) if (w == b) return true; return false; - }; - auto link = [&](std::shared_ptr inst, - const std::vector& reads, - const std::vector& writes) { + } + void link(std::shared_ptr inst, + const std::vector& reads, + const std::vector& writes) { for (int64_t b : reads) { if (is_mm_accum(inst, b, writes)) continue; // accumulator read -> handled in WRITE (UNION) auto it = writers.find(b); if (it != writers.end()) - for (auto& w : it->second) { + for (auto& w : it->second.all()) { int pct = w->get_compute_type(); // both SA ops -> occupancy (overlap on the SA pipeline); else latency. DepEvent on = (inst->get_compute_type() == MATMUL_CT && @@ -134,91 +243,73 @@ std::unique_ptr trace_to_tilegraph(const togsim::RunResult& run, } } for (int64_t b : writes) { - if (is_mm_accum(inst, b, writes)) { // UNION (commutative accumulate) - auto it = writers.find(b); - if (it != writers.end()) - for (auto& s : it->second) - if (s->get_compute_type() != MATMUL_CT) - s->add_dep(inst, DepEvent::DONE); // wait the init/bias seed only - writers[b].push_back(inst); // join; no reset, no co-matmul edge - } else { // REPLACE (normal output; resets the producer set) - writers[b] = { inst }; + auto& w = writers[b]; + if (is_mm_accum(inst, b, writes)) { // JOIN: commutative, waits only the init + if (const auto& init = w.initializer()) init->add_dep(inst, DepEvent::DONE); + w.accumulate(inst); + } else { // REPLACE: a normal output resets the set + w.replace(inst); } } tile->append_instuction(inst); - }; - - // SRAM-capacity tracking (sec 10.4): a coarse tile is one version of its buffer, - // freed once all its consumers have issued. NOT reset in flush() -- the spad is a - // physical per-core resource. v1 is single-core; multi-core would key by (core, buf). - int64_t next_alloc = 0; - std::map cur_alloc; // buf -> current version id - std::map open_ver; // buf -> version still accepting writes - struct Ver { std::vector> loads, readers; }; - std::map vers; - // Spad bytes per buffer id, from the DMA records that touch it (a compute output - // takes its footprint from its store). Pre-pass, so it is known before the compute. - auto rec_bytes = [](const TraceRec& t) { // single source of the tile footprint - size_t numel = 1; - for (auto d : t.dims) numel *= (size_t)d; - return numel * (t.elem_bits / 8); - }; - std::map buf_bytes; - for (const auto& t : run.trace) { - if (t.kind != TraceRec::DMA) continue; - const auto& bs = (t.dir == 1) ? t.read_bufs : t.write_bufs; // store reads spad, load writes spad - for (int64_t b : bs) buf_bytes[b] = rec_bytes(t); } - // Add each buffer once to the current tile's footprint (reloads in a K-loop reuse the same id). - auto note_bufs = [&](const std::vector& bufs) { - for (int64_t b : bufs) + + // ---- SRAM-capacity tracking (buffer-version allocations, sec 10.4) ------- + // A coarse tile = one version of its buffer; the fine DMAs filling it share one + // allocation, freed once every consumer has issued. Tracks DMA-loaded buffers and + // compute outputs, but not the virtual SA-weights (weight slots model those). + void note_bufs(const std::vector& bufs) { + for (int64_t b : bufs) // once per buffer: a K-loop reload reuses the same id if (cur_tile_bufs.insert(b).second) { auto it = buf_bytes.find(b); if (it != buf_bytes.end()) cur_tile_footprint += it->second; } - }; - auto sram_on_load = [&](int64_t b, const std::shared_ptr& ld) { - if (!cur_alloc.count(b) || !open_ver[b]) { // a read closed it -> new version - cur_alloc[b] = next_alloc++; - open_ver[b] = true; - vers[cur_alloc[b]] = {}; - } - ld->set_sram_alloc(cur_alloc[b]); - vers[cur_alloc[b]].loads.push_back(ld); - }; - // A compute that freshly produces b opens a version like a load, carrying b's - // footprint; its last reader frees it -- identical lifecycle to a load. - auto sram_on_write = [&](int64_t b, const std::shared_ptr& w) { - auto bb = buf_bytes.find(b); - if (bb == buf_bytes.end()) return; // size unknown (never DMA'd) -> untracked - if (!cur_alloc.count(b) || !open_ver[b]) { // a consuming read closed it -> new version - cur_alloc[b] = next_alloc++; - open_ver[b] = true; - vers[cur_alloc[b]] = {}; - w->set_sram_alloc(cur_alloc[b]); - w->set_sram_footprint(bb->second); - vers[cur_alloc[b]].loads.push_back(w); + } + // a version nothing ever reads is never freed -> leave it untracked + int64_t tracked(int64_t a) { return has_readers[a] ? a : (int64_t)-1; } + + // Run record `t`'s spad effects against the live version state and tag `inst` + // with what it allocates and frees. Same rule as sram_schedule() (sram_effects), + // so the version ids agree; this pass just also touches the Instruction. + void sram_apply(const togsim::TraceRec& t, const std::shared_ptr& inst) { + using togsim::TraceRec; + std::vector reads, opens; + sram_effects(t, reads, opens); + + for (int64_t b : reads) { // consume the current version + auto it = cur_alloc.find(b); + if (it == cur_alloc.end()) continue; // untracked buffer + // Its LAST reader frees it. sram_schedule() already found where that reader + // sits, so tag it here rather than retaining every reader to end-of-stream. + if (last_reader[it->second] == std::make_pair(item, rec)) + inst->add_sram_release(it->second); + open_ver[b] = false; // next write starts a new version } - // already-open version (further producing writes): same physical bytes, no re-add. - }; - auto sram_on_read = [&](int64_t b, const std::shared_ptr& rd) { - auto it = cur_alloc.find(b); - if (it == cur_alloc.end()) return; // not a load buffer -> untracked - vers[it->second].readers.push_back(rd); - open_ver[b] = false; // next write starts a new version - }; - auto sram_finalize = [&]() { // tag only each version's LAST reader - for (auto& kv : vers) { - auto& v = kv.second; - if (v.readers.empty()) { // no consumer -> never freed: untrack - for (auto& ld : v.loads) ld->set_sram_alloc(-1); - continue; + for (int64_t b : opens) { + const bool fresh = !cur_alloc.count(b) || !open_ver[b]; // a read closed it + if (fresh) { + cur_alloc[b] = next_alloc++; + open_ver[b] = true; + } + if (t.kind == TraceRec::DMA) { + // Every fine DMA that fills the buffer carries the version id, so the + // reloads of a reduction's K-loop share one allocation. Its footprint is + // derived from the tile it moves, so nothing to set here. + inst->set_sram_alloc(tracked(cur_alloc[b])); + } else if (fresh) { + // A compute that freshly produces b opens a version like a load and must + // carry b's size explicitly (it was never DMA'd in). Further producing + // writes into an already-open version are the same physical bytes. + inst->set_sram_alloc(tracked(cur_alloc[b])); + inst->set_sram_footprint(buf_bytes.at(b)); } - v.readers.back()->add_sram_release(kv.first); // it frees the whole version on issue } - }; + } - for (const auto& t : run.trace) { + // ---- the record consumer (one record at a time) ------------------------- + void feed(const togsim::TraceRec& t) { + using togsim::TraceRec; + struct RecTick { size_t& r; ~RecTick() { ++r; } } tick{rec}; if (t.kind == TraceRec::TILE_BEGIN) { // togsim_dispatch opened a work-item -> new subgraph (bound to its core) + // tile. The scope runs until the matching TILE_END (the dispatch wrapper @@ -228,13 +319,13 @@ std::unique_ptr trace_to_tilegraph(const togsim::RunResult& run, sg->set_core_id(t.core); tile = std::make_shared(Tile::Status::INITIALIZED); cur_tile_group++; - continue; + return; } if (t.kind == TraceRec::TILE_END) { flush(); // close the work-item explicitly (scope = the tile fn call) - continue; + return; } - if (!tile) continue; // defensive: ops before the first TILE_BEGIN + if (!tile) return; // defensive: ops before the first TILE_BEGIN if (t.kind == TraceRec::DMA) { int64_t uniq = next_tag++; // fresh Core tag key per dma record @@ -245,21 +336,18 @@ std::unique_ptr trace_to_tilegraph(const togsim::RunResult& run, if (t.dir == 1) { // STORE // store reads the result buffer(s) -> link() JOINs all their writers. link(inst, t.read_bufs, t.write_bufs); - for (int64_t b : t.read_bufs) sram_on_read(b, inst); // store frees what it drains } else { // LOAD tile->append_instuction(inst); // async load: the CURRENT load for this (tag_id, tag_slot), with a fresh // uniq its barriers reuse. writers = the dma until its barrier overwrites it, // so consumers gate on arrival. A sync load blocks to arrival itself. if (t.is_async) current_dma[{t.tag_id, t.tag_slot}] = {uniq, inst}; - for (int64_t b : t.write_bufs) { - // No hard WAR edge: load-buffer reuse is modeled by the SRAM version/ - // capacity machinery (sec 10.4); an edge would force single-buffering. The - // accumulator is not a load buffer -- link()'s REPLACE branch handles it. - writers[b] = { inst }; - sram_on_load(b, inst); // occupy spad - } + // No hard WAR edge: load-buffer reuse is modeled by the SRAM version / + // capacity machinery (sram_apply), which caps how many versions coexist. A + // latency WAR edge would force single-buffering and kill the spad overlap. + for (int64_t b : t.write_bufs) writers[b].replace(inst); } + sram_apply(t, inst); // a store frees what it drains; a load occupies the spad } else if (t.kind == TraceRec::MEMORY_BAR) { // The explicit async-DMA sync. Pair with the CURRENT load for this (tag_id, // tag_slot), reusing its uniq: the dma releases the bar at issue, the bar parks @@ -272,8 +360,8 @@ std::unique_ptr trace_to_tilegraph(const togsim::RunResult& run, // so the buffer's consumers gate on it, instead of emitting a redundant barrier. auto bf = bar_for_load.find({t.tag_id, t.tag_slot}); if (bf != bar_for_load.end() && bf->second.first == uniq) { - for (int64_t b : t.write_bufs) writers[b] = { bf->second.second }; - continue; + for (int64_t b : t.write_bufs) writers[b].replace(bf->second.second); + return; } auto bar = make_mem_bar(t, uniq); bar->set_tile_group(cur_tile_group); @@ -281,24 +369,54 @@ std::unique_ptr trace_to_tilegraph(const togsim::RunResult& run, tile->append_instuction(bar); // the bar is the load's DONE-handle: REPLACE writers(b) with it (no WAR -- the // load already WAR'd the prior readers when it wrote). - for (int64_t b : t.write_bufs) writers[b] = { bar }; + for (int64_t b : t.write_bufs) writers[b].replace(bar); bar_for_load[{t.tag_id, t.tag_slot}] = {uniq, bar}; } else if (t.kind == TraceRec::COMPUTE) { auto inst = make_compute(t); inst->set_tile_group(cur_tile_group); link(inst, t.read_bufs, t.write_bufs); note_bufs(t.read_bufs); note_bufs(t.write_bufs); // distinct-buffer footprint for 1- vs 2-dispatch - // in-place buffers (read AND written) are version-transparent (accumulator, - // in-place vector): skip the self-read and the self-write so footprint is not - // double-counted. read_bufs/write_bufs are tiny, so a linear scan beats a set. - auto in = [](const std::vector& v, int64_t b) { - return std::find(v.begin(), v.end(), b) != v.end(); - }; - for (int64_t b : t.read_bufs) if (!in(t.write_bufs, b)) sram_on_read(b, inst); // consuming reads - for (int64_t b : t.write_bufs) if (!in(t.read_bufs, b)) sram_on_write(b, inst); // fresh outputs + sram_apply(t, inst); // consuming reads free their version; fresh outputs open one } } - flush(); - sram_finalize(); // readers per version are now final -> set each version's refcount + + // ---- materialize exactly one dispatch tile ------------------------------ + // Returns the work-item `next`'s subgraph; nullptr once the producer is + // exhausted. Called from the TileGraph's tile source, on demand. + std::shared_ptr build_one_tile() { + if (next >= prod.num_items()) return nullptr; + item = next; rec = 0; + for (const togsim::TraceRec& t : prod.run_item(next++)) feed(t); + auto out = std::move(sg_out); + sg_out.reset(); + return out; + } +}; + +} // namespace + +std::unique_ptr trace_to_tilegraph( + const char* so_path, const int64_t* shape_args, int32_t n_shape, + const uint64_t* tensor_base, int32_t n_tensors, + const int64_t* cyc, const int64_t* ovl, int32_t n_tiles, + const int32_t* partition_cores, int32_t n_partition_cores, + const std::string& name) { + using togsim::TraceRec; + auto S = std::make_shared(); + + // Index the dispatches (records each work-item's fn/iv/core) and collect each + // buffer's spad size. Builds no Instruction. + if (!S->index(so_path, shape_args, n_shape, tensor_base, n_tensors, + cyc, ovl, n_tiles, partition_cores, n_partition_cores)) + return nullptr; + + S->sram_schedule(); // buffer-version lifetimes, materializing no Instruction + + auto tg = std::make_unique(name, name); + // Empty cache plan (no L2/CMEM persistence) -- the tile source propagates it + // to each subgraph, and DMA::is_cacheable dereferences it, so it must be a + // valid (if empty) IntervalTree rather than null. + tg->init_cache_plan({}); + tg->set_tile_source([S]() { return S->build_one_tile(); }); return tg; }