diff --git a/VX_config.toml b/VX_config.toml index 64c8f9d29..1bd94fd63 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -234,12 +234,12 @@ VX_CFG_LMEM_LOG_SIZE = 14 VX_CFG_LMEM_NUM_BANKS = "expr: $VX_CFG_NUM_LSU_LANES" [tcu] -# VX_CFG_TCU_TYPE is a string enum: 'DPI'|'DSP'|'BHF'|'TFR'|'FPNEW' +# VX_CFG_TCU_TYPE is a string enum: 'DPI'|'DSP'|'BHF'|'TFR'|'TET'|'FPNEW' # DSP uses vendor FP IP (Xilinx xil_fadd/...) and is FPGA-only; ASIC flows # (yosys/synopsys) must use a pure-RTL type. Mirror VX_CFG_FPU_TYPE's -# ASIC-awareness: TFR for ASIC, DSP for FPGA. (TFR is also the non-synthesis +# ASIC-awareness: TFR for ASIC, TET for FPGA. (TFR is also the non-synthesis # RTL default below.) -VX_CFG_TCU_TYPE = "expr: ('TFR' if $ASIC else 'DSP') if $SYNTHESIS else ('DPI' if $VX_CFG_DPI_ENABLE else 'TFR')" +VX_CFG_TCU_TYPE = "expr: ('TFR' if $ASIC else 'TET') if $SYNTHESIS else ('DPI' if $VX_CFG_DPI_ENABLE else 'TFR')" VX_CFG_NUM_TCU_LANES = "expr: $VX_CFG_NUM_THREADS" VX_CFG_NUM_TCU_BLOCKS = "expr: $VX_CFG_ISSUE_WIDTH" VX_CFG_TCU_TF32_ENABLE = true @@ -254,6 +254,8 @@ VX_CFG_TCU_NVFP4_ENABLE = false VX_CFG_TCU_SPARSE_ENABLE = false VX_CFG_TCU_DSM_ENABLE = false VX_CFG_TCU_WGMMA_ENABLE = false +VX_CFG_TCU_FEDP2K = false +VX_CFG_TCU_FEDP_DELAY = 4 VX_CFG_TCU_SPARSE_ENABLED = "expr: 1 if $VX_CFG_TCU_SPARSE_ENABLE else 0" VX_CFG_TCU_META_ENABLE = "expr: $VX_CFG_TCU_MX_ENABLE or $VX_CFG_TCU_SPARSE_ENABLE" VX_CFG_TCU_WGMMA_ENABLED = "expr: 1 if $VX_CFG_TCU_WGMMA_ENABLE else 0" @@ -335,7 +337,7 @@ VX_DBG_DEBUG_LEVEL = 3 VX_CFG_XLEN = [32, 64] VX_CFG_FLEN = [32, 64] VX_CFG_FPU_TYPE = ["DPI", "DSP", "FPNEW", "STD"] -VX_CFG_TCU_TYPE = ["DPI", "DSP", "BHF", "TFR", "FPNEW"] +VX_CFG_TCU_TYPE = ["DPI", "DSP", "BHF", "TFR", "TET", "FPNEW"] [[param]] VX_CFG_DCACHE_NUM_REQS = "int" diff --git a/ci/testcases/tensor_mx.yaml b/ci/testcases/tensor_mx.yaml index 17825013d..ddf731fae 100644 --- a/ci/testcases/tensor_mx.yaml +++ b/ci/testcases/tensor_mx.yaml @@ -96,28 +96,28 @@ tests: drivers: - simx app: sgemm_tcu_mx - configs: -DVX_CFG_NUM_THREADS=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DITYPE=nvfp4 -DOTYPE=fp32 + configs: -DVX_CFG_NUM_THREADS=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DITYPE=nvfp4 -DOTYPE=fp32 -DTCU_MX_TLS args: "-k32" - id: sgemm_tcu_mx-simx-nvfp4-nt8 via: blackbox drivers: - simx app: sgemm_tcu_mx - configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DITYPE=nvfp4 -DOTYPE=fp32 + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DITYPE=nvfp4 -DOTYPE=fp32 -DTCU_MX_TLS args: "-k64" - id: sgemm_tcu_mx-simx-nvfp4-nt16 via: blackbox drivers: - simx app: sgemm_tcu_mx - configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DITYPE=nvfp4 -DOTYPE=fp32 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DITYPE=nvfp4 -DOTYPE=fp32 -DTCU_MX_TLS args: "-k64" - id: sgemm_tcu_mx-simx-nvfp4-nt32 via: blackbox drivers: - simx app: sgemm_tcu_mx - configs: -DVX_CFG_NUM_THREADS=32 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DITYPE=nvfp4 -DOTYPE=fp32 + configs: -DVX_CFG_NUM_THREADS=32 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DITYPE=nvfp4 -DOTYPE=fp32 -DTCU_MX_TLS args: "-k128" - id: sgemm_tcu_mx-simx-mxint8-nt4 via: blackbox @@ -140,7 +140,6 @@ tests: app: sgemm_tcu_mx configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DITYPE=mxint8 -DOTYPE=int32 args: "-k32" - known_issue: "mxint8 NT16 simx numerical mismatch (NT8/NT32 pass) — pending MX fix" - id: sgemm_tcu_mx-simx-mxint8-nt32 via: blackbox drivers: @@ -155,7 +154,6 @@ tests: app: sgemm_tcu_mx configs: -DVX_CFG_NUM_THREADS=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DITYPE=mxfp8 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI args: "-k32" - known_issue: "mxfp8 rtlsim DPI-path numerical mismatch (simx passes) — pending MX fix" - id: sgemm_tcu_mx-rtlsim-mxfp8-nt8 via: blackbox drivers: @@ -163,7 +161,6 @@ tests: app: sgemm_tcu_mx configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DITYPE=mxfp8 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI args: "-k32" - known_issue: "mxfp8 rtlsim DPI-path numerical mismatch (simx passes) — pending MX fix" - id: sgemm_tcu_mx-rtlsim-mxfp8-nt16 via: blackbox drivers: @@ -171,7 +168,6 @@ tests: app: sgemm_tcu_mx configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DITYPE=mxfp8 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI args: "-k32" - known_issue: "mxfp8 rtlsim DPI-path numerical mismatch (simx passes) — pending MX fix" - id: sgemm_tcu_mx-rtlsim-mxfp8-nt32 via: blackbox drivers: @@ -179,7 +175,6 @@ tests: app: sgemm_tcu_mx configs: -DVX_CFG_NUM_THREADS=32 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DITYPE=mxfp8 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI args: "-k64" - known_issue: "mxfp8 rtlsim DPI-path numerical mismatch (simx passes) — pending MX fix" - id: sgemm_tcu_mx-rtlsim-mxbf8-nt4 via: blackbox drivers: @@ -241,28 +236,28 @@ tests: drivers: - rtlsim app: sgemm_tcu_mx - configs: -DVX_CFG_NUM_THREADS=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP4_ENABLE -DVX_CFG_TCU_NVFP4_ENABLE -DITYPE=nvfp4 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI + configs: -DVX_CFG_NUM_THREADS=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP4_ENABLE -DVX_CFG_TCU_NVFP4_ENABLE -DITYPE=nvfp4 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI -DTCU_MX_TLS args: "-k32" - id: sgemm_tcu_mx-rtlsim-nvfp4-nt8 via: blackbox drivers: - rtlsim app: sgemm_tcu_mx - configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP4_ENABLE -DVX_CFG_TCU_NVFP4_ENABLE -DITYPE=nvfp4 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP4_ENABLE -DVX_CFG_TCU_NVFP4_ENABLE -DITYPE=nvfp4 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI -DTCU_MX_TLS args: "-k64" - id: sgemm_tcu_mx-rtlsim-nvfp4-nt16 via: blackbox drivers: - rtlsim app: sgemm_tcu_mx - configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP4_ENABLE -DVX_CFG_TCU_NVFP4_ENABLE -DITYPE=nvfp4 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP4_ENABLE -DVX_CFG_TCU_NVFP4_ENABLE -DITYPE=nvfp4 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI -DTCU_MX_TLS args: "-k64" - id: sgemm_tcu_mx-rtlsim-nvfp4-nt32 via: blackbox drivers: - rtlsim app: sgemm_tcu_mx - configs: -DVX_CFG_NUM_THREADS=32 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP4_ENABLE -DVX_CFG_TCU_NVFP4_ENABLE -DITYPE=nvfp4 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI + configs: -DVX_CFG_NUM_THREADS=32 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP4_ENABLE -DVX_CFG_TCU_NVFP4_ENABLE -DITYPE=nvfp4 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI -DTCU_MX_TLS args: "-k128" - id: sgemm_tcu_mx-rtlsim-mxint8-nt4 via: blackbox @@ -292,17 +287,3 @@ tests: app: sgemm_tcu_mx configs: -DVX_CFG_NUM_THREADS=32 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DITYPE=mxint8 -DOTYPE=int32 -DVX_CFG_TCU_TYPE_DPI args: "-k64" -- id: sgemm_tcu_mx-simx-nvfp4-nt4-tls - via: blackbox - drivers: - - simx - app: sgemm_tcu_mx - configs: -DVX_CFG_NUM_THREADS=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DITYPE=nvfp4 -DOTYPE=fp32 -DTCU_MX_TLS - args: "-k32" -- id: sgemm_tcu_mx-rtlsim-nvfp4-nt4-tls - via: blackbox - drivers: - - rtlsim - app: sgemm_tcu_mx - configs: -DVX_CFG_NUM_THREADS=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP4_ENABLE -DVX_CFG_TCU_NVFP4_ENABLE -DITYPE=nvfp4 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI -DTCU_MX_TLS - args: "-k32" diff --git a/ci/testcases/tensor_sp_mx.yaml b/ci/testcases/tensor_sp_mx.yaml index 572096f46..478133363 100644 --- a/ci/testcases/tensor_sp_mx.yaml +++ b/ci/testcases/tensor_sp_mx.yaml @@ -5,10 +5,6 @@ defaults: - 32 - 64 tier: fast - # Seed coverage only — the sparse-MX path has no validated reference yet, so - # every case currently mismatches. Tracked xfail until the sparse-MX result is - # validated; drop this once the cases pass. - known_issue: "sparse-MX (sgemm_tcu_sp_mx) lacks a validated reference; results mismatch" tests: - id: sgemm_tcu_sp_mx-simx-mxfp8-nt4 via: blackbox diff --git a/hw/rtl/core/VX_scoreboard.sv b/hw/rtl/core/VX_scoreboard.sv index d9433482a..d0b39a7ff 100644 --- a/hw/rtl/core/VX_scoreboard.sv +++ b/hw/rtl/core/VX_scoreboard.sv @@ -241,14 +241,14 @@ module VX_scoreboard import VX_gpu_pkg::*; #( wire [PER_ISSUE_WARPS-1:0] arb_ready_in; // FU lock: a sequence must not interleave with another warp at the same FU. - // fu_locked ('1 = open, one-hot = locked) gates arb_valid_in so only the lock - // holder is requested while it holds the lock. - reg [PER_ISSUE_WARPS-1:0] fu_locked; + // Each FU has an independent mask: '1 = open, one-hot = locked. + reg [NUM_EX_UNITS-1:0][PER_ISSUE_WARPS-1:0] fu_locked; for (genvar w = 0; w < PER_ISSUE_WARPS; ++w) begin : g_arb_data_in // operands_ready carries data-hazard + FU-congestion; fu_locked adds the // FU-lock gate so only the lock holder is requested during a sequence. - assign arb_valid_in[w] = staging_if[w].valid && operands_ready[w] && fu_locked[w]; + wire [EX_BITS-1:0] staging_ex = staging_if[w].data.ex_type; + assign arb_valid_in[w] = staging_if[w].valid && operands_ready[w] && fu_locked[staging_ex][w]; assign arb_data_in[w] = { staging_if[w].data.uuid, @@ -281,7 +281,7 @@ module VX_scoreboard import VX_gpu_pkg::*; #( VX_generic_arbiter #( .NUM_REQS (PER_ISSUE_WARPS), .TYPE ((PER_ISSUE_WARPS > 8) ? "M" : "R"), - .STICKY (1) // Greedy + .STICKY (0) ) out_arb ( .clk (clk), .reset (reset), @@ -335,9 +335,9 @@ module VX_scoreboard import VX_gpu_pkg::*; #( fu_locked <= '1; end else if (issue_fire) begin if (issue_fu_lock && ~issue_fu_unlock) begin - fu_locked <= arb_onehot; + fu_locked[issue_ex] <= arb_onehot; end else if (issue_fu_unlock) begin - fu_locked <= '1; + fu_locked[issue_ex] <= '1; end end end diff --git a/hw/rtl/tcu/VX_tcu_abuf.sv b/hw/rtl/tcu/VX_tcu_abuf.sv index fd8cd63c3..9a896b7a7 100644 --- a/hw/rtl/tcu/VX_tcu_abuf.sv +++ b/hw/rtl/tcu/VX_tcu_abuf.sv @@ -62,11 +62,13 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // Execute-side observation (req_valid is already gated to WGMMA) input wire [NW_WIDTH-1:0] req_wid, input wire req_valid, + input wire req_setup, input wire [3:0] req_step_m, input wire [3:0] req_step_n, input wire [3:0] req_step_k, input wire [`VX_CFG_XLEN-1:0] req_desc_a, input wire req_a_is_smem, + input wire req_is_sparse, input wire [UUID_WIDTH-1:0] req_uuid, // LMEM bank-parallel read port @@ -74,7 +76,7 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // Outputs output wire abuf_ready, - output wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] abuf_rs1_data + output wire [TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] abuf_rs1_data ); `UNUSED_SPARAM (INSTANCE_ID) `UNUSED_VAR (req_wid) @@ -85,10 +87,14 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( localparam BANK_SEL_BITS = $clog2(NUM_BANKS); localparam WORD_SIZE_LOG2 = $clog2(`VX_CFG_XLEN / 8); - localparam A_BLOCK_WORDS = TCU_TC_M * TCU_TC_K; + localparam A_BLOCK_WORDS = TCU_TC_M * TCU_WG_FEDP_K; + localparam A_BLOCK_WORDS_SP = TCU_TC_M * TCU_TC_K; localparam A_BLOCK_BANK_ROWS = (A_BLOCK_WORDS + NUM_BANKS - 1) / NUM_BANKS; + localparam A_BLOCK_BANK_ROWS_SP = (A_BLOCK_WORDS_SP + NUM_BANKS - 1) / NUM_BANKS; localparam BLOCK_WORDS_PADDED = A_BLOCK_BANK_ROWS * NUM_BANKS; + localparam BLOCK_WORDS_PADDED_SP = A_BLOCK_BANK_ROWS_SP * NUM_BANKS; localparam A_STRIPE_BANK_ROWS = TCU_WG_M_STEPS * A_BLOCK_BANK_ROWS; + localparam A_STRIPE_BANK_ROWS_SP = TCU_WG_M_STEPS * A_BLOCK_BANK_ROWS_SP; localparam A_STRIPE_WORDS = A_STRIPE_BANK_ROWS * NUM_BANKS; // XLEN ratio: each physical LMEM bank-row carries XLEN_RATIO logical // 32-bit bank-rows side-by-side. Smem layout is XLEN-independent, so the @@ -97,6 +103,7 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( localparam BANK_ROW_WORDS = NUM_BANKS * XLEN_RATIO; // 32-bit words per LMEM bank-row localparam BANK_ROW_WORDS_LOG2 = $clog2(BANK_ROW_WORDS); localparam A_STRIPE_LMEM_ROWS = (A_STRIPE_WORDS + BANK_ROW_WORDS - 1) / BANK_ROW_WORDS; + localparam A_STRIPE_LMEM_ROWS_SP = (A_STRIPE_BANK_ROWS_SP * NUM_BANKS + BANK_ROW_WORDS - 1) / BANK_ROW_WORDS; // Row-major path issues one LMEM read per logical row (M_STEPS*TC_M rows). localparam A_TOTAL_ROWS = TCU_WG_M_STEPS * TCU_TC_M; localparam FETCH_CTR_W_BM = `CLOG2(A_STRIPE_LMEM_ROWS + 1); @@ -107,10 +114,8 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // per-row 32-bit-word stride. Cap at 14 bits (= 64KB smem / 4B). localparam LDM_W = 14; - // Canonical-config invariant: one A-block fits one (32-bit-equivalent) - // bank-row (TC_M*TC_K == NUM_BANKS). Non-canonical configs requiring - // A_SUB_BLOCKS packing in the output mux are not supported. - `STATIC_ASSERT (A_BLOCK_BANK_ROWS == 1, ("VX_tcu_abuf assumes one A-block per bank-row")) + // Canonical configs fit an A block in one bank-row normally and two + // bank-rows with dense FEDP2K. // ----------------------------------------------------------------------- // Resident slot state @@ -125,29 +130,28 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // alongside fetch_base_r and slot_step_k_r. logic slot_row_major_r; logic [LDM_W-1:0] slot_ldm_words_r; + logic slot_is_sparse_r; wire [`UP(K_STEPS_W)-1:0] req_step_k_trunc = `UP(K_STEPS_W)'(req_step_k); if (4 > K_STEPS_W) begin : g_step_k_upper_unused `UNUSED_VAR (req_step_k[3:`UP(K_STEPS_W)]) end - // The uop expander only reads rs1 (desc_a) on uop 0 of a WGMMA expansion. - // On non-first uops, req_desc_a is garbage and cannot participate in - // the residency check. - wire is_first_uop = (req_step_m == '0) && (req_step_n == '0) && (req_step_k == '0); + // The WGMMA wrapper supplies the setup-latched descriptor on compute uops. + wire is_first_compute_uop = (req_step_m == '0) && (req_step_n == '0) && (req_step_k == '0); `UNUSED_VAR (req_step_n) // only used in is_first_uop computation - // Force a refetch on every WGMMA's first uop. A cooperative-load + // Force a refetch on every WGMMA's first compute uop. A cooperative-load // pattern (K-tile loop rewrites A_warp_smem in place, issues a fresh // WGMMA with an unchanged descriptor) would incorrectly hit the cached // stripe and serve stale A data if residency were checked by desc_a alone. - // refetched_for_first_uop_r gates stripe_resident: cleared until the - // current WGMMA's first_uop fetch completes, then set; cleared again on - // the first non-first uop so the next WGMMA always triggers a fresh fetch. - reg refetched_for_first_uop_r; + // refetched_for_first_compute_r gates stripe_resident: cleared until the + // current WGMMA's first compute fetch completes, then set; cleared again + // on later compute uops so the next WGMMA always triggers a fresh fetch. + reg refetched_for_first_compute_r; wire stripe_resident = slot_valid_r && (slot_step_k_r == req_step_k_trunc) - && (!is_first_uop || refetched_for_first_uop_r); + && (!is_first_compute_uop || refetched_for_first_compute_r); // RS mode (a_from_smem=0): A from registers, abuf bypassed → always ready. wire need_smem = req_valid && req_a_is_smem; @@ -180,14 +184,12 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( localparam STRIPE_STRIDE_BANK_ROWS = A_STRIPE_LMEM_ROWS; - // Use latched desc_a base on non-first uops (req_desc_a is garbage there - // because the uop expander gates the rs1 register read on uop 0 only). - // Without this, k-stripe-transition refills mid-WGMMA would compute the - // wrong fetch_base. + // Compute uops use the setup-latched descriptor. Without this, + // k-stripe-transition refills mid-WGMMA would compute the wrong fetch_base. wire [BANK_ADDR_WIDTH-1:0] effective_desc_a_row_base = - is_first_uop ? desc_a_row_base : slot_desc_a_row_base_r; + is_first_compute_uop ? desc_a_row_base : slot_desc_a_row_base_r; wire [LDM_W-1:0] effective_ldm_words = - is_first_uop ? desc_a_ldm_words : slot_ldm_words_r; + is_first_compute_uop ? desc_a_ldm_words : slot_ldm_words_r; wire effective_row_major = (effective_ldm_words != '0); // Block-major stripe base (unchanged): one fetch covers M_STEPS A-blocks. @@ -220,7 +222,8 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // row-major: A_TOTAL_ROWS per-row reads (one per logical row) wire [FETCH_CTR_W-1:0] target_fetches = slot_row_major_r ? FETCH_CTR_W'(A_TOTAL_ROWS) - : FETCH_CTR_W'(A_STRIPE_LMEM_ROWS); + : (slot_is_sparse_r ? FETCH_CTR_W'(A_STRIPE_LMEM_ROWS_SP) + : FETCH_CTR_W'(A_STRIPE_LMEM_ROWS)); wire all_requested = (req_ctr_r >= target_fetches); wire can_issue = in_fetch && !all_requested @@ -231,9 +234,10 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // Per-row 32-bit-word offset (relative to fetch_base_r * BANK_ROW_WORDS). // Width = LDM (row stride) + counter + a bit for step_k*TC_K headroom. localparam ROW_OFF_W = LDM_W + FETCH_CTR_W + 4; + wire [3:0] row_k_words = slot_is_sparse_r ? 4'(TCU_TC_K) : 4'(TCU_WG_FEDP_K); wire [ROW_OFF_W-1:0] row_word_off_req = ROW_OFF_W'(req_ctr_r) * ROW_OFF_W'(slot_ldm_words_r) - + ROW_OFF_W'(slot_step_k_r) * ROW_OFF_W'(TCU_TC_K); + + ROW_OFF_W'(slot_step_k_r) * ROW_OFF_W'(row_k_words); wire [BANK_ADDR_WIDTH-1:0] row_lmem_addr = fetch_base_r + BANK_ADDR_WIDTH'(row_word_off_req >> BANK_ROW_WORDS_LOG2); @@ -264,63 +268,71 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( fetch_base_r <= '0; slot_row_major_r <= 1'b0; slot_ldm_words_r <= '0; - refetched_for_first_uop_r <= 1'b0; + slot_is_sparse_r <= 1'b0; + refetched_for_first_compute_r <= 1'b0; end else begin - // Inflight tracker (single outstanding request at a time) - if (tcu_lmem_if.rsp_valid) - req_inflight_r <= 1'b0; - if (tcu_lmem_if.req_valid && tcu_lmem_if.req_ready) - req_inflight_r <= 1'b1; - - // Latch desc_a's row base + ldm_words on first uop of every WGMMA - // so non-first uops (k-stripe-transition refills) can reuse them - // without needing the gated req_desc_a bus. - if (req_valid && is_first_uop) begin - slot_desc_a_row_base_r <= desc_a_row_base; - slot_ldm_words_r <= desc_a_ldm_words; - slot_row_major_r <= (desc_a_ldm_words != '0); - end + if (req_setup) begin + fsm_state_r <= S_IDLE; + req_ctr_r <= '0; + rsp_ctr_r <= '0; + req_inflight_r <= 1'b0; + slot_valid_r <= 1'b0; + slot_fetching_r <= 1'b0; + refetched_for_first_compute_r <= 1'b0; + end else begin + // Inflight tracker (single outstanding request at a time) + if (tcu_lmem_if.rsp_valid) + req_inflight_r <= 1'b0; + if (tcu_lmem_if.req_valid && tcu_lmem_if.req_ready) + req_inflight_r <= 1'b1; + + // Latch descriptor fields on the first compute uop. + if (req_valid && is_first_compute_uop) begin + slot_desc_a_row_base_r <= desc_a_row_base; + slot_ldm_words_r <= desc_a_ldm_words; + slot_row_major_r <= (desc_a_ldm_words != '0); + end - // refetched_for_first_uop_r: set when fetch completes for - // this WGMMA's first uop; cleared on the next non-first uop. - if (last_rsp && is_first_uop) - refetched_for_first_uop_r <= 1'b1; - else if (fire && !is_first_uop) - refetched_for_first_uop_r <= 1'b0; - - case (fsm_state_r) - S_IDLE: begin - if (alloc_en) begin - fsm_state_r <= S_FETCH; - slot_fetching_r <= 1'b1; - slot_valid_r <= 1'b0; - slot_step_k_r <= req_step_k_trunc; - // Row-major base is the A_warp start bank-row (no - // step_k offset; per-row arithmetic derives the - // exact LMEM addr). Block-major base is the stripe - // origin (step_k already factored in). - fetch_base_r <= effective_row_major - ? effective_desc_a_row_base - : stripe_base; - req_ctr_r <= '0; - rsp_ctr_r <= '0; - req_inflight_r <= 1'b0; + if (last_rsp && is_first_compute_uop) + refetched_for_first_compute_r <= 1'b1; + else if (fire && !is_first_compute_uop) + refetched_for_first_compute_r <= 1'b0; + + case (fsm_state_r) + S_IDLE: begin + if (alloc_en) begin + fsm_state_r <= S_FETCH; + slot_fetching_r <= 1'b1; + slot_valid_r <= 1'b0; + slot_step_k_r <= req_step_k_trunc; + slot_is_sparse_r <= req_is_sparse; + // Row-major base is the A_warp start bank-row (no + // step_k offset; per-row arithmetic derives the + // exact LMEM addr). Block-major base is the stripe + // origin (step_k already factored in). + fetch_base_r <= effective_row_major + ? effective_desc_a_row_base + : stripe_base; + req_ctr_r <= '0; + rsp_ctr_r <= '0; + req_inflight_r <= 1'b0; + end end - end - S_FETCH: begin - if (tcu_lmem_if.req_valid && tcu_lmem_if.req_ready) - req_ctr_r <= req_ctr_r + FETCH_CTR_W'(1); - if (last_rsp) begin - fsm_state_r <= S_IDLE; - slot_fetching_r <= 1'b0; - slot_valid_r <= 1'b1; - req_inflight_r <= 1'b0; - end else if (tcu_lmem_if.rsp_valid) begin - rsp_ctr_r <= rsp_ctr_r + FETCH_CTR_W'(1); + S_FETCH: begin + if (tcu_lmem_if.req_valid && tcu_lmem_if.req_ready) + req_ctr_r <= req_ctr_r + FETCH_CTR_W'(1); + if (last_rsp) begin + fsm_state_r <= S_IDLE; + slot_fetching_r <= 1'b0; + slot_valid_r <= 1'b1; + req_inflight_r <= 1'b0; + end else if (tcu_lmem_if.rsp_valid) begin + rsp_ctr_r <= rsp_ctr_r + FETCH_CTR_W'(1); + end end - end - default: fsm_state_r <= S_IDLE; - endcase + default: fsm_state_r <= S_IDLE; + endcase + end end end @@ -337,7 +349,7 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // within the LMEM response. wire [ROW_OFF_W-1:0] row_word_off_rsp = ROW_OFF_W'(rsp_ctr_r) * ROW_OFF_W'(slot_ldm_words_r) - + ROW_OFF_W'(slot_step_k_r) * ROW_OFF_W'(TCU_TC_K); + + ROW_OFF_W'(slot_step_k_r) * ROW_OFF_W'(row_k_words); wire [BANK_ROW_WORDS_LOG2:0] row_lane_rsp = (BANK_ROW_WORDS_LOG2+1)'( row_word_off_rsp & ROW_OFF_W'(BANK_ROW_WORDS - 1)); @@ -346,13 +358,13 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( storage_wren = '0; if (in_fetch && tcu_lmem_if.rsp_valid) begin if (slot_row_major_r) begin - // Row-major: write TC_K words of row rsp_ctr_r into - // storage[rsp_ctr_r * TC_K .. + TC_K). Source words start + // Row-major: write k_words of row rsp_ctr_r into + // storage[rsp_ctr_r * k_words .. + k_words). Source words start // at row_lane_rsp inside the LMEM response. - for (int k = 0; k < TCU_TC_K; ++k) begin - automatic int dst = int'(rsp_ctr_r) * TCU_TC_K + k; + for (int k = 0; k < TCU_WG_FEDP_K; ++k) begin + automatic int dst = int'(rsp_ctr_r) * int'(row_k_words) + k; automatic int src = int'(row_lane_rsp) + k; - if (dst < A_STRIPE_WORDS && src < BANK_ROW_WORDS) begin + if (k < int'(row_k_words) && dst < A_STRIPE_WORDS && src < BANK_ROW_WORDS) begin storage_wren[dst] = 1'b1; storage_wdata[dst * 32 +: 32] = tcu_lmem_if.rsp_data.data[src * 32 +: 32]; @@ -395,17 +407,27 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( ); // ----------------------------------------------------------------------- - // Output: select A-block based on step_m, pass through as 32-bit words. - // Storage holds blocks at [m_blk * BLOCK_WORDS_PADDED .. +BLOCK_WORDS_PADDED). + // Output: select A-block based on step_m. Sparse data is stored compactly, + // then expanded into FEDP_K row stride with the unavailable tail lanes zero. // ----------------------------------------------------------------------- - logic [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] rs1_mux; + logic [TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] rs1_mux; always_comb begin rs1_mux = '0; - for (int lane = 0; lane < TCU_BLOCK_CAP; ++lane) begin - if (lane < int'(A_BLOCK_WORDS)) begin - int src_idx; - src_idx = int'(req_step_m) * BLOCK_WORDS_PADDED + lane; + for (int lane = 0; lane < TCU_WG_A_DATA_SIZE; ++lane) begin + int src_idx; + if (req_is_sparse) begin + automatic int row = lane / int'(TCU_WG_FEDP_K); + automatic int k = lane % int'(TCU_WG_FEDP_K); + if (row < int'(TCU_TC_M) && k < int'(TCU_TC_K)) begin + src_idx = int'(req_step_m) * int'(BLOCK_WORDS_PADDED_SP) + + row * int'(TCU_TC_K) + + k; + if (src_idx < int'(A_STRIPE_WORDS)) + rs1_mux[lane] = `VX_CFG_XLEN'(storage_rdata[src_idx]); + end + end else if (lane < int'(A_BLOCK_WORDS)) begin + src_idx = int'(req_step_m) * int'(BLOCK_WORDS_PADDED) + lane; if (src_idx < int'(A_STRIPE_WORDS)) rs1_mux[lane] = `VX_CFG_XLEN'(storage_rdata[src_idx]); end diff --git a/hw/rtl/tcu/VX_tcu_bbuf.sv b/hw/rtl/tcu/VX_tcu_bbuf.sv index 1755dc68e..50ef9e42a 100644 --- a/hw/rtl/tcu/VX_tcu_bbuf.sv +++ b/hw/rtl/tcu/VX_tcu_bbuf.sv @@ -53,6 +53,7 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // TB-level uop observation (req_valid is already gated to WGMMA at wrapper) input wire req_valid, + input wire req_setup, input wire req_is_first_uop, input wire req_is_sparse, input wire [3:0] req_step_m, @@ -77,7 +78,8 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( localparam BANK_SEL_BITS = $clog2(NUM_BANKS); localparam WORD_SIZE_LOG2 = $clog2(`VX_CFG_XLEN / 8); - localparam B_BLOCK_WORDS = TCU_TC_K * TCU_TC_N; + localparam B_BLOCK_WORDS = TCU_WG_FEDP_K * TCU_TC_N; + localparam B_BLOCK_WORDS_SP = TCU_TC_K * TCU_TC_N; localparam B_BUF_WORDS = NUM_BANKS; // storage per slot // (= 1 logical 32-bit bank-row) localparam LG_B_SUB_BLOCKS = $clog2(TCU_WG_B_SUB_BLOCKS); @@ -91,6 +93,7 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( localparam DENSE_BLOCKS_PER_ROW = TCU_WG_B_SUB_BLOCKS * XLEN_RATIO; localparam LG_DENSE_BLOCKS_PER_ROW = (DENSE_BLOCKS_PER_ROW > 1) ? $clog2(DENSE_BLOCKS_PER_ROW) : 0; + localparam SPARSE_ONE_FETCH = (TCU_WG_FEDP_K == (2 * TCU_TC_K)); // Canonical-config invariant: 1 logical (32-bit-equivalent) bank-row // holds B_SUB_BLOCKS blocks (the smem layout is XLEN-independent). @@ -101,7 +104,7 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // WGMMA SS-descriptor's canonical layout) fetch path. Engaged // when desc_b's stride field (bits [31:16]) is non-zero. Performs // TCU_TC_N per-N-row LMEM reads per (step_k, step_n) WGMMA uop, writing - // TCU_TC_K 32-bit words per row into storage at the b_off offset + // TCU_WG_FEDP_K 32-bit words per row into storage at the b_off offset // tcu_core's `b_off + j*tcK + k` indexing will read. Mirrors // VX_tcu_abuf.sv's row-major fetch for A. localparam LDM_W = 14; @@ -136,7 +139,7 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // blocks; XLEN_RATIO of them fit in one physical bank-row. // So advance the LMEM addr every // (B_SUB_BLOCKS * XLEN_RATIO) blocks. - // Sparse: a single sparse step_n needs TC_K*TC_N*2 = 2*B_BLOCK_WORDS + // Sparse: a single sparse step_n needs TC_K*TC_N*2 = 2*B_BLOCK_WORDS_SP // 32-bit words, split across TWO dense K-blocks at the SAME // n_blk. Those two blocks live in different physical bank-rows // separated by sp_k_stride (in bank-rows). The two bbuf slots @@ -154,15 +157,15 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( : 5'({1'b0, req_step_n} >> TOTAL_SHIFT); // Sparse K-block stride in physical LMEM bank-rows. - // = n_steps * B_BLOCK_WORDS / (NUM_BANKS * XLEN_RATIO) + // = n_steps * B_BLOCK_WORDS_SP / (NUM_BANKS * XLEN_RATIO) // n_steps = NRC / 2 for the canonical (tcM*tcN == BLOCK_CAP) configs: // NRC=8 → 4, NRC=16 → 8, NRC=32 → 16. - // Old hardcoded table only matched NT=8; NT=16 has B_BLOCK_WORDS == NUM_BANKS + // Old hardcoded table only matched NT=8; NT=16 has B_BLOCK_WORDS_SP == NUM_BANKS // and so needs strides 2× larger. NT=32 happens to match NT=8. localparam SP_STRIDE_DEN = NUM_BANKS * XLEN_RATIO; - localparam SP_STRIDE_NR8 = (4 * B_BLOCK_WORDS) / SP_STRIDE_DEN; - localparam SP_STRIDE_NR16 = (8 * B_BLOCK_WORDS) / SP_STRIDE_DEN; - localparam SP_STRIDE_NR32 = (16 * B_BLOCK_WORDS) / SP_STRIDE_DEN; + localparam SP_STRIDE_NR8 = (4 * B_BLOCK_WORDS_SP) / SP_STRIDE_DEN; + localparam SP_STRIDE_NR16 = (8 * B_BLOCK_WORDS_SP) / SP_STRIDE_DEN; + localparam SP_STRIDE_NR32 = (16 * B_BLOCK_WORDS_SP) / SP_STRIDE_DEN; logic [5:0] sp_k_stride; always_comb begin case (req_cd_nregs) @@ -181,8 +184,7 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( : SUB_HALF_W'(({27'b0, block_index} >> LG_B_SUB_BLOCKS) & ((1 << LG_XLEN_RATIO) - 1)); // Sparse within-physical-bank-row selector. Picks which of the - // DENSE_BLOCKS_PER_ROW dense blocks to extract (B_BLOCK_WORDS 32-bit - // words at offset sparse_pos * B_BLOCK_WORDS in the LMEM response). + // DENSE_BLOCKS_PER_ROW dense blocks to extract. localparam SPARSE_POS_W = (LG_DENSE_BLOCKS_PER_ROW == 0) ? 1 : LG_DENSE_BLOCKS_PER_ROW; wire [SPARSE_POS_W-1:0] sparse_pos_w = (LG_DENSE_BLOCKS_PER_ROW == 0) @@ -227,9 +229,8 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // Mode the slot pair was filled under (sparse vs dense). On mode // transition for the same warpgroup we must refill. logic slot_is_sparse_r; - // K-major mode + per-WGMMA latched fields. Latched at alloc_en so - // non-first-uop refills can re-derive addresses (rs2 bus is invalid - // on non-first uops). + // K-major mode + per-WGMMA latched fields. Descriptor fields are latched + // on setup; per-compute step fields are latched when a fetch is allocated. logic slot_row_major_r; logic [LDM_W-1:0] slot_ldm_words_r; logic [3:0] slot_step_k_r; @@ -239,19 +240,14 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( logic [KM_CTR_W-1:0] km_req_ctr_r; logic [KM_CTR_W-1:0] km_rsp_ctr_r; - // req_desc_b is only valid on the first uop of a WGMMA expansion; latch - // desc_b on first uop and use the latched base for subsequent uops. - // is_first_uop is provided by op_args.tcu (set alongside fu_lock in - // VX_tcu_uops), not re-derived here. + // The WGMMA wrapper supplies the setup-latched descriptor on compute uops. `UNUSED_VAR (req_step_m) - wire is_first_uop = req_is_first_uop; wire [BANK_ADDR_WIDTH-1:0] effective_desc_b_row_base = - is_first_uop ? desc_b_row_base : slot_desc_b_row_base_r; + req_is_first_uop ? desc_b_row_base : slot_desc_b_row_base_r; // K-major slot fields (slot_row_major_r / slot_ldm_words_r / // slot_step_k_r / slot_step_n_r) are latched at alloc_en — see the // always_ff below. The K-major addressing arithmetic reads them - // directly. desc_b_ldm_words on the bus is used only for the - // first-uop residency / mode-select comparison. + // directly. desc_b_ldm_words on the bus is used only for setup. // Per-mode fetch addresses. wire [BANK_ADDR_WIDTH-1:0] fetch_addr_dense = @@ -276,10 +272,9 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( && (slot_b_addr_r == fetch_addr_b_sparse) && (slot_b_sparse_pos_r == sparse_pos_w); - // K-major residency (dense + sparse): same (step_k, step_n) already - // in the slot pair. For sparse, BOTH slots must be filled (slot_a holds - // K-pair 0, slot_b holds K-pair 1; both written from the same S_FETCH_A - // multi-fetch — see the storage_write block). + // K-major residency (dense + sparse): same (step_k, step_n) already in + // the slot pair. For sparse, both slots must be filled because sparse B + // is split across the two dense candidate streams. wire bank_row_resident_kmajor = slot_a_valid_r && slot_row_major_r && (!slot_is_sparse_r || slot_b_valid_r) @@ -287,10 +282,10 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( && (slot_step_n_r == req_step_n); // Block-major residency is the existing dense/sparse branch; K-major - // overrides it when the LIVE descriptor (uop 0) or the latched slot mode - // (non-first uops) selects K-major. + // overrides it when the first compute descriptor or latched slot mode + // selects K-major. wire req_wants_kmajor = - is_first_uop ? (desc_b_ldm_words != '0) : slot_row_major_r; + req_is_first_uop ? (desc_b_ldm_words != '0) : slot_row_major_r; wire bank_row_resident = req_wants_kmajor ? bank_row_resident_kmajor @@ -304,23 +299,24 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // K-major address generation // ----------------------------------------------------------------------- // For row r (= km_req_ctr_r) of the current (step_k, step_n) block: - // word_off = (step_n × TCU_TC_N + r) × ldm_words + step_k × TCU_TC_K + // word_off = (step_n × TCU_TC_N + r) × ldm_words + step_k × k_words // bank_row = base + (word_off >> log2(BANK_ROW_WORDS)) // lane = word_off & (BANK_ROW_WORDS - 1) - // The fetched bank-row's `lane..lane + tcK` words are the K-pair operands - // for FEDP slot (j = r, k_pair = 0..tcK-1). + // k_words is fedpK for dense and tcK for sparse, matching SimX. + + wire [3:0] km_k_words = slot_is_sparse_r ? 4'(TCU_TC_K) : 4'(TCU_WG_FEDP_K); wire [KM_OFF_W-1:0] km_word_off_req = KM_OFF_W'(slot_step_n_r) * KM_OFF_W'(TCU_TC_N) * KM_OFF_W'(slot_ldm_words_r) + KM_OFF_W'(km_req_ctr_r) * KM_OFF_W'(slot_ldm_words_r) - + KM_OFF_W'(slot_step_k_r) * KM_OFF_W'(TCU_TC_K); + + KM_OFF_W'(slot_step_k_r) * KM_OFF_W'(km_k_words); wire [BANK_ADDR_WIDTH-1:0] km_lmem_addr = slot_desc_b_row_base_r + BANK_ADDR_WIDTH'(km_word_off_req >> BANK_ROW_WORDS_LOG2); wire [KM_OFF_W-1:0] km_word_off_rsp = KM_OFF_W'(slot_step_n_r) * KM_OFF_W'(TCU_TC_N) * KM_OFF_W'(slot_ldm_words_r) + KM_OFF_W'(km_rsp_ctr_r) * KM_OFF_W'(slot_ldm_words_r) - + KM_OFF_W'(slot_step_k_r) * KM_OFF_W'(TCU_TC_K); + + KM_OFF_W'(slot_step_k_r) * KM_OFF_W'(km_k_words); wire [BANK_ROW_WORDS_LOG2:0] km_lane_rsp = (BANK_ROW_WORDS_LOG2+1)'( km_word_off_rsp & KM_OFF_W'(BANK_ROW_WORDS - 1)); @@ -405,85 +401,85 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( km_req_ctr_r <= '0; km_rsp_ctr_r <= '0; end else begin - if (tcu_lmem_if.rsp_valid) - req_inflight_r <= 1'b0; - if (tcu_lmem_if.req_valid && tcu_lmem_if.req_ready) - req_inflight_r <= 1'b1; - - // Latch desc_b base + ldm_words on first uop of every WGMMA so - // non-first uops can re-derive fetch_addr without needing the - // gated req_desc_b bus. - if (req_valid && is_first_uop) begin - slot_desc_b_row_base_r <= desc_b_row_base; - slot_ldm_words_r <= desc_b_ldm_words; - slot_row_major_r <= (desc_b_ldm_words != '0); - end + if (req_setup) begin + slot_a_valid_r <= 1'b0; + slot_b_valid_r <= 1'b0; + slot_fetching_r <= 1'b0; + fsm_state_r <= S_IDLE; + req_inflight_r <= 1'b0; + km_req_ctr_r <= '0; + km_rsp_ctr_r <= '0; + end else begin + if (tcu_lmem_if.rsp_valid) + req_inflight_r <= 1'b0; + if (tcu_lmem_if.req_valid && tcu_lmem_if.req_ready) + req_inflight_r <= 1'b1; + + // Latch descriptor fields on the first compute uop. + if (req_valid && req_is_first_uop) begin + slot_desc_b_row_base_r <= desc_b_row_base; + slot_ldm_words_r <= desc_b_ldm_words; + slot_row_major_r <= (desc_b_ldm_words != '0); + end - // K-major req/rsp counters advance independently of FSM state - // (single-outstanding still enforced via req_inflight_r). - if (slot_row_major_r && tcu_lmem_if.req_valid && tcu_lmem_if.req_ready) - km_req_ctr_r <= km_req_ctr_r + KM_CTR_W'(1); - if (slot_row_major_r && tcu_lmem_if.rsp_valid && !km_final_rsp) - km_rsp_ctr_r <= km_rsp_ctr_r + KM_CTR_W'(1); - - case (fsm_state_r) - S_IDLE: begin - if (alloc_en) begin - fsm_state_r <= S_FETCH_A; - slot_fetching_r <= 1'b1; - slot_a_valid_r <= 1'b0; - slot_b_valid_r <= 1'b0; - slot_a_addr_r <= fetch_addr_a; - slot_a_sub_half_r <= dense_sub_half; - slot_a_sparse_pos_r <= sparse_pos_w; - slot_b_addr_r <= fetch_addr_b_sparse; - slot_b_sparse_pos_r <= sparse_pos_w; - slot_is_sparse_r <= req_is_sparse; - // K-major slot state — latched here so non-first - // uops (a different (step_k, step_n) in the same - // WGMMA) can compute their own addressing. - slot_step_k_r <= req_step_k; - slot_step_n_r <= req_step_n; - // slot_row_major_r is latched separately from - // desc_b_ldm_words above (covers refills mid-WGMMA - // where is_first_uop is false but the WGMMA's mode - // is what was set at uop 0). - km_req_ctr_r <= '0; - km_rsp_ctr_r <= '0; - req_inflight_r <= 1'b0; + // K-major req/rsp counters advance independently of FSM state + // (single-outstanding still enforced via req_inflight_r). + if (slot_row_major_r && tcu_lmem_if.req_valid && tcu_lmem_if.req_ready) + km_req_ctr_r <= km_req_ctr_r + KM_CTR_W'(1); + if (slot_row_major_r && tcu_lmem_if.rsp_valid && !km_final_rsp) + km_rsp_ctr_r <= km_rsp_ctr_r + KM_CTR_W'(1); + + case (fsm_state_r) + S_IDLE: begin + if (alloc_en) begin + fsm_state_r <= S_FETCH_A; + slot_fetching_r <= 1'b1; + slot_a_valid_r <= 1'b0; + slot_b_valid_r <= 1'b0; + slot_a_addr_r <= fetch_addr_a; + slot_a_sub_half_r <= dense_sub_half; + slot_a_sparse_pos_r <= sparse_pos_w; + slot_b_addr_r <= fetch_addr_b_sparse; + slot_b_sparse_pos_r <= sparse_pos_w; + slot_is_sparse_r <= req_is_sparse; + // K-major per-compute slot state for this + // (step_k, step_n) block. + slot_step_k_r <= req_step_k; + slot_step_n_r <= req_step_n; + // slot_row_major_r is latched from the setup + // descriptor and reused for all compute refills. + km_req_ctr_r <= '0; + km_rsp_ctr_r <= '0; + req_inflight_r <= 1'b0; + end end - end - S_FETCH_A: begin - if (last_rsp) begin - slot_a_valid_r <= 1'b1; - req_inflight_r <= 1'b0; - // K-major sparse fills BOTH slots from the same - // S_FETCH_A multi-fetch (each response carries one - // N-row's worth of K-pair 0 + K-pair 1 words; the - // storage_write block routes the second half to - // slot_b). Skip S_FETCH_B. - if (slot_is_sparse_r && slot_row_major_r) begin - slot_b_valid_r <= 1'b1; - fsm_state_r <= S_IDLE; - slot_fetching_r <= 1'b0; - end else if (slot_is_sparse_r) begin - fsm_state_r <= S_FETCH_B; - end else begin + S_FETCH_A: begin + if (last_rsp) begin + slot_a_valid_r <= 1'b1; + req_inflight_r <= 1'b0; + if (slot_is_sparse_r && (slot_row_major_r || SPARSE_ONE_FETCH)) begin + slot_b_valid_r <= 1'b1; + fsm_state_r <= S_IDLE; + slot_fetching_r <= 1'b0; + end else if (slot_is_sparse_r) begin + fsm_state_r <= S_FETCH_B; + end else begin + fsm_state_r <= S_IDLE; + slot_fetching_r <= 1'b0; + end + end + end + S_FETCH_B: begin + if (last_rsp) begin fsm_state_r <= S_IDLE; slot_fetching_r <= 1'b0; + slot_b_valid_r <= 1'b1; + req_inflight_r <= 1'b0; end end - end - S_FETCH_B: begin - if (last_rsp) begin - fsm_state_r <= S_IDLE; - slot_fetching_r <= 1'b0; - slot_b_valid_r <= 1'b1; - req_inflight_r <= 1'b0; - end - end - default: fsm_state_r <= S_IDLE; - endcase + default: fsm_state_r <= S_IDLE; + endcase + end end end @@ -497,14 +493,8 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // XLEN>32, the lower NUM_BANKS otherwise. tcu_core's // b_off then picks within those words at execute time. // - // Sparse write: copies exactly one dense block (B_BLOCK_WORDS words) - // from the physical response at offset - // sparse_pos × B_BLOCK_WORDS. The other words in the - // physical bank-row are dropped — a different step_n - // needs them and will refill on the next request. The - // upper (NUM_BANKS - B_BLOCK_WORDS) storage entries are - // unused in sparse mode but the dense width is retained - // so the LUTRAM can serve both modes. + // Sparse write: extracts the compressed candidate lanes from one or two + // dense B blocks and stores them in the compact rs2 order. // ----------------------------------------------------------------------- logic [B_BUF_WORDS*32-1:0] storage_a_wdata, storage_b_wdata; @@ -523,23 +513,27 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( ? (OFF_W'(slot_a_sparse_pos_r) * OFF_W'(B_BLOCK_WORDS)) : (OFF_W'(slot_a_sub_half_r) * OFF_W'(NUM_BANKS)); wire [OFF_W-1:0] b_off_words = slot_is_sparse_r - ? (OFF_W'(slot_b_sparse_pos_r) * OFF_W'(B_BLOCK_WORDS)) + ? (OFF_W'(slot_b_sparse_pos_r) * OFF_W'(B_BLOCK_WORDS_SP)) : '0; - wire [OFF_W-1:0] write_count = slot_is_sparse_r ? OFF_W'(B_BLOCK_WORDS) + wire [OFF_W-1:0] write_count = slot_is_sparse_r ? OFF_W'(B_BLOCK_WORDS_SP) : OFF_W'(NUM_BANKS); // Sparse storage permutation: B_smem block is J-major (n_in outer, k_word // inner), but the FEDP indexes rs2[k*tcN*2 + j*2 + cand] (K-major). // Permute on write so storage[b] holds the word the FEDP expects at slot b: // storage[k_pair*tcN*2 + j*2 + cand] = block[j*tcK + k_pair*2 + cand] - // i.e. src(b) = (b/2/tcN)*2 + (b%2) + ((b/2)%tcN)*tcK. - logic [OFF_W-1:0] sparse_src [B_BUF_WORDS]; + // i.e. src(b) = (b/2/tcN)*2 + (b%2) + ((b/2)%tcN)*k_words. + logic [OFF_W-1:0] sparse_src_a [B_BUF_WORDS]; + logic [OFF_W-1:0] sparse_src_b [B_BUF_WORDS]; always_comb begin for (int b = 0; b < B_BUF_WORDS; ++b) begin automatic int unsigned cand_b = b & 1; automatic int unsigned j_b = (b >> 1) % TCU_TC_N; automatic int unsigned k_pair_b = (b >> 1) / TCU_TC_N; - sparse_src[b] = OFF_W'(j_b * TCU_TC_K + k_pair_b * 2 + cand_b); + sparse_src_a[b] = OFF_W'(j_b * TCU_WG_FEDP_K + k_pair_b * 2 + cand_b); + sparse_src_b[b] = SPARSE_ONE_FETCH + ? OFF_W'(j_b * TCU_WG_FEDP_K + TCU_TC_K + k_pair_b * 2 + cand_b) + : OFF_W'(j_b * TCU_TC_K + k_pair_b * 2 + cand_b); end end @@ -555,21 +549,21 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // two sparse candidates adjacent — word (z*2 + cand) holds the // FEDP's bword{cand} for K-step z. // The FEDP reads rs2[k_idx*(TC_N*2) + j*2 + cand] with k_idx=z; - // slot_a||slot_b splits at B_BLOCK_WORDS (= TC_K*TC_N). + // slot_a||slot_b splits at B_BLOCK_WORDS_SP (= TC_K*TC_N). // Drive each (z,cand) word to its exact flat target and route by slot. for (int z = 0; z < TCU_TC_K; ++z) begin for (int c = 0; c < 2; ++c) begin automatic int src = int'(km_lane_rsp) + z * 2 + c; automatic int tgt = z * (TCU_TC_N * 2) + int'(km_rsp_ctr_r) * 2 + c; if (src < (NUM_BANKS * XLEN_RATIO) && in_fetch_a) begin - if (tgt < int'(B_BLOCK_WORDS)) begin + if (tgt < int'(B_BLOCK_WORDS_SP)) begin if (tgt < B_BUF_WORDS) begin storage_a_wren[tgt] = 1'b1; storage_a_wdata[tgt * 32 +: 32] = tcu_lmem_if.rsp_data.data[src * 32 +: 32]; end end else begin - automatic int tgt_b = tgt - int'(B_BLOCK_WORDS); + automatic int tgt_b = tgt - int'(B_BLOCK_WORDS_SP); if (tgt_b < B_BUF_WORDS) begin storage_b_wren[tgt_b] = 1'b1; storage_b_wdata[tgt_b * 32 +: 32] = @@ -580,12 +574,12 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( end end end else if (slot_row_major_r) begin - // K-major dense: write tcK words for this row (km_rsp_ctr_r) - // into storage[km_b_off + km_rsp_ctr_r * tcK .. + tcK), + // K-major dense: write fedpK words for this row (km_rsp_ctr_r) + // into storage[km_b_off + km_rsp_ctr_r * fedpK .. + fedpK), // sourced from the response at lane km_lane_rsp. - for (int k = 0; k < TCU_TC_K; ++k) begin + for (int k = 0; k < TCU_WG_FEDP_K; ++k) begin automatic int dst = int'(km_b_off) - + int'(km_rsp_ctr_r) * TCU_TC_K + + int'(km_rsp_ctr_r) * TCU_WG_FEDP_K + k; automatic int src = int'(km_lane_rsp) + k; if (dst < B_BUF_WORDS && src < (NUM_BANKS * XLEN_RATIO) && in_fetch_a) begin @@ -597,18 +591,23 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( end else begin for (int b = 0; b < B_BUF_WORDS; ++b) begin if (b < int'(write_count)) begin - automatic logic [OFF_W-1:0] src_off = slot_is_sparse_r - ? sparse_src[b] - : OFF_W'(b); if (in_fetch_a) begin + automatic logic [OFF_W-1:0] src_off = slot_is_sparse_r + ? sparse_src_a[b] + : OFF_W'(b); storage_a_wren[b] = 1'b1; storage_a_wdata[b * 32 +: 32] = tcu_lmem_if.rsp_data.data[(int'(a_off_words) + int'(src_off)) * 32 +: 32]; + if (slot_is_sparse_r && SPARSE_ONE_FETCH) begin + storage_b_wren[b] = 1'b1; + storage_b_wdata[b * 32 +: 32] = + tcu_lmem_if.rsp_data.data[(int'(a_off_words) + int'(sparse_src_b[b])) * 32 +: 32]; + end end - if (in_fetch_b) begin + if (in_fetch_b && !SPARSE_ONE_FETCH) begin storage_b_wren[b] = 1'b1; storage_b_wdata[b * 32 +: 32] = - tcu_lmem_if.rsp_data.data[(int'(b_off_words) + int'(src_off)) * 32 +: 32]; + tcu_lmem_if.rsp_data.data[(int'(b_off_words) + int'(sparse_src_b[b])) * 32 +: 32]; end end end @@ -648,12 +647,11 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( .clk (clk), .reset (reset), .read (1'b1), - // In K-major sparse mode both slots are written from the same - // in_fetch_a response (each response carries one N-row's K-pair - // 0 + K-pair 1 words). Block-major sparse keeps the legacy - // S_FETCH_A → S_FETCH_B sequence. + // Sparse FEDP2K block-major and K-major sparse write both slots + // from FETCH_A; legacy block-major sparse keeps S_FETCH_B. .write ((in_fetch_b && tcu_lmem_if.rsp_valid) - || (in_fetch_a && tcu_lmem_if.rsp_valid && slot_row_major_r && slot_is_sparse_r)), + || (in_fetch_a && tcu_lmem_if.rsp_valid && slot_is_sparse_r + && (slot_row_major_r || SPARSE_ONE_FETCH))), .wren (storage_b_wren), .waddr (1'b0), .wdata (storage_b_wdata), @@ -665,8 +663,8 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // Output mux. // Dense: rs2[0..NUM_BANKS-1] = storage_A (legacy). tcu_core's b_off // picks within at execute time. - // Sparse: rs2[0..B_BLOCK_WORDS-1] = storage_A[0..B-1] - // rs2[B_BLOCK_WORDS..2*B_BLOCK_WORDS-1] = storage_B[0..B-1] + // Sparse: rs2[0..B_BLOCK_WORDS_SP-1] = storage_A[0..B-1] + // rs2[B_BLOCK_WORDS_SP..2*B_BLOCK_WORDS_SP-1] = storage_B[0..B-1] // Each slot already holds exactly one dense block at the // sparse_pos selected at fetch time, so the mux is a static // concat. This matches tcu_core's sparse indexing @@ -678,11 +676,11 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( rs2_mux = '0; for (int lane = 0; lane < TCU_WG_RS2_WIDTH; ++lane) begin if (slot_is_sparse_r) begin - if (lane < int'(B_BLOCK_WORDS)) begin + if (lane < int'(B_BLOCK_WORDS_SP)) begin rs2_mux[lane] = `VX_CFG_XLEN'(storage_a_rdata[lane]); - end else if (lane < int'(2 * B_BLOCK_WORDS)) begin + end else if (lane < int'(2 * B_BLOCK_WORDS_SP)) begin rs2_mux[lane] = `VX_CFG_XLEN'( - storage_b_rdata[lane - int'(B_BLOCK_WORDS)]); + storage_b_rdata[lane - int'(B_BLOCK_WORDS_SP)]); end end else if (lane < int'(B_BUF_WORDS)) begin rs2_mux[lane] = `VX_CFG_XLEN'(storage_a_rdata[lane]); diff --git a/hw/rtl/tcu/VX_tcu_core.sv b/hw/rtl/tcu/VX_tcu_core.sv index 87da97894..1ec47a57a 100644 --- a/hw/rtl/tcu/VX_tcu_core.sv +++ b/hw/rtl/tcu/VX_tcu_core.sv @@ -22,7 +22,7 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( input wire reset, `ifdef VX_CFG_TCU_WGMMA_ENABLE - input wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tbuf_rs1_data, + input wire [TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tbuf_rs1_data, input wire [TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tbuf_rs2_data, input wire tbuf_ready, `endif @@ -43,28 +43,36 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( ); `UNUSED_SPARAM (INSTANCE_ID); + localparam FEDP_K = TCU_WG_FEDP_K; + `ifdef VX_CFG_TCU_TYPE_DSP localparam FCVT_LATENCY = 1; localparam FMUL_LATENCY = 8; localparam FADD_LATENCY = 11; - localparam FACC_LATENCY = $clog2(2 * TCU_TC_K + 1) * FADD_LATENCY; - localparam FEDP_LATENCY = FCVT_LATENCY + FMUL_LATENCY + FACC_LATENCY; + localparam FACC_LATENCY = $clog2(2 * FEDP_K) * FADD_LATENCY; + localparam FEDP_LATENCY = FCVT_LATENCY + FMUL_LATENCY + FACC_LATENCY + FADD_LATENCY; `elsif VX_CFG_TCU_TYPE_BHF localparam FMUL_LATENCY = 2; localparam FADD_LATENCY = 2; localparam FRND_LATENCY = 1; - localparam FACC_LATENCY = $clog2(2 * TCU_TC_K + 1) * (FADD_LATENCY + FRND_LATENCY); - localparam FEDP_LATENCY = (FMUL_LATENCY + FRND_LATENCY) + 1 + FACC_LATENCY; + localparam FACC_LATENCY = $clog2(2 * FEDP_K) * (FADD_LATENCY + FRND_LATENCY); + localparam FEDP_LATENCY = (FMUL_LATENCY + FRND_LATENCY) + 1 + FACC_LATENCY + (FADD_LATENCY + FRND_LATENCY); `elsif VX_CFG_TCU_TYPE_FPNEW localparam FMUL_LATENCY = 6; localparam FMUX_LATENCY = 1; localparam FADD_LATENCY = 7; - localparam FACC_LATENCY = $clog2(2 * TCU_TC_K) * FADD_LATENCY; + localparam FACC_LATENCY = $clog2(2 * FEDP_K) * FADD_LATENCY; localparam FEDP_LATENCY = FMUL_LATENCY + FMUX_LATENCY + FACC_LATENCY + FADD_LATENCY; `elsif VX_CFG_TCU_TYPE_DPI localparam FMUL_LATENCY = 2; localparam FACC_LATENCY = 2; localparam FEDP_LATENCY = FMUL_LATENCY + FACC_LATENCY; +`elsif VX_CFG_TCU_TYPE_TET + localparam FMUL_LATENCY = 2; + localparam FALN_LATENCY = 2; + localparam FACC_LATENCY = 2; + localparam FRND_LATENCY = 2; + localparam FEDP_LATENCY = FMUL_LATENCY + FALN_LATENCY + FACC_LATENCY + FRND_LATENCY; `else // VX_CFG_TCU_TYPE_TFR localparam FMUL_LATENCY = 1; localparam FALN_LATENCY = 1; @@ -79,6 +87,10 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( localparam LG_A_BS = $clog2(TCU_A_BLOCK_SIZE); localparam LG_B_BS = $clog2(TCU_B_BLOCK_SIZE); localparam OFF_W = $clog2(TCU_BLOCK_CAP); +`ifdef VX_CFG_TCU_WGMMA_ENABLE + localparam LG_WG_B_BS = $clog2(TCU_WG_B_BLOCK_SIZE); + localparam WG_B_OFF_W = $clog2(TCU_WG_RS2_WIDTH); +`endif `ifdef VX_CFG_TCU_SPARSE_ENABLE localparam LG_B_BS_SP = $clog2(TCU_B_BLOCK_SIZE_SP); @@ -90,11 +102,6 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( `endif `ifdef VX_CFG_TCU_MX_ENABLE - wire is_wmma = (execute_if.data.op_type == INST_TCU_WMMA) - `ifdef VX_CFG_TCU_SPARSE_ENABLE - || (execute_if.data.op_type == INST_TCU_WMMA_SP) - `endif - ; `ifdef VX_CFG_TCU_SPARSE_ENABLE wire mx_is_sparse = is_sparse; `else @@ -103,6 +110,7 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( localparam FEDP_SF = TCU_MX_MAX_SF; `else localparam FEDP_SF = 1; + `UNUSED_PARAM (FEDP_SF) `endif // ----------------------------------------------------------------------- @@ -112,7 +120,7 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // common interface. Downstream code uses only these wires and never // references tbuf_* or is_wgmma directly. - wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] rs1_data; + wire [TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] rs1_data; `ifdef VX_CFG_TCU_WGMMA_ENABLE wire [TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] rs2_data; `else @@ -132,9 +140,11 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // RF-side rs2_data is NUM_THREADS lanes wide; the WGMMA bbuf can be // wider (TCU_WG_RS2_WIDTH lanes). Pad/truncate to the wgmma width on // the false branch so both arms match TCU_WG_RS2_WIDTH * XLEN bits. + localparam WG_RS1_BITS = TCU_WG_A_DATA_SIZE * `VX_CFG_XLEN; localparam WG_RS2_BITS = TCU_WG_RS2_WIDTH * `VX_CFG_XLEN; + wire [WG_RS1_BITS-1:0] rs1_data_rf = WG_RS1_BITS'(execute_if.data.rs1_data); wire [WG_RS2_BITS-1:0] rs2_data_rf = WG_RS2_BITS'(execute_if.data.rs2_data); - assign rs1_data = (is_wgmma && wg_a_smem) ? tbuf_rs1_data : execute_if.data.rs1_data; + assign rs1_data = (is_wgmma && wg_a_smem) ? tbuf_rs1_data : rs1_data_rf; assign rs2_data = is_wgmma ? tbuf_rs2_data : rs2_data_rf; `ifdef VX_CFG_TCU_SPARSE_ENABLE @@ -160,6 +170,13 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( wire [4:0] fmt_d = execute_if.data.op_args.tcu.fmt_d; wire execute_fire = execute_if.valid && execute_if.ready; +`ifdef VX_CFG_TCU_WGMMA_ENABLE + wire is_wgmma_setup = is_wgmma && !execute_if.data.header.wb; +`else + wire is_wgmma_setup = 1'b0; +`endif + wire setup_enqueue = execute_fire && is_wgmma_setup; + wire fedp_enqueue = execute_fire && !is_wgmma_setup; // ----------------------------------------------------------------------- // Sparse metadata: VX_tcu_sp_meta (for WMMA_SP) + optional tile-buffer mux @@ -198,9 +215,29 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( wire mdata_queue_full; - wire result_fire = result_if.valid && result_if.ready; wire fedp_enable, fedp_done; + reg setup_valid_r; + tcu_header_t setup_header_r; + tcu_header_t mdata_queue_out; + + wire setup_result_fire = setup_valid_r && result_if.ready; + wire fedp_result_fire = fedp_done && result_if.ready && !setup_valid_r; + + always @(posedge clk) begin + if (reset) begin + setup_valid_r <= 1'b0; + end else begin + if (setup_result_fire) begin + setup_valid_r <= 1'b0; + end + if (setup_enqueue) begin + setup_valid_r <= 1'b1; + setup_header_r <= execute_if.data.header; + end + end + end + reg [PIPE_LATENCY-1:0] fedp_delay_pipe; always @(posedge clk) begin if (reset) begin @@ -209,18 +246,20 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( if (fedp_enable) begin fedp_delay_pipe <= fedp_delay_pipe >> 1; end - if (execute_fire) begin + if (fedp_enqueue) begin fedp_delay_pipe[PIPE_LATENCY-1] <= 1; end end end assign fedp_done = fedp_delay_pipe[0]; - assign result_if.valid = fedp_done; - assign fedp_enable = ~fedp_done || result_if.ready; - assign execute_if.ready = ~mdata_queue_full && fedp_enable && exe_ready_extra; + assign result_if.valid = setup_valid_r || fedp_done; + assign fedp_enable = ~fedp_done || fedp_result_fire; + assign execute_if.ready = is_wgmma_setup + ? ((~setup_valid_r || result_if.ready) && exe_ready_extra) + : (~mdata_queue_full && fedp_enable && exe_ready_extra); - wire mdata_push = execute_fire; + wire mdata_push = fedp_enqueue; VX_fifo_queue #( .DATAW ($bits(tcu_header_t)), @@ -230,9 +269,9 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( .clk (clk), .reset (reset), .push (mdata_push), - .pop (result_fire), + .pop (fedp_result_fire), .data_in(mdata_queue_in), - .data_out(result_if.data.header), + .data_out(mdata_queue_out), `UNUSED_PIN(empty), `UNUSED_PIN(alm_empty), .full (mdata_queue_full), @@ -240,17 +279,24 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( `UNUSED_PIN(size) ); + assign result_if.data.header = setup_valid_r ? setup_header_r : mdata_queue_out; + // ----------------------------------------------------------------------- // Operand offset computation // ----------------------------------------------------------------------- wire [OFF_W-1:0] a_off = (OFF_W'(step_m) & OFF_W'(TCU_A_SUB_BLOCKS-1)) << LG_A_BS; + wire [OFF_W-1:0] b_off_wm; `ifdef VX_CFG_TCU_SPARSE_ENABLE - wire [OFF_W-1:0] b_off = is_sparse + assign b_off_wm = is_sparse ? (OFF_W'(step_n) & OFF_W'(TCU_B_SUB_BLOCKS_SP-1)) << LG_B_BS_SP : (OFF_W'(step_n) & OFF_W'(TCU_B_SUB_BLOCKS-1)) << LG_B_BS; `else - wire [OFF_W-1:0] b_off = (OFF_W'(step_n) & OFF_W'(TCU_B_SUB_BLOCKS-1)) << LG_B_BS; + assign b_off_wm = (OFF_W'(step_n) & OFF_W'(TCU_B_SUB_BLOCKS-1)) << LG_B_BS; +`endif +`ifdef VX_CFG_TCU_WGMMA_ENABLE + wire [WG_B_OFF_W-1:0] b_off_wg = + (WG_B_OFF_W'(step_n) & WG_B_OFF_W'(TCU_WG_B_SUB_BLOCKS-1)) << LG_WG_B_BS; `endif // ----------------------------------------------------------------------- @@ -298,13 +344,17 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( .meta_b (mx_meta_b) ); - localparam MX_IDX_W = $clog2(TCU_TILE_M > TCU_TILE_N ? TCU_TILE_M : TCU_TILE_N); - localparam MX_K_IDX_W = `LOG2UP(TCU_TILE_K * TCU_MAX_ELT_RATIO); - localparam MX_SCALE_IDX_W = $clog2(TCU_BLOCK_CAP * 4); + localparam MX_MAX_MN = TCU_TILE_M > TCU_TILE_N ? TCU_TILE_M : TCU_TILE_N; + localparam MX_IDX_W = $clog2(MX_MAX_MN); + localparam MX_TILE_K_MAX = `MAX(TCU_TILE_K, TCU_WG_K_STEPS * TCU_WG_FEDP_K); + localparam MX_K_IDX_W = `LOG2UP(MX_TILE_K_MAX * TCU_MAX_ELT_RATIO); + localparam MX_SCALE_BLOCKS_MAX = mx_scale_blocks_k_words(TCU_NVFP4_ID, MX_TILE_K_MAX); + localparam MX_SCALE_IDX_W = $clog2(MX_MAX_MN * MX_SCALE_BLOCKS_MAX); function automatic [7:0] mx_scale_at( input logic [TCU_BLOCK_CAP-1:0][31:0] meta, input logic [4:0] fmt, + input logic [MX_SCALE_IDX_W-1:0] scale_blocks_k, input logic [MX_IDX_W-1:0] mn_idx, input logic [MX_K_IDX_W-1:0] k_base_idx ); @@ -314,8 +364,7 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( logic [1:0] byte_idx; begin scale_k = MX_SCALE_IDX_W'(k_base_idx / mx_scale_block_size(fmt)); - scale_idx = MX_SCALE_IDX_W'(mn_idx) * MX_SCALE_IDX_W'(mx_scale_blocks_k(fmt)) - + MX_SCALE_IDX_W'(scale_k); + scale_idx = MX_SCALE_IDX_W'(mn_idx) * scale_blocks_k + scale_k; word_idx = `LOG2UP(TCU_BLOCK_CAP)'(scale_idx >> 2); byte_idx = scale_idx[1:0]; mx_scale_at = meta[word_idx][byte_idx * 8 +: 8]; @@ -325,24 +374,40 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( wire [TCU_TC_M-1:0][FEDP_SF-1:0][7:0] mx_sf_a; wire [TCU_TC_N-1:0][FEDP_SF-1:0][7:0] mx_sf_b; wire [3:0] mx_elems_per_word = 4'(32 / tcu_fmt_width(fmt_s)); - wire [MX_K_IDX_W:0] mx_fedp_elems = (MX_K_IDX_W+1)'( - (MX_K_IDX_W+1)'(TCU_TC_K) * (MX_K_IDX_W+1)'(mx_elems_per_word) + wire [MX_SCALE_IDX_W-1:0] mx_scale_blocks_k_eff = + `ifdef VX_CFG_TCU_WGMMA_ENABLE + is_wgmma ? MX_SCALE_IDX_W'(mx_scale_blocks_k_words(fmt_s, TCU_WG_K_STEPS * TCU_WG_FEDP_K)) + : MX_SCALE_IDX_W'(mx_scale_blocks_k_words(fmt_s, TCU_TILE_K)); + `else + MX_SCALE_IDX_W'(mx_scale_blocks_k_words(fmt_s, TCU_TILE_K)); + `endif + wire [MX_K_IDX_W:0] mx_uop_k_words = + `ifdef VX_CFG_TCU_WGMMA_ENABLE + is_wgmma ? (MX_K_IDX_W+1)'(TCU_WG_FEDP_K) : (MX_K_IDX_W+1)'(TCU_TC_K); + `else + (MX_K_IDX_W+1)'(TCU_TC_K); + `endif + wire [MX_K_IDX_W:0] mx_uop_k_elems = (MX_K_IDX_W+1)'( + mx_uop_k_words * (MX_K_IDX_W+1)'(mx_elems_per_word) + * (MX_K_IDX_W+1)'(mx_is_sparse ? 2 : 1)); + wire [MX_K_IDX_W:0] mx_fedp_k_elems = (MX_K_IDX_W+1)'( + (MX_K_IDX_W+1)'(FEDP_K) * (MX_K_IDX_W+1)'(mx_elems_per_word) * (MX_K_IDX_W+1)'(mx_is_sparse ? 2 : 1)); - wire [MX_K_IDX_W-1:0] mx_k_base_idx = MX_K_IDX_W'(step_k * mx_fedp_elems); + wire [MX_K_IDX_W-1:0] mx_k_base_idx = MX_K_IDX_W'(step_k * mx_uop_k_elems); for (genvar i = 0; i < TCU_TC_M; ++i) begin : g_mx_sf_a_i wire [MX_IDX_W-1:0] mx_a_idx = MX_IDX_W'(step_m) * MX_IDX_W'(TCU_TC_M) + MX_IDX_W'(i); for (genvar s = 0; s < FEDP_SF; ++s) begin : g_s - wire [MX_K_IDX_W-1:0] mx_k_idx = mx_k_base_idx + MX_K_IDX_W'((s * mx_fedp_elems) / FEDP_SF); - assign mx_sf_a[i][s] = is_wmma ? mx_scale_at(mx_meta_a, fmt_s, mx_a_idx, mx_k_idx) : '0; + wire [MX_K_IDX_W-1:0] mx_k_idx = mx_k_base_idx + MX_K_IDX_W'((s * mx_fedp_k_elems) / FEDP_SF); + assign mx_sf_a[i][s] = mx_scale_at(mx_meta_a, fmt_s, mx_scale_blocks_k_eff, mx_a_idx, mx_k_idx); end end for (genvar j = 0; j < TCU_TC_N; ++j) begin : g_mx_sf_b_j wire [MX_IDX_W-1:0] mx_b_idx = MX_IDX_W'(step_n) * MX_IDX_W'(TCU_TC_N) + MX_IDX_W'(j); for (genvar s = 0; s < FEDP_SF; ++s) begin : g_s - wire [MX_K_IDX_W-1:0] mx_k_idx = mx_k_base_idx + MX_K_IDX_W'((s * mx_fedp_elems) / FEDP_SF); - assign mx_sf_b[j][s] = is_wmma ? mx_scale_at(mx_meta_b, fmt_s, mx_b_idx, mx_k_idx) : '0; + wire [MX_K_IDX_W-1:0] mx_k_idx = mx_k_base_idx + MX_K_IDX_W'((s * mx_fedp_k_elems) / FEDP_SF); + assign mx_sf_b[j][s] = mx_scale_at(mx_meta_b, fmt_s, mx_scale_blocks_k_eff, mx_b_idx, mx_k_idx); end end `endif @@ -356,18 +421,80 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( for (genvar i = 0; i < TCU_TC_M; ++i) begin : g_i for (genvar j = 0; j < TCU_TC_N; ++j) begin : g_j `ifdef VX_CFG_TCU_SPARSE_ENABLE - wire [TCU_TC_K-1:0][31:0] a_row, b_col, b_col_dense, b_col_sparse, b_col_1, b_col_2; + wire [FEDP_K-1:0][31:0] a_row, b_col, b_col_dense; + wire [TCU_TC_K-1:0][31:0] b_col_sparse, b_col_1, b_col_2; `else - wire [TCU_TC_K-1:0][31:0] a_row, b_col; + wire [FEDP_K-1:0][31:0] a_row, b_col; `endif `ifdef VX_CFG_TCU_MX_ENABLE wire [FEDP_SF-1:0][7:0] sf_a = mx_sf_a[i]; wire [FEDP_SF-1:0][7:0] sf_b = mx_sf_b[j]; `endif - for (genvar k_idx = 0; k_idx < TCU_TC_K; ++k_idx) begin : g_slice_assign - assign a_row[k_idx] = 32'(rs1_data[a_off + i * TCU_TC_K + k_idx]); - `ifdef VX_CFG_TCU_SPARSE_ENABLE - assign b_col_dense[k_idx] = 32'(rs2_data[b_off + j * TCU_TC_K + k_idx]); + for (genvar k_idx = 0; k_idx < FEDP_K; ++k_idx) begin : g_slice_assign + `ifdef VX_CFG_TCU_WGMMA_ENABLE + localparam int WG_B_IDX = j * TCU_WG_FEDP_K + k_idx; + `endif + if (k_idx < TCU_TC_K) begin : g_lo + `ifdef VX_CFG_TCU_WGMMA_ENABLE + wire [31:0] a_wgmma_smem = 32'(rs1_data[i * TCU_WG_FEDP_K + k_idx]); + wire [31:0] a_wgmma_reg = 32'(execute_if.data.rs1_data[i * TCU_TC_K + k_idx]); + assign a_row[k_idx] = is_wgmma + ? (wg_a_smem ? a_wgmma_smem : a_wgmma_reg) + : 32'(execute_if.data.rs1_data[a_off + i * TCU_TC_K + k_idx]); + `else + assign a_row[k_idx] = 32'(rs1_data[a_off + i * TCU_TC_K + k_idx]); + `endif + `ifdef VX_CFG_TCU_SPARSE_ENABLE + assign b_col_dense[k_idx] = + `ifdef VX_CFG_TCU_WGMMA_ENABLE + is_wgmma ? 32'(rs2_data[int'(b_off_wg) + WG_B_IDX]) : + `endif + 32'(rs2_data[b_off_wm + j * TCU_TC_K + k_idx]); + `else + assign b_col[k_idx] = + `ifdef VX_CFG_TCU_WGMMA_ENABLE + is_wgmma ? 32'(rs2_data[int'(b_off_wg) + WG_B_IDX]) : + `endif + 32'(rs2_data[b_off_wm + j * TCU_TC_K + k_idx]); + `endif + end else begin : g_hi + `ifdef VX_CFG_TCU_WGMMA_ENABLE + wire [31:0] a_wgmma_smem = 32'(rs1_data[i * TCU_WG_FEDP_K + k_idx]); + wire [31:0] a_wgmma_reg = + `ifdef VX_CFG_TCU_FEDP2K + `ifdef VX_CFG_TCU_SPARSE_ENABLE + is_sparse ? 32'b0 : + `endif + 32'(execute_if.data.rs2_data[i * TCU_TC_K + (k_idx - TCU_TC_K)]); + `else + 32'b0; + `endif + assign a_row[k_idx] = (is_wgmma + `ifdef VX_CFG_TCU_SPARSE_ENABLE + && !is_sparse + `endif + ) ? (wg_a_smem ? a_wgmma_smem : a_wgmma_reg) : 32'b0; + `else + assign a_row[k_idx] = 32'b0; + `endif + `ifdef VX_CFG_TCU_SPARSE_ENABLE + assign b_col_dense[k_idx] = + `ifdef VX_CFG_TCU_WGMMA_ENABLE + (is_wgmma && !is_sparse) ? 32'(rs2_data[int'(b_off_wg) + WG_B_IDX]) : + `endif + 32'b0; + `else + assign b_col[k_idx] = + `ifdef VX_CFG_TCU_WGMMA_ENABLE + is_wgmma ? 32'(rs2_data[int'(b_off_wg) + WG_B_IDX]) : + `endif + 32'b0; + `endif + end + end + + `ifdef VX_CFG_TCU_SPARSE_ENABLE + for (genvar k_idx = 0; k_idx < TCU_TC_K; ++k_idx) begin : g_sparse_slice_assign localparam J_SP = SYM_SPARSE ? (j % (TCU_TC_N / 2)) : j; // rs2_data sparse-pair layout differs by op: // WGMMA_SP: source is tbuf (shared mem), K-major → @@ -376,20 +503,18 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // idx = J_SP*(TC_K*2) + k_idx*2 + lane // The two layouts are incompatible; separate formulas are required. `ifdef VX_CFG_TCU_WGMMA_ENABLE - wire [31:0] b_col_1_wg = 32'(rs2_data[b_off + k_idx * TCU_TC_N * 2 + J_SP * 2]); - wire [31:0] b_col_2_wg = 32'(rs2_data[b_off + k_idx * TCU_TC_N * 2 + J_SP * 2 + 1]); - wire [31:0] b_col_1_wm = 32'(rs2_data[b_off + J_SP * TCU_TC_K * 2 + k_idx * 2]); - wire [31:0] b_col_2_wm = 32'(rs2_data[b_off + J_SP * TCU_TC_K * 2 + k_idx * 2 + 1]); + wire [31:0] b_col_1_wg = 32'(rs2_data[k_idx * TCU_TC_N * 2 + J_SP * 2]); + wire [31:0] b_col_2_wg = 32'(rs2_data[k_idx * TCU_TC_N * 2 + J_SP * 2 + 1]); + wire [31:0] b_col_1_wm = 32'(rs2_data[b_off_wm + J_SP * TCU_TC_K * 2 + k_idx * 2]); + wire [31:0] b_col_2_wm = 32'(rs2_data[b_off_wm + J_SP * TCU_TC_K * 2 + k_idx * 2 + 1]); assign b_col_1[k_idx] = is_wgmma ? b_col_1_wg : b_col_1_wm; assign b_col_2[k_idx] = is_wgmma ? b_col_2_wg : b_col_2_wm; `else - assign b_col_1[k_idx] = 32'(rs2_data[b_off + J_SP * TCU_TC_K * 2 + k_idx * 2]); - assign b_col_2[k_idx] = 32'(rs2_data[b_off + J_SP * TCU_TC_K * 2 + k_idx * 2 + 1]); - `endif - `else - assign b_col[k_idx] = 32'(rs2_data[b_off + j * TCU_TC_K + k_idx]); + assign b_col_1[k_idx] = 32'(rs2_data[b_off_wm + J_SP * TCU_TC_K * 2 + k_idx * 2]); + assign b_col_2[k_idx] = 32'(rs2_data[b_off_wm + J_SP * TCU_TC_K * 2 + k_idx * 2 + 1]); `endif end + `endif wire [31:0] c_val = 32'(execute_if.data.rs3_data[i * TCU_TC_N + j]); @@ -404,7 +529,13 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( .vld_mask (vld_meta_block), .b_col_out (b_col_sparse) ); - assign b_col = is_sparse ? b_col_sparse : b_col_dense; + for (genvar k_idx = 0; k_idx < FEDP_K; ++k_idx) begin : g_sparse_b_select + if (k_idx < TCU_TC_K) begin : g_lo + assign b_col[k_idx] = is_sparse ? b_col_sparse[k_idx] : b_col_dense[k_idx]; + end else begin : g_hi + assign b_col[k_idx] = is_sparse ? 32'b0 : b_col_dense[k_idx]; + end + end `ifdef VX_TCU_LD_TRACE // GATHER trace: GATHER,wid,step_m,step_n,i,k,bword0,bword1,lo,hi,gathered @@ -424,6 +555,13 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // Dual-side sparse lane mask `ifdef VX_CFG_TCU_TYPE_TFR + `define VX_TCU_TFR_LANE_MASK_ENABLE + `endif + `ifdef VX_CFG_TCU_TYPE_TET + `define VX_TCU_TFR_LANE_MASK_ENABLE + `endif + + `ifdef VX_TCU_TFR_LANE_MASK_ENABLE wire [TCU_MAX_INPUTS-1:0] vld_mask_r; `ifdef VX_CFG_TCU_DSM_ENABLE wire [TCU_MAX_INPUTS-1:0] vld_mask; @@ -447,10 +585,13 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( `else assign vld_mask_r = '1; `endif + `endif + `ifdef VX_TCU_TFR_LANE_MASK_ENABLE + `undef VX_TCU_TFR_LANE_MASK_ENABLE `endif wire [4:0] fmt_s_r, fmt_d_r; - wire [TCU_TC_K-1:0][31:0] a_row_r, b_col_r; + wire [FEDP_K-1:0][31:0] a_row_r, b_col_r; `ifdef VX_CFG_TCU_MX_ENABLE wire [FEDP_SF-1:0][7:0] sf_a_r, sf_b_r; `endif @@ -458,7 +599,7 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( `ifdef VX_CFG_TCU_MX_ENABLE VX_pipe_register #( - .DATAW (32 + 5 + 5 + TCU_TC_K * 32 + TCU_TC_K * 32 + 2 * FEDP_SF * 8) + .DATAW (32 + 5 + 5 + FEDP_K * 32 + FEDP_K * 32 + 2 * FEDP_SF * 8) ) pipe_fedp ( .clk (clk), .reset (reset), @@ -468,7 +609,7 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( ); `else VX_pipe_register #( - .DATAW (32 + 5 + 5 + TCU_TC_K * 32 + TCU_TC_K * 32) + .DATAW (32 + 5 + 5 + FEDP_K * 32 + FEDP_K * 32) ) pipe_fedp ( .clk (clk), .reset (reset), @@ -482,7 +623,7 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( VX_tcu_fedp_dpi #( .INSTANCE_ID (INSTANCE_ID), .LATENCY (FEDP_LATENCY), - .N (TCU_TC_K), + .N (FEDP_K), .SF (FEDP_SF) ) fedp ( .clk (clk), @@ -503,7 +644,7 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( VX_tcu_fedp_bhf #( .INSTANCE_ID (INSTANCE_ID), .LATENCY (FEDP_LATENCY), - .N (TCU_TC_K) + .N (FEDP_K) ) fedp ( .clk (clk), .reset (reset), @@ -519,7 +660,7 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( VX_tcu_fedp_fpnew #( .INSTANCE_ID (INSTANCE_ID), .LATENCY (FEDP_LATENCY), - .N (TCU_TC_K) + .N (FEDP_K) ) fedp ( .clk (clk), .reset (reset), @@ -535,7 +676,29 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( VX_tcu_fedp_tfr #( .INSTANCE_ID (INSTANCE_ID), .LATENCY (FEDP_LATENCY), - .N (TCU_TC_K), + .N (FEDP_K), + .SF (FEDP_SF) + ) fedp ( + .clk (clk), + .reset (reset), + .vld_mask(vld_mask_r), + .enable(fedp_enable), + .fmt_s (fmt_s_r), + .fmt_d (fmt_d_r), + .a_row (a_row_r), + .b_col (b_col_r), + `ifdef VX_CFG_TCU_MX_ENABLE + .sf_a (sf_a_r), + .sf_b (sf_b_r), + `endif + .c_val (c_val_r), + .d_val (d_val[i][j]) + ); + `elsif VX_CFG_TCU_TYPE_TET + VX_tcu_fedp_tet #( + .INSTANCE_ID (INSTANCE_ID), + .LATENCY (FEDP_LATENCY), + .N (FEDP_K), .SF (FEDP_SF) ) fedp ( .clk (clk), @@ -557,7 +720,7 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( VX_tcu_fedp_dsp #( .INSTANCE_ID (INSTANCE_ID), .LATENCY (FEDP_LATENCY), - .N (TCU_TC_K) + .N (FEDP_K) ) fedp ( .clk (clk), .reset (reset), @@ -582,9 +745,9 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( always @(posedge clk) begin if (execute_if.valid && execute_if.ready) begin `TRACE(3, ("%t: %s FEDP-enq: wid=%0d, cta_id=%0d, i=%0d, j=%0d, m=%0d, n=%0d, a_row=", $time, INSTANCE_ID, execute_if.data.header.wid, execute_if.data.header.cta_id, i, j, step_m, step_n)) - `TRACE_ARRAY1D(2, "0x%0h", a_row, TCU_TC_K) + `TRACE_ARRAY1D(2, "0x%0h", a_row, FEDP_K) `TRACE(3, (", b_col=")); - `TRACE_ARRAY1D(2, "0x%0h", b_col, TCU_TC_K) + `TRACE_ARRAY1D(2, "0x%0h", b_col, FEDP_K) `TRACE(3, (", c_val=0x%0h (#%0d)\n", c_val, execute_if.data.header.uuid)); end if (result_if.valid && result_if.ready) begin diff --git a/hw/rtl/tcu/VX_tcu_lockstep.sv b/hw/rtl/tcu/VX_tcu_lockstep.sv index 42cf9d9ed..ff3d43876 100644 --- a/hw/rtl/tcu/VX_tcu_lockstep.sv +++ b/hw/rtl/tcu/VX_tcu_lockstep.sv @@ -23,12 +23,10 @@ // drain before a new CTA enters. Same-CTA across blocks (the production // case for a warpgroup at the same uop) remains free. // -// Single-owner design: by construction only one CTA can occupy the bbuf -// at a time, so a single tcu_owner_r / tcu_owned_r pair is sufficient. -// Multiple blocks firing leader uops on the same cycle for the same CTA -// (the production warpgroup case) agree on the cta, so the priority- -// encoded pick agrees with all of them. No same-cycle multi-CTA-firing -// race exists in current configs. +// Per-block ownership matches the SimX model: a block records its CTA when +// its first compute uop enters WGMMA, and releases on its last compute uop. +// A block is deferred only while another block is mid-WGMMA for a different +// CTA. // // Contract: the consumer must AND `cta_conflict` into the request // validity presented to bbuf and into any downstream ready that gates @@ -56,29 +54,24 @@ module VX_tcu_lockstep import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // ----------------------------------------------------------------------- // State // ----------------------------------------------------------------------- - reg tcu_owned_r; - reg [NCTA_WIDTH-1:0] tcu_owner_r; - // Per-block "WGMMA expansion in progress" — set on first sub-uop fire, - // cleared on last sub-uop fire. Persists across LMEM-stall gaps so the - // gate keeps a different CTA's WGMMA from sneaking in mid-expansion - // and corrupting the shared bbuf descriptor latch. + // Per-block "WGMMA expansion in progress" — first compute uop joins the + // CTA-owned region; the last compute uop releases it. This persists + // across LMEM-stall gaps so a different CTA cannot enter mid-expansion + // and corrupt the shared bbuf state. reg [BLOCK_SIZE-1:0] in_expansion_r; + reg [BLOCK_SIZE-1:0][NCTA_WIDTH-1:0] cta_owner_r; // ----------------------------------------------------------------------- // Owner update. // - // Leader-fire detection: any block firing its first sub-uop this cycle. + // Leader-fire detection: any block joining a WGMMA expansion this cycle. // The owner cta is picked from the lowest-indexed leader-firing block. // Under same-CTA warpgroup fire (production path), every leader-firing // block carries the same cta_id, so the priority pick agrees with all. // ----------------------------------------------------------------------- wire [BLOCK_SIZE-1:0] leader_fire_b = exec_fire_b & is_first_uop_b & is_wgmma_b; - wire any_leader_fire = |leader_fire_b; - wire any_in_expansion_next; - // in_expansion_next reflects this-cycle updates (set on leader fire, - // clear on last-uop fire). Released the cycle after the last block - // exits expansion. + // clear on last-uop fire). wire [BLOCK_SIZE-1:0] in_expansion_next; for (genvar bi = 0; bi < BLOCK_SIZE; ++bi) begin : g_inexp_next wire is_wgmma_fire = exec_fire_b[bi] && is_wgmma_b[bi]; @@ -86,33 +79,16 @@ module VX_tcu_lockstep import VX_gpu_pkg::*, VX_tcu_pkg::*; #( (in_expansion_r[bi] || (is_wgmma_fire && is_first_uop_b[bi])) && !(is_wgmma_fire && is_last_uop_b[bi]); end - assign any_in_expansion_next = |in_expansion_next; - - // first_fire_cta: priority-encoded pick of the lowest leader-firing block. - logic [NCTA_WIDTH-1:0] first_fire_cta; - always_comb begin - first_fire_cta = '0; - for (int b = BLOCK_SIZE-1; b >= 0; b--) begin - if (leader_fire_b[b]) first_fire_cta = new_cta_b[b]; - end - end always @(posedge clk) begin if (reset) begin - tcu_owned_r <= 1'b0; - tcu_owner_r <= '0; in_expansion_r <= '0; + cta_owner_r <= '0; end else begin in_expansion_r <= in_expansion_next; - if (!tcu_owned_r) begin - if (any_leader_fire) begin - tcu_owned_r <= 1'b1; - tcu_owner_r <= first_fire_cta; - end - end else begin - // Release once no block is still mid-expansion. - if (!any_in_expansion_next) begin - tcu_owned_r <= 1'b0; + for (int bi = 0; bi < BLOCK_SIZE; ++bi) begin + if (leader_fire_b[bi]) begin + cta_owner_r[bi] <= new_cta_b[bi]; end end end @@ -122,9 +98,16 @@ module VX_tcu_lockstep import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // Per-block conflict — flat combinational, no propagation chain. // ----------------------------------------------------------------------- for (genvar bi = 0; bi < BLOCK_SIZE; ++bi) begin : g_conflict - assign cta_conflict[bi] = is_wgmma_b[bi] - && tcu_owned_r - && (tcu_owner_r != new_cta_b[bi]); + logic cta_conflict_r; + always_comb begin + cta_conflict_r = 1'b0; + for (int k = 0; k < BLOCK_SIZE; ++k) begin + if (k != bi && in_expansion_r[k] && (cta_owner_r[k] != new_cta_b[bi])) begin + cta_conflict_r = 1'b1; + end + end + end + assign cta_conflict[bi] = is_wgmma_b[bi] && cta_conflict_r; end // ----------------------------------------------------------------------- diff --git a/hw/rtl/tcu/VX_tcu_pkg.sv b/hw/rtl/tcu/VX_tcu_pkg.sv index 1f3203439..4683e504d 100644 --- a/hw/rtl/tcu/VX_tcu_pkg.sv +++ b/hw/rtl/tcu/VX_tcu_pkg.sv @@ -90,23 +90,29 @@ package VX_tcu_pkg; // WGMMA per-warp tile dimensions (NRA=4 fixed, NRC=NR variable). // Derived from block geometry: xtileM = 2*tcM, xtileK = 2*tcK. - // m_steps = k_steps = 2 always. + // Dense FEDP width doubles under VX_CFG_TCU_FEDP2K, matching SimX. localparam TCU_WG_TILE_M = 2 * TCU_TC_M; localparam TCU_WG_TILE_K = 2 * TCU_TC_K; +`ifdef VX_CFG_TCU_FEDP2K + localparam TCU_WG_FEDP_K = 2 * TCU_TC_K; +`else + localparam TCU_WG_FEDP_K = TCU_TC_K; +`endif localparam TCU_WG_TILE_N = (TCU_WG_NR * TCU_NT) / TCU_WG_TILE_M; // WG step counts: block geometry (TC_M/TC_N/TC_K) unchanged, tile is larger localparam TCU_WG_M_STEPS = TCU_WG_TILE_M / TCU_TC_M; localparam TCU_WG_N_STEPS = TCU_WG_TILE_N / TCU_TC_N; - localparam TCU_WG_K_STEPS = TCU_WG_TILE_K / TCU_TC_K; + localparam TCU_WG_K_STEPS = TCU_WG_TILE_K / TCU_WG_FEDP_K; localparam TCU_WG_UOPS = TCU_WG_M_STEPS * TCU_WG_N_STEPS * TCU_WG_K_STEPS; // WG A/B micro-tiling (block geometry is shared with non-WG) localparam TCU_WG_A_BLOCK_SIZE = TCU_TC_M * TCU_TC_K; + localparam TCU_WG_A_DATA_SIZE = TCU_TC_M * TCU_WG_FEDP_K; localparam TCU_WG_A_SUB_BLOCKS = TCU_BLOCK_CAP / TCU_WG_A_BLOCK_SIZE; - localparam TCU_WG_B_BLOCK_SIZE = TCU_TC_K * TCU_TC_N; + localparam TCU_WG_B_BLOCK_SIZE = TCU_WG_FEDP_K * TCU_TC_N; localparam TCU_WG_B_SUB_BLOCKS = TCU_BLOCK_CAP / TCU_WG_B_BLOCK_SIZE; // Symmetric sparse flag (NT=4, NT=16: block_em == block_en) @@ -121,9 +127,11 @@ package VX_tcu_pkg; localparam TCU_B_SUB_BLOCKS_SP = TCU_BLOCK_CAP / TCU_B_BLOCK_SIZE_SP; // WGMMA_SP always needs the full candidate lane set, regardless of SYM_SPARSE. localparam TCU_WG_B_BLOCK_SIZE_SP = TCU_TC_K * TCU_TC_N * 2; - // Width of the tbuf_rs2_data port: wider only when SPARSE is enabled (WGMMA_SP path). - // Without SPARSE, only TCU_BLOCK_CAP lanes are ever consumed, so keep the port narrow. - localparam TCU_WG_RS2_WIDTH = `VX_CFG_TCU_SPARSE_ENABLED ? TCU_WG_B_BLOCK_SIZE_SP : TCU_BLOCK_CAP; + localparam TCU_WG_RS2_WIDTH_DENSE = + (TCU_WG_B_BLOCK_SIZE > TCU_BLOCK_CAP) ? TCU_WG_B_BLOCK_SIZE : TCU_BLOCK_CAP; + localparam TCU_WG_RS2_WIDTH = `VX_CFG_TCU_SPARSE_ENABLED + ? ((TCU_WG_B_BLOCK_SIZE_SP > TCU_WG_RS2_WIDTH_DENSE) ? TCU_WG_B_BLOCK_SIZE_SP : TCU_WG_RS2_WIDTH_DENSE) + : TCU_WG_RS2_WIDTH_DENSE; localparam TCU_MIN_FMT_WIDTH = 4; //int4 localparam TCU_MAX_ELT_RATIO = 32 / TCU_MIN_FMT_WIDTH; @@ -171,7 +179,7 @@ package VX_tcu_pkg; input int unsigned block_elems ); automatic int unsigned sparse_ratio = `VX_CFG_TCU_SPARSE_ENABLED ? 2 : 1; - automatic int unsigned fedp_elems = TCU_TC_K * (32 / data_bits) * sparse_ratio; + automatic int unsigned fedp_elems = TCU_WG_FEDP_K * (32 / data_bits) * sparse_ratio; return (fedp_elems + block_elems - 1) / block_elems; endfunction @@ -303,13 +311,20 @@ package VX_tcu_pkg; endcase endfunction - function automatic int unsigned mx_scale_blocks_k(input logic [TCU_FMT_WIDTH-1:0] fmt); + function automatic int unsigned mx_scale_blocks_k_words( + input logic [TCU_FMT_WIDTH-1:0] fmt, + input int unsigned tile_k_words + ); automatic int unsigned data_bits = tcu_fmt_width(fmt); automatic int unsigned block_elems = mx_scale_block_size(fmt); - automatic int unsigned tile_elems = (data_bits != 0) ? TCU_TILE_K * (32 / data_bits) : 0; + automatic int unsigned tile_elems = (data_bits != 0) ? tile_k_words * (32 / data_bits) : 0; return (tile_elems + block_elems - 1) / block_elems; endfunction + function automatic int unsigned mx_scale_blocks_k(input logic [TCU_FMT_WIDTH-1:0] fmt); + return mx_scale_blocks_k_words(fmt, TCU_TILE_K); + endfunction + function automatic logic [4:0] meta_num_cols(input logic [TCU_FMT_WIDTH-1:0] fmt); automatic int hw = tcu_fmt_width(fmt) / 2; return 5'((TCU_BLOCK_CAP + hw - 1) / hw); @@ -424,6 +439,7 @@ package VX_tcu_pkg; // step_m/step_n/step_k at every consumer. logic is_first_uop; logic is_last_uop; + logic setup_fire; `ifdef VX_CFG_TCU_SPARSE_ENABLE logic is_sparse; `endif diff --git a/hw/rtl/tcu/VX_tcu_tbuf.sv b/hw/rtl/tcu/VX_tcu_tbuf.sv index ec01e2d9a..724866b04 100644 --- a/hw/rtl/tcu/VX_tcu_tbuf.sv +++ b/hw/rtl/tcu/VX_tcu_tbuf.sv @@ -60,7 +60,7 @@ module VX_tcu_tbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( VX_mem_bus_if.master tcu_lmem_if, // Per-block operand outputs (rs2 is broadcast — bbuf is shared) - output wire [BLOCK_SIZE-1:0][TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tbuf_rs1_data, + output wire [BLOCK_SIZE-1:0][TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tbuf_rs1_data, output wire [BLOCK_SIZE-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0]tbuf_rs2_data, output wire [BLOCK_SIZE-1:0] tbuf_ready ); @@ -84,7 +84,7 @@ module VX_tcu_tbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // Per-block abufs // ----------------------------------------------------------------------- - wire [BLOCK_SIZE-1:0][TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] abuf_rs1_data_w; + wire [BLOCK_SIZE-1:0][TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] abuf_rs1_data_w; wire [BLOCK_SIZE-1:0] abuf_ready_w; `ifdef PERF_ENABLE @@ -106,11 +106,17 @@ module VX_tcu_tbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( `endif .req_wid (req[b].wid), .req_valid (req[b].valid), + .req_setup (req[b].setup_fire), .req_step_m (req[b].step_m), .req_step_n (req[b].step_n), .req_step_k (req[b].step_k), .req_desc_a (req[b].desc_a), .req_a_is_smem (req[b].a_is_smem), + `ifdef VX_CFG_TCU_SPARSE_ENABLE + .req_is_sparse (req[b].is_sparse), + `else + .req_is_sparse (1'b0), + `endif .req_uuid (req[b].uuid), .tcu_lmem_if (lmem_masters[b]), .abuf_ready (abuf_ready_w[b]), @@ -136,6 +142,7 @@ module VX_tcu_tbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( `endif wire bbuf_req_valid; + wire bbuf_req_setup; wire bbuf_req_is_first_uop; wire bbuf_req_is_sparse; wire [3:0] bbuf_req_step_m; @@ -148,6 +155,7 @@ module VX_tcu_tbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( if (BLOCK_SIZE == 1) begin : g_bbuf_inputs_n1 assign bbuf_req_uuid = req[0].uuid; assign bbuf_req_valid = req[0].valid; + assign bbuf_req_setup = req[0].setup_fire; assign bbuf_req_is_first_uop = req[0].is_first_uop; `ifdef VX_CFG_TCU_SPARSE_ENABLE assign bbuf_req_is_sparse = req[0].is_sparse; @@ -161,8 +169,10 @@ module VX_tcu_tbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( assign bbuf_req_desc_b = req[0].desc_b; end else begin : g_bbuf_inputs_pe wire [BLOCK_SIZE-1:0] req_valid_v; + wire [BLOCK_SIZE-1:0] req_setup_v; for (genvar b = 0; b < BLOCK_SIZE; ++b) begin : g_req_valid_v assign req_valid_v[b] = req[b].valid; + assign req_setup_v[b] = req[b].setup_fire; end wire [BLOCK_SIZE-1:0] bbuf_sel_oh; @@ -213,6 +223,7 @@ module VX_tcu_tbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( assign bbuf_req_uuid = sel_uuid; assign bbuf_req_valid = bbuf_sel_valid; + assign bbuf_req_setup = |req_setup_v; assign bbuf_req_is_first_uop = sel_is_first_uop; assign bbuf_req_is_sparse = sel_is_sparse; assign bbuf_req_step_m = sel_step_m; @@ -235,6 +246,7 @@ module VX_tcu_tbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( .lmem_reads (bbuf_lmem_reads_w), `endif .req_valid (bbuf_req_valid), + .req_setup (bbuf_req_setup), .req_is_first_uop (bbuf_req_is_first_uop), .req_is_sparse (bbuf_req_is_sparse), .req_step_m (bbuf_req_step_m), diff --git a/hw/rtl/tcu/VX_tcu_unit.sv b/hw/rtl/tcu/VX_tcu_unit.sv index 26148747c..281099873 100644 --- a/hw/rtl/tcu/VX_tcu_unit.sv +++ b/hw/rtl/tcu/VX_tcu_unit.sv @@ -151,7 +151,7 @@ module VX_tcu_unit import VX_gpu_pkg::*, VX_tcu_pkg::*; #( assign exec_data_w[bi] = core_execute_if[bi].data; end - wire [BLOCK_SIZE-1:0][TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tbuf_rs1_data; + wire [BLOCK_SIZE-1:0][TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tbuf_rs1_data; wire [BLOCK_SIZE-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tbuf_rs2_data; wire [BLOCK_SIZE-1:0] tbuf_ready_eff; diff --git a/hw/rtl/tcu/VX_tcu_uops.sv b/hw/rtl/tcu/VX_tcu_uops.sv index 16599d3aa..73a7de039 100644 --- a/hw/rtl/tcu/VX_tcu_uops.sv +++ b/hw/rtl/tcu/VX_tcu_uops.sv @@ -49,11 +49,15 @@ module VX_tcu_uops import VX_tcu_pkg::*, VX_gpu_pkg::*; ( localparam IDX_BITS = LG_N + LG_M + LG_K; `ifdef VX_CFG_TCU_WGMMA_ENABLE `ifdef VX_CFG_TCU_SPARSE_ENABLE - localparam MAX_WG_UOPS_SP = TCU_WG_UOPS / 2; - localparam CTR_W_MAX_WMMA = MAX_UOPS > TCU_WG_UOPS ? MAX_UOPS : TCU_WG_UOPS; - localparam CTR_W_BASE = $clog2(CTR_W_MAX_WMMA > MAX_WG_UOPS_SP ? CTR_W_MAX_WMMA : MAX_WG_UOPS_SP); + localparam TCU_WG_K_STEPS_SP = (TCU_WG_K_STEPS > 1) ? (TCU_WG_K_STEPS / 2) : 1; + localparam MAX_WG_UOPS_SP = TCU_WG_M_STEPS * TCU_WG_N_STEPS * TCU_WG_K_STEPS_SP; + localparam MAX_WG_UOPS_DENSE = TCU_WG_UOPS + 1; + localparam MAX_WG_UOPS_SPARSE = MAX_WG_UOPS_SP + 1; + localparam CTR_W_MAX_WMMA = MAX_UOPS > MAX_WG_UOPS_DENSE ? MAX_UOPS : MAX_WG_UOPS_DENSE; + localparam CTR_W_BASE = $clog2(CTR_W_MAX_WMMA > MAX_WG_UOPS_SPARSE ? CTR_W_MAX_WMMA : MAX_WG_UOPS_SPARSE); `else - localparam CTR_W_BASE = $clog2(MAX_UOPS > TCU_WG_UOPS ? MAX_UOPS : TCU_WG_UOPS); + localparam MAX_WG_UOPS_DENSE = TCU_WG_UOPS + 1; + localparam CTR_W_BASE = $clog2(MAX_UOPS > MAX_WG_UOPS_DENSE ? MAX_UOPS : MAX_WG_UOPS_DENSE); `endif `else localparam CTR_W_BASE = $clog2(MAX_UOPS); @@ -78,11 +82,14 @@ module VX_tcu_uops import VX_tcu_pkg::*, VX_gpu_pkg::*; ( || (ibuf_in.op_type == INST_TCU_WGMMA_SP) `endif ; +`ifdef VX_CFG_TCU_SPARSE_ENABLE + wire wg_is_sparse = (ibuf_in.op_type == INST_TCU_WGMMA_SP); +`endif wire wg_a_from_smem = ibuf_in.op_args.tcu.a_from_smem; // Variable NRC based on cd_nregs: 0→8, 1→16, 2→32 // Loop order: m (inner) → n → k (outer) [K-outer] - // m_steps=2 and k_steps=2 are fixed; n varies (middle). + // m_steps=2; k_steps is 2 normally and 1 with FEDP2K. // K-outer lets independent (m,n) tiles overlap FEDP latency. localparam LG_M_WG = $clog2(TCU_WG_M_STEPS); // 1 localparam LG_K_WG = $clog2(TCU_WG_K_STEPS); // 1 @@ -93,19 +100,36 @@ module VX_tcu_uops import VX_tcu_pkg::*, VX_gpu_pkg::*; ( localparam WG_MN_NR16 = 16; // nrc=16: n_steps=8, mn=16 localparam WG_MN_NR32 = 32; // nrc=32: n_steps=16, mn=32 - // Pre-computed total uop counts for each cd_nregs value (dense) + // Pre-computed compute uop counts for each cd_nregs value (dense) localparam WG_UOPS_NR8 = WG_MN_NR8 * TCU_WG_K_STEPS; localparam WG_UOPS_NR16 = WG_MN_NR16 * TCU_WG_K_STEPS; localparam WG_UOPS_NR32 = WG_MN_NR32 * TCU_WG_K_STEPS; +`ifdef VX_CFG_TCU_SPARSE_ENABLE + localparam WG_UOPS_SP_NR8 = WG_MN_NR8 * TCU_WG_K_STEPS_SP; + localparam WG_UOPS_SP_NR16 = WG_MN_NR16 * TCU_WG_K_STEPS_SP; + localparam WG_UOPS_SP_NR32 = WG_MN_NR32 * TCU_WG_K_STEPS_SP; +`endif // Mux uop count based on cd_nregs: 0→8, 1→16, 2→32 reg [UOP_CTR_W-1:0] wg_uop_cnt; + reg [UOP_CTR_W-1:0] wg_compute_uop_cnt; always_comb begin case (ibuf_in.op_args.tcu.cd_nregs) - 2'd0: wg_uop_cnt = UOP_CTR_W'(WG_UOPS_NR8); - 2'd1: wg_uop_cnt = UOP_CTR_W'(WG_UOPS_NR16); - default: wg_uop_cnt = UOP_CTR_W'(WG_UOPS_NR32); + 2'd0: wg_compute_uop_cnt = UOP_CTR_W'(WG_UOPS_NR8); + 2'd1: wg_compute_uop_cnt = UOP_CTR_W'(WG_UOPS_NR16); + default: wg_compute_uop_cnt = UOP_CTR_W'(WG_UOPS_NR32); endcase + `ifdef VX_CFG_TCU_SPARSE_ENABLE + if (wg_is_sparse) begin + case (ibuf_in.op_args.tcu.cd_nregs) + 2'd0: wg_compute_uop_cnt = UOP_CTR_W'(WG_UOPS_SP_NR8); + 2'd1: wg_compute_uop_cnt = UOP_CTR_W'(WG_UOPS_SP_NR16); + default: wg_compute_uop_cnt = UOP_CTR_W'(WG_UOPS_SP_NR32); + endcase + wg_uop_cnt = wg_compute_uop_cnt + UOP_CTR_W'(1); + end else + `endif + wg_uop_cnt = wg_compute_uop_cnt + UOP_CTR_W'(1); end // K-outer index extraction: @@ -113,13 +137,11 @@ module VX_tcu_uops import VX_tcu_pkg::*, VX_gpu_pkg::*; ( // Sparse: ctr = n * m_steps + m (k always 0) // Since n_steps varies by cd_nregs, k-index bit position shifts. // m is always bit 0 (m_steps=2). n and k extracted via mux. - wire [`UP(CTR_W)-1:0] wg_idx_ctr = ctr; + wire is_wg_setup_uop = (ctr == '0); + wire [`UP(CTR_W)-1:0] wg_idx_ctr = is_wg_setup_uop ? '0 : (ctr - `UP(CTR_W)'(1)); wire [`UP(LG_M_WG)-1:0] wg_m_index = wg_idx_ctr[0 +: `UP(LG_M_WG)]; reg [`UP(LG_K_WG)-1:0] wg_k_index; reg [`UP(LG_N_WG_MAX)-1:0] wg_n_index; -`ifdef VX_CFG_TCU_SPARSE_ENABLE - wire wg_is_sparse = (ibuf_in.op_type == INST_TCU_WGMMA_SP); -`endif always_comb begin `ifdef VX_CFG_TCU_SPARSE_ENABLE if (wg_is_sparse) begin @@ -154,10 +176,9 @@ module VX_tcu_uops import VX_tcu_pkg::*, VX_gpu_pkg::*; ( localparam [4:0] wg_ra_base = TCU_WG_RA; // Register offsets for from-reg mode - // Dense A: rs1_off = m * k_steps + k (NRA=4 regs at ra_base: f24..f27) + // Dense A: rs1_off = m * 2 + k (NRA=4 regs at ra_base: f24..f27) // Sparse A: rs1_off = m (NRA compressed to 2 regs f24,f25; f26,f27 hold metadata) - localparam LG_WG_A_SB = $clog2(`UP(TCU_WG_A_SUB_BLOCKS)); - wire [`UP(CTR_W)-1:0] wg_rs1_reg_off_dense = ((`UP(CTR_W)'(wg_m_index) >> LG_WG_A_SB) << `UP(LG_K_WG)) | `UP(CTR_W)'(wg_k_index); + wire [`UP(CTR_W)-1:0] wg_rs1_reg_off_dense = (`UP(CTR_W)'(wg_m_index) << 1) | `UP(CTR_W)'(wg_k_index); `ifdef VX_CFG_TCU_SPARSE_ENABLE wire [`UP(CTR_W)-1:0] wg_rs1_reg_off = (wg_is_sparse && !wg_a_from_smem) ? `UP(CTR_W)'(wg_m_index) @@ -179,15 +200,11 @@ module VX_tcu_uops import VX_tcu_pkg::*, VX_gpu_pkg::*; ( ; `endif - // Sparse WGMMA (RS) expands to wg_uop_cnt/2 MMA uops; WMMA_SP expands + // Sparse WGMMA (RS) expands to max(1, k_steps/2) MMA uops plus setup; WMMA_SP expands // to TCU_UOPS (sym) or TCU_UOPS/2 (asym). assign uop_count = `ifdef VX_CFG_TCU_WGMMA_ENABLE - is_wgmma ? ( - `ifdef VX_CFG_TCU_SPARSE_ENABLE - wg_is_sparse ? (wg_uop_cnt >> 1) : - `endif - wg_uop_cnt) : + is_wgmma ? wg_uop_cnt : `endif `ifdef VX_CFG_TCU_SPARSE_ENABLE is_sparse ? (SYM_SPARSE ? UOP_CTR_W'(TCU_UOPS) : UOP_CTR_W'(TCU_UOPS / 2)) : @@ -312,25 +329,44 @@ module VX_tcu_uops import VX_tcu_pkg::*, VX_gpu_pkg::*; ( `endif `ifdef VX_CFG_TCU_WGMMA_ENABLE if (is_wgmma) begin - ibuf_r.op_args.tcu.step_m = 4'(wg_m_index); - ibuf_r.op_args.tcu.step_n = 4'(wg_n_index); - ibuf_r.op_args.tcu.step_k = 4'(wg_k_index); - ibuf_r.wb = 1'b1; + ibuf_r.op_args.tcu.step_m = is_wg_setup_uop ? 4'd0 : 4'(wg_m_index); + ibuf_r.op_args.tcu.step_n = is_wg_setup_uop ? 4'd0 : 4'(wg_n_index); + ibuf_r.op_args.tcu.step_k = is_wg_setup_uop ? 4'd0 : 4'(wg_k_index); + ibuf_r.wb = !is_wg_setup_uop; ibuf_r.rd = make_reg_num(REG_TYPE_F, TCU_WG_RC + wg_rs3_off); ibuf_r.rs3 = make_reg_num(REG_TYPE_F, TCU_WG_RC + wg_rs3_off); - ibuf_r.used_rs[2] = 1'b1; + ibuf_r.used_rs[2] = !is_wg_setup_uop; // Smem descriptors are invariant across the whole WGMMA expansion, - // so only fetch them on the first MMA uop. - if (wg_a_from_smem) begin + // so only fetch them on the setup uop. + if (is_wg_setup_uop) begin + ibuf_r.rs1 = make_reg_num(REG_TYPE_I, 5'd10); + ibuf_r.used_rs[0] = 1'b1; + end else if (wg_a_from_smem) begin ibuf_r.rs1 = make_reg_num(REG_TYPE_I, 5'd10); - ibuf_r.used_rs[0] = (wg_idx_ctr == '0); + ibuf_r.used_rs[0] = 1'b0; end else begin ibuf_r.rs1 = make_reg_num(REG_TYPE_F, wg_ra_base + 5'(wg_rs1_reg_off)); ibuf_r.used_rs[0] = 1'b1; end - // B source: always smem descriptor (x11), fetched on first MMA uop + // Setup reads desc_b from x11. Dense FEDP2K RS compute uses rs2 + // as the upper A half, matching SimX. ibuf_r.rs2 = make_reg_num(REG_TYPE_I, 5'd11); - ibuf_r.used_rs[1] = (wg_idx_ctr == '0); + if (is_wg_setup_uop) begin + ibuf_r.used_rs[1] = 1'b1; + end else if (!wg_a_from_smem + `ifdef VX_CFG_TCU_FEDP2K + `ifdef VX_CFG_TCU_SPARSE_ENABLE + && !wg_is_sparse + `endif + `else + && 1'b0 + `endif + ) begin + ibuf_r.rs2 = make_reg_num(REG_TYPE_F, wg_ra_base + 5'(wg_rs1_reg_off) + 5'd1); + ibuf_r.used_rs[1] = 1'b1; + end else begin + ibuf_r.used_rs[1] = 1'b0; + end end else `endif begin @@ -370,12 +406,12 @@ module VX_tcu_uops import VX_tcu_pkg::*, VX_gpu_pkg::*; ( end `ifdef VX_CFG_TCU_WGMMA_ENABLE if (is_wgmma) begin - ibuf_r.fu_lock = (uop_idx == '0); + ibuf_r.fu_lock = is_wg_setup_uop; ibuf_r.fu_unlock = (uop_idx == (uop_count - UOP_CTR_W'(1))); // Expose first/last-uop predicates via op_args.tcu so downstream // consumers (bbuf, lockstep) read a single source of truth. - ibuf_r.op_args.tcu.is_first_uop = (uop_idx == '0); - ibuf_r.op_args.tcu.is_last_uop = (uop_idx == (uop_count - UOP_CTR_W'(1))); + ibuf_r.op_args.tcu.is_first_uop = !is_wg_setup_uop && (wg_idx_ctr == '0); + ibuf_r.op_args.tcu.is_last_uop = !is_wg_setup_uop && (uop_idx == (uop_count - UOP_CTR_W'(1))); end else `endif begin diff --git a/hw/rtl/tcu/VX_tcu_wgmma.sv b/hw/rtl/tcu/VX_tcu_wgmma.sv index 8b9d5fc9d..b8888f25f 100644 --- a/hw/rtl/tcu/VX_tcu_wgmma.sv +++ b/hw/rtl/tcu/VX_tcu_wgmma.sv @@ -46,7 +46,7 @@ module VX_tcu_wgmma import VX_gpu_pkg::*, VX_tcu_pkg::*; #( VX_mem_bus_if.master tcu_lmem_if, // Outputs to tcu_core (consumed by the wrapper). - output wire [BLOCK_SIZE-1:0][TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] tbuf_rs1_data, + output wire [BLOCK_SIZE-1:0][TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tbuf_rs1_data, output wire [BLOCK_SIZE-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tbuf_rs2_data, output wire [BLOCK_SIZE-1:0] tbuf_ready_eff ); @@ -62,19 +62,39 @@ module VX_tcu_wgmma import VX_gpu_pkg::*, VX_tcu_pkg::*; #( wire [BLOCK_SIZE-1:0] exec_fire_b_w; wire [BLOCK_SIZE-1:0] is_first_uop_b_w; wire [BLOCK_SIZE-1:0] is_last_uop_b_w; + wire [BLOCK_SIZE-1:0] is_setup_uop_b_w; + + logic [BLOCK_SIZE-1:0][`VX_CFG_XLEN-1:0] desc_a_r; + logic [BLOCK_SIZE-1:0][`VX_CFG_XLEN-1:0] desc_b_r; + for (genvar bi = 0; bi < BLOCK_SIZE; ++bi) begin : g_lockstep_inputs wire is_wgmma_op = (exec_data[bi].op_type == INST_TCU_WGMMA) `ifdef VX_CFG_TCU_SPARSE_ENABLE || (exec_data[bi].op_type == INST_TCU_WGMMA_SP) `endif ; - assign is_wgmma_b_w[bi] = exec_valid[bi] && is_wgmma_op; + assign is_setup_uop_b_w[bi] = is_wgmma_op && !exec_data[bi].header.wb; + assign is_wgmma_b_w[bi] = exec_valid[bi] && is_wgmma_op && !is_setup_uop_b_w[bi]; assign new_cta_b_w[bi] = exec_data[bi].header.cta_id; assign exec_fire_b_w[bi] = exec_valid[bi] && exec_ready[bi]; assign is_first_uop_b_w[bi] = exec_data[bi].op_args.tcu.is_first_uop; assign is_last_uop_b_w[bi] = exec_data[bi].op_args.tcu.is_last_uop; end + always_ff @(posedge clk) begin + if (reset) begin + desc_a_r <= '0; + desc_b_r <= '0; + end else begin + for (int bi = 0; bi < BLOCK_SIZE; ++bi) begin + if (exec_valid[bi] && exec_ready[bi] && is_setup_uop_b_w[bi]) begin + desc_a_r[bi] <= exec_data[bi].rs1_data[0]; + desc_b_r[bi] <= exec_data[bi].rs2_data[0]; + end + end + end + end + VX_tcu_lockstep #( .INSTANCE_ID (`SFORMATF(("%s-lockstep", INSTANCE_ID))), .BLOCK_SIZE (BLOCK_SIZE) @@ -102,18 +122,20 @@ module VX_tcu_wgmma import VX_gpu_pkg::*, VX_tcu_pkg::*; #( || (exec_data[bi].op_type == INST_TCU_WGMMA_SP) `endif ; - assign req[bi].valid = exec_valid[bi] && is_wgmma_b && !cta_conflict[bi]; + wire is_setup_uop = is_wgmma_b && !exec_data[bi].header.wb; + assign req[bi].valid = exec_valid[bi] && is_wgmma_b && !is_setup_uop && !cta_conflict[bi]; assign req[bi].uuid = exec_data[bi].header.uuid; assign req[bi].wid = exec_data[bi].header.wid; assign req[bi].step_m = exec_data[bi].op_args.tcu.step_m; assign req[bi].step_k = exec_data[bi].op_args.tcu.step_k; assign req[bi].step_n = exec_data[bi].op_args.tcu.step_n; assign req[bi].cd_nregs = exec_data[bi].op_args.tcu.cd_nregs; - assign req[bi].desc_a = exec_data[bi].rs1_data[0]; - assign req[bi].desc_b = exec_data[bi].rs2_data[0]; + assign req[bi].desc_a = desc_a_r[bi]; + assign req[bi].desc_b = desc_b_r[bi]; assign req[bi].a_is_smem = exec_data[bi].op_args.tcu.a_from_smem; assign req[bi].is_first_uop = exec_data[bi].op_args.tcu.is_first_uop; assign req[bi].is_last_uop = exec_data[bi].op_args.tcu.is_last_uop; + assign req[bi].setup_fire = exec_valid[bi] && exec_ready[bi] && is_setup_uop; `ifdef VX_CFG_TCU_SPARSE_ENABLE assign req[bi].is_sparse = (exec_data[bi].op_type == INST_TCU_WGMMA_SP); `endif @@ -148,7 +170,7 @@ module VX_tcu_wgmma import VX_gpu_pkg::*, VX_tcu_pkg::*; #( ); for (genvar bi = 0; bi < BLOCK_SIZE; ++bi) begin : g_tbuf_eff - assign tbuf_ready_eff[bi] = tbuf_ready[bi] && !cta_conflict[bi]; + assign tbuf_ready_eff[bi] = is_setup_uop_b_w[bi] ? 1'b1 : (tbuf_ready[bi] && !cta_conflict[bi]); end // ----------------------------------------------------------------------- diff --git a/hw/rtl/tcu/tet/VX_tcu_fedp_tet.sv b/hw/rtl/tcu/tet/VX_tcu_fedp_tet.sv new file mode 100644 index 000000000..feb7dca56 --- /dev/null +++ b/hw/rtl/tcu/tet/VX_tcu_fedp_tet.sv @@ -0,0 +1,409 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +module VX_tcu_fedp_tet import VX_tcu_pkg::*; #( + parameter `STRING INSTANCE_ID = "", + parameter LANE_MASK = 0, + parameter LATENCY = 0, + parameter N = TCU_TC_K, + parameter SF = 1, + parameter W = 25 +) ( + input wire clk, + input wire reset, + input wire enable, + input wire [TCU_MAX_INPUTS-1:0] vld_mask, + input wire [4:0] fmt_s, + input wire [4:0] fmt_d, + input wire [N-1:0][31:0] a_row, + input wire [N-1:0][31:0] b_col, +`ifdef VX_CFG_TCU_MX_ENABLE + input wire [SF-1:0][7:0] sf_a, + input wire [SF-1:0][7:0] sf_b, +`endif + input wire [31:0] c_val, + output wire [31:0] d_val +); + `UNUSED_SPARAM (INSTANCE_ID) + `UNUSED_VAR (fmt_d) + +`ifndef VX_CFG_TCU_MX_ENABLE + `UNUSED_PARAM (SF) +`endif + + localparam TCK = 2 * N; + localparam EXP_W = TCU_EXP_BITS; + localparam EXC_W = $bits(fedp_excep_t); + localparam C_HI_W = 7; + localparam HR = $clog2(TCK+1); + + localparam ALN_SIG_W = W + 2; + localparam ACC_SIG_W = W + 1 + HR; + + localparam MUL0_LATENCY = 1; + localparam MUL1_LATENCY = 1; + localparam ALN0_LATENCY = 1; + localparam ALN1_LATENCY = 1; + localparam ACC0_LATENCY = 1; + localparam ACC1_LATENCY = 1; + localparam NRM0_LATENCY = 1; + localparam NRM1_LATENCY = 1; + localparam MUL_LATENCY = MUL0_LATENCY + MUL1_LATENCY; + localparam ALN_LATENCY = ALN0_LATENCY + ALN1_LATENCY; + localparam ACC_LATENCY = ACC0_LATENCY + ACC1_LATENCY; + localparam NRM_LATENCY = NRM0_LATENCY + NRM1_LATENCY; + localparam TOTAL_LATENCY = MUL_LATENCY + ALN_LATENCY + ACC_LATENCY + NRM_LATENCY; + + `STATIC_ASSERT (LATENCY == 0 || LATENCY == TOTAL_LATENCY, + ("invalid latency! expected=%0d, actual=%0d", TOTAL_LATENCY, LATENCY)) + + localparam S0_IDX = 0; + localparam S1_IDX = S0_IDX + MUL0_LATENCY; + localparam S2_IDX = S1_IDX + MUL1_LATENCY; + localparam S3_IDX = S2_IDX + ALN0_LATENCY; + localparam S4_IDX = S3_IDX + ALN1_LATENCY; + localparam S5_IDX = S4_IDX + ACC0_LATENCY; + localparam S6_IDX = S5_IDX + ACC1_LATENCY; + localparam S7_IDX = S6_IDX + NRM0_LATENCY; + localparam S8_IDX = S7_IDX + NRM1_LATENCY; + `UNUSED_PARAM(S8_IDX) + + reg [TOTAL_LATENCY-1:0] vld_pipe_r; + reg [TOTAL_LATENCY-1:0][31:0] req_pipe_r; + reg [31:0] req_id; + + wire vld_any = (|vld_mask) && (LANE_MASK != 0); + + always_ff @(posedge clk) begin + if (reset) begin + vld_pipe_r <= '0; + req_pipe_r <= '0; + req_id <= 0; + end else if (enable) begin + vld_pipe_r <= {vld_pipe_r[TOTAL_LATENCY-2:0], vld_any}; + req_pipe_r <= {req_pipe_r[TOTAL_LATENCY-2:0], req_id}; + req_id <= req_id + 32'(vld_any); + end + end + + wire [TOTAL_LATENCY:0] vld_pipe = {vld_pipe_r, (~reset && enable && vld_any)}; + wire [TOTAL_LATENCY:0][31:0] req_pipe = {req_pipe_r, req_id}; + + // ====================================================================== + // Stage 0/1: Multiply & Diff Matrix + // ====================================================================== + + wire [TCK:0][EXP_W-1:0] exponents; + wire [TCK:0] exp_sel; + wire [TCK-1:0][TCK-1:0][EXP_W:0] exp_diff_mat; + wire [TCK:0][W-1:0] raw_sigs; + fedp_excep_t exceptions; + wire [TCK-1:0] lane_mask; + + wire is_int = tcu_fmt_is_int(fmt_s); + + wire [7:0] cval_top = c_val[31:24]; + wire [6:0] cval_hi = cval_top[7:1] + 7'(cval_top[0]); + wire [C_HI_W-1:0] mul_cval_hi; + wire mul_is_int; + + VX_tcu_tet_shared_mul #( + .INSTANCE_ID (INSTANCE_ID), + .N (N), + .W (W), + .WA (ACC_SIG_W), + .EXP_W (EXP_W), + .TCK (TCK), + .SF (SF) + ) shared_mul ( + .clk (clk), + .reset (reset), + .enable (enable), + .valid_in (vld_pipe[S0_IDX]), + .req_id (req_pipe[S0_IDX]), + .vld_mask (vld_mask | TCU_MAX_INPUTS'(LANE_MASK == 0)), + .fmt_s (fmt_s), + .a_row (a_row), + .b_col (b_col), + .c_val (c_val), + `ifdef VX_CFG_TCU_MX_ENABLE + .sf_a (sf_a), + .sf_b (sf_b), + `endif + .exponents (exponents), + .exp_sel (exp_sel), + .exp_diff_mat (exp_diff_mat), + .raw_sigs (raw_sigs), + .exceptions (exceptions), + .lane_mask (lane_mask) + ); + + VX_tcu_tet_register #( + .DATAW (C_HI_W + 1), + .DEPTH (MUL0_LATENCY) + ) pipe_mul_ctrl ( + .clk (clk), + .reset (reset), + .enable (enable), + .data_in ({cval_hi, is_int}), + .data_out ({mul_cval_hi, mul_is_int}) + ); + + wire [TCK:0][EXP_W-1:0] s2_exponents; + wire [TCK:0] s2_exp_sel; + wire [TCK-1:0][TCK-1:0][EXP_W:0] s2_exp_diff_mat; + fedp_excep_t s2_exceptions; + wire [TCK-1:0] s2_lane_mask; + wire [TCK:0][W-1:0] s2_raw_sig; + wire s2_is_int; + wire [C_HI_W-1:0] s2_cval_hi; + + wire [TCK-1:0][W-1:0] pipe_mul_lane_din, pipe_mul_lane_dout; + `MAP_AOS_SOA(i, TCK, pipe_mul_lane_din[i], raw_sigs[i]) + `MAP_AOS_SOA(i, TCK, s2_raw_sig[i], pipe_mul_lane_dout[i]) + + VX_tcu_tet_pipe_register #( + .NUM_LANES (TCK), + .SHARED_DATAW(((TCK+1)*EXP_W) + (TCK+1) + (TCK*TCK*(EXP_W+1)) + EXC_W + TCK + W + C_HI_W + 1), + .LANE_DATAW (W), + .DEPTH (MUL1_LATENCY), + .LANE_MASK (LANE_MASK) + ) pipe_mul ( + .clk (clk), + .reset (reset), + .enable (enable), + .lane_mask (lane_mask), + .shared_data_in ({exponents, exp_sel, exp_diff_mat, exceptions, lane_mask, raw_sigs[TCK], mul_cval_hi, mul_is_int}), + .shared_data_out ({s2_exponents, s2_exp_sel, s2_exp_diff_mat, s2_exceptions, s2_lane_mask, s2_raw_sig[TCK], s2_cval_hi, s2_is_int}), + .lane_data_in (pipe_mul_lane_din), + .lane_data_out (pipe_mul_lane_dout) + ); + + // ====================================================================== + // Stage 2/3: Max Exp & Alignment + // ====================================================================== + + wire [TCK:0][ALN_SIG_W-1:0] s3_aln_sigs; + wire [TCK:0] s3_aln_sticky; + wire [EXP_W-1:0] s3_max_exp; + fedp_excep_t s3_exceptions; + wire [TCK-1:0] s3_lane_mask; + wire s3_is_int; + wire [C_HI_W-1:0] s3_cval_hi; + + VX_tcu_tet_align #( + .INSTANCE_ID (INSTANCE_ID), + .N (TCK+1), + .WI (W), + .WO (ALN_SIG_W) + ) align ( + .clk (clk), + .reset (reset), + .enable (enable), + .valid_in (vld_pipe[S2_IDX]), + .req_id (req_pipe[S2_IDX]), + .exponents (s2_exponents), + .sel_exp (s2_exp_sel), + .diff_mat (s2_exp_diff_mat), + .sigs_in (s2_raw_sig), + .is_int (s2_is_int), + .max_exp (s3_max_exp), + .sigs_out (s3_aln_sigs), + .sticky_bits (s3_aln_sticky) + ); + + VX_tcu_tet_register #( + .DATAW (EXC_W + TCK + C_HI_W + 1), + .DEPTH (ALN0_LATENCY) + ) pipe_aln_ctrl ( + .clk (clk), + .reset (reset), + .enable (enable), + .data_in ({s2_exceptions, s2_lane_mask, s2_cval_hi, s2_is_int}), + .data_out ({s3_exceptions, s3_lane_mask, s3_cval_hi, s3_is_int}) + ); + + wire [EXP_W-1:0] s4_max_exp; + fedp_excep_t s4_exceptions; + wire [TCK-1:0] s4_lane_mask; + wire [TCK:0][ALN_SIG_W-1:0] s4_aln_sigs; + wire [TCK:0] s4_aln_sticky; + wire s4_is_int; + wire [C_HI_W-1:0] s4_cval_hi; + + wire [TCK-1:0][(ALN_SIG_W + 1)-1:0] pipe_aln_lane_din, pipe_aln_lane_dout; + `MAP_AOS_SOA(i, TCK, pipe_aln_lane_din[i], {s3_aln_sigs[i], s3_aln_sticky[i]}) + `MAP_AOS_SOA(i, TCK, {s4_aln_sigs[i], s4_aln_sticky[i]}, pipe_aln_lane_dout[i]) + + VX_tcu_tet_pipe_register #( + .NUM_LANES (TCK), + .SHARED_DATAW(EXP_W + EXC_W + TCK + ALN_SIG_W + 1 + C_HI_W + 1), + .LANE_DATAW (ALN_SIG_W + 1), + .DEPTH (ALN1_LATENCY), + .LANE_MASK (LANE_MASK) + ) pipe_aln ( + .clk (clk), + .reset (reset), + .enable (enable), + .lane_mask (s3_lane_mask), + .shared_data_in ({s3_max_exp, s3_exceptions, s3_lane_mask, s3_aln_sigs[TCK], s3_aln_sticky[TCK], s3_cval_hi, s3_is_int}), + .shared_data_out ({s4_max_exp, s4_exceptions, s4_lane_mask, s4_aln_sigs[TCK], s4_aln_sticky[TCK], s4_cval_hi, s4_is_int}), + .lane_data_in (pipe_aln_lane_din), + .lane_data_out (pipe_aln_lane_dout) + ); + + // ====================================================================== + // Stage 4/5: Accumulation + // ====================================================================== + + wire [ACC_SIG_W-1:0] s5_acc_sum; + wire s5_acc_sticky; + wire [EXP_W-1:0] s5_max_exp; + fedp_excep_t s5_exceptions; + wire s5_is_int; + wire [C_HI_W-1:0] s5_cval_hi; + + VX_tcu_tet_acc #( + .INSTANCE_ID (INSTANCE_ID), + .N (TCK+1), + .WI (ALN_SIG_W), + .WO (ACC_SIG_W) + ) acc ( + .clk (clk), + .reset (reset), + .enable (enable), + .valid_in (vld_pipe[S4_IDX]), + .req_id (req_pipe[S4_IDX]), + .lane_mask (s4_lane_mask), + .is_int (s4_is_int), + .sigs_in (s4_aln_sigs), + .sticky_in (s4_aln_sticky), + .sig_out (s5_acc_sum), + .sticky_out (s5_acc_sticky) + ); + + VX_tcu_tet_register #( + .DATAW (EXP_W + EXC_W + C_HI_W + 1), + .DEPTH (ACC0_LATENCY) + ) pipe_acc_ctrl ( + .clk (clk), + .reset (reset), + .enable (enable), + .data_in ({s4_max_exp, s4_exceptions, s4_cval_hi, s4_is_int}), + .data_out ({s5_max_exp, s5_exceptions, s5_cval_hi, s5_is_int}) + ); + + wire [EXP_W-1:0] s6_max_exp; + wire [ACC_SIG_W-1:0] s6_acc_sum; + fedp_excep_t s6_exceptions; + wire s6_acc_sticky; + wire s6_is_int; + wire [C_HI_W-1:0] s6_cval_hi; + + VX_tcu_tet_register #( + .DATAW (EXP_W + ACC_SIG_W + EXC_W + 1 + C_HI_W + 1), + .DEPTH (ACC1_LATENCY) + ) pipe_acc ( + .clk (clk), + .reset (reset), + .enable (enable), + .data_in ({s5_max_exp, s5_acc_sum, s5_exceptions, s5_acc_sticky, s5_cval_hi, s5_is_int}), + .data_out ({s6_max_exp, s6_acc_sum, s6_exceptions, s6_acc_sticky, s6_cval_hi, s6_is_int}) + ); + + // ====================================================================== + // Stage 6/7: Normalization & rounding + // ====================================================================== + + wire [31:0] final_result; + + VX_tcu_tet_norm_round #( + .INSTANCE_ID (INSTANCE_ID), + .EXP_W (EXP_W), + .C_HI_W (C_HI_W), + .WA (ACC_SIG_W) + ) norm_round ( + .clk (clk), + .reset (reset), + .enable (enable), + .valid_in (vld_pipe[S6_IDX]), + .req_id (req_pipe[S6_IDX]), + .max_exp (s6_max_exp), + .acc_sig (s6_acc_sum), + .sticky_in (s6_acc_sticky), + .exceptions (s6_exceptions), + .cval_hi (s6_cval_hi), + .is_int (s6_is_int), + .result (final_result) + ); + + VX_tcu_tet_register #( + .DATAW (32), + .DEPTH (NRM1_LATENCY) + ) pipe_norm_round ( + .clk (clk), + .reset (reset), + .enable (enable), + .data_in (final_result), + .data_out (d_val) + ); + +`ifdef DBG_TRACE_TCU + always_ff @(posedge clk) begin + if (vld_pipe[S0_IDX]) begin + `TRACE(4, ("%t: %s FEDP-S0(%0d): fmt_s=%0d, a_row=", $time, INSTANCE_ID, req_pipe[S0_IDX], fmt_s)); + `TRACE_ARRAY1D(4, "0x%0h", a_row, N) + `TRACE(4, (", b_col=")) + `TRACE_ARRAY1D(4, "0x%0h", b_col, N) + `TRACE(4, (", c_val=0x%0h, vld_mask=%b\n", c_val, vld_mask)) + end + end + + always_ff @(posedge clk) begin + if (vld_pipe[S2_IDX]) begin + `TRACE(4, ("%t: %s FEDP-S2(%0d): is_int=%b, cval_hi=0x%0h, exponents=", $time, INSTANCE_ID, req_pipe[S2_IDX], s2_is_int, s2_cval_hi)); + `TRACE_ARRAY1D(4, "0x%0h", s2_exponents, (TCK+1)) + `TRACE(4, (", raw_sig=")) + `TRACE_ARRAY1D(4, "0x%0h", s2_raw_sig, (TCK+1)) + `TRACE(4, (", exceptions=%0b, lane_mask=%b\n", s2_exceptions, s2_lane_mask)) + end + end + + always_ff @(posedge clk) begin + if (vld_pipe[S4_IDX]) begin + `TRACE(4, ("%t: %s FEDP-S4(%0d): is_int=%b, cval_hi=0x%0h, max_exp=0x%0h, aln_sig=", + $time, INSTANCE_ID, req_pipe[S4_IDX], s4_is_int, s4_cval_hi, s4_max_exp)); + `TRACE_ARRAY1D(4, "0x%0h", s4_aln_sigs, (TCK+1)) + `TRACE(4, (", sticky_bits=")) + `TRACE_ARRAY1D(4, "0b%b", s4_aln_sticky, (TCK+1)) + `TRACE(4, (", exceptions=%0b\n", s4_exceptions)) + end + end + + always_ff @(posedge clk) begin + if (vld_pipe[S6_IDX]) begin + `TRACE(4, ("%t: %s FEDP-S6(%0d): is_int=%b, cval_hi=0x%0h, acc_sig=0x%0h, max_exp=0x%0h, sticky=%b, exceptions=%0b\n", + $time, INSTANCE_ID, req_pipe[S6_IDX], s6_is_int, s6_cval_hi, s6_acc_sum, s6_max_exp, s6_acc_sticky, s6_exceptions)); + end + end + + always_ff @(posedge clk) begin + if (vld_pipe[S8_IDX]) begin + `TRACE(4, ("%t: %s FEDP-S8(%0d): result=0x%0h\n", $time, INSTANCE_ID, req_pipe[S8_IDX], d_val)); + end + end +`endif + +endmodule diff --git a/hw/rtl/tcu/tet/VX_tcu_tet_acc.sv b/hw/rtl/tcu/tet/VX_tcu_tet_acc.sv new file mode 100644 index 000000000..ba9fecee9 --- /dev/null +++ b/hw/rtl/tcu/tet/VX_tcu_tet_acc.sv @@ -0,0 +1,129 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +module VX_tcu_tet_acc import VX_tcu_pkg::*; #( + parameter `STRING INSTANCE_ID = "", + parameter N = 5, + parameter WI = 26, + parameter WO = 30 +) ( + input wire clk, + input wire reset, + input wire enable, + input wire valid_in, + input wire [31:0] req_id, + input wire [N-2:0] lane_mask, + input wire is_int, + input wire [N-1:0][WI-1:0] sigs_in, + input wire [N-1:0] sticky_in, + output wire [WO-1:0] sig_out, + output wire sticky_out +); + `UNUSED_SPARAM (INSTANCE_ID) + + wire [N-1:0][WI-1:0] masked_sigs; + wire [N-1:0] masked_sticky; + for (genvar i = 0; i < N-1; ++i) begin : g_mask + assign masked_sigs[i] = sigs_in[i] & {WI{lane_mask[i]}}; + assign masked_sticky[i] = sticky_in[i] & lane_mask[i]; + end + + assign masked_sigs[N-1] = sigs_in[N-1]; + assign masked_sticky[N-1] = sticky_in[N-1]; + + wire [N-1:0] fp_neg_terms; + wire [N:0][WO-1:0] sig_operands; + + for (genvar i = 0; i < N; ++i) begin : g_ext + wire [WO-1:0] int_sig = $signed({{(WO-WI){masked_sigs[i][WI-1]}}, masked_sigs[i]}); + wire [WO-1:0] fp_mag = WO'(masked_sigs[i][WI-2:0]); + assign fp_neg_terms[i] = ~is_int & masked_sigs[i][WI-1]; + assign sig_operands[i] = is_int ? int_sig : (fp_neg_terms[i] ? ~fp_mag : fp_mag); + end + + wire [`CLOG2(N+1)-1:0] fp_neg_count; + VX_popcount #( + .N (N) + ) fp_neg_popcount ( + .data_in (fp_neg_terms), + .data_out (fp_neg_count) + ); + + assign sig_operands[N] = WO'(fp_neg_count); + + wire [WO-1:0] sum_vec; + wire [WO-1:0] carry_vec; + + VX_csa_tree #( + .N (N+1), + .W (WO), + .S (WO) + ) sig_csa ( + .operands (sig_operands), + .sum (sum_vec), + .carry (carry_vec) + ); + + wire sticky_w = |masked_sticky; + + wire [WO-1:0] s1_sum_vec; + wire [WO-1:0] s1_carry_vec; + wire s1_sticky; + wire s1_valid; + wire [31:0] s1_req_id; + + VX_tcu_tet_register #( + .DATAW ((2 * WO) + 1 + 1 + 32), + .DEPTH (1) + ) pipe_acc0 ( + .clk (clk), + .reset (reset), + .enable (enable), + .data_in ({sum_vec, carry_vec, sticky_w, valid_in, req_id}), + .data_out ({s1_sum_vec, s1_carry_vec, s1_sticky, s1_valid, s1_req_id}) + ); + +`ifndef DBG_TRACE_TCU + `UNUSED_VAR ({s1_valid, s1_req_id}) +`endif + + VX_ks_adder #( + .N (WO), + .BYPASS (`FORCE_BUILTIN_ADDER(WO)) + ) final_adder ( + .cin (1'b0), + .dataa (s1_sum_vec), + .datab (s1_carry_vec), + .sum (sig_out), + `UNUSED_PIN (cout) + ); + + assign sticky_out = s1_sticky; + +`ifdef DBG_TRACE_TCU + always_ff @(posedge clk) begin + if (s1_valid) begin + `TRACE(4, ("%t: %s FEDP-ACC(%0d): lane_mask=%b, sigs_in=", $time, INSTANCE_ID, s1_req_id, lane_mask)); + `TRACE_ARRAY1D(4, "0x%0h", sigs_in, N) + `TRACE(4, (", masked_sigs=")); + `TRACE_ARRAY1D(4, "0x%0h", masked_sigs, N) + `TRACE(4, (", sticky_in=")); + `TRACE_ARRAY1D(4, "%0d", sticky_in, N) + `TRACE(4, (", sig_out=0x%0h, sticky_out=%0d\n", sig_out, sticky_out)); + end + end +`endif + +endmodule diff --git a/hw/rtl/tcu/tet/VX_tcu_tet_align.sv b/hw/rtl/tcu/tet/VX_tcu_tet_align.sv new file mode 100644 index 000000000..9b82f88d4 --- /dev/null +++ b/hw/rtl/tcu/tet/VX_tcu_tet_align.sv @@ -0,0 +1,156 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +module VX_tcu_tet_align import VX_tcu_pkg::*; #( + parameter `STRING INSTANCE_ID = "", + parameter N = 5, + parameter WI = 25, + parameter WO = WI + 2 +) ( + input wire clk, + input wire reset, + input wire enable, + input wire valid_in, + input wire [31:0] req_id, + + input wire [N-1:0][TCU_EXP_BITS-1:0] exponents, + input wire [N-1:0] sel_exp, + input wire [N-2:0][N-2:0][TCU_EXP_BITS:0] diff_mat, + + input wire [N-1:0][WI-1:0] sigs_in, + input wire is_int, + output logic [TCU_EXP_BITS-1:0] max_exp, + output wire [N-1:0][WO-1:0] sigs_out, + output wire [N-1:0] sticky_bits +); + `UNUSED_SPARAM (INSTANCE_ID) + + localparam MAX_PRE_SHIFT = WI - 23; + localparam SHIFT_MAG_W = (WI - 1) + MAX_PRE_SHIFT; + + wire [TCU_EXP_BITS-1:0] or_red[N:0] /* verilator split_var */; + wire [N-1:0][7:0] shift_amts; + wire [TCU_EXP_BITS-1:0] max_exp_w; + + assign or_red[0] = {TCU_EXP_BITS{1'b0}}; + for (genvar i = 0; i < N; i++) begin : g_or_red + assign or_red[i+1] = or_red[i] | (sel_exp[i] ? exponents[i] : {TCU_EXP_BITS{1'b0}}); + end + assign max_exp_w = or_red[N]; + + for (genvar i = 0; i < N; i++) begin : g_shift_amts + wire [TCU_EXP_BITS-1:0] sh_or [N:0] /* verilator split_var */; + + assign sh_or[0] = {TCU_EXP_BITS{1'b0}}; + for (genvar k = 0; k < N; k++) begin : g_sh_mux + if (k == i) begin : g_self + assign sh_or[k+1] = sh_or[k]; + end else if (k < i) begin : g_direct + wire [TCU_EXP_BITS-1:0] diff_lane = diff_mat[k][i-1][TCU_EXP_BITS-1:0]; + assign sh_or[k+1] = sh_or[k] | (sel_exp[k] ? diff_lane : {TCU_EXP_BITS{1'b0}}); + end else begin : g_invert + wire [TCU_EXP_BITS-1:0] diff_lane = diff_mat[i][k-1][TCU_EXP_BITS-1:0]; + assign sh_or[k+1] = sh_or[k] | (sel_exp[k] ? ~diff_lane : {TCU_EXP_BITS{1'b0}}); + end + end + + wire needs_inc; + if (i == N-1) begin : g_no_inc + assign needs_inc = 1'b0; + end else begin : g_calc_inc + wire [N-2-i:0] inc_sel; + for (genvar k = i+1; k < N; k++) begin : g_inc_sel + assign inc_sel[k-i-1] = sel_exp[k]; + end + assign needs_inc = |inc_sel; + end + + if (TCU_EXP_BITS > 8) begin : g_sat + wire [7:0] shift_lo = sh_or[N][7:0] + 8'(needs_inc); + wire shift_hi = (|sh_or[N][TCU_EXP_BITS-1:8]) || (needs_inc && (&sh_or[N][7:0])); + assign shift_amts[i] = shift_hi ? 8'hFF : shift_lo; + end else begin : g_no_sat + wire [TCU_EXP_BITS-1:0] shift_full = sh_or[N] + TCU_EXP_BITS'(needs_inc); + assign shift_amts[i] = 8'(shift_full); + end + end + + wire [TCU_EXP_BITS-1:0] s1_max_exp; + wire [N-1:0][7:0] s1_shift_amts; + wire [N-1:0][WI-1:0] s1_sigs_in; + wire s1_is_int; + wire s1_valid; + wire [31:0] s1_req_id; + + VX_tcu_tet_register #( + .DATAW (TCU_EXP_BITS + (N * 8) + (N * WI) + 1 + 1 + 32), + .DEPTH (1) + ) pipe_align0 ( + .clk (clk), + .reset (reset), + .enable (enable), + .data_in ({max_exp_w, shift_amts, sigs_in, is_int, valid_in, req_id}), + .data_out ({s1_max_exp, s1_shift_amts, s1_sigs_in, s1_is_int, s1_valid, s1_req_id}) + ); + + always_comb begin + max_exp = s1_max_exp; + end + + for (genvar i = 0; i < N; ++i) begin : g_align_lanes + wire [7:0] shift_amt = s1_shift_amts[i]; + + wire in_sign = s1_sigs_in[i][WI-1]; + wire [WI-2:0] in_mag = s1_sigs_in[i][WI-2:0]; + + wire [SHIFT_MAG_W-1:0] mag_shifted; + if (i == N-1) begin : g_c_term + assign mag_shifted = { {(MAX_PRE_SHIFT - (WI - 24)){1'b0}}, in_mag, {(WI - 24){1'b0}} }; + end else begin : g_prod_term + assign mag_shifted = { in_mag, {(WI - 23){1'b0}} }; + end + + wire is_overshift = (shift_amt >= 8'(SHIFT_MAG_W)); + wire [SHIFT_MAG_W-1:0] shift_res_full = mag_shifted >> shift_amt; + wire [WO-2:0] adj_mag = is_overshift ? '0 : shift_res_full[WO-2:0]; + + wire [SHIFT_MAG_W-1:0] sticky_check_shift = mag_shifted << (8'(SHIFT_MAG_W) - shift_amt); + assign sticky_bits[i] = is_overshift ? (|mag_shifted) : (|sticky_check_shift); + + assign sigs_out[i] = s1_is_int ? WO'($signed(s1_sigs_in[i])) : {in_sign, adj_mag}; + end + +`ifndef DBG_TRACE_TCU + `UNUSED_VAR ({s1_valid, s1_req_id}) +`endif + +`ifdef DBG_TRACE_TCU + always_ff @(posedge clk) begin + if (s1_valid) begin + `TRACE(4, ("%t: %s FEDP-ALIGN(%0d): is_int=%0d", $time, INSTANCE_ID, s1_req_id, s1_is_int)); + `TRACE(4, (", max_exp=0x%0h, shift_amts=", max_exp)); + `TRACE_ARRAY1D(4, "0x%0h", s1_shift_amts, N) + `TRACE(4, (", sigs_in=")); + `TRACE_ARRAY1D(4, "0x%0h", s1_sigs_in, N) + `TRACE(4, (", sigs_out=")); + `TRACE_ARRAY1D(4, "0x%0h", sigs_out, N) + `TRACE(4, (", sticky=")); + `TRACE_ARRAY1D(4, "%0d", sticky_bits, N) + `TRACE(4, ("\n")); + end + end +`endif + +endmodule diff --git a/hw/rtl/tcu/tet/VX_tcu_tet_classifier.sv b/hw/rtl/tcu/tet/VX_tcu_tet_classifier.sv new file mode 100644 index 000000000..e22ddb7f8 --- /dev/null +++ b/hw/rtl/tcu/tet/VX_tcu_tet_classifier.sv @@ -0,0 +1,35 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +module VX_tcu_tet_classifier import VX_tcu_pkg::*; #( + parameter EXP_W = 8, + parameter MAN_W = 23 +) ( + input wire [EXP_W-1:0] exp, + input wire [MAN_W-1:0] man, + output fedp_class_t cls +); + wire exp_zero = ~|exp; + wire exp_ones = &exp; + + wire man_non_zero = |man; + wire man_zero = ~man_non_zero; + + assign cls.is_zero = exp_zero & man_zero; + assign cls.is_sub = exp_zero & man_non_zero; + assign cls.is_inf = exp_ones & man_zero; + assign cls.is_nan = exp_ones & man_non_zero; + +endmodule diff --git a/hw/rtl/tcu/tet/VX_tcu_tet_exc_reduce.sv b/hw/rtl/tcu/tet/VX_tcu_tet_exc_reduce.sv new file mode 100644 index 000000000..6e08a1b9d --- /dev/null +++ b/hw/rtl/tcu/tet/VX_tcu_tet_exc_reduce.sv @@ -0,0 +1,49 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +module VX_tcu_tet_exc_reduce import VX_tcu_pkg::*; #( + parameter TCK = 4 +) ( + input fedp_excep_t [TCK:0] exc_in, + output fedp_excep_t exc_out +); + logic [TCK-1:0] lane_nan; + logic [TCK-1:0] lane_inf; + logic [TCK-1:0] lane_sign; + + for (genvar i = 0; i < TCK; ++i) begin : g_unpack + assign lane_nan[i] = exc_in[i].is_nan; + assign lane_inf[i] = exc_in[i].is_inf; + assign lane_sign[i] = exc_in[i].sign; + end + + wire c_is_nan = exc_in[TCK].is_nan; + wire c_is_inf = exc_in[TCK].is_inf; + wire c_sign = exc_in[TCK].sign; + + wire [TCK-1:0] p_pos_inf = lane_inf & ~lane_sign; + wire [TCK-1:0] p_neg_inf = lane_inf & lane_sign; + + wire has_pos = (|p_pos_inf) | (c_is_inf & ~c_sign); + wire has_neg = (|p_neg_inf) | (c_is_inf & c_sign); + + wire res_nan = (|lane_nan) | c_is_nan | (has_pos & has_neg); + wire res_inf = (has_pos | has_neg) & ~res_nan; + + assign exc_out.is_nan = res_nan; + assign exc_out.is_inf = res_inf; + assign exc_out.sign = has_neg & ~has_pos; + +endmodule diff --git a/hw/rtl/tcu/tet/VX_tcu_tet_lane_mask.sv b/hw/rtl/tcu/tet/VX_tcu_tet_lane_mask.sv new file mode 100644 index 000000000..8126bbc57 --- /dev/null +++ b/hw/rtl/tcu/tet/VX_tcu_tet_lane_mask.sv @@ -0,0 +1,102 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +module VX_tcu_tet_lane_mask import VX_tcu_pkg::*; #( + parameter N = 2, + parameter TCK = 2 * N +) ( + input wire [TCU_MAX_INPUTS-1:0] vld_mask, + input wire [4:0] fmt_s, + output logic [TCK-1:0] lane_mask +); + `UNUSED_VAR (vld_mask) + wire [TCK-1:0] mask_32; + wire [TCK-1:0] mask_16; + wire [TCK-1:0] mask_8; + wire [TCK-1:0] mask_4; + + // 32-bit mask + for (genvar i = 0; i < TCK; ++i) begin : g_mask_32 + if ((i % 2) == 0) begin : g_even_lane + assign mask_32[i] = vld_mask[i * 4]; + end else begin : g_odd_lane + assign mask_32[i] = 1'b0; + end + end + `UNUSED_VAR (mask_32) + + // 16-bit mask + for (genvar i = 0; i < TCK; ++i) begin : g_mask_16 + assign mask_16[i] = vld_mask[i * 4]; + end + `UNUSED_VAR (mask_16) + + // 8-bit mask + for (genvar i = 0; i < TCK; ++i) begin : g_mask_8 + assign mask_8[i] = vld_mask[i * 4 + 0] | vld_mask[i * 4 + 2]; + end + `UNUSED_VAR (mask_8) + + // 4-bit mask + for (genvar i = 0; i < TCK; ++i) begin : g_mask_4 + assign mask_4[i] = vld_mask[i * 4 + 0] + | vld_mask[i * 4 + 1] + | vld_mask[i * 4 + 2] + | vld_mask[i * 4 + 3]; + end + `UNUSED_VAR (mask_4) + + // Format selection + always_comb begin + case (fmt_s) + `ifdef VX_CFG_TCU_FP16_ENABLE + TCU_FP16_ID, + TCU_BF16_ID: lane_mask = mask_16; + `endif + `ifdef VX_CFG_TCU_TF32_ENABLE + TCU_TF32_ID: lane_mask = mask_32; + `endif + `ifdef VX_CFG_TCU_FP8_ENABLE + TCU_FP8_ID, + TCU_BF8_ID: lane_mask = mask_8; + `ifdef VX_CFG_TCU_MX_ENABLE + TCU_MXFP8_ID, + TCU_MXBF8_ID:lane_mask = mask_8; + `ifdef VX_CFG_TCU_FP4_ENABLE + `ifdef VX_CFG_TCU_MXFP4_ENABLE + TCU_MXFP4_ID:lane_mask = mask_4; + `endif + `ifdef VX_CFG_TCU_NVFP4_ENABLE + TCU_NVFP4_ID:lane_mask = mask_4; + `endif + `endif + `endif + `endif + `ifdef VX_CFG_TCU_INT8_ENABLE + TCU_I8_ID, + TCU_U8_ID: lane_mask = mask_8; + `ifdef VX_CFG_TCU_MX_ENABLE + TCU_MXI8_ID: lane_mask = mask_8; + `endif + `endif + `ifdef VX_CFG_TCU_INT4_ENABLE + TCU_I4_ID, + TCU_U4_ID: lane_mask = mask_4; + `endif + default: lane_mask = '0; + endcase + end + +endmodule diff --git a/hw/rtl/tcu/tet/VX_tcu_tet_max_exp.sv b/hw/rtl/tcu/tet/VX_tcu_tet_max_exp.sv new file mode 100644 index 000000000..85c14c6f0 --- /dev/null +++ b/hw/rtl/tcu/tet/VX_tcu_tet_max_exp.sv @@ -0,0 +1,66 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +module VX_tcu_tet_max_exp import VX_tcu_pkg::*; #( + parameter N = 5, + parameter WIDTH = 8 +) ( + input wire [N-1:0][WIDTH-1:0] exponents, + output wire [N-1:0] sel_exp, + output wire [N-2:0][N-2:0][WIDTH:0] diff_mat +); + + // Signed subtractor matrix. + wire [N-2:0] sign_mat[N-2:0] /* verilator split_var */; + + for (genvar i = 0; i < N-1; i++) begin : g_row + for (genvar j = 0; j < N-1; j++) begin : g_col + if (j < i) begin : g_lower + assign sign_mat[i][j] = ~sign_mat[j][i]; + assign diff_mat[i][j] = '0; + end else begin : g_upper + assign diff_mat[i][j] = {1'b0, exponents[i]} - {1'b0, exponents[j+1]}; + assign sign_mat[i][j] = diff_mat[i][j][WIDTH]; + end + end + end + + // Find maximum exponent index based on the sign matrix + for (genvar i = 0; i < N; i++) begin : g_index + wire and_left, or_right; + if (i == 0) begin : g_first + assign and_left = 1'b1; + end else begin : g_and_left + wire [i-1:0] left_signals; + for (genvar jl = 0; jl < i; jl++) begin : g_left + assign left_signals[jl] = sign_mat[jl][i-1]; + end + assign and_left = &left_signals; + end + + if (i == N-1) begin : g_last + assign or_right = 1'b0; + end else begin : g_or_right + wire [N-2-i:0] right_signals; + for (genvar jr = i+1; jr < N; jr++) begin : g_right + assign right_signals[jr-i-1] = sign_mat[i][jr-1]; + end + assign or_right = |right_signals; + end + + assign sel_exp[i] = and_left & (~or_right); + end + +endmodule diff --git a/hw/rtl/tcu/tet/VX_tcu_tet_mul_f16.sv b/hw/rtl/tcu/tet/VX_tcu_tet_mul_f16.sv new file mode 100644 index 000000000..ca94dc394 --- /dev/null +++ b/hw/rtl/tcu/tet/VX_tcu_tet_mul_f16.sv @@ -0,0 +1,352 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +module VX_tcu_tet_mul_f16 import VX_tcu_pkg::*; +#( + parameter `STRING INSTANCE_ID = "", + parameter N = 2, + parameter TCK = 2 * N, + parameter W = 25, + parameter WA = 28, + parameter EXP_W = 10 +) ( + input wire clk, + input wire reset, + input wire enable, + input wire valid_in, + input wire [31:0] req_id, + + input wire [TCU_MAX_INPUTS-1:0] vld_mask, + input wire [3:0] fmt_f, + + input wire [N-1:0][31:0] a_row, + input wire [N-1:0][31:0] b_col, + + output logic [TCK-1:0][24:0] result_sig, + output logic [TCK-1:0][EXP_W-1:0] result_exp, + output fedp_excep_t [TCK-1:0] exceptions +); + `UNUSED_SPARAM (INSTANCE_ID) + `UNUSED_VAR ({req_id, valid_in}) + `UNUSED_VAR (vld_mask) + + localparam F32_BIAS = 127; + localparam S_FP32 = 23; + localparam S_SUPER = 22; + localparam BIAS_BASE = F32_BIAS + S_FP32 - S_SUPER - W + WA - 1 + 128; + + localparam E_TF32 = VX_tcu_pkg::exp_bits(TCU_TF32_ID); + localparam S_TF32 = VX_tcu_pkg::sign_pos(TCU_TF32_ID); + localparam B_TF32 = (1 << (E_TF32 - 1)) - 1; + localparam [7:0] BIAS_CONST_TF32 = 8'(BIAS_BASE - 2*B_TF32); + + localparam E_FP16 = VX_tcu_pkg::exp_bits(TCU_FP16_ID); + localparam S_FP16 = VX_tcu_pkg::sign_pos(TCU_FP16_ID); + localparam B_FP16 = (1 << (E_FP16 - 1)) - 1; + localparam [7:0] BIAS_CONST_FP16 = 8'(BIAS_BASE - 2*B_FP16); + + localparam E_BF16 = VX_tcu_pkg::exp_bits(TCU_BF16_ID); + localparam S_BF16 = VX_tcu_pkg::sign_pos(TCU_BF16_ID); + localparam B_BF16 = (1 << (E_BF16 - 1)) - 1; + localparam [7:0] BIAS_CONST_BF16 = 8'(BIAS_BASE - 2*B_BF16); + + `UNUSED_PARAM (BIAS_CONST_TF32) + `UNUSED_PARAM (BIAS_CONST_BF16) + `UNUSED_PARAM (S_TF32) + `UNUSED_PARAM (S_BF16) +`ifndef VX_CFG_TCU_FP16_ENABLE + `UNUSED_PARAM (BIAS_CONST_FP16) + `UNUSED_PARAM (S_FP16) +`endif + + + for (genvar i = 0; i < TCK; ++i) begin : g_lane + + wire lane_valid = vld_mask[i*4]; + localparam OFF_16 = (i % 2) * 16; +`ifndef VX_CFG_TCU_FP16_ENABLE + `UNUSED_PARAM (OFF_16) +`endif + + // Input muxing and field extraction + logic [7:0] raw_ea, raw_eb; + logic [9:0] raw_ma, raw_mb; + logic raw_sa, raw_sb; + logic [7:0] bias_sel; + + always_comb begin + case (fmt_f) + `ifdef VX_CFG_TCU_FP16_ENABLE + TCU_FP16_ID: begin + raw_ea = 8'(a_row[i/2][S_FP16-1+OFF_16 -: E_FP16]); + raw_eb = 8'(b_col[i/2][S_FP16-1+OFF_16 -: E_FP16]); + raw_ma = a_row[i/2][9+OFF_16 -: 10]; + raw_mb = b_col[i/2][9+OFF_16 -: 10]; + raw_sa = a_row[i/2][15+OFF_16]; + raw_sb = b_col[i/2][15+OFF_16]; + bias_sel = BIAS_CONST_FP16; + end + TCU_BF16_ID: begin + raw_ea = a_row[i/2][S_BF16-1+OFF_16 -: E_BF16]; + raw_eb = b_col[i/2][S_BF16-1+OFF_16 -: E_BF16]; + raw_ma = {a_row[i/2][6+OFF_16 -: 7], 3'b0}; + raw_mb = {b_col[i/2][6+OFF_16 -: 7], 3'b0}; + raw_sa = a_row[i/2][15+OFF_16]; + raw_sb = b_col[i/2][15+OFF_16]; + bias_sel = BIAS_CONST_BF16; + end + `endif + `ifdef VX_CFG_TCU_TF32_ENABLE + TCU_TF32_ID: begin + if ((i % 2) == 0) begin + raw_ea = a_row[i/2][S_TF32-1 -: E_TF32]; + raw_eb = b_col[i/2][S_TF32-1 -: E_TF32]; + raw_ma = a_row[i/2][9:0]; + raw_mb = b_col[i/2][9:0]; + raw_sa = a_row[i/2][18]; + raw_sb = b_col[i/2][18]; + bias_sel = BIAS_CONST_TF32; + end else begin + raw_ea = '0; + raw_eb = '0; + raw_ma = '0; + raw_mb = '0; + raw_sa = '0; + raw_sb = '0; + bias_sel = '0; + end + end + `endif + default: begin + raw_ea = '0; + raw_eb = '0; + raw_ma = '0; + raw_mb = '0; + raw_sa = '0; + raw_sb = '0; + bias_sel = '0; + end + endcase + end + + // Classification + fedp_class_t cls_a_f16; + VX_tcu_tet_classifier #( + .EXP_W (E_FP16), + .MAN_W (10) + ) class_a_f16 ( + .exp (raw_ea[E_FP16-1:0]), + .man (raw_ma), + .cls (cls_a_f16) + ); + + fedp_class_t cls_b_f16; + VX_tcu_tet_classifier #( + .EXP_W (E_FP16), + .MAN_W (10) + ) class_b_f16 ( + .exp (raw_eb[E_FP16-1:0]), + .man (raw_mb), + .cls (cls_b_f16) + ); + + fedp_class_t cls_a_wide; + VX_tcu_tet_classifier #( + .EXP_W (8), + .MAN_W (10) + ) class_a_wide ( + .exp (raw_ea), + .man (raw_ma), + .cls (cls_a_wide) + ); + + fedp_class_t cls_b_wide; + VX_tcu_tet_classifier #( + .EXP_W (8), + .MAN_W (10) + ) class_b_wide ( + .exp (raw_eb), + .man (raw_mb), + .cls (cls_b_wide) + ); + + fedp_class_t cls_a; + fedp_class_t cls_b; + always_comb begin + if (fmt_f == TCU_FP16_ID) begin + cls_a = cls_a_f16; + cls_b = cls_b_f16; + end else begin + cls_a = cls_a_wide; + cls_b = cls_b_wide; + end + end + + // Operand preparation + wire is_ea_zero = ~|raw_ea; + wire is_eb_zero = ~|raw_eb; + + wire [7:0] ea_sel = is_ea_zero ? 8'b1 : raw_ea; + wire [7:0] eb_sel = is_eb_zero ? 8'b1 : raw_eb; + + wire [10:0] ma_sel = {~is_ea_zero, raw_ma}; + wire [10:0] mb_sel = {~is_eb_zero, raw_mb}; + `UNUSED_VAR (cls_a.is_sub) + `UNUSED_VAR (cls_b.is_sub) + + wire sign_sel = raw_sa ^ raw_sb; + wire zero_sel = cls_a.is_zero | cls_b.is_zero; + + wire nan_in = cls_a.is_nan | cls_b.is_nan; + wire inf_z = (cls_a.is_inf & cls_b.is_zero) | (cls_a.is_zero & cls_b.is_inf); + wire inf_op = cls_a.is_inf | cls_b.is_inf; + + wire nan_sel = nan_in | inf_z; + wire inf_sel = inf_op & ~inf_z; + + wire [EXP_W-1:0] exp_final = EXP_W'(bias_sel) + EXP_W'(ea_sel) + EXP_W'(eb_sel); + + wire [6:0] mul_a_lo = ma_sel[6:0]; + wire [3:0] mul_a_hi = ma_sel[10:7]; + wire [6:0] mul_b_lo = mb_sel[6:0]; + wire [3:0] mul_b_hi = mb_sel[10:7]; + + wire [0:0][6:0] mul_ll_a = mul_a_lo; + wire [0:0][6:0] mul_ll_b = mul_b_lo; + wire [0:0][13:0] mul_ll_p; + + VX_tcu_tet_wmul #( + .N (7), + .P (14), + .USE_DSP (1) + ) wtmul_ll ( + .a (mul_ll_a), + .b (mul_ll_b), + .p (mul_ll_p) + ); + + wire [0:0][6:0] mul_lh_a = mul_a_lo; + wire [0:0][3:0] mul_lh_b = mul_b_hi; + wire [0:0][10:0] mul_lh_p; + + VX_tcu_tet_wmul #( + .N (7), + .M (4), + .P (11), + .USE_DSP (1) + ) wtmul_lh ( + .a (mul_lh_a), + .b (mul_lh_b), + .p (mul_lh_p) + ); + + wire [0:0][3:0] mul_hl_a = mul_a_hi; + wire [0:0][6:0] mul_hl_b = mul_b_lo; + wire [0:0][10:0] mul_hl_p; + + VX_tcu_tet_wmul #( + .N (4), + .M (7), + .P (11), + .USE_DSP (1) + ) wtmul_hl ( + .a (mul_hl_a), + .b (mul_hl_b), + .p (mul_hl_p) + ); + + wire [0:0][3:0] mul_hh_a = mul_a_hi; + wire [0:0][3:0] mul_hh_b = mul_b_hi; + wire [0:0][7:0] mul_hh_p; + + VX_tcu_tet_wmul #( + .N (4), + .P (8), + .USE_DSP (1) + ) wtmul_hh ( + .a (mul_hh_a), + .b (mul_hh_b), + .p (mul_hh_p) + ); + + wire [13:0] mul_ll = mul_ll_p[0]; + wire [10:0] mul_lh = mul_lh_p[0]; + wire [10:0] mul_hl = mul_hl_p[0]; + wire [7:0] mul_hh = mul_hh_p[0]; + + wire [13:0] s1_mul_ll; + wire [10:0] s1_mul_lh; + wire [10:0] s1_mul_hl; + wire [7:0] s1_mul_hh; + wire s1_sign_sel; + wire s1_zero_sel; + wire s1_nan_sel; + wire s1_inf_sel; + wire s1_lane_valid; + wire [3:0] s1_fmt_f; + wire [EXP_W-1:0] s1_exp_final; + + VX_tcu_tet_register #( + .DATAW (14 + 11 + 11 + 8 + EXP_W + 5 + 4), + .DEPTH (1) + ) pipe_mul_s0 ( + .clk (clk), + .reset (reset), + .enable (enable), + .data_in ({mul_ll, mul_lh, mul_hl, mul_hh, exp_final, sign_sel, zero_sel, nan_sel, inf_sel, lane_valid, fmt_f}), + .data_out ({s1_mul_ll, s1_mul_lh, s1_mul_hl, s1_mul_hh, s1_exp_final, s1_sign_sel, s1_zero_sel, s1_nan_sel, s1_inf_sel, s1_lane_valid, s1_fmt_f}) + ); + + wire [21:0] prod_lo = {8'b0, s1_mul_ll}; + wire [21:0] prod_lh = {4'b0, s1_mul_lh, 7'b0}; + wire [21:0] prod_hl = {4'b0, s1_mul_hl, 7'b0}; + wire [21:0] prod_hh = {s1_mul_hh, 14'b0}; + wire [21:0] man_prod = (prod_lo + prod_lh) + (prod_hl + prod_hh); + + logic s1_fmt_valid; + always_comb begin + case (s1_fmt_f) + `ifdef VX_CFG_TCU_FP16_ENABLE + TCU_FP16_ID, + TCU_BF16_ID: s1_fmt_valid = 1'b1; + `endif + `ifdef VX_CFG_TCU_TF32_ENABLE + TCU_TF32_ID: s1_fmt_valid = 1'b1; + `endif + default: s1_fmt_valid = 1'b0; + endcase + end + + assign result_exp[i] = (~s1_zero_sel && s1_lane_valid && s1_fmt_valid) ? s1_exp_final : '0; + + always_comb begin + case (s1_fmt_f) + `ifdef VX_CFG_TCU_FP16_ENABLE + TCU_FP16_ID: result_sig[i] = {s1_sign_sel, man_prod, 2'b0}; + TCU_BF16_ID: result_sig[i] = {s1_sign_sel, man_prod, 2'b0}; + `endif + `ifdef VX_CFG_TCU_TF32_ENABLE + TCU_TF32_ID: result_sig[i] = {s1_sign_sel, man_prod, 2'b0}; + `endif + default: result_sig[i] = '0; + endcase + end + + assign exceptions[i].is_nan = s1_nan_sel && s1_lane_valid && s1_fmt_valid; + assign exceptions[i].is_inf = s1_inf_sel && s1_lane_valid && s1_fmt_valid; + assign exceptions[i].sign = s1_sign_sel; + end + +endmodule diff --git a/hw/rtl/tcu/tet/VX_tcu_tet_mul_f4.sv b/hw/rtl/tcu/tet/VX_tcu_tet_mul_f4.sv new file mode 100644 index 000000000..dbb50c7a2 --- /dev/null +++ b/hw/rtl/tcu/tet/VX_tcu_tet_mul_f4.sv @@ -0,0 +1,433 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +module VX_tcu_tet_mul_f4 import VX_tcu_pkg::*; +#( + parameter `STRING INSTANCE_ID = "", + parameter N = 2, + parameter TCK = 2 * N, + parameter W = 25, + parameter WA = 28, + parameter EXP_W = 10 +) ( + input wire clk, + input wire valid_in, + input wire [31:0] req_id, + + input wire [TCU_MAX_INPUTS-1:0] vld_mask, + input wire [TCU_FMT_WIDTH-1:0] fmt_f, + + input wire [N-1:0][31:0] a_row, + input wire [N-1:0][31:0] b_col, +`ifdef VX_CFG_TCU_MX_ENABLE + input wire [7:0] sf_a, + input wire [7:0] sf_b, +`endif + + output logic [TCK-1:0][24:0] result_sig, + output logic [TCK-1:0][EXP_W-1:0] result_exp, + output fedp_excep_t [TCK-1:0] exceptions +); + `UNUSED_SPARAM (INSTANCE_ID) + `UNUSED_SPARAM (W) + `UNUSED_VAR ({clk, req_id, valid_in, fmt_f}) + +`ifdef VX_CFG_TCU_MX_ENABLE +`ifdef VX_CFG_TCU_FP4_ENABLE + +`ifdef VX_CFG_TCU_MXFP4_ENABLE + wire [TCK-1:0][24:0] result_sig_mxfp4; + wire [TCK-1:0][EXP_W-1:0] result_exp_mxfp4; + fedp_excep_t [TCK-1:0] exceptions_mxfp4; + + localparam F32_BIAS_MXFP4 = 127; + localparam S_FP32_MXFP4 = 23; + localparam S_SUPER_MXFP4 = 22; + localparam BIAS_BASE_MXFP4 = F32_BIAS_MXFP4 + 2*(S_FP32_MXFP4 - S_SUPER_MXFP4) - W + WA - 1 + 128; + + localparam SIG_SHIFT_MXFP4 = 11; + + localparam F4_EXP_BIAS_MXFP4 = 1; + localparam EXP_TERM_W_MXFP4 = 6; + localparam EXP_TERM_BIAS_MXFP4 = 1 << (EXP_TERM_W_MXFP4 - 1); + localparam EXP_COMP_MXFP4 = -(2 * F4_EXP_BIAS_MXFP4 + 10); + localparam [EXP_TERM_W_MXFP4-1:0] EXP_ADJ_MXFP4 = EXP_TERM_W_MXFP4'(EXP_TERM_BIAS_MXFP4 + EXP_COMP_MXFP4); + localparam [EXP_W-1:0] EXP_BASE_BIASED_MXFP4 = EXP_W'(BIAS_BASE_MXFP4 + EXP_COMP_MXFP4); + + for (genvar i = 0; i < TCK; ++i) begin : g_lane_mxfp4 + localparam K_WORD = i / 2; + + wire [3:0][23:0] term_mag_shifted; + wire [3:0][EXP_TERM_W_MXFP4-1:0] term_exp_biased; + wire [3:0] term_valid; + wire [3:0] term_sign; + + for (genvar j = 0; j < 4; ++j) begin : g_term + localparam OFF = (i % 2) * 16 + j * 4; + + wire lane_valid = vld_mask[i * 4 + j]; + wire [3:0] raw_a = a_row[K_WORD][OFF +: 4]; + wire [3:0] raw_b = b_col[K_WORD][OFF +: 4]; + + wire a_zero = ~|raw_a[2:0]; + wire b_zero = ~|raw_b[2:0]; + assign term_valid[j] = lane_valid && !a_zero && !b_zero; + assign term_sign[j] = raw_a[3] ^ raw_b[3]; + + wire [1:0] a_man = ~|raw_a[2:1] ? 2'b01 : {1'b1, raw_a[0]}; + wire [1:0] b_man = ~|raw_b[2:1] ? 2'b01 : {1'b1, raw_b[0]}; + + wire [1:0] a_exp, b_exp; + assign a_exp[0] = raw_a[2] & ~raw_a[1]; + assign a_exp[1] = raw_a[2] & raw_a[1]; + assign b_exp[0] = raw_b[2] & ~raw_b[1]; + assign b_exp[1] = raw_b[2] & raw_b[1]; + + wire [3:0] f4_man_prod; + VX_wallace_mul #( + .N(2), + .CPA_KS(!`FORCE_BUILTIN_ADDER(2*2)) + ) f4_wtmul ( + .a(a_man), + .b(b_man), + .p(f4_man_prod) + ); + + wire signed [9:0] sf_exp_a = $signed({1'b0, sf_a}) - 10'sd127; + wire signed [9:0] sf_exp_b = $signed({1'b0, sf_b}) - 10'sd127; + wire signed [5:0] exp_biased_raw = 6'(10'(EXP_ADJ_MXFP4) + + sf_exp_a + + sf_exp_b + + 10'(a_exp) + + 10'(b_exp)); + + assign term_exp_biased[j] = term_valid[j] ? exp_biased_raw : '0; + assign term_mag_shifted[j] = term_valid[j] ? (24'(f4_man_prod) << SIG_SHIFT_MXFP4) : 24'd0; + end + + wire [EXP_TERM_W_MXFP4-1:0] max_exp_01 = (term_exp_biased[0] >= term_exp_biased[1]) ? term_exp_biased[0] : term_exp_biased[1]; + wire [EXP_TERM_W_MXFP4-1:0] max_exp_23 = (term_exp_biased[2] >= term_exp_biased[3]) ? term_exp_biased[2] : term_exp_biased[3]; + wire [EXP_TERM_W_MXFP4-1:0] max_exp_biased = (max_exp_01 >= max_exp_23) ? max_exp_01 : max_exp_23; + + wire [3:0][26:0] term_signed; + for (genvar j = 0; j < 4; ++j) begin : g_align + wire [EXP_TERM_W_MXFP4-1:0] shift_amt; + VX_ks_adder #( + .N(EXP_TERM_W_MXFP4), + .BYPASS(`FORCE_BUILTIN_ADDER(EXP_TERM_W_MXFP4)) + ) shift_ksa ( + .dataa(max_exp_biased), + .datab(~term_exp_biased[j]), + .cin(1'b1), + .sum(shift_amt), + `UNUSED_PIN(cout) + ); + + wire [23:0] aligned_mag = (shift_amt >= EXP_TERM_W_MXFP4'(24)) ? 24'd0 : (term_mag_shifted[j] >> shift_amt[4:0]); + wire [26:0] aligned_ext = {3'b0, aligned_mag}; + + wire [26:0] neg_term; + VX_ks_adder #( + .N(27), + .BYPASS(`FORCE_BUILTIN_ADDER(27)) + ) term_neg_ksa ( + .dataa(~aligned_ext), + .datab(27'd0), + .cin(1'b1), + .sum(neg_term), + `UNUSED_PIN(cout) + ); + + assign term_signed[j] = term_sign[j] ? neg_term : aligned_ext; + end + + wire [26:0] sum_vec, carry_vec; + VX_csa_tree #( + .N(4), + .W(27), + .S(27) + ) term_csa ( + .operands (term_signed), + .sum (sum_vec), + .carry (carry_vec) + ); + + wire [26:0] signed_sum; + VX_ks_adder #( + .N(27), + .BYPASS(`FORCE_BUILTIN_ADDER(27)) + ) sum_ksa ( + .dataa(sum_vec), + .datab(carry_vec), + .cin(1'b0), + .sum(signed_sum), + `UNUSED_PIN(cout) + ); + + wire sum_sign = signed_sum[26]; + wire [25:0] neg_sum_raw; + VX_ks_adder #( + .N(26), + .BYPASS(`FORCE_BUILTIN_ADDER(26)) + ) sum_neg_ksa ( + .dataa(~signed_sum[25:0]), + .datab(26'd0), + .cin(1'b1), + .sum(neg_sum_raw), + `UNUSED_PIN(cout) + ); + + wire [25:0] abs_sum = sum_sign ? neg_sum_raw : signed_sum[25:0]; + wire is_zero_out = ~|abs_sum; + + assign result_sig_mxfp4[i] = {sum_sign & ~is_zero_out, abs_sum[23:0]}; + assign result_exp_mxfp4[i] = is_zero_out ? '0 : (EXP_W'(max_exp_biased) + EXP_W'(EXP_BASE_BIASED_MXFP4)); + + assign exceptions_mxfp4[i].is_nan = 1'b0; + assign exceptions_mxfp4[i].is_inf = 1'b0; + assign exceptions_mxfp4[i].sign = sum_sign & ~is_zero_out; + end +`endif // VX_CFG_TCU_MXFP4_ENABLE + +`ifdef VX_CFG_TCU_NVFP4_ENABLE + wire [TCK-1:0][24:0] result_sig_nvfp4; + wire [TCK-1:0][EXP_W-1:0] result_exp_nvfp4; + fedp_excep_t [TCK-1:0] exceptions_nvfp4; + + localparam F32_BIAS = 127; + localparam S_FP32 = 23; + localparam S_SUPER = 22; + localparam BIAS_BASE = F32_BIAS + 2*(S_FP32 - S_SUPER) - W + WA - 1 + 128; + + localparam SIG_SHIFT = 11; + + localparam F4_EXP_BIAS = 1; + localparam SF_EXP_BIAS = 7; + localparam SF_MAN_BITS = 3; + localparam EXP_TERM_W = 6; + localparam EXP_TERM_BIAS = 1 << (EXP_TERM_W - 1); + // fp4 = man * 2^(exp - 1), e4m3 scale = man * 2^(exp - 7 - 3). + localparam EXP_COMP_NVFP4 = -(2 * F4_EXP_BIAS + 2 * (SF_EXP_BIAS + SF_MAN_BITS)); + localparam [5:0] EXP_ADJ_NVFP4 = 6'(EXP_TERM_BIAS + EXP_COMP_NVFP4); + localparam [EXP_W-1:0] EXP_BASE_BIASED = EXP_W'(BIAS_BASE + EXP_COMP_NVFP4); + + for (genvar i = 0; i < TCK; ++i) begin : g_lane_nvfp4 + localparam K_WORD = i / 2; + `UNUSED_VAR ({sf_a[7], sf_b[7]}) + + // e4m3 scale factor mantissa mul + wire [3:0] sf_man_a = {1'b1, sf_a[2:0]}; + wire [3:0] sf_man_b = {1'b1, sf_b[2:0]}; + wire [3:0] sf_exp_a = sf_a[6:3]; + wire [3:0] sf_exp_b = sf_b[6:3]; + + wire [7:0] sf_man_prod; + VX_wallace_mul #( + .N(4), + .CPA_KS(!`FORCE_BUILTIN_ADDER(4*2)) + ) sf_wtmul ( + .a(sf_man_a), + .b(sf_man_b), + .p(sf_man_prod) + ); + + wire [3:0][23:0] term_mag_shifted; + wire [3:0][5:0] term_exp_biased; + wire [3:0] term_valid; + wire [3:0] term_sign; + + for (genvar j = 0; j < 4; ++j) begin : g_term + localparam OFF = (i % 2) * 16 + j * 4; + + wire lane_valid = vld_mask[i * 4 + j]; + wire [3:0] raw_a = a_row[K_WORD][OFF +: 4]; + wire [3:0] raw_b = b_col[K_WORD][OFF +: 4]; + + wire a_zero = ~|raw_a[2:0]; + wire b_zero = ~|raw_b[2:0]; + assign term_valid[j] = lane_valid && !a_zero && !b_zero; + assign term_sign[j] = raw_a[3] ^ raw_b[3]; + + wire [1:0] a_man = ~|raw_a[2:1] ? 2'b01 : {1'b1, raw_a[0]}; + wire [1:0] b_man = ~|raw_b[2:1] ? 2'b01 : {1'b1, raw_b[0]}; + + wire [1:0] a_exp, b_exp; + assign a_exp[0] = raw_a[2] & ~raw_a[1]; + assign a_exp[1] = raw_a[2] & raw_a[1]; + assign b_exp[0] = raw_b[2] & ~raw_b[1]; + assign b_exp[1] = raw_b[2] & raw_b[1]; + + wire [3:0] f4_man_prod; + VX_wallace_mul #( + .N(2), + .CPA_KS(!`FORCE_BUILTIN_ADDER(2*2)) + ) f4_wtmul ( + .a(a_man), + .b(b_man), + .p(f4_man_prod) + ); + + wire [15:0] term_man_prod_full; + `UNUSED_VAR (term_man_prod_full[15:11]) + VX_wallace_mul #( + .N(8), + .CPA_KS(!`FORCE_BUILTIN_ADDER(8*2)) + ) term_wtmul ( + .a({4'b0, f4_man_prod}), + .b(sf_man_prod), + .p(term_man_prod_full) + ); + + wire [5:0] exp_sum_vec, exp_carry_vec; + VX_csa_tree #( + .N(5), + .W(6), + .S(6) + ) exp_csa ( + .operands ({EXP_ADJ_NVFP4, 6'(sf_exp_a), 6'(sf_exp_b), 6'(a_exp), 6'(b_exp)}), + .sum (exp_sum_vec), + .carry (exp_carry_vec) + ); + + wire [5:0] exp_biased_raw; + VX_ks_adder #( + .N(6), + .BYPASS(`FORCE_BUILTIN_ADDER(6)) + ) exp_ksa ( + .dataa(exp_sum_vec), + .datab(exp_carry_vec), + .cin(1'b0), + .sum(exp_biased_raw), + `UNUSED_PIN(cout) + ); + + assign term_exp_biased[j] = term_valid[j] ? exp_biased_raw : 6'd0; + assign term_mag_shifted[j] = term_valid[j] ? (24'(term_man_prod_full[10:0]) << SIG_SHIFT) : 24'd0; + end + + wire [5:0] max_exp_01 = (term_exp_biased[0] >= term_exp_biased[1]) ? term_exp_biased[0] : term_exp_biased[1]; + wire [5:0] max_exp_23 = (term_exp_biased[2] >= term_exp_biased[3]) ? term_exp_biased[2] : term_exp_biased[3]; + wire [5:0] max_exp_biased = (max_exp_01 >= max_exp_23) ? max_exp_01 : max_exp_23; + + wire [3:0][26:0] term_signed; + for (genvar j = 0; j < 4; ++j) begin : g_align + wire [5:0] shift_amt; + VX_ks_adder #( + .N(6), + .BYPASS(`FORCE_BUILTIN_ADDER(6)) + ) shift_ksa ( + .dataa(max_exp_biased), + .datab(~term_exp_biased[j]), + .cin(1'b1), + .sum(shift_amt), + `UNUSED_PIN(cout) + ); + + wire [23:0] aligned_mag = (shift_amt >= 6'd24) ? 24'd0 : (term_mag_shifted[j] >> shift_amt[4:0]); + wire [26:0] aligned_ext = {3'b0, aligned_mag}; + + wire [26:0] neg_term; + VX_ks_adder #( + .N(27), + .BYPASS(`FORCE_BUILTIN_ADDER(27)) + ) term_neg_ksa ( + .dataa(~aligned_ext), + .datab(27'd0), + .cin(1'b1), + .sum(neg_term), + `UNUSED_PIN(cout) + ); + + assign term_signed[j] = term_sign[j] ? neg_term : aligned_ext; + end + + wire [26:0] sum_vec, carry_vec; + VX_csa_tree #( + .N(4), + .W(27), + .S(27) + ) term_csa ( + .operands (term_signed), + .sum (sum_vec), + .carry (carry_vec) + ); + + wire [26:0] signed_sum; + VX_ks_adder #( + .N(27), + .BYPASS(`FORCE_BUILTIN_ADDER(27)) + ) sum_ksa ( + .dataa(sum_vec), + .datab(carry_vec), + .cin(1'b0), + .sum(signed_sum), + `UNUSED_PIN(cout) + ); + + wire sum_sign = signed_sum[26]; + wire [25:0] neg_sum_raw; + VX_ks_adder #( + .N(26), + .BYPASS(`FORCE_BUILTIN_ADDER(26)) + ) sum_neg_ksa ( + .dataa(~signed_sum[25:0]), + .datab(26'd0), + .cin(1'b1), + .sum(neg_sum_raw), + `UNUSED_PIN(cout) + ); + + wire [25:0] abs_sum = sum_sign ? neg_sum_raw : signed_sum[25:0]; + wire is_zero_out = ~|abs_sum; + + assign result_sig_nvfp4[i] = {sum_sign & ~is_zero_out, abs_sum[23:0]}; + assign result_exp_nvfp4[i] = is_zero_out ? '0 : (EXP_W'(max_exp_biased) + EXP_W'(EXP_BASE_BIASED)); + + assign exceptions_nvfp4[i].is_nan = 1'b0; + assign exceptions_nvfp4[i].is_inf = 1'b0; + assign exceptions_nvfp4[i].sign = sum_sign & ~is_zero_out; + end +`endif // VX_CFG_TCU_NVFP4_ENABLE + + always_comb begin + result_sig = '0; + result_exp = '0; + exceptions = '0; + case (fmt_f) + `ifdef VX_CFG_TCU_MXFP4_ENABLE + TCU_MXFP4_ID: begin + result_sig = result_sig_mxfp4; + result_exp = result_exp_mxfp4; + exceptions = exceptions_mxfp4; + end + `endif + `ifdef VX_CFG_TCU_NVFP4_ENABLE + TCU_NVFP4_ID: begin + result_sig = result_sig_nvfp4; + result_exp = result_exp_nvfp4; + exceptions = exceptions_nvfp4; + end + `endif + default: begin + result_sig = '0; + result_exp = '0; + exceptions = '0; + end + endcase + end + +`endif // VX_CFG_TCU_FP4_ENABLE +`endif // VX_CFG_TCU_MX_ENABLE +endmodule diff --git a/hw/rtl/tcu/tet/VX_tcu_tet_mul_f8.sv b/hw/rtl/tcu/tet/VX_tcu_tet_mul_f8.sv new file mode 100644 index 000000000..9f1b9d20a --- /dev/null +++ b/hw/rtl/tcu/tet/VX_tcu_tet_mul_f8.sv @@ -0,0 +1,363 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +module VX_tcu_tet_mul_f8 import VX_tcu_pkg::*; +#( + parameter `STRING INSTANCE_ID = "", + parameter N = 2, + parameter TCK = 2 * N, + parameter W = 25, + parameter WA = 28, + parameter EXP_W = 10 +) ( + input wire clk, + input wire reset, + input wire enable, + input wire valid_in, + input wire [31:0] req_id, + + input wire [TCU_MAX_INPUTS-1:0] vld_mask, + input wire [3:0] fmt_f, + + input wire [N-1:0][31:0] a_row, + input wire [N-1:0][31:0] b_col, +`ifdef VX_CFG_TCU_MX_ENABLE + input wire [7:0] sf_a, + input wire [7:0] sf_b, +`endif + + output logic [TCK-1:0][24:0] result_sig, + output logic [TCK-1:0][EXP_W-1:0] result_exp, + output fedp_excep_t [TCK-1:0] exceptions +); + `UNUSED_SPARAM (INSTANCE_ID) + `UNUSED_VAR ({clk, req_id, valid_in}) + `UNUSED_VAR (vld_mask) + + localparam F32_BIAS = 127; + localparam S_FP32 = 23; + localparam S_SUPER = 22; + localparam BIAS_BASE = F32_BIAS + 2*(S_FP32 - S_SUPER) - W + WA - 1 + 128; + + localparam E_FP8 = VX_tcu_pkg::exp_bits(TCU_FP8_ID); + localparam S_FP8 = VX_tcu_pkg::sign_pos(TCU_FP8_ID); + localparam B_FP8 = (1 << (E_FP8 - 1)) - 1; + localparam [7:0] BIAS_CONST_FP8 = 8'(BIAS_BASE - 2*B_FP8); + + localparam E_BF8 = VX_tcu_pkg::exp_bits(TCU_BF8_ID); + localparam S_BF8 = VX_tcu_pkg::sign_pos(TCU_BF8_ID); + localparam B_BF8 = (1 << (E_BF8 - 1)) - 1; + localparam [7:0] BIAS_CONST_BF8 = 8'(BIAS_BASE - 2*B_BF8); + + for (genvar i = 0; i < TCK; ++i) begin : g_lane + + wire [1:0] lane_valid = {vld_mask[i * 4 + 2], vld_mask[i * 4 + 0]}; + wire is_bfloat = tcu_fmt_is_bfloat(fmt_f); + + wire [1:0][5:0] pre_sum; + wire [1:0][7:0] man_prod; + wire [1:0] zero_sel, sign_sel, nan_sel, inf_sel; + + for (genvar j = 0; j < 2; ++j) begin : g_extract + localparam OFF = (i % 2) * 16 + j * 8; + + wire [7:0] raw_a = a_row[i/2][OFF +: 8]; + wire [7:0] raw_b = b_col[i/2][OFF +: 8]; + + logic [4:0] raw_ea, raw_eb; + logic [2:0] raw_ma, raw_mb; + logic raw_sa, raw_sb; + + always_comb begin + if (is_bfloat) begin + raw_ea = 5'(raw_a[S_BF8-1 -: E_BF8]); + raw_eb = 5'(raw_b[S_BF8-1 -: E_BF8]); + raw_ma = {raw_a[1:0], 1'b0}; + raw_mb = {raw_b[1:0], 1'b0}; + raw_sa = raw_a[7]; + raw_sb = raw_b[7]; + end else begin + raw_ea = 5'(raw_a[S_FP8-1 -: E_FP8]); + raw_eb = 5'(raw_b[S_FP8-1 -: E_FP8]); + raw_ma = raw_a[2:0]; + raw_mb = raw_b[2:0]; + raw_sa = raw_a[7]; + raw_sb = raw_b[7]; + end + end + + fedp_class_t cls_a; + VX_tcu_tet_classifier #( + .EXP_W (5), + .MAN_W (3) + ) class_a ( + .exp (raw_ea), + .man (raw_ma), + .cls (cls_a) + ); + + fedp_class_t cls_b; + VX_tcu_tet_classifier #( + .EXP_W (5), + .MAN_W (3) + ) class_b ( + .exp (raw_eb), + .man (raw_mb), + .cls (cls_b) + ); + + wire is_ea_zero = (raw_ea == 0); + wire is_eb_zero = (raw_eb == 0); + + wire [4:0] ea_sel = is_ea_zero ? 5'b1 : raw_ea; + wire [4:0] eb_sel = is_eb_zero ? 5'b1 : raw_eb; + wire [3:0] ma_sel = {~is_ea_zero, raw_ma}; + wire [3:0] mb_sel = {~is_eb_zero, raw_mb}; + `UNUSED_VAR (cls_a.is_sub) + `UNUSED_VAR (cls_b.is_sub) + + assign sign_sel[j] = raw_sa ^ raw_sb; + assign zero_sel[j] = cls_a.is_zero | cls_b.is_zero; + + wire a_is_inf = is_bfloat ? cls_a.is_inf : 1'b0; + wire b_is_inf = is_bfloat ? cls_b.is_inf : 1'b0; + wire a_is_nan = is_bfloat ? cls_a.is_nan : (raw_ea == 5'h0F) && (raw_ma == 3'b111); + wire b_is_nan = is_bfloat ? cls_b.is_nan : (raw_eb == 5'h0F) && (raw_mb == 3'b111); + + wire nan_in = a_is_nan | b_is_nan; + wire inf_z = (a_is_inf & cls_b.is_zero) | (cls_a.is_zero & b_is_inf); + wire inf_op = a_is_inf | b_is_inf; + + assign nan_sel[j] = nan_in | inf_z; + assign inf_sel[j] = inf_op & ~inf_z; + + VX_ks_adder #( + .N (6), + .BYPASS (`FORCE_BUILTIN_ADDER(6)) + ) exp_add ( + .dataa (6'(ea_sel)), + .datab (6'(eb_sel)), + .cin (1'b0), + .sum (pre_sum[j]), + `UNUSED_PIN(cout) + ); + + wire [0:0][3:0] wmul_a = ma_sel; + wire [0:0][3:0] wmul_b = mb_sel; + wire [0:0][7:0] wmul_p; + + VX_tcu_tet_wmul #( + .N (4), + .P (8), + .USE_DSP (1) + ) wtmul ( + .a (wmul_a), + .b (wmul_b), + .p (wmul_p) + ); + + assign man_prod[j] = wmul_p[0]; + end + + wire v0_w = ~zero_sel[0] && lane_valid[0]; + wire v1_w = ~zero_sel[1] && lane_valid[1]; + + wire [6:0] diff_0_minus_1_w = {1'b0, pre_sum[0]} - {1'b0, pre_sum[1]}; + wire [6:0] diff_1_minus_0_w = {1'b0, pre_sum[1]} - {1'b0, pre_sum[0]}; + `UNUSED_VAR (diff_1_minus_0_w[6]) + + wire term0_ge_term1_w = ~diff_0_minus_1_w[6]; + wire term0_is_max_w = (v0_w & ~v1_w) || (v1_w & term0_ge_term1_w); + wire [5:0] diff_abs_w = term0_is_max_w ? diff_0_minus_1_w[5:0] : diff_1_minus_0_w[5:0]; + wire [5:0] max_pre_sum_w = term0_is_max_w ? pre_sum[0] : pre_sum[1]; + wire pre_sum_eq_w = (pre_sum[0] == pre_sum[1]); + + wire [5:0] s1_max_pre_sum; + wire [5:0] s1_diff_abs; + wire [1:0][7:0] s1_man_prod; + wire [1:0] s1_lane_valid; + wire [1:0] s1_zero_sel, s1_sign_sel, s1_nan_sel, s1_inf_sel; + wire s1_diff_sign; + wire s1_pre_sum_eq; + wire s1_is_bfloat; + `ifdef VX_CFG_TCU_MX_ENABLE + wire s1_fmt_is_mx; + wire [7:0] s1_sf_a, s1_sf_b; + `endif + + VX_tcu_tet_register #( + `ifdef VX_CFG_TCU_MX_ENABLE + .DATAW ((2 * 6) + (2 * 8) + (4 * 2) + 2 + 2 + 1 + 1 + 16), + `else + .DATAW ((2 * 6) + (2 * 8) + (4 * 2) + 2 + 2 + 1), + `endif + .DEPTH (1) + ) pipe_mul_s0 ( + .clk (clk), + .reset (reset), + .enable (enable), + `ifdef VX_CFG_TCU_MX_ENABLE + .data_in ({max_pre_sum_w, diff_abs_w, man_prod, lane_valid, zero_sel, sign_sel, nan_sel, inf_sel, term0_is_max_w, pre_sum_eq_w, is_bfloat, fmt_f[3], sf_a, sf_b}), + .data_out ({s1_max_pre_sum, s1_diff_abs, s1_man_prod, s1_lane_valid, s1_zero_sel, s1_sign_sel, s1_nan_sel, s1_inf_sel, s1_diff_sign, s1_pre_sum_eq, s1_is_bfloat, s1_fmt_is_mx, s1_sf_a, s1_sf_b}) + `else + .data_in ({max_pre_sum_w, diff_abs_w, man_prod, lane_valid, zero_sel, sign_sel, nan_sel, inf_sel, term0_is_max_w, pre_sum_eq_w, is_bfloat}), + .data_out ({s1_max_pre_sum, s1_diff_abs, s1_man_prod, s1_lane_valid, s1_zero_sel, s1_sign_sel, s1_nan_sel, s1_inf_sel, s1_diff_sign, s1_pre_sum_eq, s1_is_bfloat}) + `endif + ); + + wire v0 = ~s1_zero_sel[0] && s1_lane_valid[0]; + wire v1 = ~s1_zero_sel[1] && s1_lane_valid[1]; + + wire [7:0] bias_sel = s1_is_bfloat ? BIAS_CONST_BF8 : BIAS_CONST_FP8; + + wire [EXP_W-1:0] bias_sel_cpa; + `ifdef VX_CFG_TCU_MX_ENABLE + wire [EXP_W-1:0] bias_cpa_sum, bias_cpa_carry; + wire [3*EXP_W-1:0] sf_comp = s1_fmt_is_mx ? {EXP_W'(s1_sf_a), EXP_W'(s1_sf_b), -EXP_W'(254)} : (3*EXP_W)'(0); + VX_csa_tree #( + .N (4), + .W (EXP_W), + .S (EXP_W) + ) exp_sf_csa ( + .operands ({EXP_W'(bias_sel), sf_comp}), + .sum (bias_cpa_sum), + .carry (bias_cpa_carry) + ); + VX_ks_adder #( + .N (EXP_W), + .BYPASS (`FORCE_BUILTIN_ADDER(EXP_W)) + ) exp_bias_add ( + .dataa (bias_cpa_sum), + .datab (bias_cpa_carry), + .cin (1'b0), + .sum (bias_sel_cpa), + `UNUSED_PIN(cout) + ); + `else + assign bias_sel_cpa = EXP_W'(bias_sel); + `endif + + wire [EXP_W-1:0] final_exp; + VX_ks_adder #( + .N (EXP_W), + .BYPASS (`FORCE_BUILTIN_ADDER(EXP_W)) + ) exp_final_add ( + .dataa (EXP_W'(s1_max_pre_sum)), + .datab (bias_sel_cpa), + .cin (1'b0), + .sum (final_exp), + `UNUSED_PIN(cout) + ); + + wire [7:0] man_prod0_v = s1_man_prod[0] & {8{s1_lane_valid[0]}}; + wire [7:0] man_prod1_v = s1_man_prod[1] & {8{s1_lane_valid[1]}}; + + wire [7:0] prod_max = s1_diff_sign ? man_prod0_v : man_prod1_v; + wire [7:0] prod_min = s1_diff_sign ? man_prod1_v : man_prod0_v; + wire sign_max = s1_diff_sign ? s1_sign_sel[0] : s1_sign_sel[1]; + wire sign_min = s1_diff_sign ? s1_sign_sel[1] : s1_sign_sel[0]; + + wire [23:0] sig_max = {prod_max, 16'b0}; + wire diff_ge_32 = s1_diff_abs[5]; + wire [4:0] shamt = s1_diff_abs[4:0]; + wire [23:0] sig_min_shifted = diff_ge_32 ? 24'b0 : ({prod_min, 16'b0} >> shamt); + + wire [24:0] sig_max_ext = {1'b0, sig_max}; + wire [24:0] sig_min_ext = {1'b0, sig_min_shifted}; + + wire do_sub = s1_sign_sel[0] ^ s1_sign_sel[1]; + + wire [24:0] add_raw; + VX_ks_adder #( + .N (25), + .BYPASS (`FORCE_BUILTIN_ADDER(25)) + ) add_adder ( + .dataa (sig_max_ext), + .datab (sig_min_ext), + .cin (1'b0), + .sum (add_raw), + `UNUSED_PIN(cout) + ); + + wire [24:0] sub_raw; + VX_ks_adder #( + .N (25), + .BYPASS (`FORCE_BUILTIN_ADDER(25)) + ) sub_adder ( + .dataa (sig_max_ext), + .datab (~sig_min_ext), + .cin (1'b1), + .sum (sub_raw), + `UNUSED_PIN(cout) + ); + + wire [24:0] sub_rev_raw; + VX_ks_adder #( + .N (25), + .BYPASS (`FORCE_BUILTIN_ADDER(25)) + ) sub_rev_adder ( + .dataa (sig_min_ext), + .datab (~sig_max_ext), + .cin (1'b1), + .sum (sub_rev_raw), + `UNUSED_PIN(cout) + ); + + wire sub_neg = sub_raw[24]; + wire [24:0] sub_abs = sub_neg ? sub_rev_raw : sub_raw; + wire [24:0] sig_add_raw = do_sub ? sub_abs : add_raw; + + wire [23:0] sig_add = sig_add_raw[24:1]; + `UNUSED_VAR (sig_add_raw[0]) + + wire mag_is_equal = s1_pre_sum_eq && (man_prod0_v == man_prod1_v); + wire is_zero_out = do_sub && mag_is_equal; + + wire sig_sign_raw = sub_neg ? sign_min : sign_max; + wire sig_sign = is_zero_out ? 1'b0 : sig_sign_raw; + + assign result_sig[i] = {sig_sign, sig_add}; + assign result_exp[i] = ((v0 || v1) && !is_zero_out) ? final_exp : '0; + + wire pos_inf_0 = s1_inf_sel[0] && ~s1_sign_sel[0] && s1_lane_valid[0]; + wire neg_inf_0 = s1_inf_sel[0] && s1_sign_sel[0] && s1_lane_valid[0]; + wire pos_inf_1 = s1_inf_sel[1] && ~s1_sign_sel[1] && s1_lane_valid[1]; + wire neg_inf_1 = s1_inf_sel[1] && s1_sign_sel[1] && s1_lane_valid[1]; + + wire add_nan = (pos_inf_0 && neg_inf_1) || (neg_inf_0 && pos_inf_1); + + wire any_nan = (|(s1_nan_sel & s1_lane_valid)) || add_nan; + wire any_inf = (|(s1_inf_sel & s1_lane_valid)) && ~any_nan; + + wire final_sign_inf = s1_inf_sel[0] ? s1_sign_sel[0] : s1_sign_sel[1]; + + assign exceptions[i].is_nan = any_nan; + assign exceptions[i].is_inf = any_inf; + assign exceptions[i].sign = any_inf ? final_sign_inf : sig_sign; + + `ifdef DBG_TRACE_TCU + always_ff @(posedge clk) begin + if (valid_in && s1_lane_valid != 0) begin + `TRACE(4, ("%t: %s FEDP-REDUCE(%0d) lane=%0d: ", $time, INSTANCE_ID, req_id, i)); + `TRACE(4, ("max=(%0d, 0x%0h, %0d), ", sign_max, sig_max, s1_max_pre_sum)); + `TRACE(4, ("min=(%0d, 0x%0h) ", sign_min, sig_min_shifted)); + `TRACE(4, ("-> s=%0d, P=0x%0h, E=%0d\n", sig_sign, sig_add, final_exp)); + end + end + `endif + + end + +endmodule diff --git a/hw/rtl/tcu/tet/VX_tcu_tet_mul_i4.sv b/hw/rtl/tcu/tet/VX_tcu_tet_mul_i4.sv new file mode 100644 index 000000000..50bb28116 --- /dev/null +++ b/hw/rtl/tcu/tet/VX_tcu_tet_mul_i4.sv @@ -0,0 +1,85 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +module VX_tcu_tet_mul_i4 import VX_tcu_pkg::*; #( + parameter `STRING INSTANCE_ID = "", + parameter N = 2, + parameter TCK = 2 * N +) ( + input wire clk, + input wire valid_in, + input wire [31:0] req_id, + + input wire [TCU_MAX_INPUTS-1:0] vld_mask, + input wire [3:0] fmt_i, + + input wire [N-1:0][31:0] a_row, + input wire [N-1:0][31:0] b_col, + + output logic [TCK-1:0][24:0] result +); + `UNUSED_SPARAM (INSTANCE_ID) + `UNUSED_VAR ({clk, req_id, valid_in}) + + wire is_signed_int = fmt_i[3] || tcu_fmt_is_signed_int(fmt_i); + + // Multiplication and accumulation + for (genvar i = 0; i < TCK; ++i) begin : g_lane + + wire signed [3:0][9:0] y_prod_i4; + for (genvar j = 0; j < 4; ++j) begin : g_i4 + wire lane_valid = vld_mask[i * 4 + j]; + wire [3:0] raw_a = a_row[i/2][(i%2)*16 + j*4 +: 4]; + wire [3:0] raw_b = b_col[i/2][(i%2)*16 + j*4 +: 4]; + wire signed [4:0] s_a = is_signed_int ? $signed({raw_a[3], raw_a}) : $signed({1'b0, raw_a}); + wire signed [4:0] s_b = is_signed_int ? $signed({raw_b[3], raw_b}) : $signed({1'b0, raw_b}); + wire signed [9:0] prod_full = s_a * s_b; + assign y_prod_i4[j] = prod_full & {10{lane_valid}}; + end + + wire [9:0] y_i4_sum, y_i4_carry; + VX_csa_tree #( + .N (4), + .W (10), + .S (10) + ) i4_csa ( + .operands (y_prod_i4), + .sum (y_i4_sum), + .carry (y_i4_carry) + ); + + wire [9:0] y_i4_add_res; + VX_ks_adder #( + .N (10), + .BYPASS (`FORCE_BUILTIN_ADDER(10)) + ) i4_ksa ( + .cin (1'b0), + .dataa (y_i4_sum), + .datab (y_i4_carry), + .sum (y_i4_add_res), + `UNUSED_PIN(cout) + ); + + // Output muxing + always_comb begin + case ({1'b1, fmt_i}) + TCU_I4_ID: result[i] = 25'($signed(y_i4_add_res)); + TCU_U4_ID: result[i] = {15'b0, y_i4_add_res}; + default: result[i] = '0; + endcase + end + end + +endmodule diff --git a/hw/rtl/tcu/tet/VX_tcu_tet_mul_i8.sv b/hw/rtl/tcu/tet/VX_tcu_tet_mul_i8.sv new file mode 100644 index 000000000..df09de0ae --- /dev/null +++ b/hw/rtl/tcu/tet/VX_tcu_tet_mul_i8.sv @@ -0,0 +1,147 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +module VX_tcu_tet_mul_i8 import VX_tcu_pkg::*; #( + parameter `STRING INSTANCE_ID = "", + parameter N = 2, + parameter TCK = 2 * N +) ( + input wire clk, + input wire reset, + input wire enable, + input wire valid_in, + input wire [31:0] req_id, + + input wire [TCU_MAX_INPUTS-1:0] vld_mask, + input wire [3:0] fmt_i, + + input wire [N-1:0][31:0] a_row, + input wire [N-1:0][31:0] b_col, +`ifdef VX_CFG_TCU_MX_ENABLE + input wire [7:0] sf_a, + input wire [7:0] sf_b, +`endif + + output logic [TCK-1:0][24:0] result +); + `UNUSED_SPARAM (INSTANCE_ID) + `UNUSED_VAR ({req_id, valid_in}) + for (genvar i = 1; i < TCU_MAX_INPUTS; i += 2) begin : g_unused_vld + `UNUSED_VAR (vld_mask[i]) + end + + wire is_signed_int = fmt_i[3] || tcu_fmt_is_signed_int(fmt_i); + + // Multiplication and accumulation + for (genvar i = 0; i < TCK; ++i) begin : g_lane + + wire signed [16:0] y_prod_i8 [2]; + for (genvar j = 0; j < 2; ++j) begin : g_i8 + wire lane_valid = vld_mask[i * 4 + j * 2]; + wire [7:0] raw_a = a_row[i/2][(i%2)*16 + j*8 +: 8]; + wire [7:0] raw_b = b_col[i/2][(i%2)*16 + j*8 +: 8]; + wire signed [8:0] s_a = is_signed_int ? $signed({raw_a[7], raw_a}) : $signed({1'b0, raw_a}); + wire signed [8:0] s_b = is_signed_int ? $signed({raw_b[7], raw_b}) : $signed({1'b0, raw_b}); + (* use_dsp = "yes" *) wire signed [16:0] prod_full = s_a * s_b; + assign y_prod_i8[j] = prod_full & {17{lane_valid}}; + end + + wire signed [16:0] s1_y_prod_i8 [2]; + wire s1_is_i8; + wire s1_is_u8; + `ifdef VX_CFG_TCU_MX_ENABLE + wire signed [8:0] combined_sf = $signed({1'b0, sf_a}) + $signed({1'b0, sf_b}) - 9'sd266; + wire s1_is_mxi8; + wire signed [8:0] s1_combined_sf; + `endif + + VX_tcu_tet_register #( + `ifdef VX_CFG_TCU_MX_ENABLE + .DATAW ((2 * 17) + 9 + 3), + `else + .DATAW ((2 * 17) + 2), + `endif + .DEPTH (1) + ) pipe_i8_s0 ( + .clk (clk), + .reset (reset), + .enable (enable), + `ifdef VX_CFG_TCU_MX_ENABLE + .data_in ({y_prod_i8[0], y_prod_i8[1], combined_sf, (fmt_i == 4'(TCU_MXI8_ID)), (fmt_i == 4'(TCU_U8_ID)), (fmt_i == 4'(TCU_I8_ID))}), + .data_out ({s1_y_prod_i8[0], s1_y_prod_i8[1], s1_combined_sf, s1_is_mxi8, s1_is_u8, s1_is_i8}) + `else + .data_in ({y_prod_i8[0], y_prod_i8[1], (fmt_i == 4'(TCU_U8_ID)), (fmt_i == 4'(TCU_I8_ID))}), + .data_out ({s1_y_prod_i8[0], s1_y_prod_i8[1], s1_is_u8, s1_is_i8}) + `endif + ); + + wire [16:0] y_i8_add_res; + VX_ks_adder #( + .N(17), + .BYPASS (`FORCE_BUILTIN_ADDER(17)) + ) i8_ksa ( + .cin (1'b0), + .dataa (s1_y_prod_i8[0]), + .datab (s1_y_prod_i8[1]), + .sum (y_i8_add_res), + `UNUSED_PIN(cout) + ); + +`ifdef VX_CFG_TCU_MX_ENABLE + wire is_right_shift = s1_combined_sf[8]; + wire shift_overflow = (s1_combined_sf > 9'sd24) || (s1_combined_sf < -9'sd24); + wire [4:0] shift_amount = is_right_shift ? (-s1_combined_sf[4:0]) : s1_combined_sf[4:0]; + + wire signed [24:0] y_mxi8_scaled [2]; + for (genvar j = 0; j < 2; ++j) begin : g_mxi8 + wire signed [24:0] raw_prod = {{8{s1_y_prod_i8[j][16]}}, s1_y_prod_i8[j]}; + wire [24:0] abs_prod = raw_prod[24] ? -raw_prod : raw_prod; + wire signed [24:0] right_shifted = raw_prod[24] ? -25'($signed(abs_prod >> shift_amount)) + : 25'($signed(abs_prod >> shift_amount)); + assign y_mxi8_scaled[j] = shift_overflow ? 25'sd0 + : is_right_shift ? right_shifted + : (raw_prod <<< shift_amount); + end + + wire [24:0] y_mxi8_add_res; + VX_ks_adder #( + .N (25), + .BYPASS (`FORCE_BUILTIN_ADDER(25)) + ) mxi8_ksa ( + .cin (1'b0), + .dataa (y_mxi8_scaled[0]), + .datab (y_mxi8_scaled[1]), + .sum (y_mxi8_add_res), + `UNUSED_PIN(cout) + ); +`endif + + // Output muxing + always_comb begin + if (s1_is_i8) begin + result[i] = 25'($signed(y_i8_add_res)); + end else if (s1_is_u8) begin + result[i] = {8'b0, y_i8_add_res}; + `ifdef VX_CFG_TCU_MX_ENABLE + end else if (s1_is_mxi8) begin + result[i] = y_mxi8_add_res; + `endif + end else begin + result[i] = '0; + end + end + end + +endmodule diff --git a/hw/rtl/tcu/tet/VX_tcu_tet_mul_join.sv b/hw/rtl/tcu/tet/VX_tcu_tet_mul_join.sv new file mode 100644 index 000000000..c764bc2d6 --- /dev/null +++ b/hw/rtl/tcu/tet/VX_tcu_tet_mul_join.sv @@ -0,0 +1,190 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +module VX_tcu_tet_mul_join import VX_tcu_pkg::*; #( + parameter `STRING INSTANCE_ID = "", + parameter N = 2, + parameter TCK = 2 * N, + parameter W = 25, + parameter WA = 28, + parameter EXP_W = 10 +) ( + input wire clk, + input wire valid_in, + input wire [31:0] req_id, + + input wire [4:0] fmt_s, + input wire [31:0] c_val, + +`ifdef VX_CFG_TCU_FP16_ENABLE +`define TET_JOIN_F16_ENABLE +`elsif VX_CFG_TCU_TF32_ENABLE +`define TET_JOIN_F16_ENABLE +`endif + +`ifdef TET_JOIN_F16_ENABLE + input wire [TCK-1:0][24:0] sig_f16, + input wire [TCK-1:0][EXP_W-1:0] exp_f16, + input fedp_excep_t [TCK-1:0] exc_f16, +`endif + +`ifdef VX_CFG_TCU_FP8_ENABLE + input wire [TCK-1:0][24:0] sig_f8, + input wire [TCK-1:0][EXP_W-1:0] exp_f8, + input fedp_excep_t [TCK-1:0] exc_f8, +`endif + +`ifdef VX_CFG_TCU_MX_ENABLE +`ifdef VX_CFG_TCU_FP4_ENABLE + input wire [TCK-1:0][24:0] sig_f4, + input wire [TCK-1:0][EXP_W-1:0] exp_f4, + input fedp_excep_t [TCK-1:0] exc_f4, +`endif +`endif + +`ifdef VX_CFG_TCU_INT8_ENABLE + input wire [TCK-1:0][24:0] sig_int8, +`endif +`ifdef VX_CFG_TCU_INT4_ENABLE + input wire [TCK-1:0][24:0] sig_int4, +`endif + + output logic [TCK:0][24:0] sig_out, + output logic [TCK:0][EXP_W-1:0] exp_out, + output fedp_excep_t [TCK:0] exc_out +); + `UNUSED_SPARAM (INSTANCE_ID) + `UNUSED_VAR ({clk, req_id, valid_in}) + + logic [TCK-1:0][24:0] sig_sel; + logic [TCK-1:0][EXP_W-1:0] exp_sel; + fedp_excep_t [TCK-1:0] exc_sel; + + // Path selection + always_comb begin + case (fmt_s) + `ifdef TET_JOIN_F16_ENABLE + `ifdef VX_CFG_TCU_FP16_ENABLE + TCU_FP16_ID, + TCU_BF16_ID: begin + sig_sel = sig_f16; + exp_sel = exp_f16; + exc_sel = exc_f16; + end + `endif + `ifdef VX_CFG_TCU_TF32_ENABLE + TCU_TF32_ID: begin + sig_sel = sig_f16; + exp_sel = exp_f16; + exc_sel = exc_f16; + end + `endif + `endif + + `ifdef VX_CFG_TCU_FP8_ENABLE + TCU_FP8_ID, TCU_BF8_ID + `ifdef VX_CFG_TCU_MX_ENABLE + , TCU_MXFP8_ID, TCU_MXBF8_ID + `endif + : begin + sig_sel = sig_f8; + exp_sel = exp_f8; + exc_sel = exc_f8; + end + `endif + + `ifdef VX_CFG_TCU_MX_ENABLE + `ifdef VX_CFG_TCU_FP4_ENABLE + `ifdef VX_CFG_TCU_MXFP4_ENABLE + TCU_MXFP4_ID: begin + sig_sel = sig_f4; + exp_sel = exp_f4; + exc_sel = exc_f4; + end + `endif + `ifdef VX_CFG_TCU_NVFP4_ENABLE + TCU_NVFP4_ID: begin + sig_sel = sig_f4; + exp_sel = exp_f4; + exc_sel = exc_f4; + end + `endif + `endif + `endif + + `ifdef VX_CFG_TCU_INT8_ENABLE + TCU_I8_ID, TCU_U8_ID + `ifdef VX_CFG_TCU_MX_ENABLE + , TCU_MXI8_ID + `endif + : begin + sig_sel = sig_int8; + exp_sel = '0; + exc_sel = '0; + end + `endif + `ifdef VX_CFG_TCU_INT4_ENABLE + TCU_I4_ID, TCU_U4_ID: begin + sig_sel = sig_int4; + exp_sel = '0; + exc_sel = '0; + end + `endif + default: begin + sig_sel = '0; + exp_sel = '0; + exc_sel = '0; + end + endcase + end + + wire c_is_int = tcu_fmt_is_int(fmt_s); + + // C-term handling + fedp_class_t cls_c; + VX_tcu_tet_classifier #( + .EXP_W (8), + .MAN_W (23) + ) class_c ( + .exp (c_val[30:23]), + .man (c_val[22:0]), + .cls (cls_c) + ); + + wire c_sign = c_val[31]; + + wire [24:0] c_sig_final = c_is_int ? c_val[24:0] : (cls_c.is_zero ? 25'd0 : {c_val[31], (cls_c.is_sub ? 1'b0 : 1'b1), c_val[22:0]}); + + wire [7:0] c_exp_raw = (cls_c.is_sub || cls_c.is_zero) ? 8'd1 : c_val[30:23]; + wire [EXP_W-1:0] c_exp_adj = EXP_W'(c_exp_raw) - EXP_W'(W-1) + EXP_W'(WA-1) + 128; + wire [EXP_W-1:0] c_exp_final = cls_c.is_zero ? '0 : c_exp_adj; + + // Output aggregation + assign sig_out = {c_sig_final, sig_sel}; + assign exp_out = {c_exp_final, exp_sel}; + + for (genvar i = 0; i < TCK; ++i) begin : g_exc + assign exc_out[i] = exc_sel[i]; + end + + assign exc_out[TCK].is_nan = cls_c.is_nan; + assign exc_out[TCK].is_inf = cls_c.is_inf; + assign exc_out[TCK].sign = c_sign; + +endmodule + +`ifdef TET_JOIN_F16_ENABLE +`undef TET_JOIN_F16_ENABLE +`endif diff --git a/hw/rtl/tcu/tet/VX_tcu_tet_norm_round.sv b/hw/rtl/tcu/tet/VX_tcu_tet_norm_round.sv new file mode 100644 index 000000000..eb725f101 --- /dev/null +++ b/hw/rtl/tcu/tet/VX_tcu_tet_norm_round.sv @@ -0,0 +1,243 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +module VX_tcu_tet_norm_round import VX_tcu_pkg::*; #( + parameter `STRING INSTANCE_ID = "", + parameter WA = 30, + parameter EXP_W = 10, + parameter C_HI_W = 8 +) ( + input wire clk, + input wire reset, + input wire enable, + input wire valid_in, + input wire [31:0] req_id, + input wire [EXP_W-1:0] max_exp, + input wire [WA-1:0] acc_sig, + input wire [C_HI_W-1:0] cval_hi, + input wire is_int, + input wire sticky_in, + input fedp_excep_t exceptions, + output wire [31:0] result +); + `UNUSED_SPARAM (INSTANCE_ID) + +`ifdef VX_CFG_TCU_INT8_ENABLE +`define TET_NORM_INT_ENABLE +`elsif VX_CFG_TCU_INT4_ENABLE +`define TET_NORM_INT_ENABLE +`endif + + wire sum_sign = acc_sig[WA-1]; + wire [WA-1:0] abs_sum = sum_sign ? (~acc_sig + WA'(1)) : acc_sig; + wire zero_sum = ~|abs_sum; + + wire [$clog2(WA)-1:0] lz_count_pred; + VX_lzc #( + .N(WA) + ) lzc_inst ( + .data_in (abs_sum), + .data_out (lz_count_pred), + `UNUSED_PIN (valid_out) + ); + + wire signed [EXP_W-1:0] norm_exp_base; + wire signed [EXP_W-1:0] norm_exp_plus1; + wire signed [EXP_W-1:0] norm_exp_plus2; + + wire [EXP_W-1:0] sub_term = EXP_W'({1'b1, 2'b00, lz_count_pred}); + VX_ks_adder #( + .N(EXP_W), + .BYPASS (`FORCE_BUILTIN_ADDER(EXP_W)) + ) exp_sub ( + .cin (1'b1), + .dataa (max_exp), + .datab (~sub_term), + .sum (norm_exp_base), + `UNUSED_PIN (cout) + ); + + assign norm_exp_plus1 = norm_exp_base + EXP_W'(1'b1); + assign norm_exp_plus2 = norm_exp_base + EXP_W'(2'd2); + + wire [WA:0] abs_sum_ext = {1'b0, abs_sum}; + wire [WA:0] shifted_sum_raw = abs_sum_ext << lz_count_pred; + + wire overshift = shifted_sum_raw[WA]; + + wire [26:0] aligned_bits = overshift ? shifted_sum_raw[WA -: 27] + : shifted_sum_raw[WA-1 -: 27]; + + wire [23:0] norm_man = aligned_bits[26:3]; + wire guard_bit = aligned_bits[2]; + wire round_bit = aligned_bits[1]; + wire sticky_bit = aligned_bits[0] | (|shifted_sum_raw[WA-27:0]) | sticky_in; + wire lsb_bit = norm_man[0]; + wire round_up = guard_bit && (round_bit || sticky_bit || lsb_bit); + +`ifdef TET_NORM_INT_ENABLE + wire [6:0] ext_acc_int = 7'($signed(acc_sig[WA-1:25])); + wire [6:0] int_hi; + VX_ks_adder #( + .N (7), + .BYPASS (`FORCE_BUILTIN_ADDER(7)) + ) int_adder ( + .cin (0), + .dataa (ext_acc_int), + .datab (cval_hi), + .sum (int_hi), + `UNUSED_PIN (cout) + ); + + wire [31:0] int_result = {int_hi, acc_sig[24:0]}; +`else + `UNUSED_VAR ({cval_hi, is_int}) +`endif + + wire [23:0] s1_norm_man; + wire s1_round_up; + wire s1_overshift; + wire signed [EXP_W-1:0] s1_norm_exp_base; + wire signed [EXP_W-1:0] s1_norm_exp_plus1; + wire signed [EXP_W-1:0] s1_norm_exp_plus2; + wire s1_sum_sign; + wire s1_zero_sum; + fedp_excep_t s1_exceptions; + wire s1_is_int; +`ifdef TET_NORM_INT_ENABLE + wire [31:0] s1_int_result; +`endif + wire [WA-1:0] s1_abs_sum; + wire [$clog2(WA)-1:0] s1_lz_count_pred; + wire [WA:0] s1_shifted_sum_raw; + wire s1_round_bit; + wire s1_sticky_bit; + wire s1_valid; + wire [31:0] s1_req_id; + + VX_tcu_tet_register #( + `ifdef TET_NORM_INT_ENABLE + .DATAW (24 + 1 + 1 + (3 * EXP_W) + 1 + 1 + $bits(fedp_excep_t) + 1 + 32 + WA + $clog2(WA) + (WA+1) + 1 + 1 + 1 + 32), + `else + .DATAW (24 + 1 + 1 + (3 * EXP_W) + 1 + 1 + $bits(fedp_excep_t) + 1 + WA + $clog2(WA) + (WA+1) + 1 + 1 + 1 + 32), + `endif + .DEPTH (1) + ) pipe_norm0 ( + .clk (clk), + .reset (reset), + .enable (enable), + `ifdef TET_NORM_INT_ENABLE + .data_in ({norm_man, round_up, overshift, norm_exp_base, norm_exp_plus1, norm_exp_plus2, sum_sign, zero_sum, exceptions, is_int, int_result, abs_sum, lz_count_pred, shifted_sum_raw, round_bit, sticky_bit, valid_in, req_id}), + .data_out ({s1_norm_man, s1_round_up, s1_overshift, s1_norm_exp_base, s1_norm_exp_plus1, s1_norm_exp_plus2, s1_sum_sign, s1_zero_sum, s1_exceptions, s1_is_int, s1_int_result, s1_abs_sum, s1_lz_count_pred, s1_shifted_sum_raw, s1_round_bit, s1_sticky_bit, s1_valid, s1_req_id}) + `else + .data_in ({norm_man, round_up, overshift, norm_exp_base, norm_exp_plus1, norm_exp_plus2, sum_sign, zero_sum, exceptions, is_int, abs_sum, lz_count_pred, shifted_sum_raw, round_bit, sticky_bit, valid_in, req_id}), + .data_out ({s1_norm_man, s1_round_up, s1_overshift, s1_norm_exp_base, s1_norm_exp_plus1, s1_norm_exp_plus2, s1_sum_sign, s1_zero_sum, s1_exceptions, s1_is_int, s1_abs_sum, s1_lz_count_pred, s1_shifted_sum_raw, s1_round_bit, s1_sticky_bit, s1_valid, s1_req_id}) + `endif + ); + +`ifndef DBG_TRACE_TCU + `UNUSED_VAR ({s1_req_id, s1_valid, s1_round_bit, s1_sticky_bit, s1_abs_sum, s1_lz_count_pred, s1_shifted_sum_raw}) + `ifndef TET_NORM_INT_ENABLE + `UNUSED_VAR (s1_is_int) + `endif +`endif + + wire [24:0] man_plus_zero = {1'b0, s1_norm_man}; + wire [24:0] rounded_sig_full; + VX_ks_adder #( + .N(25), + .BYPASS (`FORCE_BUILTIN_ADDER(25)) + ) round_adder ( + .cin (s1_round_up), + .dataa (man_plus_zero), + .datab (25'd0), + .sum (rounded_sig_full), + `UNUSED_PIN (cout) + ); + + wire carry_out = rounded_sig_full[24]; + wire [22:0] final_man = carry_out ? rounded_sig_full[23:1] : rounded_sig_full[22:0]; + + logic signed [EXP_W-1:0] final_exp_s; + always_comb begin + case ({s1_overshift, carry_out}) + 2'b00: final_exp_s = s1_norm_exp_base; + 2'b01: final_exp_s = s1_norm_exp_plus1; + 2'b10: final_exp_s = s1_norm_exp_plus1; + 2'b11: final_exp_s = s1_norm_exp_plus2; + endcase + end + + logic [7:0] packed_exp; + logic exp_overflow, exp_underflow; + always_comb begin + if (final_exp_s >= 255) begin + packed_exp = 8'hFF; + exp_overflow = 1'b1; + exp_underflow = 1'b0; + end else if (final_exp_s <= 0) begin + packed_exp = 8'h00; + exp_overflow = 1'b0; + exp_underflow = 1'b1; + end else begin + packed_exp = final_exp_s[7:0]; + exp_overflow = 1'b0; + exp_underflow = 1'b0; + end + end + + wire [31:0] fp_nan_result = {1'b0, 8'hFF, 1'b1, 22'd0}; + wire [31:0] fp_inf_result = {s1_exceptions.sign, 8'hFF, 23'd0}; + wire [31:0] fp_zero_result = {s1_sum_sign, 8'd0, 23'd0}; + wire [31:0] fp_overflow_result = {s1_sum_sign, 8'hFF, 23'd0}; + wire [31:0] fp_normal_result = {s1_sum_sign, packed_exp, final_man}; + + logic [31:0] fp_result; + always_comb begin + if (s1_exceptions.is_nan) begin + fp_result = fp_nan_result; + end else if (s1_exceptions.is_inf) begin + fp_result = fp_inf_result; + end else begin + if (s1_zero_sum || exp_underflow) begin + fp_result = fp_zero_result; + end else if (exp_overflow) begin + fp_result = fp_overflow_result; + end else begin + fp_result = fp_normal_result; + end + end + end + +`ifdef TET_NORM_INT_ENABLE + assign result = s1_is_int ? s1_int_result : fp_result; +`else + assign result = fp_result; +`endif + +`ifdef DBG_TRACE_TCU + always_ff @(posedge clk) begin + if (s1_valid) begin + `TRACE(4, ("%t: %s FEDP-NORM(%0d): is_int=%b, abs_sum=0x%0h, sign=%b, lzc=%0d, norm_exp=%0d, shifted=0x%0h, R=%b, S=%b, Rup=%b, carry=%b, final_exp=%0d, result=0x%0h\n", + $time, INSTANCE_ID, s1_req_id, s1_is_int, s1_abs_sum, s1_sum_sign, s1_lz_count_pred, s1_norm_exp_base, s1_shifted_sum_raw, s1_round_bit, s1_sticky_bit, s1_round_up, carry_out, final_exp_s, result)); + end + end +`endif + +endmodule + +`ifdef TET_NORM_INT_ENABLE +`undef TET_NORM_INT_ENABLE +`endif diff --git a/hw/rtl/tcu/tet/VX_tcu_tet_pipe_register.sv b/hw/rtl/tcu/tet/VX_tcu_tet_pipe_register.sv new file mode 100644 index 000000000..c2c234794 --- /dev/null +++ b/hw/rtl/tcu/tet/VX_tcu_tet_pipe_register.sv @@ -0,0 +1,85 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +module VX_tcu_tet_pipe_register #( + parameter SHARED_DATAW = 1, + parameter LANE_DATAW = 1, + parameter NUM_LANES = 1, + parameter DEPTH = 1, + parameter LANE_MASK = 0 +) ( + input wire clk, + input wire reset, + input wire enable, + input wire [NUM_LANES-1:0] lane_mask, + + input wire [SHARED_DATAW-1:0] shared_data_in, + output wire [SHARED_DATAW-1:0] shared_data_out, + + input wire [NUM_LANES*LANE_DATAW-1:0] lane_data_in, + output wire [NUM_LANES*LANE_DATAW-1:0] lane_data_out +); + + if (LANE_MASK == 0) begin : g_merged + (* shreg_extract = "no" *) reg [DEPTH-1:0][SHARED_DATAW + (NUM_LANES * LANE_DATAW)-1:0] pipe; + + always_ff @(posedge clk) begin + if (reset) begin + pipe <= '0; + end else if (enable) begin + pipe[0] <= {shared_data_in, lane_data_in}; + for (int i = 1; i < DEPTH; ++i) begin + pipe[i] <= pipe[i-1]; + end + end + end + + assign {shared_data_out, lane_data_out} = pipe[DEPTH-1]; + `UNUSED_VAR (lane_mask) + end else begin : g_split + (* shreg_extract = "no" *) reg [DEPTH-1:0][SHARED_DATAW-1:0] pipe_shared; + (* shreg_extract = "no" *) reg [NUM_LANES-1:0][DEPTH-1:0][LANE_DATAW-1:0] pipe_lane; + + always_ff @(posedge clk) begin + if (reset) begin + pipe_shared <= '0; + end else if (enable) begin + pipe_shared[0] <= shared_data_in; + for (int i = 1; i < DEPTH; ++i) begin + pipe_shared[i] <= pipe_shared[i-1]; + end + end + end + + for (genvar lane = 0; lane < NUM_LANES; ++lane) begin : g_lanes + always_ff @(posedge clk) begin + if (reset) begin + pipe_lane[lane] <= '0; + end else if (enable) begin + pipe_lane[lane][0] <= lane_data_in[lane*LANE_DATAW +: LANE_DATAW]; + for (int i = 1; i < DEPTH; ++i) begin + pipe_lane[lane][i] <= pipe_lane[lane][i-1]; + end + end + end + + assign lane_data_out[lane*LANE_DATAW +: LANE_DATAW] = pipe_lane[lane][DEPTH-1]; + end + + assign shared_data_out = pipe_shared[DEPTH-1]; + `UNUSED_VAR (lane_mask) + end + +endmodule diff --git a/hw/rtl/tcu/tet/VX_tcu_tet_register.sv b/hw/rtl/tcu/tet/VX_tcu_tet_register.sv new file mode 100644 index 000000000..14ccb9db7 --- /dev/null +++ b/hw/rtl/tcu/tet/VX_tcu_tet_register.sv @@ -0,0 +1,41 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +module VX_tcu_tet_register #( + parameter DATAW = 1, + parameter DEPTH = 1 +) ( + input wire clk, + input wire reset, + input wire enable, + input wire [DATAW-1:0] data_in, + output wire [DATAW-1:0] data_out +); + (* shreg_extract = "no" *) reg [DEPTH-1:0][DATAW-1:0] pipe; + + always_ff @(posedge clk) begin + if (reset) begin + pipe <= '0; + end else if (enable) begin + pipe[0] <= data_in; + for (int i = 1; i < DEPTH; ++i) begin + pipe[i] <= pipe[i-1]; + end + end + end + + assign data_out = pipe[DEPTH-1]; + +endmodule diff --git a/hw/rtl/tcu/tet/VX_tcu_tet_shared_mul.sv b/hw/rtl/tcu/tet/VX_tcu_tet_shared_mul.sv new file mode 100644 index 000000000..9cd40940e --- /dev/null +++ b/hw/rtl/tcu/tet/VX_tcu_tet_shared_mul.sv @@ -0,0 +1,361 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +module VX_tcu_tet_shared_mul import VX_tcu_pkg::*; #( + parameter `STRING INSTANCE_ID = "", + parameter N = 2, + parameter W = 25, + parameter WA = 28, + parameter EXP_W = 10, + parameter TCK = 2 * N, + parameter SF = 1 +) ( + input wire clk, + input wire reset, + input wire enable, + input wire valid_in, + input wire [31:0] req_id, + + input wire [TCU_MAX_INPUTS-1:0] vld_mask, + + input wire [4:0] fmt_s, + + input wire [N-1:0][31:0] a_row, + input wire [N-1:0][31:0] b_col, + input wire [31:0] c_val, +`ifdef VX_CFG_TCU_MX_ENABLE + input wire [SF-1:0][7:0] sf_a, + input wire [SF-1:0][7:0] sf_b, +`endif + output wire [TCK:0][EXP_W-1:0] exponents, + output wire [TCK:0] exp_sel, + output wire [TCK-1:0][TCK-1:0][EXP_W:0] exp_diff_mat, + + output wire [TCK:0][W-1:0] raw_sigs, + output wire fedp_excep_t exceptions, + output wire [TCK-1:0] lane_mask +); + `UNUSED_SPARAM (INSTANCE_ID) + +`ifndef VX_CFG_TCU_MX_ENABLE + `UNUSED_PARAM (SF) +`endif + +`ifdef VX_CFG_TCU_FP16_ENABLE +`define TET_MUL_F16_ENABLE +`elsif VX_CFG_TCU_TF32_ENABLE +`define TET_MUL_F16_ENABLE +`endif + +`ifdef TET_MUL_F16_ENABLE + wire [TCK-1:0][24:0] mul_f16_sig; + wire [TCK-1:0][EXP_W-1:0] mul_f16_exp; + fedp_excep_t [TCK-1:0] mul_f16_exc; + + VX_tcu_tet_mul_f16 #( + .INSTANCE_ID (INSTANCE_ID), + .N (N), + .TCK (TCK), + .W (W), + .WA (WA), + .EXP_W (EXP_W) + ) mul_f16 ( + .clk (clk), + .reset (reset), + .enable (enable), + .valid_in (valid_in), + .req_id (req_id), + .vld_mask (vld_mask), + .fmt_f (fmt_s[3:0]), + .a_row (a_row), + .b_col (b_col), + .result_sig (mul_f16_sig), + .result_exp (mul_f16_exp), + .exceptions (mul_f16_exc) + ); + + wire [TCK-1:0][24:0] s1_mul_f16_sig = mul_f16_sig; + wire [TCK-1:0][EXP_W-1:0] s1_mul_f16_exp = mul_f16_exp; + fedp_excep_t [TCK-1:0] s1_mul_f16_exc; + assign s1_mul_f16_exc = mul_f16_exc; +`endif + +`ifdef VX_CFG_TCU_FP8_ENABLE + wire [TCK-1:0][24:0] mul_f8_sig; + wire [TCK-1:0][EXP_W-1:0] mul_f8_exp; + fedp_excep_t [TCK-1:0] mul_f8_exc; + + wire [SF-1:0][TCK-1:0][24:0] mul_f8_sig_s; + wire [SF-1:0][TCK-1:0][EXP_W-1:0] mul_f8_exp_s; + fedp_excep_t [SF-1:0][TCK-1:0] mul_f8_exc_s; + + for (genvar s = 0; s < SF; ++s) begin : g_mul_f8_sf + VX_tcu_tet_mul_f8 #( + .INSTANCE_ID (INSTANCE_ID), + .N (N), + .TCK (TCK), + .W (W), + .WA (WA), + .EXP_W (EXP_W) + ) mul_f8 ( + .reset (reset), + .enable (enable), + .clk (clk), + .valid_in (valid_in), + .req_id (req_id), + .vld_mask (vld_mask), + .fmt_f (fmt_s[3:0]), + .a_row (a_row), + .b_col (b_col), + `ifdef VX_CFG_TCU_MX_ENABLE + .sf_a (sf_a[s]), + .sf_b (sf_b[s]), + `endif + .result_sig (mul_f8_sig_s[s]), + .result_exp (mul_f8_exp_s[s]), + .exceptions (mul_f8_exc_s[s]) + ); + end + + for (genvar i = 0; i < TCK; ++i) begin : g_mul_f8_lane + localparam SF_SLOT = (i * SF) / TCK; + assign mul_f8_sig[i] = mul_f8_sig_s[SF_SLOT][i]; + assign mul_f8_exp[i] = mul_f8_exp_s[SF_SLOT][i]; + assign mul_f8_exc[i] = mul_f8_exc_s[SF_SLOT][i]; + end + + wire [TCK-1:0][24:0] s1_mul_f8_sig = mul_f8_sig; + wire [TCK-1:0][EXP_W-1:0] s1_mul_f8_exp = mul_f8_exp; + fedp_excep_t [TCK-1:0] s1_mul_f8_exc; + assign s1_mul_f8_exc = mul_f8_exc; +`endif + +`ifdef VX_CFG_TCU_MX_ENABLE +`ifdef VX_CFG_TCU_FP4_ENABLE + wire [TCK-1:0][24:0] mul_f4_sig; + wire [TCK-1:0][EXP_W-1:0] mul_f4_exp; + fedp_excep_t [TCK-1:0] mul_f4_exc; + + wire [SF-1:0][TCK-1:0][24:0] mul_f4_sig_s; + wire [SF-1:0][TCK-1:0][EXP_W-1:0] mul_f4_exp_s; + fedp_excep_t [SF-1:0][TCK-1:0] mul_f4_exc_s; + + for (genvar s = 0; s < SF; ++s) begin : g_mul_f4_sf + VX_tcu_tet_mul_f4 #( + .INSTANCE_ID (INSTANCE_ID), + .N (N), + .TCK (TCK), + .W (W), + .WA (WA), + .EXP_W (EXP_W) + ) mul_f4 ( + .clk (clk), + .valid_in (valid_in), + .req_id (req_id), + .vld_mask (vld_mask), + .fmt_f (fmt_s), + .a_row (a_row), + .b_col (b_col), + .sf_a (sf_a[s]), + .sf_b (sf_b[s]), + .result_sig (mul_f4_sig_s[s]), + .result_exp (mul_f4_exp_s[s]), + .exceptions (mul_f4_exc_s[s]) + ); + end + + for (genvar i = 0; i < TCK; ++i) begin : g_mul_f4_lane + localparam SF_SLOT = (i * SF) / TCK; + assign mul_f4_sig[i] = mul_f4_sig_s[SF_SLOT][i]; + assign mul_f4_exp[i] = mul_f4_exp_s[SF_SLOT][i]; + assign mul_f4_exc[i] = mul_f4_exc_s[SF_SLOT][i]; + end + + wire [TCK-1:0][24:0] s1_mul_f4_sig; + wire [TCK-1:0][EXP_W-1:0] s1_mul_f4_exp; + fedp_excep_t [TCK-1:0] s1_mul_f4_exc; + + VX_tcu_tet_register #( + .DATAW ((TCK * 25) + (TCK * EXP_W) + (TCK * $bits(fedp_excep_t))), + .DEPTH (1) + ) pipe_mul_f4 ( + .clk (clk), + .reset (reset), + .enable (enable), + .data_in ({mul_f4_sig, mul_f4_exp, mul_f4_exc}), + .data_out ({s1_mul_f4_sig, s1_mul_f4_exp, s1_mul_f4_exc}) + ); +`endif +`endif + +`ifdef VX_CFG_TCU_INT8_ENABLE + wire [TCK-1:0][24:0] mul_int8_sig; + wire [SF-1:0][TCK-1:0][24:0] mul_int8_sig_s; + for (genvar s = 0; s < SF; ++s) begin : g_mul_i8_sf + VX_tcu_tet_mul_i8 #( + .INSTANCE_ID (INSTANCE_ID), + .N (N), + .TCK (TCK) + ) mul_int8 ( + .reset (reset), + .enable (enable), + .clk (clk), + .valid_in (valid_in), + .req_id (req_id), + .vld_mask (vld_mask), + .fmt_i (fmt_s[3:0]), + .a_row (a_row), + .b_col (b_col), + `ifdef VX_CFG_TCU_MX_ENABLE + .sf_a (sf_a[s]), + .sf_b (sf_b[s]), + `endif + .result (mul_int8_sig_s[s]) + ); + end + + for (genvar i = 0; i < TCK; ++i) begin : g_mul_i8_lane + localparam SF_SLOT = (i * SF) / TCK; + assign mul_int8_sig[i] = mul_int8_sig_s[SF_SLOT][i]; + end + + wire [TCK-1:0][24:0] s1_mul_int8_sig = mul_int8_sig; +`endif + +`ifdef VX_CFG_TCU_INT4_ENABLE + wire [TCK-1:0][24:0] mul_int4_sig; + VX_tcu_tet_mul_i4 #( + .INSTANCE_ID (INSTANCE_ID), + .N (N), + .TCK (TCK) + ) mul_int4 ( + .clk (clk), + .valid_in (valid_in), + .req_id (req_id), + .vld_mask (vld_mask), + .fmt_i (fmt_s[3:0]), + .a_row (a_row), + .b_col (b_col), + .result (mul_int4_sig) + ); + + wire [TCK-1:0][24:0] s1_mul_int4_sig; + VX_tcu_tet_register #( + .DATAW (TCK * 25), + .DEPTH (1) + ) pipe_mul_i4 ( + .clk (clk), + .reset (reset), + .enable (enable), + .data_in (mul_int4_sig), + .data_out (s1_mul_int4_sig) + ); +`endif + + wire s1_valid; + wire [31:0] s1_req_id; + wire [TCU_MAX_INPUTS-1:0] s1_vld_mask; + wire [4:0] s1_fmt_s; + wire [31:0] s1_c_val; + + VX_tcu_tet_register #( + .DATAW (1 + 32 + TCU_MAX_INPUTS + 5 + 32), + .DEPTH (1) + ) pipe_mul_ctrl ( + .clk (clk), + .reset (reset), + .enable (enable), + .data_in ({valid_in, req_id, vld_mask, fmt_s, c_val}), + .data_out ({s1_valid, s1_req_id, s1_vld_mask, s1_fmt_s, s1_c_val}) + ); + + fedp_excep_t [TCK:0] join_exceptions; + + VX_tcu_tet_mul_join #( + .INSTANCE_ID (INSTANCE_ID), + .N (N), + .TCK (TCK), + .W (W), + .WA (WA), + .EXP_W (EXP_W) + ) join_stage ( + .clk (clk), + .valid_in (s1_valid), + .req_id (s1_req_id), + .fmt_s (s1_fmt_s), + .c_val (s1_c_val), + + `ifdef TET_MUL_F16_ENABLE + .sig_f16 (s1_mul_f16_sig), + .exp_f16 (s1_mul_f16_exp), + .exc_f16 (s1_mul_f16_exc), + `endif + + `ifdef VX_CFG_TCU_FP8_ENABLE + .sig_f8 (s1_mul_f8_sig), + .exp_f8 (s1_mul_f8_exp), + .exc_f8 (s1_mul_f8_exc), + `endif + + `ifdef VX_CFG_TCU_MX_ENABLE + `ifdef VX_CFG_TCU_FP4_ENABLE + .sig_f4 (s1_mul_f4_sig), + .exp_f4 (s1_mul_f4_exp), + .exc_f4 (s1_mul_f4_exc), + `endif + `endif + + `ifdef VX_CFG_TCU_INT8_ENABLE + .sig_int8 (s1_mul_int8_sig), + `endif + `ifdef VX_CFG_TCU_INT4_ENABLE + .sig_int4 (s1_mul_int4_sig), + `endif + + .sig_out (raw_sigs), + .exp_out (exponents), + .exc_out (join_exceptions) + ); + + VX_tcu_tet_exc_reduce #( + .TCK (TCK) + ) exc_reduce ( + .exc_in (join_exceptions), + .exc_out (exceptions) + ); + + VX_tcu_tet_max_exp #( + .N (TCK+1), + .WIDTH (EXP_W) + ) find_diff_mat ( + .exponents (exponents), + .sel_exp (exp_sel), + .diff_mat (exp_diff_mat) + ); + + VX_tcu_tet_lane_mask #( + .N (N), + .TCK (TCK) + ) lane_mask_inst ( + .vld_mask (s1_vld_mask), + .fmt_s (s1_fmt_s), + .lane_mask (lane_mask) + ); + +endmodule + +`ifdef TET_MUL_F16_ENABLE +`undef TET_MUL_F16_ENABLE +`endif diff --git a/hw/rtl/tcu/tet/VX_tcu_tet_wmul.sv b/hw/rtl/tcu/tet/VX_tcu_tet_wmul.sv new file mode 100644 index 000000000..c429dd0dd --- /dev/null +++ b/hw/rtl/tcu/tet/VX_tcu_tet_wmul.sv @@ -0,0 +1,74 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +module VX_tcu_tet_wmul #( + parameter N = 4, + parameter M = N, + parameter LANES = 1, + parameter SHARED_B = 0, + parameter P = N + M, + parameter USE_DSP = 0 +) ( + input wire [LANES-1:0][N-1:0] a, + input wire [LANES-1:0][M-1:0] b, + output wire [LANES-1:0][P-1:0] p +); + `STATIC_ASSERT (LANES == 1 || LANES == 2, ("VX_tcu_tet_wmul: LANES must be 1 or 2")) + + if (USE_DSP == 0) begin : g_lut + for (genvar i = 0; i < LANES; ++i) begin : g_mul + localparam BI = (SHARED_B != 0) ? 0 : i; + if (N == M) begin : g_wal + VX_wallace_mul #( + .N (N), + .P (P), + .CPA_KS (!`FORCE_BUILTIN_ADDER(N+M)) + ) u_mul ( + .a (a[i]), + .b (b[BI]), + .p (p[i]) + ); + end else begin : g_inf + assign p[i] = a[i] * b[BI]; + end + end + if (SHARED_B != 0 && LANES > 1) begin : g_unused + `UNUSED_VAR (b[1]) + end + end else if (LANES == 1) begin : g_dsp1 + (* use_dsp = "yes" *) wire [P-1:0] prod = a[0] * b[0]; + assign p[0] = prod; + end else if (SHARED_B != 0) begin : g_dsp_shared + localparam K = P; + localparam AW = K + N; + wire [AW-1:0] pa = {a[1], {(K-N){1'b0}}, a[0]}; + (* use_dsp = "yes" *) wire [AW+M-1:0] prod = pa * b[0]; + assign p[0] = prod[P-1:0]; + assign p[1] = prod[K +: P]; + `UNUSED_VAR (b[1]) + `UNUSED_VAR (prod) + end else begin : g_dsp_indep + localparam K = P + 1; + localparam AW = K + N; + localparam BW = K + M; + wire [AW-1:0] pa = {a[1], {(K-N){1'b0}}, a[0]}; + wire [BW-1:0] pb = {b[1], {(K-M){1'b0}}, b[0]}; + (* use_dsp = "yes" *) wire [AW+BW-1:0] prod = pa * pb; + assign p[0] = prod[P-1:0]; + assign p[1] = prod[2*K +: P]; + `UNUSED_VAR (prod) + end + +endmodule diff --git a/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f4.sv b/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f4.sv index a3e16b83a..ac215d550 100644 --- a/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f4.sv +++ b/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f4.sv @@ -27,7 +27,7 @@ module VX_tcu_tfr_mul_f4 import VX_tcu_pkg::*; input wire [31:0] req_id, input wire [TCU_MAX_INPUTS-1:0] vld_mask, - input wire [3:0] fmt_f, + input wire [TCU_FMT_WIDTH-1:0] fmt_f, input wire [N-1:0][31:0] a_row, input wire [N-1:0][31:0] b_col, @@ -407,14 +407,14 @@ module VX_tcu_tfr_mul_f4 import VX_tcu_pkg::*; exceptions = '0; case (fmt_f) `ifdef VX_CFG_TCU_MXFP4_ENABLE - 4'(TCU_MXFP4_ID): begin + TCU_MXFP4_ID: begin result_sig = result_sig_mxfp4; result_exp = result_exp_mxfp4; exceptions = exceptions_mxfp4; end `endif `ifdef VX_CFG_TCU_NVFP4_ENABLE - 4'(TCU_NVFP4_ID): begin + TCU_NVFP4_ID: begin result_sig = result_sig_nvfp4; result_exp = result_exp_nvfp4; exceptions = exceptions_nvfp4; diff --git a/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f8.sv b/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f8.sv index c79303549..943c06638 100644 --- a/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f8.sv +++ b/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f8.sv @@ -189,20 +189,31 @@ module VX_tcu_tfr_mul_f8 import VX_tcu_pkg::*; wire [EXP_W-1:0] final_exp; `ifdef VX_CFG_TCU_MX_ENABLE + wire [EXP_W-1:0] bias_cpa_sum, bias_cpa_carry; wire [3*EXP_W-1:0] sf_comp = fmt_f[3] ? {EXP_W'(sf_a), EXP_W'(sf_b), -EXP_W'(254)} : (3*EXP_W)'(0); VX_csa_tree #( - .N(5), + .N(4), .W(EXP_W), .S(EXP_W) ) exp_sf_csa ( - .operands ({EXP_W'(max_pre_sum), EXP_W'(bias_sel), sf_comp}), - .sum (max_pre_sum_cpa), - .carry (bias_sel_cpa) + .operands ({EXP_W'(bias_sel), sf_comp}), + .sum (bias_cpa_sum), + .carry (bias_cpa_carry) + ); + VX_ks_adder #( + .N(EXP_W), + .BYPASS(`FORCE_BUILTIN_ADDER(EXP_W)) + ) exp_bias_add ( + .dataa(bias_cpa_sum), + .datab(bias_cpa_carry), + .cin(1'b0), + .sum(bias_sel_cpa), + `UNUSED_PIN(cout) ); `else - assign max_pre_sum_cpa = EXP_W'(max_pre_sum); assign bias_sel_cpa = EXP_W'(bias_sel); `endif + assign max_pre_sum_cpa = EXP_W'(max_pre_sum); VX_ks_adder #( .N(EXP_W), .BYPASS(`FORCE_BUILTIN_ADDER(EXP_W)) diff --git a/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_i8.sv b/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_i8.sv index 153737686..5f8a96016 100644 --- a/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_i8.sv +++ b/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_i8.sv @@ -69,7 +69,7 @@ module VX_tcu_tfr_mul_i8 import VX_tcu_pkg::*; #( ); `ifdef VX_CFG_TCU_MX_ENABLE - wire signed [8:0] combined_sf = $signed(sf_a + sf_b - 9'd266); + wire signed [8:0] combined_sf = $signed({1'b0, sf_a}) + $signed({1'b0, sf_b}) - 9'sd266; wire is_right_shift = combined_sf[8]; wire shift_overflow = (combined_sf > 9'sd24) || (combined_sf < -9'sd24); wire [4:0] shift_amount = is_right_shift ? (-combined_sf[4:0]) : combined_sf[4:0]; @@ -77,8 +77,11 @@ module VX_tcu_tfr_mul_i8 import VX_tcu_pkg::*; #( wire signed [24:0] y_mxi8_scaled [2]; for (genvar j = 0; j < 2; ++j) begin : g_mxi8 wire signed [24:0] raw_prod = {{8{y_prod_i8[j][16]}}, y_prod_i8[j]}; + wire [24:0] abs_prod = raw_prod[24] ? -raw_prod : raw_prod; + wire signed [24:0] right_shifted = raw_prod[24] ? -25'($signed(abs_prod >> shift_amount)) + : 25'($signed(abs_prod >> shift_amount)); assign y_mxi8_scaled[j] = shift_overflow ? 25'sd0 - : is_right_shift ? (raw_prod >>> shift_amount) + : is_right_shift ? right_shifted : (raw_prod <<< shift_amount); end diff --git a/hw/rtl/tcu/tfr/VX_tcu_tfr_shared_mul.sv b/hw/rtl/tcu/tfr/VX_tcu_tfr_shared_mul.sv index 2e2c0bd01..5c418209a 100644 --- a/hw/rtl/tcu/tfr/VX_tcu_tfr_shared_mul.sv +++ b/hw/rtl/tcu/tfr/VX_tcu_tfr_shared_mul.sv @@ -150,7 +150,7 @@ module VX_tcu_tfr_shared_mul import VX_tcu_pkg::*; #( .valid_in (valid_in), .req_id (req_id), .vld_mask (vld_mask), - .fmt_f (fmt_s[3:0]), + .fmt_f (fmt_s), .a_row (a_row), .b_col (b_col), .sf_a (sf_a[s]), diff --git a/hw/syn/altera/dut/core/Makefile b/hw/syn/altera/dut/core/Makefile index c98c33fee..187c85768 100644 --- a/hw/syn/altera/dut/core/Makefile +++ b/hw/syn/altera/dut/core/Makefile @@ -36,6 +36,9 @@ ifneq (,$(filter -DVX_CFG_EXT_TCU_ENABLE, $(XCONFIGS))) ifneq (,$(filter -DVX_CFG_TCU_TYPE_TFR, $(XCONFIGS))) RTL_INCLUDE += -I$(RTL_DIR)/tcu/tfr endif + ifneq (,$(filter -DVX_CFG_TCU_TYPE_TET, $(XCONFIGS))) + RTL_INCLUDE += -I$(RTL_DIR)/tcu/tet + endif endif # Add V extension sources diff --git a/hw/syn/altera/dut/top/Makefile b/hw/syn/altera/dut/top/Makefile index 30fa3ff42..22c30ac6f 100644 --- a/hw/syn/altera/dut/top/Makefile +++ b/hw/syn/altera/dut/top/Makefile @@ -42,6 +42,9 @@ ifneq (,$(filter -DVX_CFG_EXT_TCU_ENABLE, $(XCONFIGS))) ifneq (,$(filter -DVX_CFG_TCU_TYPE_TFR, $(XCONFIGS))) RTL_INCLUDE += -I$(RTL_DIR)/tcu/tfr endif + ifneq (,$(filter -DVX_CFG_TCU_TYPE_TET, $(XCONFIGS))) + RTL_INCLUDE += -I$(RTL_DIR)/tcu/tet + endif endif # Add DXA extension sources diff --git a/hw/syn/altera/dut/vortex/Makefile b/hw/syn/altera/dut/vortex/Makefile index 579039b75..2c21067f3 100644 --- a/hw/syn/altera/dut/vortex/Makefile +++ b/hw/syn/altera/dut/vortex/Makefile @@ -38,6 +38,9 @@ ifneq (,$(filter -DVX_CFG_EXT_TCU_ENABLE, $(XCONFIGS))) ifneq (,$(filter -DVX_CFG_TCU_TYPE_TFR, $(XCONFIGS))) RTL_INCLUDE += -I$(RTL_DIR)/tcu/tfr endif + ifneq (,$(filter -DVX_CFG_TCU_TYPE_TET, $(XCONFIGS))) + RTL_INCLUDE += -I$(RTL_DIR)/tcu/tet + endif endif # Add DXA extension sources diff --git a/hw/syn/altera/opae/Makefile b/hw/syn/altera/opae/Makefile index c59f71df3..26d3f0ff4 100644 --- a/hw/syn/altera/opae/Makefile +++ b/hw/syn/altera/opae/Makefile @@ -110,6 +110,9 @@ ifneq (,$(filter -DVX_CFG_EXT_TCU_ENABLE, $(XCONFIGS))) ifneq (,$(filter -DVX_CFG_TCU_TYPE_TFR, $(XCONFIGS))) RTL_INCLUDE += -I$(RTL_DIR)/tcu/tfr endif + ifneq (,$(filter -DVX_CFG_TCU_TYPE_TET, $(XCONFIGS))) + RTL_INCLUDE += -I$(RTL_DIR)/tcu/tet + endif endif # Add DXA extension sources diff --git a/hw/syn/synopsys/Makefile b/hw/syn/synopsys/Makefile index ea890e504..9aad5dd0d 100644 --- a/hw/syn/synopsys/Makefile +++ b/hw/syn/synopsys/Makefile @@ -143,6 +143,9 @@ ifneq (,$(filter -DVX_CFG_EXT_TCU_ENABLE, $(XCONFIGS))) ifneq (,$(filter -DVX_CFG_TCU_TYPE_TFR, $(XCONFIGS))) RTL_INCLUDE += -I$(RTL_DIR)/tcu/tfr endif + ifneq (,$(filter -DVX_CFG_TCU_TYPE_TET, $(XCONFIGS))) + RTL_INCLUDE += -I$(RTL_DIR)/tcu/tet + endif endif # Add DXA extension sources diff --git a/hw/syn/xilinx/dut/core/Makefile b/hw/syn/xilinx/dut/core/Makefile index 0852538ae..5ee7807f3 100644 --- a/hw/syn/xilinx/dut/core/Makefile +++ b/hw/syn/xilinx/dut/core/Makefile @@ -38,6 +38,9 @@ ifneq (,$(filter -DVX_CFG_EXT_TCU_ENABLE, $(XCONFIGS))) ifneq (,$(filter -DVX_CFG_TCU_TYPE_TFR, $(XCONFIGS))) RTL_INCLUDE += -I$(RTL_DIR)/tcu/tfr endif + ifneq (,$(filter -DVX_CFG_TCU_TYPE_TET, $(XCONFIGS))) + RTL_INCLUDE += -I$(RTL_DIR)/tcu/tet + endif endif # Add DXA extension sources diff --git a/hw/syn/xilinx/dut/tcu/Makefile b/hw/syn/xilinx/dut/tcu/Makefile index fb308a3f4..621e1d020 100644 --- a/hw/syn/xilinx/dut/tcu/Makefile +++ b/hw/syn/xilinx/dut/tcu/Makefile @@ -31,3 +31,6 @@ endif ifneq (,$(filter -DVX_CFG_TCU_TYPE_TFR, $(XCONFIGS))) RTL_INCLUDE += -I$(RTL_DIR)/tcu/tfr endif +ifneq (,$(filter -DVX_CFG_TCU_TYPE_TET, $(XCONFIGS))) +RTL_INCLUDE += -I$(RTL_DIR)/tcu/tet +endif diff --git a/hw/syn/xilinx/dut/top/Makefile b/hw/syn/xilinx/dut/top/Makefile index 54d2b534d..e17d0f1c6 100644 --- a/hw/syn/xilinx/dut/top/Makefile +++ b/hw/syn/xilinx/dut/top/Makefile @@ -41,6 +41,9 @@ ifneq (,$(filter -DVX_CFG_EXT_TCU_ENABLE, $(XCONFIGS))) ifneq (,$(filter -DVX_CFG_TCU_TYPE_TFR, $(XCONFIGS))) RTL_INCLUDE += -I$(RTL_DIR)/tcu/tfr endif + ifneq (,$(filter -DVX_CFG_TCU_TYPE_TET, $(XCONFIGS))) + RTL_INCLUDE += -I$(RTL_DIR)/tcu/tet + endif endif # Add DXA extension sources diff --git a/hw/syn/xilinx/dut/vortex/Makefile b/hw/syn/xilinx/dut/vortex/Makefile index a999ad566..4e94098b0 100644 --- a/hw/syn/xilinx/dut/vortex/Makefile +++ b/hw/syn/xilinx/dut/vortex/Makefile @@ -39,6 +39,9 @@ ifneq (,$(filter -DVX_CFG_EXT_TCU_ENABLE, $(XCONFIGS))) ifneq (,$(filter -DVX_CFG_TCU_TYPE_TFR, $(XCONFIGS))) RTL_INCLUDE += -I$(RTL_DIR)/tcu/tfr endif + ifneq (,$(filter -DVX_CFG_TCU_TYPE_TET, $(XCONFIGS))) + RTL_INCLUDE += -I$(RTL_DIR)/tcu/tet + endif endif # Add DXA extension sources diff --git a/hw/syn/xilinx/xrt/Makefile b/hw/syn/xilinx/xrt/Makefile index 85cc26b20..0f8562c6f 100644 --- a/hw/syn/xilinx/xrt/Makefile +++ b/hw/syn/xilinx/xrt/Makefile @@ -131,6 +131,9 @@ ifneq (,$(filter -DVX_CFG_EXT_TCU_ENABLE, $(XCONFIGS))) ifneq (,$(filter -DVX_CFG_TCU_TYPE_TFR, $(XCONFIGS))) RTL_INCLUDE += -I$(RTL_DIR)/tcu/tfr endif + ifneq (,$(filter -DVX_CFG_TCU_TYPE_TET, $(XCONFIGS))) + RTL_INCLUDE += -I$(RTL_DIR)/tcu/tet + endif endif # Add DXA extension sources diff --git a/hw/syn/yosys/Makefile b/hw/syn/yosys/Makefile index f13a0eae4..957af1820 100644 --- a/hw/syn/yosys/Makefile +++ b/hw/syn/yosys/Makefile @@ -100,6 +100,9 @@ ifneq (,$(filter -DVX_CFG_EXT_TCU_ENABLE, $(XCONFIGS))) ifneq (,$(filter -DVX_CFG_TCU_TYPE_TFR, $(XCONFIGS))) RTL_INCLUDE += -I$(RTL_DIR)/tcu/tfr endif + ifneq (,$(filter -DVX_CFG_TCU_TYPE_TET, $(XCONFIGS))) + RTL_INCLUDE += -I$(RTL_DIR)/tcu/tet + endif endif # Add DXA extension sources diff --git a/hw/unittest/tcu_fedp/Makefile b/hw/unittest/tcu_fedp/Makefile index 47d7d3531..2dfbada09 100644 --- a/hw/unittest/tcu_fedp/Makefile +++ b/hw/unittest/tcu_fedp/Makefile @@ -2,7 +2,11 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk NUM_REGS ?= 2 +ifneq (,$(filter -DVX_CFG_TCU_TYPE_TET, $(CONFIGS))) +LATENCY ?= 8 +else LATENCY ?= 4 +endif PROJECT := tcu_fedp @@ -81,6 +85,13 @@ PARAMS += -GLANE_MASK=1 CXXFLAGS += -DVX_CFG_TCU_TYPE_TFR endif +ifneq (,$(filter -DVX_CFG_TCU_TYPE_TET, $(XCONFIGS))) +RTL_INCLUDE += -I$(RTL_DIR)/tcu/tet +TOP := VX_tcu_fedp_tet +PARAMS += -GLANE_MASK=1 +CXXFLAGS += -DVX_CFG_TCU_TYPE_TET +endif + # test fp16 OPTS ?= --fmt=1 diff --git a/hw/unittest/tcu_fedp/fedp.h b/hw/unittest/tcu_fedp/fedp.h index d74555646..0b7e87dce 100644 --- a/hw/unittest/tcu_fedp/fedp.h +++ b/hw/unittest/tcu_fedp/fedp.h @@ -81,6 +81,54 @@ class FEDP { return f32FromBits(out); } + float run_product_terms(const std::vector& raw_sigs, const std::vector& exponents, float c) { + assert(raw_sigs.size() == exponents.size()); + + const auto c_enc = bitsFromF32(c); + const auto c_dec = decode_input(c_enc, 8, 23); + const auto c_term = decodeC_to_common(c_dec); + + mul_res_t mul_res; + mul_res.terms.reserve(raw_sigs.size() + 1); + std::vector eps; + eps.reserve(exponents.size() + 1); + + for (size_t i = 0; i < raw_sigs.size(); ++i) { + mul_res.terms.push_back({int((raw_sigs[i] >> 24) & 0x1), int(raw_sigs[i] & 0xffffff)}); + eps.push_back(exponents[i]); + } + + bool c_is_zero = (c_term.cls == 0 && c_term.sp == 0); + mul_res.flags = ((c_term.sp == 3) ? FL_NAN : 0) + | ((c_term.sp == 1) ? FL_PINF : 0) + | ((c_term.sp == 2) ? FL_NINF : 0); + + int Ec = c_term.Ec + F32_BIAS + EXP_POS_BIAS + 24 - W_; + if (c_is_zero) { + Ec = 0; + } + + mul_res.terms.push_back({c_term.sign, c_term.Mc}); + eps.push_back(Ec); + + mul_res.L = *std::max_element(eps.begin(), eps.end()); + mul_res.shifts.reserve(eps.size()); + for (size_t i = 0; i < eps.size(); ++i) { + mul_res.shifts.push_back(mul_res.L - eps[i]); + } + + HR_ = 32 - lzcN(mul_res.terms.size() - 1, 32); + + const auto aln = alignment(mul_res); + const auto acc = accumulate(aln); + const auto nrm = normalize(acc); + const auto out = rounding(nrm); + + ++req_id_; + + return f32FromBits(out); + } + private: enum FRM_TYPE { FRM_RNE = 0, FRM_RTZ = 1, FRM_RDN = 2, FRM_RUP = 3, FRM_RMM = 4 }; @@ -578,4 +626,4 @@ class FEDP { int exp_bits_, sig_bits_, frm_, W_, HR_; bool renorm_, no_window_; uint64_t req_id_ = 0; -}; \ No newline at end of file +}; diff --git a/hw/unittest/tcu_fedp/main.cpp b/hw/unittest/tcu_fedp/main.cpp index 3e11bd1dd..21ff78b84 100644 --- a/hw/unittest/tcu_fedp/main.cpp +++ b/hw/unittest/tcu_fedp/main.cpp @@ -11,7 +11,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#if defined(VX_CFG_TCU_TYPE_TFR) +#if defined(VX_CFG_TCU_TYPE_TET) +#include "VVX_tcu_fedp_tet.h" +#define MODULE VVX_tcu_fedp_tet +#elif defined(VX_CFG_TCU_TYPE_TFR) #include "VVX_tcu_fedp_tfr.h" #define MODULE VVX_tcu_fedp_tfr #elif defined(VX_CFG_TCU_TYPE_BHF) @@ -42,10 +45,12 @@ #endif #include +#include #include #include #include #include +#include #include #include #include @@ -53,6 +58,7 @@ #include #include #include +#include "rvfloats.h" #include "softfloat_ext.h" #ifdef USE_FEDP @@ -265,6 +271,7 @@ static int int_fmt_width(int fmt) { case 18: return 8; // uint8 case 19: return 4; // int4 case 20: return 4; // uint4 + case 24: return 8; // mxint8 default: std::cerr << "Unsupported integer format: " << fmt << std::endl; std::abort(); @@ -278,12 +285,201 @@ static int int_fmt_sign(int fmt) { case 18: return false; // uint8 case 19: return true; // int4 case 20: return false; // uint4 + case 24: return true; // mxint8 default: std::cerr << "Unsupported integer format: " << fmt << std::endl; std::abort(); } } +static bool mx_fmt_is_float(uint32_t fmt) { + switch (fmt) { + case 8: // mxfp8 + case 9: // mxbf8 + case 10: // mxfp4 + case 11: // nvfp4 + return true; + default: + return false; + } +} + +static bool mx_fmt_is_int(uint32_t fmt) { + return fmt == 24; // mxint8 +} + +static int mx_fmt_width(uint32_t fmt) { + switch (fmt) { + case 8: // mxfp8 + case 9: // mxbf8 + case 24: // mxint8 + return 8; + case 10: // mxfp4 + case 11: // nvfp4 + return 4; + default: + std::cerr << "Unsupported MX format: " << fmt << std::endl; + std::abort(); + } +} + +static uint8_t mx_default_scale(uint32_t fmt) { + if (fmt == 11) { + return rv_ftoe4m3_s(bit_cast(1.0f), 0, nullptr); + } + return 127; +} + +static uint32_t mx_float_to_bits(uint32_t fmt, float value, uint8_t sf) { + uint32_t value_bits = bit_cast(value); + switch (fmt) { + case 8: return rv_ftomxfp8_s(value_bits, sf, 0, nullptr); + case 9: return rv_ftomxbf8_s(value_bits, sf, 0, nullptr); + case 10: return rv_ftomxfp4_s(value_bits, sf, 0, nullptr) & 0xf; + case 11: return rv_ftonvfp4_s(value_bits, sf, 0, nullptr) & 0xf; + default: + std::cerr << "Unsupported MX float format: " << fmt << std::endl; + std::abort(); + } +} + +static float mx_bits_to_float(uint32_t fmt, uint32_t value, uint8_t sf) { + uint32_t value_bits; + switch (fmt) { + case 8: value_bits = rv_mxfp8tof_s(value & 0xff, sf, 0, nullptr); break; + case 9: value_bits = rv_mxbf8tof_s(value & 0xff, sf, 0, nullptr); break; + case 10: value_bits = rv_mxfp4tof_s(value & 0xf, sf, 0, nullptr); break; + case 11: value_bits = rv_nvfp4tof_s(value & 0xf, sf, 0, nullptr); break; + default: + std::cerr << "Unsupported MX float format: " << fmt << std::endl; + std::abort(); + } + return bit_cast(value_bits); +} + +static int32_t mxint8_scaled_product(int32_t a, int32_t b, uint8_t sf_a, uint8_t sf_b) { + int32_t combined_sf = static_cast(sf_a) + static_cast(sf_b) - 266; + if (combined_sf > 24 || combined_sf < -24) { + return 0; + } + + int32_t product = a * b; + if (combined_sf >= 0) { + return product << combined_sf; + } + + int32_t shift = -combined_sf; + int32_t abs_product = product < 0 ? -product : product; + int32_t shifted = abs_product >> shift; + return product < 0 ? -shifted : shifted; +} + +static int ceil_log2(uint32_t value) { + int width = 0; + uint32_t v = value - 1; + while (v != 0) { + v >>= 1; + ++width; + } + return width; +} + +static void mx_fp4_decode_term(uint32_t raw, uint32_t *mantissa, uint32_t *exponent, bool *is_zero) { + *is_zero = ((raw & 0x7) == 0); + *mantissa = ((raw & 0x6) == 0) ? 1 : (2 | (raw & 0x1)); + *exponent = ((raw & 0x4) != 0) ? (((raw & 0x2) != 0) ? 2 : 1) : 0; +} + +static void mx_fp4_product_terms( + uint32_t fmt, + const std::vector& a_values, + const std::vector& b_values, + uint32_t vld_mask, + uint8_t sf_a, + uint8_t sf_b, + int W, + int total_elements, + std::vector *raw_sigs, + std::vector *exponents) { + constexpr int SIG_SHIFT = 11; + constexpr int EXP_TERM_BIAS = 32; + constexpr int EXP_COMP_MXFP4 = -(2 * 1 + 10); + constexpr int EXP_COMP_NVFP4 = -(2 * 1 + 2 * (7 + 3)); + + int tck = total_elements / 4; + int HR = ceil_log2(tck + 1); + int WA = W + 1 + HR; + int bias_base = 127 + 2 * (23 - 22) - W + WA - 1 + 128; + int exp_comp = (fmt == 10) ? EXP_COMP_MXFP4 : EXP_COMP_NVFP4; + int exp_base_biased = bias_base + exp_comp; + + raw_sigs->assign(tck, 0); + exponents->assign(tck, 0); + + uint32_t sf_man_prod = 1; + uint32_t sf_exp_a = 0; + uint32_t sf_exp_b = 0; + if (fmt == 11) { + uint32_t sf_man_a = 8 | (sf_a & 0x7); + uint32_t sf_man_b = 8 | (sf_b & 0x7); + sf_man_prod = sf_man_a * sf_man_b; + sf_exp_a = (sf_a >> 3) & 0xf; + sf_exp_b = (sf_b >> 3) & 0xf; + } + + for (int i = 0; i < tck; ++i) { + uint32_t term_mag_shifted[4] = {}; + uint32_t term_exp_biased[4] = {}; + bool term_sign[4] = {}; + + for (int j = 0; j < 4; ++j) { + int e = i * 4 + j; + uint32_t raw_a = a_values[e] & 0xf; + uint32_t raw_b = b_values[e] & 0xf; + uint32_t a_man, b_man, a_exp, b_exp; + bool a_zero, b_zero; + mx_fp4_decode_term(raw_a, &a_man, &a_exp, &a_zero); + mx_fp4_decode_term(raw_b, &b_man, &b_exp, &b_zero); + + bool valid = ((vld_mask >> e) & 0x1) && !a_zero && !b_zero; + term_sign[j] = ((raw_a ^ raw_b) >> 3) & 0x1; + if (valid) { + uint32_t man_prod = a_man * b_man; + if (fmt == 10) { + term_exp_biased[j] = uint32_t((EXP_TERM_BIAS + EXP_COMP_MXFP4) + + int(sf_a) - 127 + + int(sf_b) - 127 + + int(a_exp) + int(b_exp)) & 0x3f; + term_mag_shifted[j] = (man_prod << SIG_SHIFT) & 0xffffff; + } else { + uint32_t full_prod = (man_prod * sf_man_prod) & 0x7ff; + term_exp_biased[j] = uint32_t((EXP_TERM_BIAS + EXP_COMP_NVFP4) + + int(sf_exp_a) + int(sf_exp_b) + + int(a_exp) + int(b_exp)) & 0x3f; + term_mag_shifted[j] = (full_prod << SIG_SHIFT) & 0xffffff; + } + } + } + + uint32_t max_exp_01 = std::max(term_exp_biased[0], term_exp_biased[1]); + uint32_t max_exp_23 = std::max(term_exp_biased[2], term_exp_biased[3]); + uint32_t max_exp_biased = std::max(max_exp_01, max_exp_23); + int32_t signed_sum = 0; + for (int j = 0; j < 4; ++j) { + uint32_t shift_amt = (max_exp_biased - term_exp_biased[j]) & 0x3f; + uint32_t aligned_mag = (shift_amt >= 24) ? 0 : (term_mag_shifted[j] >> shift_amt); + signed_sum += term_sign[j] ? -int32_t(aligned_mag) : int32_t(aligned_mag); + } + + bool sum_sign = signed_sum < 0; + uint32_t abs_sum = sum_sign ? uint32_t(-signed_sum) : uint32_t(signed_sum); + if (abs_sum != 0) { + (*raw_sigs)[i] = (uint32_t(sum_sign) << 24) | (abs_sum & 0xffffff); + (*exponents)[i] = int(max_exp_biased) + exp_base_biased - (WA - 1); + } + } +} + static void pack_elements(const std::vector &elements, int element_bits, int num_words, uint32_t *packed) { int elements_per_word = 32 / element_bits; int elements_mask = (1 << element_bits) - 1; @@ -486,6 +682,40 @@ class Testbench { return value; } + int32_t generate_mxint8_value(int test_id) { + std::uniform_int_distribution int_dist(-127, 127); + + if (test_id == -1) { + return int_dist(rng_); + } + + switch (test_id % 5) { + case 0: return 0; + case 1: return -127; + case 2: return 127; + case 3: return 1; + default: return int_dist(rng_); + } + } + + float generate_mx_float_value(uint32_t fmt, int test_id) { + float range = (fmt == 10 || fmt == 11) ? 6.0f : 16.0f; + std::uniform_real_distribution value_dist(-range, range); + + if (test_id == -1) { + return value_dist(rng_); + } + + switch (test_id % 6) { + case 0: return 0.0f; + case 1: return -range; + case 2: return range; + case 3: return 1.0f; + case 4: return -1.0f; + default: return value_dist(rng_); + } + } + uint32_t generate_fp_value(const std::string &feature, uint32_t exp_bits, uint32_t sig_bits, uint32_t test_id) { const uint32_t all_exp = (exp_bits == 32 ? 0xFFFFFFFFu : ((1u << exp_bits) - 1u)); const uint32_t max_frac = (sig_bits == 32 ? 0xFFFFFFFFu : ((1u << sig_bits) - 1u)); @@ -613,11 +843,15 @@ class Testbench { dut_->clk = 0; dut_->reset = 0; dut_->enable = 0; - #ifdef VX_CFG_TCU_TYPE_TFR + #if defined(VX_CFG_TCU_TYPE_TFR) || defined(VX_CFG_TCU_TYPE_TET) dut_->vld_mask = 0; #endif dut_->fmt_s = config_.fmt_s; dut_->fmt_d = config_.fmt_d; +#ifdef VX_CFG_TCU_MX_ENABLE + dut_->sf_a = mx_default_scale(config_.fmt_s); + dut_->sf_b = mx_default_scale(config_.fmt_s); +#endif for (int i = 0; i < NUM_REGS; i++) { WRITE_WDATA(dut_->a_row, i, 0); WRITE_WDATA(dut_->b_col, i, 0); @@ -690,14 +924,14 @@ class Testbench { dut_->c_val = c_value; dut_->fmt_s = config_.fmt_s; dut_->enable = 1; - #ifdef VX_CFG_TCU_TYPE_TFR + #if defined(VX_CFG_TCU_TYPE_TFR) || defined(VX_CFG_TCU_TYPE_TET) dut_->vld_mask = current_vld_mask; #endif // Run for latency cycles for (int i = 0; i < LATENCY; i++) { tick(); - #ifdef VX_CFG_TCU_TYPE_TFR + #if defined(VX_CFG_TCU_TYPE_TFR) || defined(VX_CFG_TCU_TYPE_TET) dut_->vld_mask = 0; #endif } @@ -722,6 +956,226 @@ class Testbench { return true; } + bool test_mxint8() { + std::cout << "Testing MX integer format" << std::endl; + + int element_bits = mx_fmt_width(config_.fmt_s); + int elements_per_word = 32 / element_bits; + int total_elements = NUM_REGS * elements_per_word; + int vld_bits_per_elem = 32 / elements_per_word / 4; + uint32_t elem_vld_mask = (1 << vld_bits_per_elem) - 1; + uint8_t sf_a_value = mx_default_scale(config_.fmt_s); + uint8_t sf_b_value = mx_default_scale(config_.fmt_s); + + std::cout << " elements_per_word=" << elements_per_word << ", total_elements=" << total_elements + << ", sf_a=0x" << std::hex << uint32_t(sf_a_value) + << ", sf_b=0x" << uint32_t(sf_b_value) << std::dec << std::endl; + + std::uniform_int_distribution sparsity_dist(0, 99); + + for (int test_id = 0; test_id < config_.num_tests; test_id++) { + std::vector a_values(total_elements), b_values(total_elements); + std::vector a_signed(total_elements), b_signed(total_elements); + + bool a_enable = (test_id % 3) == 0; + bool b_enable = (test_id % 3) == 1; + bool c_enable = (test_id % 3) == 2; + uint32_t current_vld_mask = 0; + + for (int i = 0; i < total_elements; i++) { + bool is_sparse = (config_.sparsity > 0) && (sparsity_dist(rng_) < config_.sparsity); + + if (is_sparse) { + a_signed[i] = 0; + a_values[i] = 0; + } else { + a_signed[i] = generate_mxint8_value((a_enable && i == 0) ? test_id : -1); + a_values[i] = uint8_t(a_signed[i]); + current_vld_mask |= (elem_vld_mask << (i * vld_bits_per_elem)); + } + + b_signed[i] = generate_mxint8_value((b_enable && i == 0) ? test_id : -1); + b_values[i] = uint8_t(b_signed[i]); + } + + int32_t c_value = generate_int_value(true, 32, c_enable ? test_id : -1); + + uint32_t a_packed[NUM_REGS], b_packed[NUM_REGS]; + pack_elements(a_values, element_bits, NUM_REGS, a_packed); + pack_elements(b_values, element_bits, NUM_REGS, b_packed); + + for (int i = 0; i < NUM_REGS; i++) { + WRITE_WDATA(dut_->a_row, i, a_packed[i]); + WRITE_WDATA(dut_->b_col, i, b_packed[i]); + } + dut_->c_val = c_value; + dut_->fmt_s = config_.fmt_s; + #ifdef VX_CFG_TCU_MX_ENABLE + dut_->sf_a = sf_a_value; + dut_->sf_b = sf_b_value; + #endif + dut_->enable = 1; + #if defined(VX_CFG_TCU_TYPE_TFR) || defined(VX_CFG_TCU_TYPE_TET) + dut_->vld_mask = current_vld_mask; + #endif + + for (int i = 0; i < LATENCY; i++) { + tick(); + #if defined(VX_CFG_TCU_TYPE_TFR) || defined(VX_CFG_TCU_TYPE_TET) + dut_->vld_mask = 0; + #endif + } + dut_->enable = 0; + tick(); + + int32_t expected = c_value; + for (int i = 0; i < total_elements; i++) { + expected += mxint8_scaled_product(a_signed[i], b_signed[i], sf_a_value, sf_b_value); + } + + int32_t dut_result = dut_->d_val; + if (dut_result != expected) { + std::cout << "Test:" << test_id << " failed:" << std::endl; + print_format(" a_values=", a_values, true); + print_format(" b_values=", b_values, true); + print_format(" c_value=", c_value, true); + print_format(" expected=", expected, true); + print_format(" actual=", dut_result, true); + return false; + } + } + + return true; + } + + bool test_mx_floating_points() { + std::cout << "Testing MX floating-point format" << std::endl; + + int element_bits = mx_fmt_width(config_.fmt_s); + int elements_per_word = 32 / element_bits; + int total_elements = NUM_REGS * elements_per_word; + int vld_bits_per_elem = 32 / elements_per_word / 4; + uint32_t elem_vld_mask = (1 << vld_bits_per_elem) - 1; + uint8_t sf_a_value = mx_default_scale(config_.fmt_s); + uint8_t sf_b_value = mx_default_scale(config_.fmt_s); + + std::cout << " elements_per_word=" << elements_per_word << ", total_elements=" << total_elements + << ", sf_a=0x" << std::hex << uint32_t(sf_a_value) + << ", sf_b=0x" << uint32_t(sf_b_value) << std::dec << std::endl; + + #ifdef USE_FEDP + std::unique_ptr mx_fedp; + if (config_.fmt_s == 8 || config_.fmt_s == 9 || config_.fmt_s == 10 || config_.fmt_s == 11) { + int exp_bits = (config_.fmt_s == 8) ? 4 : 5; + int sig_bits = (config_.fmt_s == 8) ? 3 : 2; + mx_fedp = std::make_unique(exp_bits, sig_bits, (int)config_.frm, config_.W, config_.renorm); + } + #endif + + std::uniform_int_distribution sparsity_dist(0, 99); + + for (int test_id = 0; test_id < config_.num_tests; test_id++) { + std::vector a_values_float(total_elements), b_values_float(total_elements); + std::vector a_value_hex(total_elements), b_value_hex(total_elements); + + bool a_enable = (test_id % 3) == 0; + bool b_enable = (test_id % 3) == 1; + bool c_enable = (test_id % 3) == 2; + uint32_t current_vld_mask = 0; + + for (int i = 0; i < total_elements; i++) { + bool is_sparse = (config_.sparsity > 0) && (sparsity_dist(rng_) < config_.sparsity); + + if (is_sparse) { + a_value_hex[i] = 0; + a_values_float[i] = 0.0f; + } else { + float a_src = generate_mx_float_value(config_.fmt_s, (a_enable && (i & 0x1) == 0) ? test_id : -1); + a_value_hex[i] = mx_float_to_bits(config_.fmt_s, a_src, sf_a_value); + a_values_float[i] = mx_bits_to_float(config_.fmt_s, a_value_hex[i], sf_a_value); + current_vld_mask |= (elem_vld_mask << (i * vld_bits_per_elem)); + } + + float b_src = generate_mx_float_value(config_.fmt_s, (b_enable && (i & 0x1) == 0) ? test_id : -1); + b_value_hex[i] = mx_float_to_bits(config_.fmt_s, b_src, sf_b_value); + b_values_float[i] = mx_bits_to_float(config_.fmt_s, b_value_hex[i], sf_b_value); + } + + float c_value_float = generate_mx_float_value(config_.fmt_s, c_enable ? test_id : -1); + uint32_t c_value_hex = bit_cast(c_value_float); + + if (config_.test_id >= 0 && test_id != config_.test_id) + continue; + + std::vector a_packed(NUM_REGS), b_packed(NUM_REGS); + pack_elements(a_value_hex, element_bits, NUM_REGS, a_packed.data()); + pack_elements(b_value_hex, element_bits, NUM_REGS, b_packed.data()); + + for (int i = 0; i < NUM_REGS; i++) { + WRITE_WDATA(dut_->a_row, i, a_packed[i]); + WRITE_WDATA(dut_->b_col, i, b_packed[i]); + } + dut_->c_val = c_value_hex; + dut_->fmt_s = config_.fmt_s; + #ifdef VX_CFG_TCU_MX_ENABLE + dut_->sf_a = sf_a_value; + dut_->sf_b = sf_b_value; + #endif + dut_->enable = 1; + #if defined(VX_CFG_TCU_TYPE_TFR) || defined(VX_CFG_TCU_TYPE_TET) + dut_->vld_mask = current_vld_mask; + #endif + + for (int i = 0; i < LATENCY; i++) { + tick(); + #if defined(VX_CFG_TCU_TYPE_TFR) || defined(VX_CFG_TCU_TYPE_TET) + dut_->vld_mask = 0; + #endif + } + dut_->enable = 0; + tick(); + + float expected; + #ifdef USE_FEDP + if (mx_fedp && (config_.fmt_s == 10 || config_.fmt_s == 11)) { + std::vector raw_sigs; + std::vector exponents; + mx_fp4_product_terms(config_.fmt_s, a_value_hex, b_value_hex, current_vld_mask, + sf_a_value, sf_b_value, config_.W, total_elements, &raw_sigs, &exponents); + expected = mx_fedp->run_product_terms(raw_sigs, exponents, c_value_float); + } else if (mx_fedp) { + expected = (*mx_fedp)(a_packed.data(), b_packed.data(), c_value_float, NUM_REGS); + } else + #endif + { + long double expected_acc = c_value_float; + for (int i = 0; i < total_elements; i++) { + expected_acc += static_cast(a_values_float[i]) * static_cast(b_values_float[i]); + } + expected = static_cast(expected_acc); + } + + uint32_t dut_result_bits = dut_->d_val; + float dut_result = bit_cast(dut_result_bits); + + int delta = approximately_equal(dut_result, expected); + if (abs(delta) > config_.ulp) { + std::cout << "Test #" << test_id << " failed:" << std::endl; + print_float(" af_values=", a_values_float, true); + print_format(" ax_values=", a_value_hex, true); + print_float(" bf_values=", b_values_float, true); + print_format(" bx_values=", b_value_hex, true); + print_float(" c_value=", c_value_float, true); + print_float(" expected=", expected, true); + print_float(" actual=", dut_result, true); + std::cout << " delta=" << delta << std::endl; + return false; + } + } + + return true; + } + bool test_floating_points(const std::vector &features_to_test) { if (features_to_test.empty()) return true; @@ -801,14 +1255,14 @@ class Testbench { dut_->c_val = c_value_hex; dut_->fmt_s = config_.fmt_s; dut_->enable = 1; - #ifdef VX_CFG_TCU_TYPE_TFR + #if defined(VX_CFG_TCU_TYPE_TFR) || defined(VX_CFG_TCU_TYPE_TET) dut_->vld_mask = current_vld_mask; #endif // Run for latency cycles for (int i = 0; i < LATENCY; i++) { tick(); - #ifdef VX_CFG_TCU_TYPE_TFR + #if defined(VX_CFG_TCU_TYPE_TFR) || defined(VX_CFG_TCU_TYPE_TET) dut_->vld_mask = 0; #endif } @@ -865,7 +1319,15 @@ class Testbench { bool run_tests() { this->reset(); - if (config_.fmt_s >= 16) { + if (mx_fmt_is_float(config_.fmt_s)) { + if (!test_mx_floating_points()) { + return false; + } + } else if (mx_fmt_is_int(config_.fmt_s)) { + if (!test_mxint8()) { + return false; + } + } else if (config_.fmt_s >= 16) { if (!test_integers()) { return false; } diff --git a/hw/unittest/tcu_unit/Makefile b/hw/unittest/tcu_unit/Makefile index c1bc33f3d..b61a0d406 100644 --- a/hw/unittest/tcu_unit/Makefile +++ b/hw/unittest/tcu_unit/Makefile @@ -62,6 +62,12 @@ TOP := VX_tcu_fedp_tfr PARAMS += -GLANE_MASK=1 endif +ifneq (,$(filter -DVX_CFG_TCU_TYPE_TET, $(XCONFIGS))) +RTL_INCLUDE += -I$(RTL_DIR)/tcu/tet +TOP := VX_tcu_fedp_tet +PARAMS += -GLANE_MASK=1 +endif + # default test: fp16 OPTS ?= --fmt=1 diff --git a/perf/results/dse/wgmma/desc/sgemm_tcu_wg_fedp2k_desc_uop.csv b/perf/results/dse/wgmma/desc/sgemm_tcu_wg_fedp2k_desc_uop.csv new file mode 100644 index 000000000..b463e5db1 --- /dev/null +++ b/perf/results/dse/wgmma/desc/sgemm_tcu_wg_fedp2k_desc_uop.csv @@ -0,0 +1,13 @@ +model,fedp_latency,mode,threads,warps,issue_width,itype,otype,idle_pct,occupancy,occupancy_pct,simt_util,simt_util_pct,stall_fetch_pct,stall_ibuf_pct,stall_scrb_pct,stall_opds_pct,stall_alu_pct,stall_lsu_pct,stall_sfu_pct,stall_fpu_pct,stall_tcu_pct,mix_alu_pct,mix_lsu_pct,mix_sfu_pct,mix_fpu_pct,mix_tcu_pct,branches,divergent_branches,divergence_pct,ifetch_latency,load_latency,loads,stores,instrs,cycles,ipc,status +baseline,4,RS,8,8,4,fp16,fp32,32,8.0,100,8.0,100,0,0,77,2,6,1,0,0,2,84,8,3,0,5,6336,0,0,4.02,9.39,59904,45568,168032,234768,0.716,PASS +baseline,4,RS,32,8,4,fp16,fp32,43,8.0,100,32.0,100,0,0,66,1,5,4,1,0,2,83,8,3,1,5,816,0,0,4.68,22.53,31232,25088,21568,36126,0.597,PASS +baseline,4,SS,8,8,4,fp16,fp32,32,8.0,100,8.0,100,0,0,73,2,6,0,0,0,3,84,7,3,0,5,6336,0,0,4.02,9.88,43520,45568,160864,224501,0.717,PASS +baseline,4,SS,32,8,4,fp16,fp32,44,8.0,100,32.0,100,0,0,69,1,5,4,1,0,4,83,7,3,1,5,816,0,0,4.71,27.09,23040,25088,20672,35326,0.585,PASS +fedp2k,4,RS,8,8,4,fp16,fp32,31,8.0,100,8.0,100,0,0,71,3,5,2,1,0,1,79,12,4,1,4,6336,0,0,4.1,9.69,59904,45568,110944,155274,0.715,PASS +fedp2k,4,RS,32,8,4,fp16,fp32,47,8.0,100,32.0,100,0,0,56,3,4,6,1,0,1,78,12,5,1,4,816,0,0,5.02,22.5,31232,25088,14208,25786,0.551,PASS +fedp2k,4,SS,8,8,4,fp16,fp32,30,8.0,100,8.0,100,0,0,71,3,5,1,0,0,3,79,11,5,1,5,6336,0,0,4.03,9.86,43520,45568,100128,137848,0.726,PASS +fedp2k,4,SS,32,8,4,fp16,fp32,48,7.9,99,32.0,100,0,0,53,1,3,6,1,0,4,78,11,5,1,4,816,0,0,5.1,27.13,23040,25088,13104,24392,0.537,PASS +fedp2k,8,RS,8,8,4,fp16,fp32,31,8.0,100,8.0,100,0,0,73,3,5,1,1,0,2,79,12,4,1,4,6336,0,0,4.1,9.61,59904,45568,110944,155220,0.715,PASS +fedp2k,8,RS,32,8,4,fp16,fp32,47,8.0,100,32.0,100,0,0,56,3,4,6,1,0,1,78,12,5,1,4,816,0,0,5.02,22.5,31232,25088,14208,25786,0.551,PASS +fedp2k,8,SS,8,8,4,fp16,fp32,30,8.0,100,8.0,100,0,0,68,3,5,1,1,0,4,79,11,5,1,5,6336,0,0,4.03,10.05,43520,45568,100128,137867,0.726,PASS +fedp2k,8,SS,32,8,4,fp16,fp32,48,8.0,99,32.0,100,0,0,61,1,3,6,1,0,4,78,11,5,1,4,816,0,0,5.1,27.26,23040,25088,13104,24291,0.539,PASS diff --git a/perf/results/dse/wgmma/desc/sgemm_tcu_wg_fedp2k_tcu_meta_ld.csv b/perf/results/dse/wgmma/desc/sgemm_tcu_wg_fedp2k_tcu_meta_ld.csv new file mode 100644 index 000000000..cd1cd4e52 --- /dev/null +++ b/perf/results/dse/wgmma/desc/sgemm_tcu_wg_fedp2k_tcu_meta_ld.csv @@ -0,0 +1,13 @@ +model,fedp_latency,mode,threads,warps,issue_width,itype,otype,idle_pct,occupancy,occupancy_pct,simt_util,simt_util_pct,stall_fetch_pct,stall_ibuf_pct,stall_scrb_pct,stall_opds_pct,stall_alu_pct,stall_lsu_pct,stall_sfu_pct,stall_fpu_pct,stall_tcu_pct,mix_alu_pct,mix_lsu_pct,mix_sfu_pct,mix_fpu_pct,mix_tcu_pct,branches,divergent_branches,divergence_pct,ifetch_latency,load_latency,loads,stores,instrs,cycles,ipc,status +baseline,4,RS,8,8,4,fp16,fp32,32,8.0,100,7.9,99,0,0,69,1,5,1,0,0,2,82,8,5,0,5,7936,512,6,4.02,9.32,59904,46080,172960,243040,0.712,PASS +baseline,4,RS,32,8,4,fp16,fp32,43,7.9,99,31.5,99,0,0,62,1,5,4,1,0,2,81,8,5,1,5,1024,64,6,4.67,21.85,31232,25152,22240,37409,0.595,PASS +baseline,4,SS,8,8,4,fp16,fp32,33,8.0,100,7.8,98,0,0,70,1,6,0,0,0,3,82,7,5,0,6,7936,512,6,4.02,9.94,43520,46592,167392,239417,0.699,PASS +baseline,4,SS,32,8,4,fp16,fp32,43,8.0,99,31.3,98,0,0,68,1,5,4,1,0,4,81,8,5,1,5,1024,64,6,4.69,26.6,23040,25216,21552,36545,0.59,PASS +fedp2k,4,RS,8,8,4,fp16,fp32,34,8.0,100,7.8,98,0,0,66,3,5,1,1,0,1,77,12,7,0,4,7936,512,6,4.1,9.6,59904,46080,115872,169531,0.683,PASS +fedp2k,4,RS,32,8,4,fp16,fp32,47,8.0,100,31.3,98,0,0,54,2,3,5,1,0,1,76,12,7,1,4,1024,64,6,4.98,22.05,31232,25152,14880,27394,0.543,PASS +fedp2k,4,SS,8,8,4,fp16,fp32,30,7.9,99,7.7,97,0,0,61,3,4,1,1,0,2,76,11,7,1,5,7936,512,6,4.03,9.48,43520,46592,106656,147193,0.725,PASS +fedp2k,4,SS,32,8,4,fp16,fp32,48,7.9,99,31.0,97,0,0,50,2,3,5,1,0,4,75,12,8,1,5,1024,64,6,5.04,26.94,23040,25216,13984,26141,0.535,PASS +fedp2k,8,RS,8,8,4,fp16,fp32,33,8.0,100,7.8,98,0,0,66,3,5,1,1,0,2,77,12,7,0,4,7936,512,6,4.11,9.7,59904,46080,115872,167617,0.691,PASS +fedp2k,8,RS,32,8,4,fp16,fp32,47,8.0,100,31.3,98,0,0,54,2,3,5,1,0,2,76,12,7,1,4,1024,64,6,4.98,22.02,31232,25152,14880,27285,0.545,PASS +fedp2k,8,SS,8,8,4,fp16,fp32,30,7.9,99,7.7,97,0,0,60,3,4,0,1,0,3,76,11,7,1,5,7936,512,6,4.03,9.52,43520,46592,106656,147023,0.725,PASS +fedp2k,8,SS,32,8,4,fp16,fp32,48,7.9,99,31.0,97,0,0,51,2,3,5,1,0,5,75,12,8,1,5,1024,64,6,5.04,26.75,23040,25216,13984,26256,0.533,PASS diff --git a/perf/results/dse/wgmma/sp_rs_ss/sgemm_tcu_wg_sp_fedp2k_fp16.csv b/perf/results/dse/wgmma/sp_rs_ss/sgemm_tcu_wg_sp_fedp2k_fp16.csv new file mode 100644 index 000000000..9e740871f --- /dev/null +++ b/perf/results/dse/wgmma/sp_rs_ss/sgemm_tcu_wg_sp_fedp2k_fp16.csv @@ -0,0 +1,13 @@ +model,fedp_latency,mode,threads,warps,issue_width,itype,otype,idle_pct,occupancy,occupancy_pct,simt_util,simt_util_pct,stall_fetch_pct,stall_ibuf_pct,stall_scrb_pct,stall_opds_pct,stall_alu_pct,stall_lsu_pct,stall_sfu_pct,stall_fpu_pct,stall_tcu_pct,mix_alu_pct,mix_lsu_pct,mix_sfu_pct,mix_fpu_pct,mix_tcu_pct,branches,divergent_branches,divergence_pct,ifetch_latency,load_latency,loads,stores,instrs,cycles,ipc,status +baseline,4,RS,8,8,4,fp16,fp32,60,8.0,99,8.0,100,0,0,57,0,1,9,1,0,1,71,9,16,0,4,13056,0,0,4.23,41.85,61440,36864,138592,335658,0.413,PASS +baseline,4,SS,8,8,4,fp16,fp32,61,8.0,100,8.0,100,0,0,57,0,0,9,1,0,1,70,9,17,0,4,13056,0,0,4.23,46.96,53248,36864,132384,331183,0.400,PASS +baseline,4,RS,32,8,4,fp16,fp32,90,7.9,99,32.0,100,0,0,66,0,0,17,3,0,0,69,10,16,1,4,1664,0,0,12.75,173.62,35328,23552,17760,169411,0.105,PASS +baseline,4,SS,32,8,4,fp16,fp32,90,7.9,99,32.0,100,0,0,66,0,0,18,3,0,0,68,10,17,1,4,1664,0,0,13.08,208.59,31232,23552,16992,168734,0.101,PASS +fedp2k,4,RS,8,8,4,fp16,fp32,65,8.0,100,8.0,100,0,0,59,1,0,8,2,0,0,69,10,17,0,4,13056,0,0,4.29,45.74,66048,37376,131104,360829,0.363,PASS +fedp2k,4,SS,8,8,4,fp16,fp32,68,8.0,100,8.0,100,0,0,64,0,0,9,4,0,1,68,10,18,0,4,13056,0,0,4.29,54.27,57856,37376,124960,382144,0.327,PASS +fedp2k,4,RS,32,8,4,fp16,fp32,90,7.9,98,32.0,100,0,0,64,0,0,16,4,0,0,68,11,17,1,4,1664,0,0,12.97,171.89,35328,23552,17376,169277,0.103,PASS +fedp2k,4,SS,32,8,4,fp16,fp32,91,7.8,98,32.0,100,0,0,67,0,0,18,2,0,0,67,10,18,1,4,1664,0,0,13.30,208.78,31232,23552,16608,170203,0.098,PASS +fedp2k,8,RS,8,8,4,fp16,fp32,64,7.9,99,8.0,100,0,0,60,1,0,9,2,0,1,69,10,17,0,4,13056,0,0,4.28,45.83,66048,37376,131104,356623,0.368,PASS +fedp2k,8,SS,8,8,4,fp16,fp32,68,8.0,100,8.0,100,0,0,64,0,0,9,4,0,1,68,10,18,0,4,13056,0,0,4.29,54.27,57856,37376,124960,382144,0.327,PASS +fedp2k,8,RS,32,8,4,fp16,fp32,90,7.9,98,32.0,100,0,0,64,0,0,16,4,0,0,68,11,17,1,4,1664,0,0,12.97,172.00,35328,23552,17376,169287,0.103,PASS +fedp2k,8,SS,32,8,4,fp16,fp32,91,7.9,98,32.0,100,0,0,67,0,0,17,2,0,0,67,10,18,1,4,1664,0,0,13.30,209.51,31232,23552,16608,170430,0.097,PASS diff --git a/perf/results/dse/wgmma/sp_rs_ss/sgemm_tcu_wg_sp_fedp2k_fp8.csv b/perf/results/dse/wgmma/sp_rs_ss/sgemm_tcu_wg_sp_fedp2k_fp8.csv new file mode 100644 index 000000000..72313478f --- /dev/null +++ b/perf/results/dse/wgmma/sp_rs_ss/sgemm_tcu_wg_sp_fedp2k_fp8.csv @@ -0,0 +1,13 @@ +model,fedp_latency,mode,threads,warps,issue_width,itype,otype,idle_pct,occupancy,occupancy_pct,simt_util,simt_util_pct,stall_fetch_pct,stall_ibuf_pct,stall_scrb_pct,stall_opds_pct,stall_alu_pct,stall_lsu_pct,stall_sfu_pct,stall_fpu_pct,stall_tcu_pct,mix_alu_pct,mix_lsu_pct,mix_sfu_pct,mix_fpu_pct,mix_tcu_pct,branches,divergent_branches,divergence_pct,ifetch_latency,load_latency,loads,stores,instrs,cycles,ipc,status +baseline,4,RS,8,8,4,fp8,fp32,59,7.9,99,8.0,100,0,0,56,1,0,9,1,0,0,71,11,15,1,3,8192,0,0,4.35,36.23,45056,34816,90336,214859,0.420,PASS +baseline,4,SS,8,8,4,fp8,fp32,59,7.9,99,8.0,100,0,0,56,1,0,9,1,0,1,71,11,15,1,3,8192,0,0,4.36,39.24,40960,34816,88032,211130,0.417,PASS +baseline,4,RS,32,8,4,fp8,fp32,88,7.7,96,32.0,100,0,0,60,0,0,16,1,0,0,69,12,15,1,3,1056,0,0,14.43,138.98,24064,21504,11840,94625,0.125,PASS +baseline,4,SS,32,8,4,fp8,fp32,88,7.7,97,32.0,100,0,0,58,0,0,16,1,0,0,69,12,15,1,3,1056,0,0,14.61,138.39,22016,21504,11552,95324,0.121,PASS +fedp2k,4,RS,8,8,4,fp8,fp32,60,7.9,99,8.0,100,0,0,53,1,0,9,2,0,0,70,11,15,1,3,8192,0,0,4.36,36.50,45056,34816,87264,211279,0.413,PASS +fedp2k,4,SS,8,8,4,fp8,fp32,60,8.0,99,8.0,100,0,0,55,1,0,9,1,0,1,70,11,16,1,3,8192,0,0,4.40,39.41,40960,34816,84960,209156,0.406,PASS +fedp2k,4,RS,32,8,4,fp8,fp32,88,7.9,99,32.0,100,0,0,61,0,0,17,1,0,0,69,12,15,1,3,1056,0,0,14.43,128.49,24064,21504,11664,94822,0.123,PASS +fedp2k,4,SS,32,8,4,fp8,fp32,88,7.9,99,32.0,100,0,0,63,0,0,17,0,0,0,69,12,15,1,3,1056,0,0,14.74,145.60,22016,21504,11376,93752,0.121,PASS +fedp2k,8,RS,8,8,4,fp8,fp32,60,7.9,99,8.0,100,0,0,52,1,0,9,2,0,1,70,11,15,1,3,8192,0,0,4.36,35.99,45056,34816,87264,212092,0.411,PASS +fedp2k,8,SS,8,8,4,fp8,fp32,60,7.9,99,8.0,100,0,0,54,1,0,9,1,0,1,70,11,16,1,3,8192,0,0,4.40,38.96,40960,34816,84960,206133,0.412,PASS +fedp2k,8,RS,32,8,4,fp8,fp32,88,7.9,99,32.0,100,0,0,61,0,0,17,1,0,0,69,12,15,1,3,1056,0,0,14.43,128.49,24064,21504,11664,94866,0.123,PASS +fedp2k,8,SS,32,8,4,fp8,fp32,88,7.9,99,32.0,100,0,0,63,0,0,17,0,0,0,69,12,15,1,3,1056,0,0,14.74,147.17,22016,21504,11376,93702,0.121,PASS diff --git a/perf/results/tcu/sgemm_tcu_perf.md b/perf/results/tcu/sgemm_tcu_perf.md index f7a4b5837..f03ce925d 100644 --- a/perf/results/tcu/sgemm_tcu_perf.md +++ b/perf/results/tcu/sgemm_tcu_perf.md @@ -1,6 +1,6 @@ # TCU Perf Snapshot -Last updated: 20-06-2026 +Last updated: 01-07-2026 ## SimX Commands @@ -42,7 +42,9 @@ CONFIGS="-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -D CONFIGS="-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_TCU_MX_ENABLE -DITYPE=mxint8 -DOTYPE=int32" ./ci/blackbox.sh --driver=simx --app=sgemm_tcu_sp_mx --args="-m64 -n64 -k64" --perf=1 ``` -## RTLsim (DPI) Commands +## RTLsim Commands + +Swap out -DVX_CFG_TCU_TYPE_DPI backend for -DVX_CFG_TCU_TYPE_TFR or -DVX_CFG_TCU_TYPE_TET as required ```bash # sgemm_tcu @@ -87,6 +89,7 @@ CONFIGS="-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -D Target Clock Frequency: 300 MHz Post-Implementation results reported +TFR runs: ```bash # tcu CONFIGS="-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_TFR -DVX_CFG_NUM_THREADS=16 -DVX_CFG_TCU_NUM_WARPS=16" @@ -101,3 +104,12 @@ CONFIGS="-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_TFR -DVX_CFG_NUM_THREADS=16 - CONFIGS="-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_TFR -DVX_CFG_NUM_THREADS=16 -DVX_CFG_TCU_NUM_WARPS=16 -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_TCU_MX_ENABLE" ``` + +TET runs: +```bash +# tcu +CONFIGS="-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_TET -DVX_CFG_NUM_THREADS=16 -DVX_CFG_TCU_NUM_WARPS=16" + +# tcu_mx +CONFIGS="-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_TET -DVX_CFG_NUM_THREADS=16 -DVX_CFG_TCU_NUM_WARPS=16 -DVX_CFG_TCU_MX_ENABLE" +``` \ No newline at end of file diff --git a/perf/results/tcu/sgemm_tcu_tfr_tet.csv b/perf/results/tcu/sgemm_tcu_tfr_tet.csv new file mode 100644 index 000000000..f93d70f8f --- /dev/null +++ b/perf/results/tcu/sgemm_tcu_tfr_tet.csv @@ -0,0 +1,17 @@ +test,backend,warps,threads,itype,otype,idle_pct,occupancy,occupancy_pct,simt_util,simt_util_pct,stall_fetch_pct,stall_ibuf_pct,stall_scrb_pct,stall_opds_pct,stall_alu_pct,stall_lsu_pct,stall_sfu_pct,stall_fpu_pct,stall_tcu_pct,mix_alu_pct,mix_lsu_pct,mix_sfu_pct,mix_fpu_pct,mix_tcu_pct,branches,divergent_branches,divergence_pct,ifetch_latency,load_latency,loads,stores,instrs,cycles,ipc,status +sgemm_tcu,tfr,8,8,fp16,fp32,71,7.9,99,8.0,100,0,14,96,1,0,0,0,0,0,39,21,4,2,34,448,0,0,4.70,14.83,35328,4608,24096,51656,0.466,PASS +sgemm_tcu,tfr,8,8,fp8,fp32,72,7.9,98,8.0,100,0,3,95,5,0,0,0,0,0,38,22,5,4,31,320,0,0,4.41,13.72,18944,4608,13408,28613,0.469,PASS +sgemm_tcu,tfr,8,8,int8,int32,72,7.9,98,8.0,100,0,3,95,5,0,0,0,0,0,38,22,5,4,31,320,0,0,4.41,13.72,18944,4608,13408,28613,0.469,PASS +sgemm_tcu,tfr,8,8,int4,int32,72,7.9,98,8.0,100,0,3,95,5,0,0,0,0,0,38,22,5,4,31,320,0,0,4.41,13.72,18944,4608,13408,28613,0.469,PASS +sgemm_tcu_mx,tfr,8,8,mxfp8,fp32,74,7.9,99,8.0,100,0,21,97,3,0,0,0,0,0,41,20,6,4,29,320,0,0,4.98,16.17,22016,4608,15264,38792,0.393,PASS +sgemm_tcu_mx,tfr,8,8,mxfp4,fp32,78,7.9,99,8.0,100,0,17,96,4,0,0,0,0,0,42,21,7,6,23,256,0,0,5.14,21.51,12800,4608,9568,28382,0.337,PASS +sgemm_tcu_mx,tfr,8,8,nvfp4,fp32,78,7.9,99,8.0,100,0,17,96,4,0,0,0,0,0,42,21,7,6,23,256,0,0,5.14,21.51,12800,4608,9568,28382,0.337,PASS +sgemm_tcu_mx,tfr,8,8,mxint8,int32,74,7.9,99,8.0,100,0,21,97,3,0,0,0,0,0,41,20,6,4,29,320,0,0,4.98,16.17,22016,4608,15264,38792,0.393,PASS +sgemm_tcu,tet,8,8,fp16,fp32,70,7.9,99,8.0,100,0,9,95,1,0,0,0,0,0,39,21,4,2,34,448,0,0,4.53,14.76,35328,4608,24096,51379,0.469,PASS +sgemm_tcu,tet,8,8,fp8,fp32,75,7.8,98,8.0,100,0,19,94,4,0,0,0,0,0,38,22,5,4,31,320,0,0,5.07,19.16,18944,4608,13408,35768,0.375,PASS +sgemm_tcu,tet,8,8,int8,int32,75,7.8,98,8.0,100,0,19,94,4,0,0,0,0,0,38,22,5,4,31,320,0,0,5.07,19.16,18944,4608,13408,35768,0.375,PASS +sgemm_tcu,tet,8,8,int4,int32,75,7.8,98,8.0,100,0,19,94,4,0,0,0,0,0,38,22,5,4,31,320,0,0,5.07,19.16,18944,4608,13408,35768,0.375,PASS +sgemm_tcu_mx,tet,8,8,mxfp8,fp32,74,7.9,99,8.0,100,0,19,97,4,0,0,0,0,0,41,20,6,4,29,320,0,0,4.94,16.01,22016,4608,15264,38324,0.398,PASS +sgemm_tcu_mx,tet,8,8,mxfp4,fp32,77,7.9,99,8.0,100,0,29,96,4,0,0,0,0,0,42,21,7,6,23,256,0,0,5.47,21.87,12800,4608,9568,28416,0.337,PASS +sgemm_tcu_mx,tet,8,8,nvfp4,fp32,77,7.9,99,8.0,100,0,29,96,4,0,0,0,0,0,42,21,7,6,23,256,0,0,5.47,21.87,12800,4608,9568,28416,0.337,PASS +sgemm_tcu_mx,tet,8,8,mxint8,int32,74,7.9,99,8.0,100,0,19,97,4,0,0,0,0,0,41,20,6,4,29,320,0,0,4.94,16.01,22016,4608,15264,38324,0.398,PASS diff --git a/perf/results/tcu/tcu_synth.csv b/perf/results/tcu/tcu_synth.csv index df2750ce2..4691ac8da 100644 --- a/perf/results/tcu/tcu_synth.csv +++ b/perf/results/tcu/tcu_synth.csv @@ -1,5 +1,7 @@ -design,target_freq,wns,fmax,lut,dsp,ff,static,vectorless_dynamic,total_pwr -tcu,300,-1.184,221.4,192154,0,33977,3.397,5.728,9.125 -tcu_sp,300,-1.851,192.9,198217,0,28101,3.388,5.289,8.677 -tcu_mx,300,-1.799,194.9,240713,0,32572,3.453,8.139,11.592 -tcu_sp_mx,300,-1.840,193.3,242051,0,30610,3.462,8.539,12.002 +design,backend,target_freq,wns,fmax,lut,dsp,ff,static,vectorless_dynamic,total_pwr +tcu,tfr,300,-1.184,221.4,192154,0,33977,3.397,5.728,9.125 +tcu_sp,tfr,300,-1.851,192.9,198217,0,28101,3.388,5.289,8.677 +tcu_mx,tfr,300,-1.799,194.9,240713,0,32572,3.453,8.139,11.592 +tcu_sp_mx,tfr,300,-1.840,193.3,242051,0,30610,3.462,8.539,12.002 +tcu,tet,300,0.023,302.1,124885,1024,45365,3.378,4.842,8.219 +tcu_mx,tet,300,-0.257,278.6,192344,1024,55021,3.450,7.996,11.446 \ No newline at end of file diff --git a/sim/opaesim/Makefile b/sim/opaesim/Makefile index 4839df7aa..e2230d33c 100644 --- a/sim/opaesim/Makefile +++ b/sim/opaesim/Makefile @@ -87,6 +87,8 @@ ifneq (,$(filter -DVX_CFG_EXT_TCU_ENABLE, $(XCONFIGS))) RTL_PKGS += $(THIRD_PARTY_DIR)/cvfpu/src/fpnew_pkg.sv $(THIRD_PARTY_DIR)/cvfpu/src/common_cells/src/cf_math_pkg.sv RTL_INCLUDE += -I$(RTL_DIR)/tcu/fpnew RTL_INCLUDE += -I$(THIRD_PARTY_DIR)/cvfpu/src/common_cells/include -I$(THIRD_PARTY_DIR)/cvfpu/src/common_cells/src -I$(THIRD_PARTY_DIR)/cvfpu/src + else ifneq (,$(filter -DVX_CFG_TCU_TYPE_TET, $(XCONFIGS))) + RTL_INCLUDE += -I$(RTL_DIR)/tcu/tet else ifneq (,$(filter -DVX_CFG_TCU_TYPE_TFR, $(XCONFIGS))) RTL_INCLUDE += -I$(RTL_DIR)/tcu/tfr else diff --git a/sim/rtlsim/Makefile b/sim/rtlsim/Makefile index 53487a8e3..f2836c485 100644 --- a/sim/rtlsim/Makefile +++ b/sim/rtlsim/Makefile @@ -75,13 +75,24 @@ ifneq (,$(filter -DVX_CFG_EXT_TCU_ENABLE, $(XCONFIGS))) RTL_INCLUDE += -I$(RTL_DIR)/tcu/dsp else ifneq (,$(filter -DVX_CFG_TCU_TYPE_BHF, $(XCONFIGS))) RTL_INCLUDE += -I$(RTL_DIR)/tcu/bhf - RTL_INCLUDE += -J$(THIRD_PARTY_DIR)/hardfloat/source/RISCV + RTL_PRE_SRCS += $(THIRD_PARTY_DIR)/hardfloat/source/fNToRecFN.v + RTL_PRE_SRCS += $(THIRD_PARTY_DIR)/hardfloat/source/addRecFN.v + RTL_PRE_SRCS += $(THIRD_PARTY_DIR)/hardfloat/source/bsg_hardfloat_pkg.sv + RTL_PRE_SRCS += $(THIRD_PARTY_DIR)/hardfloat/source/HardFloat_primitives.v + RTL_PRE_SRCS += $(THIRD_PARTY_DIR)/hardfloat/source/HardFloat_rawFN.v + RTL_PRE_SRCS += $(THIRD_PARTY_DIR)/hardfloat/source/isSigNaNRecFN.v + RTL_PRE_SRCS += $(THIRD_PARTY_DIR)/hardfloat/source/mulRecFN.v + RTL_PRE_SRCS += $(THIRD_PARTY_DIR)/hardfloat/source/recFNToFN.v + RTL_PRE_SRCS += $(THIRD_PARTY_DIR)/hardfloat/source/recFNToRecFN.v + RTL_INCLUDE += -I$(THIRD_PARTY_DIR)/hardfloat/source/RISCV RTL_INCLUDE += -I$(THIRD_PARTY_DIR)/hardfloat/source else ifneq (,$(filter -DVX_CFG_TCU_TYPE_FPNEW, $(XCONFIGS))) RTL_INCLUDE += -I$(RTL_DIR)/tcu/fpnew RTL_INCLUDE += -I$(THIRD_PARTY_DIR)/cvfpu/src/common_cells/include RTL_INCLUDE += -I$(THIRD_PARTY_DIR)/cvfpu/src/common_cells/src RTL_INCLUDE += -I$(THIRD_PARTY_DIR)/cvfpu/src + else ifneq (,$(filter -DVX_CFG_TCU_TYPE_TET, $(XCONFIGS))) + RTL_INCLUDE += -I$(RTL_DIR)/tcu/tet else ifneq (,$(filter -DVX_CFG_TCU_TYPE_TFR, $(XCONFIGS))) RTL_INCLUDE += -I$(RTL_DIR)/tcu/tfr endif @@ -137,6 +148,7 @@ VL_FLAGS += $(CONFIGS) VL_FLAGS += $(TCU_META_DCFG) VL_FLAGS += $(RTL_INCLUDE) VL_FLAGS += $(RTL_PKGS) +VL_FLAGS += $(RTL_PRE_SRCS) VL_FLAGS += --cc $(TOP) --top-module $(TOP) # Extract RTL directories from include directories diff --git a/sim/simx/decode.cpp b/sim/simx/decode.cpp index 7d2e7af56..70a825496 100644 --- a/sim/simx/decode.cpp +++ b/sim/simx/decode.cpp @@ -889,7 +889,7 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { uint32_t fmt_d = rd, fmt_s = rs1; bool is_sparse = (rs2 & 1) != 0; instr->set_op_type(is_sparse ? TcuType::WMMA_SP : TcuType::WMMA); - instr->set_args(IntrTcuArgs{0, 0, fmt_s, fmt_d, 0, 0, 0, 0, 0}); + instr->set_args(IntrTcuArgs{0, 0, fmt_s, fmt_d, 0, 0, 0, 0, 0, 0}); instr->set_macro_op(); instr->set_wstall(true); } break; @@ -900,7 +900,7 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { uint32_t cd_nregs = (rs2 >> 1) & 0x3; bool is_a_smem = (rs2 >> 3) & 1; instr->set_op_type(is_sparse ? TcuType::WGMMA_SP : TcuType::WGMMA); - instr->set_args(IntrTcuArgs{is_a_smem ? 1u : 0u, cd_nregs, fmt_s, fmt_d, 0, 0, 0, 0, 0}); + instr->set_args(IntrTcuArgs{is_a_smem ? 1u : 0u, cd_nregs, fmt_s, fmt_d, 0, 0, 0, 0, 0, 0}); instr->set_macro_op(); instr->set_wstall(true); } break; @@ -910,7 +910,7 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { uint32_t fmt_s = rs2; uint32_t slot = rd; instr->set_op_type(TcuType::TCU_LD); - instr->set_args(IntrTcuArgs{0, 0, fmt_s, slot, 0, 0, 0, 0, 0}); + instr->set_args(IntrTcuArgs{0, 0, fmt_s, slot, 0, 0, 0, 0, 0, 0}); // rs1 holds the warp-broadcast base address (real I-reg read). instr->set_src_reg(0, rs1, RegType::Integer); } break; diff --git a/sim/simx/tcu/tcu_unit.cpp b/sim/simx/tcu/tcu_unit.cpp index aa551dfd4..41811ef1f 100644 --- a/sim/simx/tcu/tcu_unit.cpp +++ b/sim/simx/tcu/tcu_unit.cpp @@ -32,6 +32,7 @@ using namespace vortex; namespace vt = vortex::tensor; using cfg = vt::wmma_config_t; using wg_cfg = vt::wgmma_config_t; +static constexpr uint32_t kFedpWords = wg_cfg::fedpK; inline uint64_t nan_box(uint32_t value) { return value | 0xffffffff00000000; @@ -148,11 +149,15 @@ template struct FEDP { using itype = typename It::dtype; static uint32_t eval(const reg_data_t *a_row, const reg_data_t *b_col, uint32_t c_val) { + return eval_n(a_row, b_col, c_val, cfg::tcK); + } + + static uint32_t eval_n(const reg_data_t *a_row, const reg_data_t *b_col, uint32_t c_val, uint32_t k_words) { constexpr uint32_t i_ratio = sizeof(uint32_t) / sizeof(itype); static_assert(i_ratio * sizeof(itype) == sizeof(uint32_t), "FEDP: tcK * i_ratio must be <= 32"); if constexpr (std::is_same_v) { uint32_t acc = 0; - for (uint32_t z = 0; z < cfg::tcK; ++z) { + for (uint32_t z = 0; z < k_words; ++z) { auto a = reinterpret_cast(&a_row[z].u32); auto b = reinterpret_cast(&b_col[z].u32); uint32_t prod = 0; @@ -164,7 +169,7 @@ struct FEDP { return rv_fadd_s(c_val, acc, 0, nullptr); } else { uint32_t acc = c_val; - for (uint32_t z = 0; z < cfg::tcK; ++z) { + for (uint32_t z = 0; z < k_words; ++z) { auto a = reinterpret_cast(&a_row[z].u32); auto b = reinterpret_cast(&b_col[z].u32); for (uint32_t i = 0; i < i_ratio; ++i) { @@ -179,8 +184,12 @@ struct FEDP { template <> struct FEDP{ static uint32_t eval(const reg_data_t *a_row, const reg_data_t *b_col, uint32_t c_val) { + return eval_n(a_row, b_col, c_val, cfg::tcK); + } + + static uint32_t eval_n(const reg_data_t *a_row, const reg_data_t *b_col, uint32_t c_val, uint32_t k_words) { auto acc = bit_cast(c_val); - for (uint32_t z = 0; z < cfg::tcK; ++z) { + for (uint32_t z = 0; z < k_words; ++z) { auto a = a_row[z].u32; auto b = b_col[z].u32; for (uint32_t i = 0; i < 8; ++i) { // 8 * 4 bits = 32 bits @@ -202,8 +211,12 @@ struct FEDP{ template <> struct FEDP{ static uint32_t eval(const reg_data_t *a_row, const reg_data_t *b_col, uint32_t c_val) { + return eval_n(a_row, b_col, c_val, cfg::tcK); + } + + static uint32_t eval_n(const reg_data_t *a_row, const reg_data_t *b_col, uint32_t c_val, uint32_t k_words) { auto acc = bit_cast(c_val); - for (uint32_t z = 0; z < cfg::tcK; ++z) { + for (uint32_t z = 0; z < k_words; ++z) { auto a = a_row[z].u32; auto b = b_col[z].u32; for (uint32_t i = 0; i < 8; ++i) { // 8 * 4 bits = 32 bits @@ -216,91 +229,65 @@ struct FEDP{ } }; -using PFN_FEDP = uint32_t (*)(const reg_data_t*, const reg_data_t*, uint32_t); +using PFN_FEDP_N = uint32_t (*)(const reg_data_t*, const reg_data_t*, uint32_t, uint32_t); -static PFN_FEDP select_FEDP(uint32_t IT, uint32_t OT) { +static PFN_FEDP_N select_FEDP_N(uint32_t IT, uint32_t OT) { switch (OT) { case vt::fp32::id: switch (IT) { case vt::fp16::id: - return FEDP::eval; + return FEDP::eval_n; case vt::bf16::id: - return FEDP::eval; + return FEDP::eval_n; case vt::fp8::id: - return FEDP::eval; + return FEDP::eval_n; case vt::bf8::id: - return FEDP::eval; + return FEDP::eval_n; case vt::tf32::id: - return FEDP::eval; + return FEDP::eval_n; default: - std::cout << "Error: unsupported mma format: " << IT << " -> " << OT << "!" << std::endl; - std::abort(); + break; } break; case vt::fp16::id: - switch (IT) { - case vt::fp16::id: - return FEDP::eval; - default: - std::cout << "Error: unsupported mma format: " << IT << " -> " << OT << "!" << std::endl; - std::abort(); - } + if (IT == vt::fp16::id) + return FEDP::eval_n; break; case vt::bf16::id: - switch (IT) { - case vt::bf16::id: - return FEDP::eval; - default: - std::cout << "Error: unsupported mma format: " << IT << " -> " << OT << "!" << std::endl; - std::abort(); - } + if (IT == vt::bf16::id) + return FEDP::eval_n; break; case vt::fp8::id: - switch (IT) { - case vt::fp8::id: - return FEDP::eval; - default: - std::cout << "Error: unsupported mma format: " << IT << " -> " << OT << "!" << std::endl; - std::abort(); - } + if (IT == vt::fp8::id) + return FEDP::eval_n; break; case vt::bf8::id: - switch (IT) { - case vt::bf8::id: - return FEDP::eval; - default: - std::cout << "Error: unsupported mma format: " << IT << " -> " << OT << "!" << std::endl; - std::abort(); - } + if (IT == vt::bf8::id) + return FEDP::eval_n; break; case vt::tf32::id: - switch (IT) { - case vt::tf32::id: - return FEDP::eval; - default: - std::cout << "Error: unsupported mma format: " << IT << " -> " << OT << "!" << std::endl; - std::abort(); - } + if (IT == vt::tf32::id) + return FEDP::eval_n; break; case vt::int32::id: switch (IT) { case vt::int8::id: - return FEDP::eval; + return FEDP::eval_n; case vt::uint8::id: - return FEDP::eval; + return FEDP::eval_n; case vt::int4::id: - return FEDP::eval; + return FEDP::eval_n; case vt::uint4::id: - return FEDP::eval; + return FEDP::eval_n; default: - std::cout << "Error: unsupported mma format: " << IT << " -> " << OT << "!" << std::endl; - std::abort(); + break; } break; default: - std::cout << "Error: unsupported output type: " << OT << "!" << std::endl; - std::abort(); + break; } + std::cout << "Error: unsupported mma format: " << IT << " -> " << OT << "!" << std::endl; + std::abort(); } // Format-agnostic sparse gather: for each bword, iterate over its elem_count packed @@ -355,6 +342,7 @@ class TcuUnit::Impl { exec_done_.fill(false); wgmma_planned_warps_.fill(0); in_wgmma_.fill(false); + wgmma_desc_.fill({0, 0}); } ~Impl() {} @@ -375,6 +363,8 @@ class TcuUnit::Impl { exec_done_.fill(false); wgmma_planned_warps_.fill(0); in_wgmma_.fill(false); + lmem_desc_.clear(); + wgmma_desc_.fill({0, 0}); cta_owner_a_.fill(-1); cta_owner_b_ = -1; cur_block_ = 0; @@ -391,28 +381,21 @@ class TcuUnit::Impl { if (input.empty()) continue; auto trace = input.peek(); if (!tcu_is_wgmma(std::get(trace->op_type))) continue; - wgmma_active |= (1u << b); uint32_t wid = trace->wid; uint64_t wid_bit = (uint64_t(1) << wid); - if (wgmma_planned_warps_.at(b) & wid_bit) continue; + int32_t new_cta = (int32_t)core_->scheduler().warp(wid).cta_csrs.cta_id; auto& instr = *trace->instr_ptr; auto tpuArgs = std::get(instr.get_args()); - if (!(tpuArgs.step_m == 0 && tpuArgs.step_n == 0 && tpuArgs.step_k == 0)) { - // Non-first uop arrived without a prior plan: first uop already drained. - // Mark planned and continue (descriptors persist in lmem_desc_[wid]). - wgmma_planned_warps_.at(b) |= wid_bit; + + if (tpuArgs.is_setup_uop) continue; - } - auto& rs1_data = trace->src_data[0]; - auto& rs2_data = trace->src_data[1]; - uint32_t a_desc = rs1_data.empty() ? 0 : rs1_data.at(0).u32; - uint32_t b_desc = rs2_data.empty() ? 0 : rs2_data.at(0).u32; + + wgmma_active |= (1u << b); // CTA-overlap fence — defer this block's WGMMA if any other block // is mid-flight with a different CTA. The shared B buffer assumes // single-CTA occupancy across all blocks. - int32_t new_cta = (int32_t)core_->scheduler().warp(wid).cta_csrs.cta_id; bool block_other_cta_inflight = false; for (uint32_t k = 0; k < VX_CFG_NUM_TCU_BLOCKS; ++k) { if (k == b) continue; @@ -426,6 +409,16 @@ class TcuUnit::Impl { continue; } + if (wgmma_planned_warps_.at(b) & wid_bit) continue; + if (!(tpuArgs.step_m == 0 && tpuArgs.step_n == 0 && tpuArgs.step_k == 0)) { + // Non-first uop arrived without a prior plan: first uop already drained. + // Mark planned and continue (descriptors persist in lmem_desc_[wid]). + wgmma_planned_warps_.at(b) |= wid_bit; + continue; + } + uint32_t a_desc = wgmma_desc_[wid][0]; + uint32_t b_desc = wgmma_desc_[wid][1]; + // Drop the shared B buffer only when no other block is mid-WGMMA — // otherwise we'd evict their resident bytes mid-flight. bool any_in_wgmma = false; @@ -476,19 +469,31 @@ class TcuUnit::Impl { continue; auto trace = input.peek(); auto tcu_type = std::get(trace->op_type); + auto tpuArgs = std::get(trace->instr_ptr->get_args()); - #ifdef VX_CFG_TCU_WGMMA_ENABLE + #ifdef VX_CFG_TCU_WGMMA_ENABLE // CTA-overlap fence deferred this block — skip until pass 1 plans it. - if (tcu_is_wgmma(tcu_type) && + if (tcu_is_wgmma(tcu_type) && !tpuArgs.is_setup_uop && !(wgmma_planned_warps_.at(b) & (uint64_t(1) << trace->wid))) continue; + if (tcu_is_wgmma(tcu_type) && !tpuArgs.is_setup_uop) { + int32_t this_cta = (int32_t)core_->scheduler().warp(trace->wid).cta_csrs.cta_id; + bool block_other_cta_inflight = false; + for (uint32_t k = 0; k < VX_CFG_NUM_TCU_BLOCKS; ++k) { + if (k == b) continue; + if (in_wgmma_.at(k) && cta_owner_a_.at(k) != this_cta) { + block_other_cta_inflight = true; + break; + } + } + if (block_other_cta_inflight) + continue; + } #endif // Execute once per trace; results persist across backpressure retries // via exec_done_[b]. if (!exec_done_.at(b)) { - auto& instr = *trace->instr_ptr; - auto tpuArgs = std::get(instr.get_args()); uint32_t wid = trace->wid; uint32_t num_threads = VX_CFG_NUM_THREADS; auto& rs1_data = trace->src_data[0]; @@ -513,7 +518,7 @@ class TcuUnit::Impl { cur_block_ = b; // CTA lockstep invariant: no block may execute a WGMMA uop for a // different cta_id while another block is mid-WGMMA. - { + if (!tpuArgs.is_setup_uop) { int32_t this_cta = (int32_t)core_->scheduler().warp(wid).cta_csrs.cta_id; for (uint32_t k = 0; k < VX_CFG_NUM_TCU_BLOCKS; ++k) { if (k == b) continue; @@ -528,9 +533,9 @@ class TcuUnit::Impl { } this->wgmma(wid, tpuArgs.fmt_s, tpuArgs.fmt_d, tpuArgs.step_m, tpuArgs.step_n, tpuArgs.step_k, - a_desc, b_desc, rs1_data, rs3_data, rd_data, + a_desc, b_desc, rs1_data, rs2_data, rs3_data, rd_data, tcu_is_sparse(tcu_type), - tpuArgs.cd_nregs, tpuArgs.is_a_smem); + tpuArgs.cd_nregs, tpuArgs.is_a_smem, tpuArgs.is_setup_uop); } break; #endif #ifdef VX_CFG_TCU_META_ENABLE @@ -552,7 +557,7 @@ class TcuUnit::Impl { case TcuType::WMMA_SP: case TcuType::WGMMA: case TcuType::WGMMA_SP: - delay = 4; + delay = tpuArgs.is_setup_uop ? 0 : VX_CFG_TCU_FEDP_DELAY; break; #ifdef VX_CFG_TCU_META_ENABLE case TcuType::TCU_LD: @@ -591,18 +596,16 @@ class TcuUnit::Impl { uint32_t fmt_s = args.fmt_s; bool is_a_smem = args.is_a_smem; uint32_t e_bits = elem_bits(fmt_s); - if (e_bits < 8) return; - uint32_t e_bytes = e_bits / 8; // NRC: cd_nregs 0/1/2 → 8/16/32; xtileN = NRC * NT / xtileM. uint32_t nrc = (args.cd_nregs == 0) ? 8 : (args.cd_nregs == 1) ? 16 : 32; uint32_t xtile_n = (nrc * VX_CFG_NUM_THREADS) / wg_cfg::xtileM; lmem_desc_t sd_a{}, sd_b{}; if (is_a_smem) { - sd_a = {uint64_t(VX_MEM_LMEM_BASE_ADDR) + (a_desc & 0xFFFF), (a_desc >> 16) / e_bytes, false}; + sd_a = {uint64_t(VX_MEM_LMEM_BASE_ADDR) + (a_desc & 0xFFFF), (a_desc >> 16) * 8 / e_bits, false}; lmem_desc_[wid][0] = sd_a; } - sd_b = {uint64_t(VX_MEM_LMEM_BASE_ADDR) + (b_desc & 0xFFFF), (b_desc >> 16) / e_bytes, false}; + sd_b = {uint64_t(VX_MEM_LMEM_BASE_ADDR) + (b_desc & 0xFFFF), (b_desc >> 16) * 8 / e_bits, false}; lmem_desc_[wid][1] = sd_b; // tileK = xtileK × ratio (ratio = 32/e_bits); sparse compresses K on A only. @@ -611,9 +614,11 @@ class TcuUnit::Impl { uint32_t a_k = is_sparse ? (tile_k / 2) : tile_k; // ldm==0 → block-major layout; ldm!=0 → row-major (stride in elements). - uint32_t k_blk_dim = cfg::tcK * ratio; - uint32_t a_blk_elems = cfg::tcM * k_blk_dim; - uint32_t b_blk_elems = k_blk_dim * cfg::tcN; + uint32_t fedp_words = kFedpWords; + uint32_t b_k_blk_dim = fedp_words * ratio; + uint32_t a_k_blk_dim = is_sparse ? (cfg::tcK * ratio) : b_k_blk_dim; + uint32_t a_blk_elems = cfg::tcM * a_k_blk_dim; + uint32_t b_blk_elems = b_k_blk_dim * cfg::tcN; uint32_t n_steps = xtile_n / cfg::tcN; auto& tbuf = simobject_->tbuf(); @@ -629,14 +634,14 @@ class TcuUnit::Impl { if (a_block_major) { uint32_t m_blk = r / cfg::tcM; uint32_t i_in = r % cfg::tcM; - uint32_t k_blk = c / k_blk_dim; - uint32_t k_in = c % k_blk_dim; + uint32_t k_blk = c / a_k_blk_dim; + uint32_t k_in = c % a_k_blk_dim; elem_off = (k_blk * wg_cfg::m_steps + m_blk) * a_blk_elems - + i_in * k_blk_dim + k_in; + + i_in * a_k_blk_dim + k_in; } else { elem_off = uint64_t(r) * sd_a.ldm + c; } - uint64_t addr = sd_a.base + elem_off * e_bytes; + uint64_t addr = sd_a.base + elem_off * e_bits / 8; a_lines.push_back(addr & ~uint64_t(VX_CFG_MEM_BLOCK_SIZE - 1)); } } @@ -654,17 +659,17 @@ class TcuUnit::Impl { for (uint32_t c = 0; c < xtile_n; ++c) { uint64_t elem_off; if (b_block_major) { - uint32_t k_blk = r / k_blk_dim; - uint32_t r_in = r % k_blk_dim; + uint32_t k_blk = r / b_k_blk_dim; + uint32_t r_in = r % b_k_blk_dim; uint32_t n_blk = c / cfg::tcN; uint32_t n_in = c % cfg::tcN; // Within-block layout: N outer, K inner. elem_off = (k_blk * n_steps + n_blk) * b_blk_elems - + n_in * k_blk_dim + r_in; + + n_in * b_k_blk_dim + r_in; } else { elem_off = uint64_t(c) * sd_b.ldm + r; } - uint64_t addr = sd_b.base + elem_off * e_bytes; + uint64_t addr = sd_b.base + elem_off * e_bits / 8; b_lines.push_back(addr & ~uint64_t(VX_CFG_MEM_BLOCK_SIZE - 1)); } } @@ -763,16 +768,15 @@ class TcuUnit::Impl { ? (step_n % cfg::b_sub_blocks_sp) * cfg::b_block_size_sp : (step_n % cfg::b_sub_blocks) * cfg::b_block_size; - // Prepare A tile [tcM][tcK] - reg_data_t a_tile[cfg::tcM * cfg::tcK]; + // WMMA occupies the low half of a widened FEDP and zeros the unused half. + reg_data_t a_tile[cfg::tcM * kFedpWords] = {}; for (uint32_t i = 0; i < cfg::tcM; ++i) { for (uint32_t z = 0; z < cfg::tcK; ++z) { - a_tile[i * cfg::tcK + z] = rs1_data.at(a_off + i * cfg::tcK + z); + a_tile[i * kFedpWords + z] = rs1_data.at(a_off + i * cfg::tcK + z); } } - // Prepare B tile [tcM][tcN][tcK] - reg_data_t b_tile[cfg::tcM * cfg::tcN * cfg::tcK]; + reg_data_t b_tile[cfg::tcM * cfg::tcN * kFedpWords] = {}; if (is_sparse) { constexpr uint32_t kCompression = 2; uint32_t ebits = elem_bits(fmt_s); @@ -792,7 +796,7 @@ class TcuUnit::Impl { lo |= meta_bit(row_base + meta_bits * z + b) << b; hi |= meta_bit(row_base + meta_bits * (cfg::tcK + z) + b) << b; } - b_tile[(i * cfg::tcN + j) * cfg::tcK + z].u32 = + b_tile[(i * cfg::tcN + j) * kFedpWords + z].u32 = gather_sparse(rs2_data.at(b_idx).u32, rs2_data.at(b_idx + 1).u32, lo, hi, ebits); } } @@ -801,14 +805,16 @@ class TcuUnit::Impl { for (uint32_t i = 0; i < cfg::tcM; ++i) { for (uint32_t j = 0; j < cfg::tcN; ++j) { for (uint32_t z = 0; z < cfg::tcK; ++z) { - b_tile[(i * cfg::tcN + j) * cfg::tcK + z] = rs2_data.at(b_off + j * cfg::tcK + z); + b_tile[(i * cfg::tcN + j) * kFedpWords + z] = rs2_data.at(b_off + j * cfg::tcK + z); } } } } fedp_tile(wid, step_m, step_n, step_k, fmt_s, fmt_d, - a_tile, b_tile, rs3_data, rd_data, is_sparse, true); + a_tile, b_tile, rs3_data, rd_data, + is_sparse, kFedpWords, cfg::tcK, + cfg::k_steps * cfg::tcK); } void wgmma(uint32_t wid, @@ -820,12 +826,13 @@ class TcuUnit::Impl { uint32_t a_desc, uint32_t b_desc, const std::vector& rs1_data, + const std::vector& rs2_data, const std::vector& rs3_data, std::vector& rd_data, bool is_sparse, uint32_t cd_nregs, - uint32_t is_a_smem) { - __unused(cd_nregs); + uint32_t is_a_smem, + uint32_t is_setup_uop) { if (is_sparse && !vt::sparse_format_supported(fmt_s)) { std::cout << "Error: WGMMA_SP unsupported input format: " << vt::fmt_string(fmt_s) << " (id=" << fmt_s << ")" << std::endl; @@ -833,46 +840,51 @@ class TcuUnit::Impl { } uint32_t ratio = elem_ratio(fmt_s); - uint32_t k_words = cfg::tcK; - uint32_t e_bytes = elem_bits(fmt_s) / 8; + uint32_t k_words = is_sparse ? cfg::tcK : kFedpWords; + uint32_t e_bits = elem_bits(fmt_s); // Decode smem descriptors (B always from smem, A optionally). - lmem_desc_t sd_a, sd_b; - if (step_k == 0 && step_m == 0 && step_n == 0) { + if (is_setup_uop) { + wgmma_desc_[wid][0] = a_desc; + wgmma_desc_[wid][1] = b_desc; if (is_a_smem) { - sd_a = {uint64_t(VX_MEM_LMEM_BASE_ADDR) + (a_desc & 0xFFFF), (a_desc >> 16) / e_bytes, false}; - lmem_desc_[wid][0] = sd_a; + lmem_desc_[wid][0] = {uint64_t(VX_MEM_LMEM_BASE_ADDR) + (a_desc & 0xFFFF), (a_desc >> 16) * 8 / e_bits, false}; } - sd_b = {uint64_t(VX_MEM_LMEM_BASE_ADDR) + (b_desc & 0xFFFF), (b_desc >> 16) / e_bytes, false}; - lmem_desc_[wid][1] = sd_b; - } else { - sd_a = lmem_desc_[wid][0]; - sd_b = lmem_desc_[wid][1]; + lmem_desc_[wid][1] = {uint64_t(VX_MEM_LMEM_BASE_ADDR) + (b_desc & 0xFFFF), (b_desc >> 16) * 8 / e_bits, false}; + uint32_t nrc = (cd_nregs == 0) ? 8 : (cd_nregs == 1) ? 16 : 32; + cur_xtile_n_ = (nrc * VX_CFG_NUM_THREADS) / wg_cfg::xtileM; + return; } + + lmem_desc_t sd_a = lmem_desc_[wid][0]; + lmem_desc_t sd_b = lmem_desc_[wid][1]; // load_lmem_word distinguishes A from B by descriptor base. cur_a_desc_base_ = is_a_smem ? sd_a.base : ~uint64_t(0); + cur_is_sparse_ = is_sparse; // NRC: cd_nregs 0/1/2 → 8/16/32; xtileN = NRC * NT / xtileM. { uint32_t nrc = (cd_nregs == 0) ? 8 : (cd_nregs == 1) ? 16 : 32; cur_xtile_n_ = (nrc * VX_CFG_NUM_THREADS) / wg_cfg::xtileM; } - // Prepare A tile [tcM][tcK] - reg_data_t a_tile[cfg::tcM * cfg::tcK]; + reg_data_t a_tile[cfg::tcM * kFedpWords] = {}; for (uint32_t i = 0; i < cfg::tcM; ++i) { uint32_t a_row_idx = step_m * cfg::tcM + i; for (uint32_t z = 0; z < k_words; ++z) { if (is_a_smem) { uint32_t k_elem = (step_k * k_words + z) * ratio; - a_tile[i * cfg::tcK + z].u32 = load_lmem_word(sd_a, a_row_idx, k_elem, fmt_s, false); + a_tile[i * k_words + z].u32 = load_lmem_word(sd_a, a_row_idx, k_elem, fmt_s, false); } else { - a_tile[i * cfg::tcK + z] = rs1_data.at(i * cfg::tcK + z); + if (z < cfg::tcK) { + a_tile[i * k_words + z] = rs1_data.at(i * cfg::tcK + z); + } else { + a_tile[i * k_words + z] = rs2_data.at(i * cfg::tcK + (z - cfg::tcK)); + } } } } - // Prepare B tile [tcM][tcN][tcK] - reg_data_t b_tile[cfg::tcM * cfg::tcN * cfg::tcK]; + reg_data_t b_tile[cfg::tcM * cfg::tcN * kFedpWords] = {}; if (is_sparse) { uint32_t ebits = elem_bits(fmt_s); uint32_t rtl_i_ratio = 32 / ebits; @@ -907,12 +919,12 @@ class TcuUnit::Impl { uint32_t bword0 = load_lmem_word(sd_b, k_elem_b0, b_col_idx, fmt_s, true); uint32_t bword1 = load_lmem_word(sd_b, k_elem_b0 + ratio, b_col_idx, fmt_s, true); uint32_t gathered = gather_sparse(bword0, bword1, lo, hi, ebits); - b_tile[(i * cfg::tcN + j) * cfg::tcK + z].u32 = gathered; + b_tile[(i * cfg::tcN + j) * k_words + z].u32 = gathered; // Trace: B-gather (CSV: wid,step_m,step_n,i,lane,bword0,bword1,lo,hi,gathered). if (const char* p = std::getenv("VORTEX_TCU_TRACE")) { if (p[0] == '1') { fprintf(stderr, "GATHER,%u,%u,%u,%u,%u,0x%08x,0x%08x,%u,%u,0x%08x\n", - wid, step_m, step_n, i, j*cfg::tcK+z, + wid, step_m, step_n, i, j*k_words+z, bword0, bword1, lo, hi, gathered); } } @@ -925,7 +937,7 @@ class TcuUnit::Impl { uint32_t b_col_idx = step_n * cfg::tcN + j; for (uint32_t z = 0; z < k_words; ++z) { uint32_t k_elem = (step_k * k_words + z) * ratio; - b_tile[(i * cfg::tcN + j) * cfg::tcK + z].u32 = + b_tile[(i * cfg::tcN + j) * k_words + z].u32 = load_lmem_word(sd_b, k_elem, b_col_idx, fmt_s, true); } } @@ -933,7 +945,9 @@ class TcuUnit::Impl { } fedp_tile(wid, step_m, step_n, step_k, fmt_s, fmt_d, - a_tile, b_tile, rs3_data, rd_data, is_sparse, false); + a_tile, b_tile, rs3_data, rd_data, + is_sparse, k_words, k_words, + wg_cfg::k_steps * wg_cfg::fedpK); __unused(b_desc); } @@ -975,10 +989,10 @@ class TcuUnit::Impl { return 32 / elem_bits(fmt_s); } - uint32_t mx_tile_scale_blocks(uint32_t fmt_s) const { - uint32_t logical_tile_k = cfg::k_steps * cfg::tcK * elem_ratio(fmt_s); + uint32_t mx_tile_scale_blocks(uint32_t fmt_s, uint32_t tile_k_words) const { + uint32_t logical_tile_k = tile_k_words * elem_ratio(fmt_s); uint32_t block_size = vt::mx_scale_block_size(fmt_s); - return std::max(1u, logical_tile_k / block_size); + return std::max(1u, (logical_tile_k + block_size - 1) / block_size); } static uint8_t meta_byte(const std::vector& words, uint32_t index) { @@ -997,13 +1011,15 @@ class TcuUnit::Impl { uint32_t eval_mx_fedp(uint32_t wid, uint32_t fmt_s, uint32_t fmt_d, uint32_t step_m, uint32_t step_n, uint32_t step_k, uint32_t i, uint32_t j, const reg_data_t* a_row, - const reg_data_t* b_col, uint32_t c_val, bool is_sparse) const { + const reg_data_t* b_col, uint32_t c_val, bool is_sparse, + uint32_t fedp_words, uint32_t tile_k_words) const { #ifndef VX_CFG_TCU_MX_ENABLE - __unused(wid, fmt_s, fmt_d, step_m, step_n, step_k, i, j, a_row, b_col, is_sparse); + __unused(wid, fmt_s, fmt_d, step_m, step_n, step_k, i, j, a_row, b_col, + is_sparse, fedp_words, tile_k_words); return c_val; #else uint32_t ratio = elem_ratio(fmt_s); - uint32_t scale_blocks_k = mx_tile_scale_blocks(fmt_s); + uint32_t scale_blocks_k = mx_tile_scale_blocks(fmt_s, tile_k_words); uint32_t block_size = vt::mx_scale_block_size(fmt_s); auto scale_a = [&](uint32_t elem_k) { uint32_t row = step_m * cfg::tcM + i; @@ -1016,10 +1032,10 @@ class TcuUnit::Impl { if (fmt_d == vt::fp32::id) { uint32_t acc = c_val; - for (uint32_t z = 0; z < cfg::tcK; ++z) { + for (uint32_t z = 0; z < fedp_words; ++z) { for (uint32_t e = 0; e < ratio; ++e) { uint32_t sparse_ratio = is_sparse ? 2 : 1; - uint32_t elem_k = ((step_k * cfg::tcK + z) * ratio + e) * sparse_ratio; + uint32_t elem_k = ((step_k * fedp_words + z) * ratio + e) * sparse_ratio; uint32_t xa, xb; if (fmt_s == vt::mxfp8::id) { xa = rv_mxfp8tof_s((a_row[z].u32 >> (8 * e)) & 0xff, scale_a(elem_k), 0, nullptr); @@ -1044,9 +1060,9 @@ class TcuUnit::Impl { if (fmt_s == vt::mxint8::id && fmt_d == vt::int32::id) { int32_t acc = bit_cast(c_val); - for (uint32_t z = 0; z < cfg::tcK; ++z) { + for (uint32_t z = 0; z < fedp_words; ++z) { for (uint32_t e = 0; e < ratio; ++e) { - uint32_t elem_k = ((step_k * cfg::tcK + z) * ratio + e) * (is_sparse ? 2 : 1); + uint32_t elem_k = ((step_k * fedp_words + z) * ratio + e) * (is_sparse ? 2 : 1); auto a = static_cast((a_row[z].u32 >> (8 * e)) & 0xff); auto b = static_cast((b_col[z].u32 >> (8 * e)) & 0xff); int32_t shift = int32_t(scale_a(elem_k)) + int32_t(scale_b(elem_k)) - 266; @@ -1069,19 +1085,21 @@ class TcuUnit::Impl { template uint32_t gather_word(ReadLine read_line, const lmem_desc_t& desc, uint32_t row, uint32_t col, - uint32_t fmt_s, bool pack_along_row) const { + uint32_t fmt_s, bool pack_along_row, + bool sparse_a_layout = false) const { uint32_t e_bits = elem_bits(fmt_s); uint32_t ratio = (e_bits >= 32) ? 1 : (32 / e_bits); - uint32_t e_bytes = (e_bits >= 8) ? (e_bits / 8) : 1; uint32_t result = 0; for (uint32_t r = 0; r < ratio; ++r) { uint32_t cur_row = pack_along_row ? (row + r) : row; uint32_t cur_col = pack_along_row ? col : (col + r); - uint64_t byte_addr; + uint64_t elem_off; if (desc.ldm == 0) { // Block-major SMEM. K dimension is along col for A (pack_along_row // false) and along row for B (pack_along_row true). - uint32_t k_blk_dim = cfg::tcK * ratio; + uint32_t k_blk_dim = (sparse_a_layout && !pack_along_row) + ? (cfg::tcK * ratio) + : (kFedpWords * ratio); if (pack_along_row) { // B: r is K coord, c is N coord. Within-block layout: N outer, K inner. uint32_t k_blk = cur_row / k_blk_dim; @@ -1090,9 +1108,8 @@ class TcuUnit::Impl { uint32_t n_in = cur_col % cfg::tcN; uint32_t b_blk_elems = k_blk_dim * cfg::tcN; uint32_t n_steps = cur_xtile_n_ / cfg::tcN; - uint64_t off = (k_blk * n_steps + n_blk) * b_blk_elems - + n_in * k_blk_dim + r_in; - byte_addr = desc.base + off * e_bytes; + elem_off = (k_blk * n_steps + n_blk) * b_blk_elems + + n_in * k_blk_dim + r_in; } else { // A: r is M coord, c is K coord. uint32_t m_blk = cur_row / cfg::tcM; @@ -1100,20 +1117,20 @@ class TcuUnit::Impl { uint32_t k_blk = cur_col / k_blk_dim; uint32_t k_in = cur_col % k_blk_dim; uint32_t a_blk_elems = cfg::tcM * k_blk_dim; - uint64_t off = (k_blk * wg_cfg::m_steps + m_blk) * a_blk_elems - + i_in * k_blk_dim + k_in; - byte_addr = desc.base + off * e_bytes; + elem_off = (k_blk * wg_cfg::m_steps + m_blk) * a_blk_elems + + i_in * k_blk_dim + k_in; } } else if (desc.col_major) { - byte_addr = desc.base + (uint64_t(cur_col) * desc.ldm + cur_row) * e_bytes; + elem_off = uint64_t(cur_col) * desc.ldm + cur_row; } else if (pack_along_row) { // B: K-major (N-outer K-inner). cur_row=K, cur_col=N; // ldm = stride in elements between N rows. - byte_addr = desc.base + (uint64_t(cur_col) * desc.ldm + cur_row) * e_bytes; + elem_off = uint64_t(cur_col) * desc.ldm + cur_row; } else { // A: row-major (M-outer K-inner). cur_row=M, cur_col=K. - byte_addr = desc.base + (uint64_t(cur_row) * desc.ldm + cur_col) * e_bytes; + elem_off = uint64_t(cur_row) * desc.ldm + cur_col; } + uint64_t byte_addr = desc.base + elem_off * e_bits / 8; auto line = read_line(byte_addr); if (!line) { std::cout << "Error: TCU buffer miss at 0x" << std::hex << byte_addr @@ -1132,9 +1149,10 @@ class TcuUnit::Impl { result |= (val & 0xFFFF) << (r * 16); } else if (e_bits == 8) { result |= uint32_t((*line)[off]) << (r * 8); + } else if (e_bits == 4) { + uint32_t val = (uint32_t((*line)[off]) >> (4 * (elem_off & 1))) & 0xf; + result |= val << (r * 4); } else { - // 4-bit (int4/uint4) not supported. - std::cout << "Error: TCU 4-bit operand gather not supported" << std::endl; std::abort(); } } @@ -1148,7 +1166,7 @@ class TcuUnit::Impl { if (desc.base == cur_a_desc_base_) { uint32_t b = cur_block_; return gather_word([&](uint64_t addr) { return tbuf->read_a(b, addr); }, - desc, row, col, fmt_s, pack_along_row); + desc, row, col, fmt_s, pack_along_row, cur_is_sparse_); } return gather_word([&](uint64_t addr) { return tbuf->read_b(addr); }, desc, row, col, fmt_s, pack_along_row); @@ -1159,8 +1177,8 @@ class TcuUnit::Impl { static constexpr uint32_t kMaxMetaCols = VX_CFG_NUM_THREADS / 2; // FEDP tile computation for both WMMA and WGMMA. - // a_tile: [tcM][tcK] flat array of pre-loaded A operand words. - // b_tile: [tcM][tcN][tcK] flat array of pre-loaded B operand words + // a_tile: [tcM][k_words] flat array of pre-loaded A operand words. + // b_tile: [tcM][tcN][k_words] flat array of pre-loaded B operand words // (for dense, each i-slice is identical; for sparse, already gathered). void fedp_tile(uint32_t wid, uint32_t step_m, uint32_t step_n, uint32_t step_k, @@ -1170,23 +1188,23 @@ class TcuUnit::Impl { const std::vector& rs3_data, std::vector& rd_data, bool is_sparse, - bool allow_mx) { - if (!allow_mx && vt::mx_scale_format(fmt_s)) { - std::cout << "Error: MX formats are supported only by WMMA." << std::endl; - std::abort(); - } - PFN_FEDP fedp = vt::mx_scale_format(fmt_s) ? nullptr : select_FEDP(fmt_s, fmt_d); + uint32_t k_words = cfg::tcK, + uint32_t fedp_words = cfg::tcK, + uint32_t tile_k_words = cfg::k_steps * cfg::tcK) { + PFN_FEDP_N fedp = vt::mx_scale_format(fmt_s) ? nullptr : select_FEDP_N(fmt_s, fmt_d); for (uint32_t i = 0; i < cfg::tcM; ++i) { for (uint32_t j = 0; j < cfg::tcN; ++j) { auto t = i * cfg::tcN + j; auto c_val = rs3_data.at(t).u32; - auto a_row = &a_tile[i * cfg::tcK]; - auto b_col = &b_tile[(i * cfg::tcN + j) * cfg::tcK]; + auto a_row = &a_tile[i * k_words]; + auto b_col = &b_tile[(i * cfg::tcN + j) * k_words]; - uint32_t d_val = (allow_mx && vt::mx_scale_format(fmt_s)) - ? eval_mx_fedp(wid, fmt_s, fmt_d, step_m, step_n, step_k, i, j, a_row, b_col, c_val, is_sparse) - : fedp(a_row, b_col, c_val); + uint32_t d_val = vt::mx_scale_format(fmt_s) + ? eval_mx_fedp(wid, fmt_s, fmt_d, step_m, step_n, step_k, + i, j, a_row, b_col, c_val, is_sparse, + fedp_words, tile_k_words) + : fedp(a_row, b_col, c_val, k_words); rd_data.at(t).u64 = nan_box(d_val); DTH(3, simobject_->name() << " FEDP" @@ -1207,6 +1225,7 @@ class TcuUnit::Impl { std::vector> mx_meta_b_; #endif std::unordered_map lmem_desc_; + std::array, VX_CFG_NUM_WARPS> wgmma_desc_; mutable PerfStats perf_stats_; // Per-block guard: execute already happened for this trace; reset on pop(). std::array exec_done_; @@ -1218,6 +1237,7 @@ class TcuUnit::Impl { uint32_t cur_block_ = 0; // A-descriptor base for the current wgmma(); distinguishes A from B in load_lmem_word. uint64_t cur_a_desc_base_ = ~uint64_t(0); + bool cur_is_sparse_ = false; // xtileN for the active WGMMA (derived from NRC). uint32_t cur_xtile_n_ = 8; // CTA owner per block's A buffer and the shared B buffer (-1 = unowned). @@ -1277,9 +1297,9 @@ uint32_t TcuUopGen::uop_count(const Instr& instr) { if (tcu_is_wgmma(tcu_type)) { bool is_sparse = tcu_is_sparse(tcu_type); uint32_t nrc = (args.cd_nregs == 0) ? 8 : (args.cd_nregs == 1) ? 16 : 32; - uint32_t k_count = is_sparse ? (wg_cfg::k_steps / 2) : wg_cfg::k_steps; + uint32_t k_count = is_sparse ? std::max(1u, wg_cfg::k_steps / 2) : wg_cfg::k_steps; uint32_t mma_uops = k_count * nrc; - return mma_uops; + return mma_uops + 1; } #endif @@ -1337,7 +1357,7 @@ Instr::Ptr TcuUopGen::get(const Instr& macro_instr, uint32_t uop_index) { uint32_t actual_n = lg_k ? (n_sp >> lg_k) : n_sp; uint32_t reg_rs3 = rc_base + (mma_idx >> 1); uop_instr->set_op_type(TcuType::WMMA_SP); - uop_instr->set_args(IntrTcuArgs{0, 0, fmt_s, fmt_d, m_sp, actual_n, 0, 0, 0}); + uop_instr->set_args(IntrTcuArgs{0, 0, fmt_s, fmt_d, m_sp, actual_n, 0, 0, 0, 0}); uop_instr->set_dest_reg(reg_rs3, RegType::Float); uop_instr->set_src_reg(0, ra_base + m_sp, RegType::Float); uop_instr->set_src_reg(1, rb_base + n_sp, RegType::Float); @@ -1393,7 +1413,7 @@ Instr::Ptr TcuUopGen::get(const Instr& macro_instr, uint32_t uop_index) { reg_rs3 = rc_base + c_off; } uop_instr->set_op_type(is_sparse ? TcuType::WMMA_SP : TcuType::WMMA); - uop_instr->set_args(IntrTcuArgs{0, 0, fmt_s, fmt_d, m, n, k, 0, 0}); + uop_instr->set_args(IntrTcuArgs{0, 0, fmt_s, fmt_d, m, n, k, 0, 0, 0}); uop_instr->set_dest_reg(reg_rs3, RegType::Float); uop_instr->set_src_reg(0, reg_rs1, RegType::Float); uop_instr->set_src_reg(1, reg_rs2, RegType::Float); @@ -1405,52 +1425,62 @@ Instr::Ptr TcuUopGen::get(const Instr& macro_instr, uint32_t uop_index) { else if (tcu_is_wgmma(tcu_type)) { constexpr uint32_t m_steps = wg_cfg::m_steps; constexpr uint32_t k_steps = wg_cfg::k_steps; + constexpr uint32_t a_reg_k_steps = 2; uint32_t fmt_s = args.fmt_s; uint32_t fmt_d = args.fmt_d; bool is_sparse = tcu_is_sparse(tcu_type); bool is_a_smem = args.is_a_smem; uint32_t cd_nregs = args.cd_nregs; - uint32_t k_count = is_sparse ? (k_steps / 2) : k_steps; + uint32_t k_count = is_sparse ? std::max(1u, k_steps / 2) : k_steps; constexpr uint32_t a0 = 10, a1 = 11; { + uop_instr->set_op_type(is_sparse ? TcuType::WGMMA_SP : TcuType::WGMMA); + + if (uop_index == 0) { + uop_instr->set_args(IntrTcuArgs{is_a_smem ? 1u : 0u, cd_nregs, + fmt_s, fmt_d, 0, 0, 0, 1, 0, 1}); + uop_instr->set_src_reg(0, a0, RegType::Integer); + uop_instr->set_src_reg(1, a1, RegType::Integer); + uop_instr->set_fu_lock(true); + uop_instr->set_fu_unlock(false); + return uop_instr; + } + // MMA phase - uint32_t mma_idx = uop_index; + uint32_t mma_idx = uop_index - 1; uint32_t ra_base = is_a_smem ? 10 : 24; // Loop order: m (inner) -> n (middle) -> k (outer). K-outer maximizes // per-block A-buffer reuse: each A_w[m,k] is consumed across the entire // (n,m) inner sweep, and each shared B[k,n] is consumed for m_steps // consecutive uops. - uint32_t mn = total / k_count; + uint32_t compute_total = total - 1; + uint32_t mn = compute_total / k_count; uint32_t k = mma_idx / mn; uint32_t rem = mma_idx % mn; uint32_t n = rem / m_steps; uint32_t m = rem % m_steps; uint32_t r = n * m_steps + m; - uop_instr->set_op_type(is_sparse ? TcuType::WGMMA_SP : TcuType::WGMMA); - bool first = (uop_index == 0); + bool first = (mma_idx == 0); bool last = (uop_index == (total - 1)); uop_instr->set_args(IntrTcuArgs{is_a_smem ? 1u : 0u, cd_nregs, - fmt_s, fmt_d, m, n, k, first ? 1u : 0u, last ? 1u : 0u}); + fmt_s, fmt_d, m, n, k, first ? 1u : 0u, last ? 1u : 0u, 0}); uop_instr->set_dest_reg(r, RegType::Float); - if (mma_idx == 0) { - if (is_a_smem) { - uop_instr->set_src_reg(0, a0, RegType::Integer); - } else { - uint32_t rs1_off = m * k_count + k; - uop_instr->set_src_reg(0, ra_base + rs1_off, RegType::Float); - } - uop_instr->set_src_reg(1, a1, RegType::Integer); - } else if (!is_a_smem) { - uint32_t rs1_off = m * k_count + k; + if (!is_a_smem) { + uint32_t rs1_off = is_sparse ? m : (m * a_reg_k_steps + k); uop_instr->set_src_reg(0, ra_base + rs1_off, RegType::Float); + if constexpr (k_steps == 1) { + if (!is_sparse) { + uop_instr->set_src_reg(1, ra_base + rs1_off + 1, RegType::Float); + } + } } uop_instr->set_src_reg(2, r, RegType::Float); } // fu_lock on first uop, fu_unlock on last uop - uop_instr->set_fu_lock(uop_index == 0); + uop_instr->set_fu_lock(false); uop_instr->set_fu_unlock(uop_index == (total - 1)); } #endif @@ -1507,20 +1537,22 @@ void TcuUnit::wmma(uint32_t wid, } void TcuUnit::wgmma(uint32_t wid, - uint32_t fmt_s, - uint32_t fmt_d, - uint32_t step_m, - uint32_t step_n, - uint32_t step_k, - uint32_t a_desc, - uint32_t b_desc, - const std::vector& rs1_data, - const std::vector& rs3_data, - std::vector& rd_data, - bool is_sparse, - uint32_t cd_nregs, - uint32_t is_a_smem) { + uint32_t fmt_s, + uint32_t fmt_d, + uint32_t step_m, + uint32_t step_n, + uint32_t step_k, + uint32_t a_desc, + uint32_t b_desc, + const std::vector& rs1_data, + const std::vector& rs2_data, + const std::vector& rs3_data, + std::vector& rd_data, + bool is_sparse, + uint32_t cd_nregs, + uint32_t is_a_smem, + uint32_t is_setup_uop) { impl_->wgmma(wid, fmt_s, fmt_d, step_m, step_n, step_k, a_desc, b_desc, - rs1_data, rs3_data, rd_data, - is_sparse, cd_nregs, is_a_smem); + rs1_data, rs2_data, rs3_data, rd_data, + is_sparse, cd_nregs, is_a_smem, is_setup_uop); } diff --git a/sim/simx/tcu/tcu_unit.h b/sim/simx/tcu/tcu_unit.h index 8b8dacad7..2728e977b 100644 --- a/sim/simx/tcu/tcu_unit.h +++ b/sim/simx/tcu/tcu_unit.h @@ -90,11 +90,13 @@ class TcuUnit : public FuncUnit { uint32_t a_desc, uint32_t b_desc, const std::vector& rs1_data, + const std::vector& rs2_data, const std::vector& rs3_data, std::vector& rd_data, bool is_sparse, uint32_t cd_nregs, - uint32_t is_a_smem); + uint32_t is_a_smem, + uint32_t is_setup_uop); // Tile-buffer subsystem (owns abuf×Q + bbuf + LMEM arb). // Exposed so that `Core` can bind its single LMEM port pair. diff --git a/sim/simx/types.h b/sim/simx/types.h index 37904edfd..516d91278 100644 --- a/sim/simx/types.h +++ b/sim/simx/types.h @@ -711,6 +711,7 @@ struct IntrTcuArgs { uint32_t step_k : 4; uint32_t is_first_uop : 1; // set per-uop by tcu_uops expansion (C4) uint32_t is_last_uop : 1; + uint32_t is_setup_uop : 1; // WGMMA descriptor setup uop, no FEDP compute }; // Helper: is_sparse derived from op_type (no per-uop bit). diff --git a/sim/xrtsim/Makefile b/sim/xrtsim/Makefile index 8d2735ed5..d5f16191b 100644 --- a/sim/xrtsim/Makefile +++ b/sim/xrtsim/Makefile @@ -86,6 +86,8 @@ ifneq (,$(filter -DVX_CFG_EXT_TCU_ENABLE, $(XCONFIGS))) RTL_PKGS += $(THIRD_PARTY_DIR)/cvfpu/src/fpnew_pkg.sv $(THIRD_PARTY_DIR)/cvfpu/src/common_cells/src/cf_math_pkg.sv RTL_INCLUDE += -I$(RTL_DIR)/tcu/fpnew RTL_INCLUDE += -I$(THIRD_PARTY_DIR)/cvfpu/src/common_cells/include -I$(THIRD_PARTY_DIR)/cvfpu/src/common_cells/src -I$(THIRD_PARTY_DIR)/cvfpu/src + else ifneq (,$(filter -DVX_CFG_TCU_TYPE_TET, $(XCONFIGS))) + RTL_INCLUDE += -I$(RTL_DIR)/tcu/tet else ifneq (,$(filter -DVX_CFG_TCU_TYPE_TFR, $(XCONFIGS))) RTL_INCLUDE += -I$(RTL_DIR)/tcu/tfr else diff --git a/sw/common/tensor_cfg.h b/sw/common/tensor_cfg.h index 18e5116fa..d856ccc7b 100644 --- a/sw/common/tensor_cfg.h +++ b/sw/common/tensor_cfg.h @@ -345,7 +345,7 @@ struct wmma_config_t { // All geometry derived from NT and NRC alone (NRA=4 fixed): // tcM = 2^ceil(log2(NT)/2), tcN = tcK = 2^floor(log2(NT)/2) // xtileM = 2*tcM, xtileN = NRC*NT/xtileM, xtileK = 2*tcK -// m_steps = k_steps = 2 (always) +// m_steps = 2; k_steps is 2 normally, or 1 with a doubled-K WGMMA FEDP. template struct wgmma_config_t { private: @@ -360,12 +360,18 @@ struct wgmma_config_t { static constexpr uint32_t tcM = 1u << ((lg_NT + 1) / 2); static constexpr uint32_t tcN = 1u << (lg_NT / 2); static constexpr uint32_t tcK = tcN; + static constexpr uint32_t fedpK = +#ifdef VX_CFG_TCU_FEDP2K + 2 * tcK; +#else + tcK; +#endif static constexpr uint32_t xtileM = 2 * tcM; static constexpr uint32_t xtileN = (NRC_ * NT) / xtileM; static constexpr uint32_t xtileK = 2 * tcK; static constexpr uint32_t tileK = xtileK * i_ratio; static constexpr uint32_t m_steps = 2; - static constexpr uint32_t k_steps = 2; + static constexpr uint32_t k_steps = xtileK / fedpK; static constexpr uint32_t NRC = NRC_; }; diff --git a/sw/kernel/include/vx_tensor.h b/sw/kernel/include/vx_tensor.h index f067ea041..61e977289 100644 --- a/sw/kernel/include/vx_tensor.h +++ b/sw/kernel/include/vx_tensor.h @@ -715,6 +715,9 @@ struct wgmma_context { // Fragments using fragment_acc = typename ctx_c::fragment_acc; using fragment_a = typename ctx_a::fragment_a; + using fragment_b = typename ctx_c::fragment_b; + + static constexpr bool input_is_mx = mx_scale_format(It::id); // Block (micro-tile) geometry — derived from NT alone static constexpr uint32_t i_ratio = XB / sizeof(typename It::dtype); @@ -722,19 +725,27 @@ struct wgmma_context { static constexpr uint32_t tcN = 1u << (lg_NT / 2); static constexpr uint32_t tcK = tcN; - // Per-warp tile geometry — m_steps = k_steps = 2 always + // Per-warp tile geometry. WGMMA-2K keeps the logical tile K fixed and + // consumes both K halves inside one widened FEDP uop. + static constexpr uint32_t fedpK = +#ifdef VX_CFG_TCU_FEDP2K + 2 * tcK; +#else + tcK; +#endif static constexpr uint32_t m_steps = 2; - static constexpr uint32_t k_steps = 2; + static constexpr uint32_t k_steps = (2 * tcK) / fedpK; + static constexpr uint32_t a_reg_k_steps = 2; static constexpr uint32_t xtileM = m_steps * tcM; static constexpr uint32_t xtileN = (NRC_ * NT) / xtileM; - static constexpr uint32_t tileK = k_steps * tcK * i_ratio; + static constexpr uint32_t tileK = k_steps * fedpK * i_ratio; static constexpr uint32_t n_steps = xtileN / tcN; // Block-major SMEM constants. // BLOCK = micro-tile (tcM × tcK or tcK × tcN), measured in input_t element units. - static constexpr uint32_t a_blk_elems = tcM * tcK * i_ratio; // elements per A block + static constexpr uint32_t a_blk_elems = tcM * fedpK * i_ratio; // elements per A block static constexpr uint32_t a_warp_elems = xtileM * tileK; // elements per warp's A slice - static constexpr uint32_t b_blk_elems = tcK * i_ratio * tcN; // elements per B block + static constexpr uint32_t b_blk_elems = fedpK * i_ratio * tcN; // elements per B block // Cooperative-load index into A_smem for an (r, c) target in the // row-major-equivalent A view (r ∈ [0, cta_M), c ∈ [0, tileK)). @@ -744,11 +755,11 @@ struct wgmma_context { uint32_t r_in_warp = r % xtileM; uint32_t m_blk = r_in_warp / tcM; uint32_t i_in = r_in_warp % tcM; - uint32_t k_blk = c / (tcK * i_ratio); - uint32_t k_in = c % (tcK * i_ratio); + uint32_t k_blk = c / (fedpK * i_ratio); + uint32_t k_in = c % (fedpK * i_ratio); return warp_idx * a_warp_elems + (k_blk * m_steps + m_blk) * a_blk_elems - + i_in * (tcK * i_ratio) + k_in; + + i_in * (fedpK * i_ratio) + k_in; } // Cooperative-load index into B_smem for an (r, c) target in the @@ -756,12 +767,12 @@ struct wgmma_context { // Within-block layout: N outer, K inner — each 32-bit word packs i_ratio // K-elements at one (j, k_word) cell, matching tcu_core's b_off + j*TC_K + k. static __attribute__((always_inline)) uint32_t b_blockmajor_idx(uint32_t r, uint32_t c) { - uint32_t k_blk = r / (tcK * i_ratio); - uint32_t r_in = r % (tcK * i_ratio); + uint32_t k_blk = r / (fedpK * i_ratio); + uint32_t r_in = r % (fedpK * i_ratio); uint32_t n_blk = c / tcN; uint32_t n_in = c % tcN; return (k_blk * n_steps + n_blk) * b_blk_elems - + n_in * (tcK * i_ratio) + r_in; + + n_in * (fedpK * i_ratio) + r_in; } // Cooperative-load index into per-warp sparse A_smem_w for an (r, c) target @@ -769,11 +780,12 @@ struct wgmma_context { // Sparse A is K/2 compressed; same per-block shape as dense A but only // K_STEPS/2 half-k blocks. Caller passes a per-warp pointer. static __attribute__((always_inline)) uint32_t a_sp_blockmajor_idx(uint32_t r, uint32_t c) { + constexpr uint32_t sparse_blk_elems = tcM * tcK * i_ratio; uint32_t half_k_blk = c / (tcK * i_ratio); uint32_t k_in_elem = c % (tcK * i_ratio); uint32_t m_blk = r / tcM; uint32_t i_in = r % tcM; - return (half_k_blk * m_steps + m_blk) * a_blk_elems + return (half_k_blk * m_steps + m_blk) * sparse_blk_elems + i_in * (tcK * i_ratio) + k_in_elem; } @@ -784,7 +796,8 @@ struct wgmma_context { // so wg_meta_total_bytes = NT*4. Unused SRAM cells are packed with zero // to keep the load shape uniform. static constexpr uint32_t sp_rtl_i_ratio = 32 / It::bits; - static constexpr uint32_t wg_meta_banks = m_steps * (k_steps / 2); + static constexpr uint32_t sparse_k_steps = (k_steps > 1) ? (k_steps / 2) : 1; + static constexpr uint32_t wg_meta_banks = m_steps * sparse_k_steps; static constexpr uint32_t wg_meta_row_bits = tcK * 2 * sp_rtl_i_ratio; static constexpr uint32_t wg_meta_stride_words = (tcM * wg_meta_row_bits + 31) / 32; static constexpr uint32_t wg_meta_stride_bytes = wg_meta_stride_words * 4; @@ -816,6 +829,7 @@ struct wgmma_context { static_assert(tcM * tcK == NT, "wgmma block-major load assumes canonical config (TC_M*TC_K == NT)"); constexpr uint32_t k_row_elems = tcK * i_ratio; + constexpr uint32_t sparse_blk_elems = tcM * k_row_elems; uint32_t lane = vx_thread_id(); uint32_t i_in = lane / tcK; uint32_t k_in_elem = (lane % tcK) * i_ratio; @@ -824,7 +838,8 @@ struct wgmma_context { uint32_t elem_off; if (ldm == 0) { // Block-major SMEM: blocks contiguous, k-block outer. - elem_off = (k_blk * m_steps + m_blk) * a_blk_elems + constexpr uint32_t block_elems = is_sparse ? sparse_blk_elems : a_blk_elems; + elem_off = (k_blk * m_steps + m_blk) * block_elems + i_in * k_row_elems + k_in_elem; } else { @@ -837,7 +852,7 @@ struct wgmma_context { }; if constexpr (is_sparse) { - constexpr uint32_t sp_k_steps_local = k_steps / 2; + constexpr uint32_t sp_k_steps_local = sparse_k_steps; constexpr uint32_t a_regs = m_steps * sp_k_steps_local; detail::unroll_for([&](auto r) { uint32_t m_blk = r / sp_k_steps_local; @@ -846,9 +861,24 @@ struct wgmma_context { }); } else { detail::unroll_for([&](auto r) { - uint32_t m_blk = r / k_steps; - uint32_t k_blk = r % k_steps; - dst.data[r] = load_reg(m_blk, k_blk); + uint32_t m_blk = r / a_reg_k_steps; + uint32_t k_blk = r % a_reg_k_steps; + if constexpr (k_steps == 1) { + uint32_t elem_off; + if (ldm == 0) { + elem_off = m_blk * a_blk_elems + + i_in * (fedpK * i_ratio) + + k_blk * k_row_elems + + k_in_elem; + } else { + elem_off = (m_blk * tcM + i_in) * uint32_t(ldm) + + (k_blk * k_row_elems + k_in_elem); + } + dst.data[r] = *reinterpret_cast( + reinterpret_cast(src) + elem_off); + } else { + dst.data[r] = load_reg(m_blk, k_blk); + } }); } } @@ -876,6 +906,11 @@ struct wgmma_context { ); } + template + static __attribute__((always_inline)) void load_mx_metadata(Frag& frag, const void* meta_mx_ptr) { + ctx_c::load_mx_metadata(frag, meta_mx_ptr); + } + // Store accumulator with n-major register layout: r = n * m_steps + m template static __attribute__((always_inline)) void store_matrix_sync(void *dst, const Frag &src, size_t ldm) { @@ -1040,6 +1075,7 @@ struct wgmma_context { // --- RS path: A from registers, B from smem (NRC <= 16, NRA = 4) --- else if constexpr (!a_is_smem && b_is_smem) { static_assert(OpA::NR == 4, "WGMMA RS requires NRA=4"); + register uint32_t ra __asm__("a0") = 0; register uint32_t rb __asm__("a1") = op_b.value; if constexpr (NRC_ == 16) { @@ -1072,7 +1108,7 @@ struct wgmma_context { "+f"(fd12), "+f"(fd13), "+f"(fd14), "+f"(fd15) : [insn]"i"(RISCV_CUSTOM0), [fmd]"i"(Ot::id), [fms]"i"(It::id), [flags]"i"(flags), "f"(fa0), "f"(fa1), "f"(fa2), "f"(fa3), - "r"(rb) + "r"(ra), "r"(rb) ); frag_d.data = { @@ -1099,7 +1135,7 @@ struct wgmma_context { "+f"(fd4), "+f"(fd5), "+f"(fd6), "+f"(fd7) : [insn]"i"(RISCV_CUSTOM0), [fmd]"i"(Ot::id), [fms]"i"(It::id), [flags]"i"(flags), "f"(fa0), "f"(fa1), "f"(fa2), "f"(fa3), - "r"(rb) + "r"(ra), "r"(rb) ); frag_d.data = {fd0, fd1, fd2, fd3, fd4, fd5, fd6, fd7}; diff --git a/tests/regression/Makefile b/tests/regression/Makefile index d11fca954..be60b6363 100644 --- a/tests/regression/Makefile +++ b/tests/regression/Makefile @@ -8,7 +8,7 @@ TESTS := \ vecadd vecadd_v1 sgemm sgemmx sgemm_v1 conv3 relu sgemv \ sgemm2 sgemm2_v1 madmax stencil3d raycast bfs jacobi pathfinder \ softmax \ - sgemm_tcu sgemm_tcu_mx sgemm_tcu_sp_mx sgemm_tcu_wg sgemm_tcu_sp sgemm_tcu_wg_sp \ + sgemm_tcu sgemm_tcu_mx sgemm_tcu_sp_mx sgemm_tcu_wg sgemm_tcu_wg_mx sgemm_tcu_sp sgemm_tcu_wg_sp \ sgemm2_dxa sgemm2_tcu sgemm_tcu_wg_dxa sgemm_tcu_wg_sp_dxa \ sgemm2_dxa_mcast sgemm_tcu_wg_dxa_mcast \ dxa_copy dxa_copy_mcast dxa_kmajor_check \ diff --git a/tests/regression/sgemm_tcu_mx/main.cpp b/tests/regression/sgemm_tcu_mx/main.cpp index 8887c6014..fd7edc87a 100644 --- a/tests/regression/sgemm_tcu_mx/main.cpp +++ b/tests/regression/sgemm_tcu_mx/main.cpp @@ -59,10 +59,8 @@ static void pack_mx_a_metadata(std::vector &packed, uint32_t logical_tileK = cfg::tileK * kElemsPerByte; uint32_t num_k_tiles = K_logical / logical_tileK; uint32_t scale_blocks_k = K_logical / vt::ITYPE::ele_block; - uint32_t tile_scale_blocks_k = logical_tileK / vt::ITYPE::ele_block; - if (tile_scale_blocks_k == 0) { - tile_scale_blocks_k = 1; - } + uint32_t tile_scale_blocks_k = + std::max(1u, (logical_tileK + vt::ITYPE::ele_block - 1) / vt::ITYPE::ele_block); packed.assign(num_tile_rows * num_k_tiles * VX_CFG_NUM_THREADS, 0); for (uint32_t tr = 0; tr < num_tile_rows; ++tr) { @@ -91,10 +89,8 @@ static void pack_mx_b_metadata(std::vector &packed, uint32_t num_tile_cols = N / cfg::tileN; uint32_t logical_tileK = cfg::tileK * kElemsPerByte; uint32_t num_k_tiles = K_logical / logical_tileK; - uint32_t tile_scale_blocks_k = logical_tileK / vt::ITYPE::ele_block; - if (tile_scale_blocks_k == 0) { - tile_scale_blocks_k = 1; - } + uint32_t tile_scale_blocks_k = + std::max(1u, (logical_tileK + vt::ITYPE::ele_block - 1) / vt::ITYPE::ele_block); packed.assign(num_tile_cols * num_k_tiles * VX_CFG_NUM_THREADS, 0); for (uint32_t tc = 0; tc < num_tile_cols; ++tc) { diff --git a/tests/regression/sgemm_tcu_wg/Makefile b/tests/regression/sgemm_tcu_wg/Makefile index 66934cf47..d67265b4c 100644 --- a/tests/regression/sgemm_tcu_wg/Makefile +++ b/tests/regression/sgemm_tcu_wg/Makefile @@ -13,11 +13,9 @@ VX_SRCS := $(SRC_DIR)/kernel.cpp CONFIGS := $(if $(findstring -DVX_CFG_NUM_WARPS=,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_NUM_WARPS=4) CONFIGS := $(if $(findstring -DVX_CFG_ISSUE_WIDTH=,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_ISSUE_WIDTH=4) -CONFIGS += -DVX_CFG_TCU_WGMMA_ENABLE -DWGMMA_RS +CONFIGS += -DVX_CFG_TCU_WGMMA_ENABLE +CONFIGS := $(if $(findstring -DWGMMA_SS,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DWGMMA_RS) -# RTLsim is roughly two orders of magnitude slower than SimX; keep the -# default matrix size small enough for a 60s sweep budget. Override -# -m/-n/-k at the command line for full-size correctness sweeps. OPTS := $(OPTS) $(if $(findstring -m,$(OPTS)),,-m 16) OPTS := $(OPTS) $(if $(findstring -n,$(OPTS)),,-n 16) OPTS := $(OPTS) $(if $(findstring -k,$(OPTS)),,-k 16) diff --git a/tests/regression/sgemm_tcu_wg_mx/Makefile b/tests/regression/sgemm_tcu_wg_mx/Makefile new file mode 100644 index 000000000..7b985e493 --- /dev/null +++ b/tests/regression/sgemm_tcu_wg_mx/Makefile @@ -0,0 +1,23 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +CONFIGS := $(if $(findstring -DVX_CFG_EXT_TCU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_TCU_ENABLE) +CONFIGS := $(if $(findstring -DVX_CFG_TCU_MX_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_TCU_MX_ENABLE) +CONFIGS := $(if $(findstring -DVX_CFG_NUM_WARPS=,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_NUM_WARPS=4) +CONFIGS := $(if $(findstring -DVX_CFG_ISSUE_WIDTH=,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_ISSUE_WIDTH=4) +CONFIGS += -DVX_CFG_TCU_WGMMA_ENABLE +CONFIGS := $(if $(findstring -DWGMMA_SS,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DWGMMA_RS) + +PROJECT := sgemm_tcu_wg_mx +SRC_DIR := $(VORTEX_HOME)/tests/regression/$(PROJECT) +SRCS := $(SRC_DIR)/main.cpp $(SW_COMMON_DIR)/rvfloats.cpp $(SW_COMMON_DIR)/softfloat_ext.cpp +VX_SRCS := $(SRC_DIR)/kernel.cpp + +OPTS := $(OPTS) $(if $(findstring -m,$(OPTS)),,-m 64) +OPTS := $(OPTS) $(if $(findstring -n,$(OPTS)),,-n 64) +OPTS := $(OPTS) $(if $(findstring -k,$(OPTS)),,-k 64) +CXXFLAGS += -I$(THIRD_PARTY_DIR)/softfloat/source/include +LDFLAGS += $(THIRD_PARTY_DIR)/softfloat/build/Linux-x86_64-GCC/softfloat.a +KERNEL_LIB := vortex2 + +include ../common.mk diff --git a/tests/regression/sgemm_tcu_wg_mx/common.h b/tests/regression/sgemm_tcu_wg_mx/common.h new file mode 100644 index 000000000..7120f73aa --- /dev/null +++ b/tests/regression/sgemm_tcu_wg_mx/common.h @@ -0,0 +1,25 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +#ifndef WGMMA_NRC +#define WGMMA_NRC 8 +#endif +#ifndef ITYPE +#define ITYPE mxfp8 +#endif +#ifndef OTYPE +#define OTYPE fp32 +#endif + +typedef struct { + uint32_t M, N, K; + uint64_t A_addr, B_addr, C_addr; + uint64_t MX_A_addr, MX_B_addr; +#ifdef VX_CFG_TCU_MX_TLS + uint64_t A_tensor_scale_addr, B_tensor_scale_addr; +#endif +} kernel_arg_t; + +#endif diff --git a/tests/regression/sgemm_tcu_wg_mx/kernel.cpp b/tests/regression/sgemm_tcu_wg_mx/kernel.cpp new file mode 100644 index 000000000..fe4b67ad7 --- /dev/null +++ b/tests/regression/sgemm_tcu_wg_mx/kernel.cpp @@ -0,0 +1,83 @@ +#include "common.h" +#include +#include +#include + +namespace vt = vortex::tensor; +using ctx = vt::wgmma_context; + +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + static_assert(sizeof(ctx::input_t) == 1, "MX WGMMA inputs use byte storage"); + auto pA = reinterpret_cast(arg->A_addr); + auto pB = reinterpret_cast(arg->B_addr); + auto pC = reinterpret_cast(arg->C_addr); + auto pMxA = reinterpret_cast(arg->MX_A_addr); + auto pMxB = reinterpret_cast(arg->MX_B_addr); +#ifdef VX_CFG_TCU_MX_TLS + auto pATensorScale = reinterpret_cast(arg->A_tensor_scale_addr); + auto pBTensorScale = reinterpret_cast(arg->B_tensor_scale_addr); +#endif + + uint32_t N = arg->N; + uint32_t K = arg->K; + uint32_t tid = threadIdx.x; + uint32_t num_threads = blockDim.x; + uint32_t warp_rank = tid / VX_CFG_NUM_THREADS; + uint32_t num_warps = num_threads / VX_CFG_NUM_THREADS; + uint32_t cta_m = num_warps * ctx::xtileM; + uint32_t tile_row = blockIdx.y * cta_m; + uint32_t tile_col = blockIdx.x * ctx::xtileN; + uint32_t num_k_tiles = K / ctx::tileK; + + auto A_smem = reinterpret_cast(__local_mem()); + auto B_smem = A_smem + cta_m * ctx::tileK; + ctx::fragment_a fragA; + ctx::fragment_b fragB; + ctx::fragment_acc fragC; + ctx::fill_fragment(fragC, 0); + + for (uint32_t k = 0; k < K; k += ctx::tileK) { + uint32_t k_tile = k / ctx::tileK; + uint32_t a_bytes = cta_m * ctx::tileK; + for (uint32_t idx = tid; idx < a_bytes; idx += num_threads) { + uint32_t row = idx / ctx::tileK; + uint32_t col = idx % ctx::tileK; + A_smem[idx] = pA[(tile_row + row) * K + k + col]; + } + uint32_t b_bytes = ctx::xtileN * ctx::tileK; + for (uint32_t idx = tid; idx < b_bytes; idx += num_threads) { + uint32_t col = idx / ctx::tileK; + uint32_t k_byte = idx % ctx::tileK; + B_smem[idx] = pB[(tile_col + col) * K + k + k_byte]; + } + __syncthreads(); + + uint32_t tile_m = blockIdx.y * num_warps + warp_rank; + auto pTileMxA = pMxA + (tile_m * num_k_tiles + k_tile) * VX_CFG_NUM_THREADS; + auto pTileMxB = pMxB + (blockIdx.x * num_k_tiles + k_tile) * VX_CFG_NUM_THREADS; + ctx::load_mx_metadata(fragA, pTileMxA); + ctx::load_mx_metadata(fragB, pTileMxB); + + auto A_warp = A_smem + warp_rank * ctx::xtileM * ctx::tileK; + auto desc_b = vt::vx_make_smem_desc(B_smem, ctx::tileK); +#if defined(WGMMA_RS) && (WGMMA_NRC <= 16) + ctx::load_matrix_sync(fragA, A_warp, ctx::tileK); + ctx::wgmma_sync(fragC, fragA, desc_b, fragC); +#else + auto desc_a = vt::vx_make_smem_desc(A_warp, ctx::tileK); + ctx::wgmma_sync(fragC, desc_a, desc_b, fragC); +#endif + __syncthreads(); + } + +#ifdef VX_CFG_TCU_MX_TLS + if constexpr (std::is_same::value) { + float tensor_scale = (*pATensorScale) * (*pBTensorScale); + for (uint32_t r = 0; r < ctx::fragment_acc::NR; ++r) + fragC.data[r] *= tensor_scale; + } +#endif + auto out = pC + (tile_row + warp_rank * ctx::xtileM) * N + tile_col; + ctx::store_matrix_sync(out, fragC, N); +} diff --git a/tests/regression/sgemm_tcu_wg_mx/main.cpp b/tests/regression/sgemm_tcu_wg_mx/main.cpp new file mode 100644 index 000000000..5b3e80164 --- /dev/null +++ b/tests/regression/sgemm_tcu_wg_mx/main.cpp @@ -0,0 +1,465 @@ +#include "common.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MAX_ERRORS 100 + +#define RT_CHECK(_expr) \ + do { \ + int _ret = _expr; \ + if (0 == _ret) \ + break; \ + printf("Error: '%s' returned %d!\n", #_expr, (int)_ret); \ + cleanup(); \ + exit(-1); \ + } while (false) + +using namespace vortex; +namespace vt = tensor; + +using cfg = vt::wgmma_config_t; +static constexpr uint32_t kTileM = cfg::xtileM; +static constexpr uint32_t kTileN = cfg::xtileN; +using itype_t = typename vt::ITYPE::dtype; +using otype_t = typename vt::OTYPE::dtype; + +static constexpr uint32_t kElemsPerByte = (vt::ITYPE::bits < 8) ? (8 / vt::ITYPE::bits) : 1; +static constexpr uint32_t kScalePack = 4; + +static uint8_t read_nibble(const uint8_t *ptr, uint32_t offset) { + uint8_t value = ptr[offset / 2]; + return (offset & 1) ? (value >> 4) : (value & 0x0f); +} + +static uint32_t pack4(const std::vector &scales, uint32_t start) { + uint32_t word = 0; + for (uint32_t i = 0; i < kScalePack; ++i) { + if ((start + i) < scales.size()) { + word |= static_cast(scales[start + i]) << (8 * i); + } + } + return word; +} + +static void pack_mx_a_metadata(std::vector &packed, + const std::vector &scales, + uint32_t M, + uint32_t K_logical) { + uint32_t num_tile_rows = M / kTileM; + uint32_t logical_tileK = cfg::tileK * kElemsPerByte; + uint32_t num_k_tiles = K_logical / logical_tileK; + uint32_t scale_blocks_k = K_logical / vt::ITYPE::ele_block; + uint32_t tile_scale_blocks_k = + std::max(1u, (logical_tileK + vt::ITYPE::ele_block - 1) / vt::ITYPE::ele_block); + + packed.assign(num_tile_rows * num_k_tiles * VX_CFG_NUM_THREADS, 0); + for (uint32_t tr = 0; tr < num_tile_rows; ++tr) { + for (uint32_t kt = 0; kt < num_k_tiles; ++kt) { + std::vector tile_scales; + tile_scales.reserve(kTileM * tile_scale_blocks_k); + uint32_t first_block = (kt * logical_tileK) / vt::ITYPE::ele_block; + for (uint32_t m = 0; m < kTileM; ++m) { + uint32_t row = tr * kTileM + m; + for (uint32_t kb = 0; kb < tile_scale_blocks_k; ++kb) { + tile_scales.push_back(scales[row * scale_blocks_k + first_block + kb]); + } + } + uint32_t base = (tr * num_k_tiles + kt) * VX_CFG_NUM_THREADS; + for (uint32_t w = 0; w < VX_CFG_NUM_THREADS && (w * kScalePack) < tile_scales.size(); ++w) { + packed[base + w] = pack4(tile_scales, w * kScalePack); + } + } + } +} + +static void pack_mx_b_metadata(std::vector &packed, + const std::vector &scales, + uint32_t N, + uint32_t K_logical) { + uint32_t num_tile_cols = N / kTileN; + uint32_t logical_tileK = cfg::tileK * kElemsPerByte; + uint32_t num_k_tiles = K_logical / logical_tileK; + uint32_t tile_scale_blocks_k = + std::max(1u, (logical_tileK + vt::ITYPE::ele_block - 1) / vt::ITYPE::ele_block); + + packed.assign(num_tile_cols * num_k_tiles * VX_CFG_NUM_THREADS, 0); + for (uint32_t tc = 0; tc < num_tile_cols; ++tc) { + for (uint32_t kt = 0; kt < num_k_tiles; ++kt) { + std::vector tile_scales; + tile_scales.reserve(kTileN * tile_scale_blocks_k); + uint32_t first_block = (kt * logical_tileK) / vt::ITYPE::ele_block; + for (uint32_t n = 0; n < kTileN; ++n) { + uint32_t col = tc * kTileN + n; + for (uint32_t kb = 0; kb < tile_scale_blocks_k; ++kb) { + tile_scales.push_back(scales[(first_block + kb) * N + col]); + } + } + uint32_t base = (tc * num_k_tiles + kt) * VX_CFG_NUM_THREADS; + for (uint32_t w = 0; w < VX_CFG_NUM_THREADS && (w * kScalePack) < tile_scales.size(); ++w) { + packed[base + w] = pack4(tile_scales, w * kScalePack); + } + } + } +} + +static float dequantize_mx_value(const itype_t *data, + const std::vector &scales, + uint32_t offset, + uint32_t scale_index, + float tensor_scale) { + uint8_t sf = scales[scale_index]; + switch (vt::ITYPE::id) { + case vt::mxfp8::id: + return bit_cast(rv_mxfp8tof_s(data[offset], sf, 0, nullptr)); + case vt::mxbf8::id: + return bit_cast(rv_mxbf8tof_s(data[offset], sf, 0, nullptr)); + case vt::mxint8::id: + { + float scale = std::ldexp(1.0f, static_cast(sf) - 127); + return (static_cast(data[offset]) / 64.0f) * scale; + } + case vt::mxfp4::id: + { + uint8_t q = read_nibble(reinterpret_cast(data), offset); + return bit_cast(rv_mxfp4tof_s(q, sf, 0, nullptr)); + } + case vt::nvfp4::id: + { + uint8_t q = read_nibble(reinterpret_cast(data), offset); + return bit_cast(rv_nvfp4tof_s(q, sf, 0, nullptr)) * tensor_scale; + } + default: + std::abort(); + } +} + +static int32_t trunc_shift(int32_t value, int32_t shift) { + if (shift >= 0) { + return value << shift; + } + uint32_t abs_shift = static_cast(-shift); + uint32_t mag = value < 0 ? static_cast(-value) : static_cast(value); + int32_t scaled = static_cast(mag >> abs_shift); + return value < 0 ? -scaled : scaled; +} + +static void matmul_cpu(otype_t *C, + const itype_t *A, + const itype_t *B, + const std::vector &scale_a, + const std::vector &scale_b, + float tensor_scale_a, + float tensor_scale_b, + uint32_t M, + uint32_t N, + uint32_t K_logical) { + uint32_t scale_blocks_k = K_logical / vt::ITYPE::ele_block; + for (uint32_t m = 0; m < M; ++m) { + for (uint32_t n = 0; n < N; ++n) { + if constexpr (std::is_same::value) { + int32_t sum = 0; + for (uint32_t k = 0; k < K_logical; ++k) { + uint32_t scale_k = k / vt::ITYPE::ele_block; + uint8_t sf_a = scale_a[m * scale_blocks_k + scale_k]; + uint8_t sf_b = scale_b[scale_k * N + n]; + int32_t shift = static_cast(sf_a) - 133 + static_cast(sf_b) - 133; + int32_t product = static_cast(A[m * K_logical + k]) * static_cast(B[n * K_logical + k]); + sum += trunc_shift(product, shift); + } + C[m * N + n] = sum; + } else { + float sum = 0.0f; + for (uint32_t k = 0; k < K_logical; ++k) { + uint32_t scale_k = k / vt::ITYPE::ele_block; + auto a = dequantize_mx_value(A, scale_a, m * K_logical + k, + m * scale_blocks_k + scale_k, tensor_scale_a); + auto b = dequantize_mx_value(B, scale_b, n * K_logical + k, + scale_k * N + n, tensor_scale_b); + sum += a * b; + } + C[m * N + n] = sum; + } + } + } +} + +template +static bool quantize_inputs(typename FormatT::dtype *A, + typename FormatT::dtype *B, + std::vector &scale_a, + std::vector &scale_b, + float &A_tensor_scale, + float &B_tensor_scale, + const float *A_dense, + const float *B_dense, + uint32_t M, + uint32_t N, + uint32_t K_logical) { + if constexpr (std::is_same::value) { + return vt::quantize_mx_a_rowmajor( + reinterpret_cast(A), scale_a, A_tensor_scale, + A_dense, M, K_logical) + && vt::quantize_mx_b_colmajor( + reinterpret_cast(B), scale_b, B_tensor_scale, + B_dense, K_logical, N); + } else { + return vt::quantize_mx_a_rowmajor( + A, scale_a, A_dense, M, K_logical) + && vt::quantize_mx_b_colmajor( + B, scale_b, B_dense, K_logical, N); + } +} + +const char *kernel_file = "kernel.vxbin"; + +uint32_t xm = 32; +uint32_t xn = 32; +uint32_t xk = 32; + +vx_device_h device = nullptr; +vx_queue_h queue = nullptr; +vx_module_h module_ = nullptr; +vx_kernel_h kernel = nullptr; +vx_buffer_h A_buffer = nullptr; +vx_buffer_h B_buffer = nullptr; +vx_buffer_h C_buffer = nullptr; +vx_buffer_h MX_A_buffer = nullptr; +vx_buffer_h MX_B_buffer = nullptr; +#ifdef VX_CFG_TCU_MX_TLS +vx_buffer_h A_tensor_scale_buffer = nullptr; +vx_buffer_h B_tensor_scale_buffer = nullptr; +#endif +kernel_arg_t kernel_arg = {}; + +static void show_usage() { + std::cout << "Vortex SGEMM TCU MX Test." << std::endl; + std::cout << "Usage: [-m M] [-n N] [-k K] [-h]" << std::endl; +} + +static void parse_args(int argc, char **argv) { + int c; + while ((c = getopt(argc, argv, "m:n:k:h")) != -1) { + switch (c) { + case 'm': xm = atoi(optarg); break; + case 'n': xn = atoi(optarg); break; + case 'k': xk = atoi(optarg); break; + case 'h': + show_usage(); + exit(0); + default: + show_usage(); + exit(-1); + } + } +} + +void cleanup() { + if (device) { + vx_mem_free(A_buffer); + vx_mem_free(B_buffer); + vx_mem_free(C_buffer); + vx_mem_free(MX_A_buffer); + vx_mem_free(MX_B_buffer); +#ifdef VX_CFG_TCU_MX_TLS + vx_mem_free(A_tensor_scale_buffer); + vx_mem_free(B_tensor_scale_buffer); +#endif + if (kernel) vx_kernel_release(kernel); + if (module_) vx_module_release(module_); + if (queue) vx_queue_release(queue); + vx_dump_perf(device, stdout); + vx_dev_close(device); + } +} + +int main(int argc, char *argv[]) { + parse_args(argc, argv); + std::srand(50); + + RT_CHECK(vx_dev_open(&device)); + vx_queue_info_t qi = {sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0}; + RT_CHECK(vx_queue_create(device, &qi, &queue)); + + uint64_t isa_flags; + RT_CHECK(vx_dev_caps(device, VX_CAPS_ISA_FLAGS, &isa_flags)); + if ((isa_flags & VX_ISA_EXT_TCU) == 0) { + std::cout << "TCU extension not supported!" << std::endl; + cleanup(); + return -1; + } + + uint64_t NT; + RT_CHECK(vx_dev_caps(device, VX_CAPS_NUM_THREADS, &NT)); + if (NT != VX_CFG_NUM_THREADS) { + std::cout << "Error: device warp size (" << NT << ") must match VX_CFG_NUM_THREADS=" << VX_CFG_NUM_THREADS << "!" << std::endl; + return -1; + } + + uint32_t M = xm; + uint32_t N = xn; + uint32_t K_logical = xk; + uint32_t logical_tileK = cfg::tileK * kElemsPerByte; + uint32_t K_storage = K_logical / kElemsPerByte; + + uint32_t warps_per_cta = VX_CFG_ISSUE_WIDTH; + uint32_t cta_m = warps_per_cta * kTileM; + if ((M % cta_m) != 0 || (N % kTileN) != 0 || (K_logical % logical_tileK) != 0) { + std::cout << "Error: M/N/K must be multiples of CTA M=" << cta_m + << " N=" << kTileN << " K=" << logical_tileK << std::endl; + return -1; + } + if ((K_logical % vt::ITYPE::ele_block) != 0) { + std::cout << "Error: K must be a multiple of MX block size=" << vt::ITYPE::ele_block << std::endl; + return -1; + } + + size_t sizeA = M * K_storage; + size_t sizeB = K_storage * N; + size_t sizeC = M * N; + uint32_t grid_dim[2] = {N / kTileN, M / cta_m}; + uint32_t block_dim[2] = {warps_per_cta * (uint32_t)NT, 1}; + + std::cout << "input data type: " << vt::ITYPE::name << " (id=" << vt::ITYPE::id << ")" << std::endl; + std::cout << "output data type: " << vt::OTYPE::name << " (id=" << vt::OTYPE::id << ")" << std::endl; + std::cout << "MMA Tile Dimension: M=" << kTileM << ", N=" << kTileN + << ", K(logical)=" << logical_tileK << ", K(storage)=" << cfg::tileK << std::endl; + + kernel_arg.M = M; + kernel_arg.N = N; + kernel_arg.K = K_storage; +#ifdef VX_CFG_TCU_MX_TLS + kernel_arg.A_tensor_scale_addr = 0; + kernel_arg.B_tensor_scale_addr = 0; +#endif + float A_tensor_scale = 1.0f; + float B_tensor_scale = 1.0f; + + RT_CHECK(vx_mem_alloc(device, sizeA * sizeof(itype_t), VX_MEM_READ, &A_buffer)); + RT_CHECK(vx_mem_address(A_buffer, &kernel_arg.A_addr)); + RT_CHECK(vx_mem_alloc(device, sizeB * sizeof(itype_t), VX_MEM_READ, &B_buffer)); + RT_CHECK(vx_mem_address(B_buffer, &kernel_arg.B_addr)); + RT_CHECK(vx_mem_alloc(device, sizeC * sizeof(otype_t), VX_MEM_WRITE, &C_buffer)); + RT_CHECK(vx_mem_address(C_buffer, &kernel_arg.C_addr)); + + std::vector h_A_dense(M * K_logical); + std::vector h_B_dense(K_logical * N); + for (auto &v : h_A_dense) { + v = (static_cast(std::rand()) / RAND_MAX) * 2.0f - 1.0f; + } + for (auto &v : h_B_dense) { + v = (static_cast(std::rand()) / RAND_MAX) * 2.0f - 1.0f; + } + + std::vector h_A(sizeA); + std::vector h_B(sizeB); + std::vector scale_a; + std::vector scale_b; + + bool ok = quantize_inputs( + h_A.data(), h_B.data(), scale_a, scale_b, A_tensor_scale, B_tensor_scale, + h_A_dense.data(), h_B_dense.data(), M, N, K_logical); + if (!ok) { + std::cout << "Error: MX quantization failed!" << std::endl; + return -1; + } + + std::vector h_mx_a; + std::vector h_mx_b; + pack_mx_a_metadata(h_mx_a, scale_a, M, K_logical); + pack_mx_b_metadata(h_mx_b, scale_b, N, K_logical); + + RT_CHECK(vx_mem_alloc(device, h_mx_a.size() * sizeof(uint32_t), VX_MEM_READ, &MX_A_buffer)); + RT_CHECK(vx_mem_address(MX_A_buffer, &kernel_arg.MX_A_addr)); + RT_CHECK(vx_mem_alloc(device, h_mx_b.size() * sizeof(uint32_t), VX_MEM_READ, &MX_B_buffer)); + RT_CHECK(vx_mem_address(MX_B_buffer, &kernel_arg.MX_B_addr)); +#ifdef VX_CFG_TCU_MX_TLS + if constexpr (std::is_same::value) { + RT_CHECK(vx_mem_alloc(device, sizeof(float), VX_MEM_READ, &A_tensor_scale_buffer)); + RT_CHECK(vx_mem_address(A_tensor_scale_buffer, &kernel_arg.A_tensor_scale_addr)); + RT_CHECK(vx_mem_alloc(device, sizeof(float), VX_MEM_READ, &B_tensor_scale_buffer)); + RT_CHECK(vx_mem_address(B_tensor_scale_buffer, &kernel_arg.B_tensor_scale_addr)); + } +#endif + + RT_CHECK(vx_copy_to_dev(A_buffer, h_A.data(), 0, sizeA * sizeof(itype_t))); + RT_CHECK(vx_copy_to_dev(B_buffer, h_B.data(), 0, sizeB * sizeof(itype_t))); + RT_CHECK(vx_copy_to_dev(MX_A_buffer, h_mx_a.data(), 0, h_mx_a.size() * sizeof(uint32_t))); + RT_CHECK(vx_copy_to_dev(MX_B_buffer, h_mx_b.data(), 0, h_mx_b.size() * sizeof(uint32_t))); +#ifdef VX_CFG_TCU_MX_TLS + if constexpr (std::is_same::value) { + RT_CHECK(vx_copy_to_dev(A_tensor_scale_buffer, &A_tensor_scale, 0, sizeof(float))); + RT_CHECK(vx_copy_to_dev(B_tensor_scale_buffer, &B_tensor_scale, 0, sizeof(float))); + } +#endif + + RT_CHECK(vx_module_load_file(device, kernel_file, &module_)); + RT_CHECK(vx_module_get_kernel(module_, "main", &kernel)); + + auto time_start = std::chrono::high_resolution_clock::now(); + vx_launch_info_t li = {}; + li.struct_size = sizeof(li); + li.kernel = kernel; + li.args_host = &kernel_arg; + li.args_size = sizeof(kernel_arg); + li.ndim = 2; + li.grid_dim[0] = grid_dim[0]; + li.grid_dim[1] = grid_dim[1]; + li.block_dim[0] = block_dim[0]; + li.block_dim[1] = block_dim[1]; + li.lmem_size = (cta_m + kTileN) * cfg::tileK * sizeof(itype_t); + + vx_event_h launch_ev = nullptr; + RT_CHECK(vx_enqueue_launch(queue, &li, 0, nullptr, &launch_ev)); + RT_CHECK(vx_event_wait_value(launch_ev, 1, VX_TIMEOUT_INFINITE)); + vx_event_release(launch_ev); + auto time_end = std::chrono::high_resolution_clock::now(); + double elapsed = std::chrono::duration_cast(time_end - time_start).count(); + printf("Elapsed time: %lg ms\n", elapsed); + + std::vector h_C(sizeC); + RT_CHECK(vx_copy_from_dev(h_C.data(), C_buffer, 0, sizeC * sizeof(otype_t))); + + std::vector h_ref(sizeC); + matmul_cpu(h_ref.data(), h_A.data(), h_B.data(), scale_a, scale_b, + A_tensor_scale, B_tensor_scale, M, N, K_logical); + + int errors = 0; + float rel_tol = std::is_same::value ? 0.0f : + (std::is_same::value || std::is_same::value) ? 0.25f : 0.05f; + for (uint32_t i = 0; i < h_ref.size(); ++i) { + float actual = static_cast(h_C[i]); + float expected = static_cast(h_ref[i]); + float diff = std::abs(actual - expected); + float tol = std::is_same::value ? 0.0f : + std::max(1.0e-3f, std::abs(expected) * rel_tol); + if (!std::isfinite(actual) || !std::isfinite(expected) || !std::isfinite(diff) || diff > tol) { + if (errors < MAX_ERRORS) { + printf("*** error: [%d] expected=%f, actual=%f, diff=%f, tol=%f\n", + i, expected, actual, diff, tol); + } + ++errors; + } + } + + cleanup(); + if (errors != 0) { + std::cout << "Found " << errors << " / " << sizeC << " errors!" << std::endl; + std::cout << "FAILED!" << std::endl; + return errors; + } + + std::cout << "PASSED!" << std::endl; + return 0; +} diff --git a/tests/regression/sgemm_tcu_wg_sp/Makefile b/tests/regression/sgemm_tcu_wg_sp/Makefile index fa25578b9..54bb27c1a 100644 --- a/tests/regression/sgemm_tcu_wg_sp/Makefile +++ b/tests/regression/sgemm_tcu_wg_sp/Makefile @@ -13,7 +13,10 @@ VX_SRCS := $(SRC_DIR)/kernel.cpp CONFIGS := $(if $(findstring -DVX_CFG_NUM_WARPS=,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_NUM_WARPS=4) CONFIGS := $(if $(findstring -DVX_CFG_ISSUE_WIDTH=,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_ISSUE_WIDTH=4) -CONFIGS += -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_RS +CONFIGS += -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE +CONFIGS := $(if $(findstring -DITYPE=,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DITYPE=fp16) +CONFIGS := $(if $(findstring -DOTYPE=,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DOTYPE=fp32) +CONFIGS := $(if $(findstring -DWGMMA_SS,$(CONFIGS)),$(CONFIGS),$(if $(findstring -DWGMMA_RS,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DWGMMA_RS)) CXXFLAGS += -I$(THIRD_PARTY_DIR)/softfloat/source/include diff --git a/tests/regression/sgemm_tcu_wg_sp/main.cpp b/tests/regression/sgemm_tcu_wg_sp/main.cpp index 372f5338a..30b2a1324 100644 --- a/tests/regression/sgemm_tcu_wg_sp/main.cpp +++ b/tests/regression/sgemm_tcu_wg_sp/main.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -36,7 +37,7 @@ static constexpr uint32_t kTcK = wg_cfg_t::tcK; static constexpr uint32_t kTcM = wg_cfg_t::tcM; static constexpr uint32_t kMSteps = wg_cfg_t::m_steps; static constexpr uint32_t kKSteps = wg_cfg_t::k_steps; -static constexpr uint32_t kHalfKSteps = kKSteps / 2; +static constexpr uint32_t kHalfKSteps = (kKSteps > 1) ? (kKSteps / 2) : 1; static constexpr uint32_t kMetaRowBits = kTcK * 2 * kRtlIRatio; static constexpr uint32_t kMetaStrWords = (kTcM * kMetaRowBits + 31) / 32; static constexpr uint32_t kWgMetaBanks = kMSteps * kHalfKSteps; @@ -60,6 +61,67 @@ static constexpr uint32_t kDensePerSpStep = kTcK * kRtlIRatio * 2; using itype_t = vt::ITYPE::dtype; using otype_t = vt::OTYPE::dtype; +template +struct type_ops { + using dtype = typename T::dtype; + + static dtype generate() { + return dtype(float(rand()) / RAND_MAX); + } + + static float to_float(dtype value) { + return static_cast(value); + } +}; + +template <> +struct type_ops { + static uint16_t generate() { + auto value = float(rand()) / RAND_MAX; + return rv_ftoh_s(bit_cast(value), 0, nullptr); + } + + static float to_float(uint16_t value) { + return bit_cast(rv_htof_s(value, 0, nullptr)); + } +}; + +template <> +struct type_ops { + static uint16_t generate() { + auto value = float(rand()) / RAND_MAX; + return rv_ftob_s(bit_cast(value), 0, nullptr); + } + + static float to_float(uint16_t value) { + return bit_cast(rv_btof_s(value, 0, nullptr)); + } +}; + +template <> +struct type_ops { + static uint8_t generate() { + auto value = float(rand()) / RAND_MAX; + return rv_ftoe4m3_s(bit_cast(value), 0, nullptr); + } + + static float to_float(uint8_t value) { + return bit_cast(rv_e4m3tof_s(value, 0, nullptr)); + } +}; + +template <> +struct type_ops { + static uint8_t generate() { + auto value = float(rand()) / RAND_MAX; + return rv_ftoe5m2_s(bit_cast(value), 0, nullptr); + } + + static float to_float(uint8_t value) { + return bit_cast(rv_e5m2tof_s(value, 0, nullptr)); + } +}; + // CPU reference matmul using pruned (zero-padded) A static void matmul_cpu(otype_t *C, const itype_t *A_pruned, const itype_t *B, uint32_t M, uint32_t N, uint32_t K) { @@ -69,8 +131,8 @@ static void matmul_cpu(otype_t *C, const itype_t *A_pruned, const itype_t *B, for (uint32_t k = 0; k < K; ++k) { auto a = A_pruned[m * K + k]; auto b = B[k * N + n]; - auto fa = bit_cast(rv_htof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_htof_s(b, 0, nullptr)); + auto fa = type_ops::to_float(a); + auto fb = type_ops::to_float(b); sum += fa * fb; } C[m * N + n] = sum; @@ -289,12 +351,10 @@ int main(int argc, char *argv[]) { std::vector h_A_full(sizeA_full); std::vector h_B(sizeB); for (uint32_t i = 0; i < sizeA_full; ++i) { - auto fv = float(rand()) / RAND_MAX; - h_A_full[i] = rv_ftoh_s(bit_cast(fv), 0, nullptr); + h_A_full[i] = type_ops::generate(); } for (uint32_t i = 0; i < sizeB; ++i) { - auto fv = float(rand()) / RAND_MAX; - h_B[i] = rv_ftoh_s(bit_cast(fv), 0, nullptr); + h_B[i] = type_ops::generate(); } // prune A in-place (2:4) then compress @@ -380,12 +440,20 @@ int main(int argc, char *argv[]) { int errors = 0; for (uint32_t i = 0; i < sizeC; ++i) { - union fi_t { float f; int32_t i; }; - fi_t fa, fb; - fa.f = h_C[i]; - fb.f = h_ref[i]; - auto d = std::abs(fa.i - fb.i); - if (d > FLOAT_ULP) { + bool match; + if constexpr (std::is_same::value || + std::is_same::value) { + match = (h_C[i] == 0.0f && h_ref[i] == 0.0f) + || (std::abs((h_C[i] - h_ref[i]) / h_ref[i]) < 0.01f); + } else { + union fi_t { float f; int32_t i; }; + fi_t fa, fb; + fa.f = h_C[i]; + fb.f = h_ref[i]; + auto d = std::abs(fa.i - fb.i); + match = d <= FLOAT_ULP; + } + if (!match) { if (errors < MAX_ERRORS) { printf("*** error: [%u] expected=%f, actual=%f\n", i, h_ref[i], h_C[i]); }