From fc09f729a3ead5efcdd7b7ff579776820e0f5443 Mon Sep 17 00:00:00 2001 From: Yvan Tortorella Date: Tue, 5 Mar 2024 14:41:29 +0100 Subject: [PATCH] Add clic virtualization and LLC partitioning. --- Bender.lock | 2 +- Bender.yml | 2 +- cheshire.mk | 17 +++++ hw/bootrom/cheshire_bootrom.S | 1 + hw/cheshire_pkg.sv | 20 ++++-- hw/cheshire_soc.sv | 42 ++++++++++-- target/sim/src/vip_cheshire_soc.sv | 83 ++++++++++++++++++++++++ target/xilinx/src/cheshire_top_xilinx.sv | 6 ++ 8 files changed, 163 insertions(+), 10 deletions(-) diff --git a/Bender.lock b/Bender.lock index 8a9ef0ed..8313c66d 100644 --- a/Bender.lock +++ b/Bender.lock @@ -92,7 +92,7 @@ packages: Git: https://github.com/pulp-platform/common_verification.git dependencies: [] cva6: - revision: 72f3219fc57e158e41adede05cd2e65a42db97bc + revision: dbaecc59b99e3bb4d89fed546d75080f39bfe4f1 version: null source: Git: https://github.com/pulp-platform/cva6.git diff --git a/Bender.yml b/Bender.yml index c6347fb1..642b33fd 100644 --- a/Bender.yml +++ b/Bender.yml @@ -22,7 +22,7 @@ dependencies: clint: { git: "https://github.com/pulp-platform/clint.git", version: 0.2.0 } common_cells: { git: "https://github.com/pulp-platform/common_cells.git", version: 1.32.0 } common_verification: { git: "https://github.com/pulp-platform/common_verification.git", version: 0.2.0 } - cva6: { git: "https://github.com/pulp-platform/cva6.git", rev: "72f3219" } # branch: astral-new + cva6: { git: "https://github.com/pulp-platform/cva6.git", rev: "dbaecc5" } # branch: astral iDMA: { git: "https://github.com/pulp-platform/iDMA.git", version: 0.5.1 } irq_router: { git: "https://github.com/pulp-platform/irq_router.git", version: 0.0.1-beta.1 } opentitan_peripherals: { git: "https://github.com/pulp-platform/opentitan_peripherals.git", version: 0.4.0 } diff --git a/cheshire.mk b/cheshire.mk index 43cc8282..fe1b2d6e 100644 --- a/cheshire.mk +++ b/cheshire.mk @@ -11,11 +11,16 @@ BENDER ?= bender VLOG_ARGS ?= -suppress 2583 -suppress 13314 VSIM ?= vsim +MAXPARTITION ?= 16 +CACHE_PARTITION ?= 1 + + # Define used paths (prefixed to avoid name conflicts) CHS_ROOT ?= $(shell $(BENDER) path cheshire) CHS_REG_DIR := $(shell $(BENDER) path register_interface) CHS_SLINK_DIR := $(shell $(BENDER) path serial_link) CHS_LLC_DIR := $(shell $(BENDER) path axi_llc) +CHS_TAGGER_DIR := $(shell $(BENDER) path tagger) # Define paths used in dependencies OTPROOT := $(shell $(BENDER) path opentitan_peripherals) @@ -103,12 +108,24 @@ $(CHS_SLINK_DIR)/.generated: $(CHS_ROOT)/hw/serial_link.hjson cp $< $(dir $@)/src/regs/serial_link_single_channel.hjson flock -x $@ $(MAKE) -C $(CHS_SLINK_DIR) update-regs BENDER="$(BENDER)" && touch $@ +# LLC partitioning configuration +$(CHS_LLC_DIR)/.generated: + $(MAKE) -C $(CHS_LLC_DIR) REGWIDTH=64 CACHENUMLINES=256 MAXPARTITION=$(MAXPARTITION) CACHE_PARTITION=$(CACHE_PARTITION) regs + @touch $@ + +# Tagger configuration +$(CHS_TAGGER_DIR)/.generated: + $(MAKE) -C $(CHS_TAGGER_DIR) REGWIDTH=32 MAXPARTITION=$(MAXPARTITION) PATID_LEN=5 regs + @touch $@ + CHS_HW_ALL += $(CHS_ROOT)/hw/regs/cheshire_reg_pkg.sv $(CHS_ROOT)/hw/regs/cheshire_reg_top.sv CHS_HW_ALL += $(CLINTROOT)/.generated CHS_HW_ALL += $(OTPROOT)/.generated CHS_HW_ALL += $(AXIRTROOT)/.generated CHS_HW_ALL += $(AXI_VGA_ROOT)/.generated CHS_HW_ALL += $(CHS_SLINK_DIR)/.generated +CHS_HW_ALL += $(CHS_LLC_DIR)/.generated +CHS_HW_ALL += $(CHS_TAGGER_DIR)/.generated ##################### # Generate Boot ROM # diff --git a/hw/bootrom/cheshire_bootrom.S b/hw/bootrom/cheshire_bootrom.S index 22bbeb9f..89e388da 100644 --- a/hw/bootrom/cheshire_bootrom.S +++ b/hw/bootrom/cheshire_bootrom.S @@ -62,6 +62,7 @@ _start: andi t0, t0, 2 // regs.HW_FEATURES.llc beqz t0, _prom_check_run la t0, __base_llc + // Only configure half of LLC as SPM _wait_llc_bist: lw t1, 72(t0) // llc.BIST_STATUS_DONE_BIT beqz t1, _wait_llc_bist diff --git a/hw/cheshire_pkg.sv b/hw/cheshire_pkg.sv index 24dca50d..6353c484 100644 --- a/hw/cheshire_pkg.sv +++ b/hw/cheshire_pkg.sv @@ -85,6 +85,11 @@ package cheshire_pkg; byte_bt NumExtOutIntrTgts; shrt_bt NumExtOutIntrs; shrt_bt ClicIntCtlBits; + bit ClicUseSMode; + bit ClicUseUMode; + bit ClicUseVsMode; + bit ClicUseVsModePrio; + byte_bt ClicNumVsCtxts; shrt_bt NumExtIntrSyncs; // AXI parameters aw_bt AddrWidth; @@ -279,9 +284,7 @@ package cheshire_pkg; localparam doub_bt AmLlc = 'h0300_1000; localparam doub_bt AmSlink = 'h0300_6000; localparam doub_bt AmBusErr = 'h0300_8000; - // Address 0x0300_A000 is occupied by the tagger in Carfield. - // Removed for the moment to isolate the contribution of this PR. - // Leaving this comment and the free scope to keep track. + localparam doub_bt AmTagger = 'h0300_A000; localparam doub_bt AmHmrUnit = 'h0300_B000; localparam doub_bt AmSpm = 'h1000_0000; // Cached region at bottom, uncached on top localparam doub_bt AmClic = 'h0800_0000; @@ -399,6 +402,7 @@ package cheshire_pkg; aw_bt irq_router; aw_bt [2**MaxCoresWidth-1:0] bus_err; aw_bt [2**MaxCoresWidth-1:0] clic; + aw_bt [2**MaxCoresWidth-1:0] tagger; aw_bt hmr_unit; aw_bt ext_base; aw_bt num_out; @@ -428,6 +432,9 @@ package cheshire_pkg; if (cfg.BusErr) for (int j = 0; j < 2 + cfg.NumCores; j++) begin i++; ret.bus_err[j] = i; r++; ret.map[r] = '{i, AmBusErr + j*'h40, AmBusErr + (j+1)*'h40}; end + if (cfg.LlcCachePartition) for (int j = 0; j < cfg.NumCores; j++) begin + i++; ret.tagger[j] = i; r++; ret.map[r] = '{i, AmTagger + j*'h100, AmTagger + (j+1)*'h100}; + end if (cfg.HmrUnit) begin i++; ret.hmr_unit = i; r++; ret.map[r] = '{i, AmHmrUnit, AmHmrUnit+'h400}; end @@ -580,11 +587,16 @@ package cheshire_pkg; NumExtOutIntrTgts : 0, NumExtOutIntrs : 0, ClicIntCtlBits : 8, + ClicUseSMode : 0, + ClicUseUMode : 0, + ClicUseVsMode : 0, + ClicUseVsModePrio : 0, + ClicNumVsCtxts : 0, NumExtIntrSyncs : 2, // Interconnect AddrWidth : 48, AxiDataWidth : 64, - AxiUserWidth : 2, // AMO(2) + AxiUserWidth : 2, // Convention: bit 0 for core(s), bit 1 for serial link AxiMstIdWidth : 2, AxiMaxMstTrans : 24, AxiMaxSlvTrans : 24, diff --git a/hw/cheshire_soc.sv b/hw/cheshire_soc.sv index c86bfa99..13b8f4e8 100644 --- a/hw/cheshire_soc.sv +++ b/hw/cheshire_soc.sv @@ -492,6 +492,7 @@ module cheshire_soc import cheshire_pkg::*; #( // This is necessary for routing in the LLC-internal interconnect. always_comb begin axi_llc_remap_req = axi_llc_cut_req; + if (axi_llc_cut_req.aw.addr & ~AmSpmRegionMask == AmSpmBaseUncached & ~AmSpmRegionMask) axi_llc_remap_req.aw.addr = AmSpm | (AmSpmRegionMask & axi_llc_cut_req.aw.addr); if (axi_llc_cut_req.ar.addr & ~AmSpmRegionMask == AmSpmBaseUncached & ~AmSpmRegionMask) @@ -583,6 +584,9 @@ module cheshire_soc import cheshire_pkg::*; #( assign intr.intn.bus_err.cores = core_bus_err_intr_comb; + axi_mst_req_t [AxiIn.num_in-1:0] tagger_req; + axi_mst_rsp_t [AxiIn.num_in-1:0] tagger_rsp; + axi_cva6_req_t [NumIntHarts-1:0] core_out_req, core_ur_req; axi_cva6_rsp_t [NumIntHarts-1:0] core_out_rsp, core_ur_rsp; @@ -695,8 +699,11 @@ module cheshire_soc import cheshire_pkg::*; #( .INTCTLBITS ( Cfg.ClicIntCtlBits ), .reg_req_t ( reg_req_t ), .reg_rsp_t ( reg_rsp_t ), - .SSCLIC ( 1 ), - .USCLIC ( 0 ) + .SSCLIC ( Cfg.ClicUseSMode ), + .USCLIC ( Cfg.ClicUseUMode ), + .VSCLIC ( Cfg.ClicUseVsMode ), + .VSPRIO ( Cfg.ClicUseVsModePrio ), + .N_VSCTXTS ( Cfg.ClicNumVsCtxts ) ) i_clic ( .clk_i, .rst_ni, @@ -764,9 +771,36 @@ module cheshire_soc import cheshire_pkg::*; #( .rst_ni, .slv_req_i ( core_ur_req[i] ), .slv_resp_o ( core_ur_rsp[i] ), - .mst_req_o ( axi_in_req[AxiIn.cores[i]] ), - .mst_resp_i ( axi_in_rsp[AxiIn.cores[i]] ) + .mst_req_o ( tagger_req[i] ), + .mst_resp_i ( tagger_rsp[i] ) ); + + if (Cfg.LlcCachePartition) begin : gen_tagger + tagger #( + .DATA_WIDTH ( Cfg.AxiDataWidth ), + .ADDR_WIDTH ( Cfg.AddrWidth ), + .MAXPARTITION ( Cfg.LlcMaxPartition ), + .AXI_USER_ID_MSB ( Cfg.LlcUserMsb ), + .AXI_USER_ID_LSB ( Cfg.LlcUserLsb ), + .TAGGER_GRAN ( 3 ), + .axi_req_t ( axi_mst_req_t ), + .axi_rsp_t ( axi_mst_rsp_t ), + .reg_req_t ( reg_req_t ), + .reg_rsp_t ( reg_rsp_t ) + ) i_tagger ( + .clk_i, + .rst_ni, + .slv_req_i ( tagger_req[i] ), + .slv_rsp_o ( tagger_rsp[i] ), + .mst_req_o ( axi_in_req[AxiIn.cores[i]] ), + .mst_rsp_i ( axi_in_rsp[AxiIn.cores[i]] ), + .cfg_req_i ( reg_out_req[RegOut.tagger[i]] ), + .cfg_rsp_o ( reg_out_rsp[RegOut.tagger[i]] ) + ); + end else begin : gen_no_tagger + assign axi_in_req[AxiIn.cores[i]] = tagger_req[i]; + assign tagger_rsp[i] = axi_in_rsp[AxiIn.cores[i]]; + end end ///////////////////////// diff --git a/target/sim/src/vip_cheshire_soc.sv b/target/sim/src/vip_cheshire_soc.sv index 5d736f16..d8f70e5b 100644 --- a/target/sim/src/vip_cheshire_soc.sv +++ b/target/sim/src/vip_cheshire_soc.sv @@ -256,6 +256,89 @@ module vip_cheshire_soc import cheshire_pkg::*; #( $display("[JTAG] Initialization success"); endtask + // Halt the core and preload a binary + task automatic jtag_elf_halt_load(input string binary, output doub_bt entry ); + dm::dmstatus_t status; + // Wait until bootrom initialized LLC + if (DutCfg.LlcNotBypass) begin + word_bt regval; + $display("[JTAG] Wait for LLC configuration"); + jtag_poll_bit0(AmLlc + axi_llc_reg_pkg::AXI_LLC_CFG_SPM_LOW_OFFSET, regval, 20); + end + // Halt hart 0 + jtag_write(dm::DMControl, dm::dmcontrol_t'{haltreq: 1, dmactive: 1, default: '0}); + do jtag_dbg.read_dmi_exp_backoff(dm::DMStatus, status); + while (~status.allhalted); + $display("[JTAG] Halted hart 0"); + // Preload binary + jtag_elf_preload(binary, entry); + endtask + + task automatic jtag_write_reg32(input logic [31:0] start_addr, logic [63:0] value); + logic [31:0] rdata; + + automatic dm::sbcs_t sbcs = dm::sbcs_t'{sbreadonaddr: 1'b1, sbaccess: 2, default: '0}; + + $display("[JTAG] Start writing at %x ", start_addr); + jtag_write(dm::SBCS, sbcs, 0, 1); + + // Write address + $display("[JTAG] Start writing address"); + jtag_write(dm::SBAddress1, 'h0); + jtag_write(dm::SBAddress0, start_addr); + + // Write data + $display("[JTAG] Start writing data"); + jtag_write(dm::SBData0, value[31:0]); + + endtask + + task automatic jtag_read_reg32( + input logic [31:0] addr, + output logic [31:0] data, + input int unsigned idle_cycles + ); + automatic dm::sbcs_t sbcs = dm::sbcs_t'{sbreadonaddr: 1'b1, sbaccess: 2, default: '0}; + jtag_write(dm::SBCS, sbcs, 0, 1); + jtag_write(dm::SBAddress1, 'h0); + jtag_write(dm::SBAddress0, addr[31:0]); + jtag_dbg.wait_idle(idle_cycles); + jtag_dbg.read_dmi_exp_backoff(dm::SBData0, data); + endtask + + task automatic jtag_write_reg(input logic [31:0] start_addr, input doub_bt value); + logic [63:0] rdata; + + $display("[JTAG] Start writing at %x ", start_addr); + jtag_write(dm::SBCS, JtagInitSbcs, 1, 1); + // Write address + jtag_write(dm::SBAddress0, start_addr); + // Write data + jtag_write(dm::SBData1, value[63:32]); + jtag_write(dm::SBData0, value[31:0]); + + //Check correctess + jtag_read_reg(start_addr, rdata, 20); + if(rdata!=value) begin + $fatal(1,"rdata at %x: %x" , start_addr, rdata); + end else begin + $display("W/R sanity check at %x ok! : %x", start_addr, rdata); + end + endtask + + task automatic jtag_read_reg( + input doub_bt addr, + output word_bt data, + input int unsigned idle_cycles + ); + automatic dm::sbcs_t sbcs = dm::sbcs_t'{sbreadonaddr: 1'b1, sbaccess: 2, default: '0}; + jtag_write(dm::SBCS, sbcs, 0, 1); + jtag_write(dm::SBAddress1, addr[63:32]); + jtag_write(dm::SBAddress0, addr[31:0]); + jtag_dbg.wait_idle(idle_cycles); + jtag_dbg.read_dmi_exp_backoff(dm::SBData0, data); + endtask + // Load a binary task automatic jtag_elf_preload(input string binary, output doub_bt entry); longint sec_addr, sec_len; diff --git a/target/xilinx/src/cheshire_top_xilinx.sv b/target/xilinx/src/cheshire_top_xilinx.sv index aad319b8..338aa39f 100644 --- a/target/xilinx/src/cheshire_top_xilinx.sv +++ b/target/xilinx/src/cheshire_top_xilinx.sv @@ -123,6 +123,12 @@ module cheshire_top_xilinx LlcOutConnect : 1, LlcOutRegionStart : 'h8000_0000, LlcOutRegionEnd : 'h1_0000_0000, + LlcUserMsb : 0, + LlcUserLsb : 0, + // LLC partitioning + LlcCachePartition : 0, + LlcMaxPartition : 0, + LlcRemapHash : axi_llc_pkg::Modulo, // VGA: RGB332 VgaRedWidth : 5, VgaGreenWidth : 6,