1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.mem 18 19import org.chipsalliance.cde.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import utils._ 23import utility._ 24import xiangshan._ 25import xiangshan.backend.fu.fpu.FPU 26import xiangshan.backend.rob.RobLsqIO 27import xiangshan.cache._ 28import xiangshan.cache.mmu._ 29import xiangshan.frontend.FtqPtr 30import xiangshan.ExceptionNO._ 31import xiangshan.mem.mdp._ 32import xiangshan.backend.rob.RobPtr 33 34class LqPtr(implicit p: Parameters) extends CircularQueuePtr[LqPtr]( 35 p => p(XSCoreParamsKey).VirtualLoadQueueSize 36){ 37} 38 39object LqPtr { 40 def apply(f: Bool, v: UInt)(implicit p: Parameters): LqPtr = { 41 val ptr = Wire(new LqPtr) 42 ptr.flag := f 43 ptr.value := v 44 ptr 45 } 46} 47 48trait HasLoadHelper { this: XSModule => 49 def rdataHelper(uop: MicroOp, rdata: UInt): UInt = { 50 val fpWen = uop.ctrl.fpWen 51 LookupTree(uop.ctrl.fuOpType, List( 52 LSUOpType.lb -> SignExt(rdata(7, 0) , XLEN), 53 LSUOpType.lh -> SignExt(rdata(15, 0), XLEN), 54 /* 55 riscv-spec-20191213: 12.2 NaN Boxing of Narrower Values 56 Any operation that writes a narrower result to an f register must write 57 all 1s to the uppermost FLEN−n bits to yield a legal NaN-boxed value. 58 */ 59 LSUOpType.lw -> Mux(fpWen, FPU.box(rdata, FPU.S), SignExt(rdata(31, 0), XLEN)), 60 LSUOpType.ld -> Mux(fpWen, FPU.box(rdata, FPU.D), SignExt(rdata(63, 0), XLEN)), 61 LSUOpType.lbu -> ZeroExt(rdata(7, 0) , XLEN), 62 LSUOpType.lhu -> ZeroExt(rdata(15, 0), XLEN), 63 LSUOpType.lwu -> ZeroExt(rdata(31, 0), XLEN), 64 )) 65 } 66} 67 68class LqEnqIO(implicit p: Parameters) extends XSBundle { 69 val canAccept = Output(Bool()) 70 val sqCanAccept = Input(Bool()) 71 val needAlloc = Vec(exuParameters.LsExuCnt, Input(Bool())) 72 val req = Vec(exuParameters.LsExuCnt, Flipped(ValidIO(new MicroOp))) 73 val resp = Vec(exuParameters.LsExuCnt, Output(new LqPtr)) 74} 75 76class LqTriggerIO(implicit p: Parameters) extends XSBundle { 77 val hitLoadAddrTriggerHitVec = Input(Vec(3, Bool())) 78 val lqLoadAddrTriggerHitVec = Output(Vec(3, Bool())) 79} 80 81class LoadQueueTopDownIO(implicit p: Parameters) extends XSBundle { 82 val robHeadVaddr = Flipped(Valid(UInt(VAddrBits.W))) 83 val robHeadTlbReplay = Output(Bool()) 84 val robHeadTlbMiss = Output(Bool()) 85 val robHeadLoadVio = Output(Bool()) 86 val robHeadLoadMSHR = Output(Bool()) 87 val robHeadMissInDTlb = Input(Bool()) 88 val robHeadOtherReplay = Output(Bool()) 89} 90 91class LoadQueue(implicit p: Parameters) extends XSModule 92 with HasDCacheParameters 93 with HasCircularQueuePtrHelper 94 with HasLoadHelper 95 with HasPerfEvents 96{ 97 val io = IO(new Bundle() { 98 val redirect = Flipped(Valid(new Redirect)) 99 val enq = new LqEnqIO 100 val ldu = new Bundle() { 101 val stld_nuke_query = Vec(LoadPipelineWidth, Flipped(new LoadNukeQueryIO)) // from load_s2 102 val ldld_nuke_query = Vec(LoadPipelineWidth, Flipped(new LoadNukeQueryIO)) // from load_s2 103 val ldin = Vec(LoadPipelineWidth, Flipped(Decoupled(new LqWriteBundle))) // from load_s3 104 } 105 val sta = new Bundle() { 106 val storeAddrIn = Vec(StorePipelineWidth, Flipped(Valid(new LsPipelineBundle))) // from store_s1 107 } 108 val std = new Bundle() { 109 val storeDataIn = Vec(StorePipelineWidth, Flipped(Valid(new ExuOutput))) // from store_s0, store data, send to sq from rs 110 } 111 val sq = new Bundle() { 112 val stAddrReadySqPtr = Input(new SqPtr) 113 val stAddrReadyVec = Input(Vec(StoreQueueSize, Bool())) 114 val stDataReadySqPtr = Input(new SqPtr) 115 val stDataReadyVec = Input(Vec(StoreQueueSize, Bool())) 116 val stIssuePtr = Input(new SqPtr) 117 val sqEmpty = Input(Bool()) 118 } 119 val ldout = Vec(LoadPipelineWidth, DecoupledIO(new ExuOutput)) 120 val ld_raw_data = Vec(LoadPipelineWidth, Output(new LoadDataFromLQBundle)) 121 val replay = Vec(LoadPipelineWidth, Decoupled(new LsPipelineBundle)) 122 val refill = Flipped(ValidIO(new Refill)) 123 val tl_d_channel = Input(new DcacheToLduForwardIO) 124 val release = Flipped(Valid(new Release)) 125 val nuke_rollback = Output(Valid(new Redirect)) 126 val nack_rollback = Output(Valid(new Redirect)) 127 val rob = Flipped(new RobLsqIO) 128 val uncache = new UncacheWordIO 129 val exceptionAddr = new ExceptionAddrIO 130 val lqFull = Output(Bool()) 131 val lqDeq = Output(UInt(log2Up(CommitWidth + 1).W)) 132 val lqCancelCnt = Output(UInt(log2Up(VirtualLoadQueueSize+1).W)) 133 val lq_rep_full = Output(Bool()) 134 val tlbReplayDelayCycleCtrl = Vec(4, Input(UInt(ReSelectLen.W))) 135 val l2_hint = Input(Valid(new L2ToL1Hint())) 136 val tlb_hint = Flipped(new TlbHintIO) 137 val lqEmpty = Output(Bool()) 138 val debugTopDown = new LoadQueueTopDownIO 139 }) 140 141 val loadQueueRAR = Module(new LoadQueueRAR) // read-after-read violation 142 val loadQueueRAW = Module(new LoadQueueRAW) // read-after-write violation 143 val loadQueueReplay = Module(new LoadQueueReplay) // enqueue if need replay 144 val virtualLoadQueue = Module(new VirtualLoadQueue) // control state 145 val exceptionBuffer = Module(new LqExceptionBuffer) // exception buffer 146 val uncacheBuffer = Module(new UncacheBuffer) // uncache buffer 147 148 /** 149 * LoadQueueRAR 150 */ 151 loadQueueRAR.io.redirect <> io.redirect 152 loadQueueRAR.io.release <> io.release 153 loadQueueRAR.io.ldWbPtr <> virtualLoadQueue.io.ldWbPtr 154 for (w <- 0 until LoadPipelineWidth) { 155 loadQueueRAR.io.query(w).req <> io.ldu.ldld_nuke_query(w).req // from load_s1 156 loadQueueRAR.io.query(w).resp <> io.ldu.ldld_nuke_query(w).resp // to load_s2 157 loadQueueRAR.io.query(w).revoke := io.ldu.ldld_nuke_query(w).revoke // from load_s3 158 } 159 160 /** 161 * LoadQueueRAW 162 */ 163 loadQueueRAW.io.redirect <> io.redirect 164 loadQueueRAW.io.storeIn <> io.sta.storeAddrIn 165 loadQueueRAW.io.stAddrReadySqPtr <> io.sq.stAddrReadySqPtr 166 loadQueueRAW.io.stIssuePtr <> io.sq.stIssuePtr 167 for (w <- 0 until LoadPipelineWidth) { 168 loadQueueRAW.io.query(w).req <> io.ldu.stld_nuke_query(w).req // from load_s1 169 loadQueueRAW.io.query(w).resp <> io.ldu.stld_nuke_query(w).resp // to load_s2 170 loadQueueRAW.io.query(w).revoke := io.ldu.stld_nuke_query(w).revoke // from load_s3 171 } 172 173 /** 174 * VirtualLoadQueue 175 */ 176 virtualLoadQueue.io.redirect <> io.redirect 177 virtualLoadQueue.io.enq <> io.enq 178 virtualLoadQueue.io.ldin <> io.ldu.ldin // from load_s3 179 virtualLoadQueue.io.lqFull <> io.lqFull 180 virtualLoadQueue.io.lqDeq <> io.lqDeq 181 virtualLoadQueue.io.lqCancelCnt <> io.lqCancelCnt 182 virtualLoadQueue.io.lqEmpty <> io.lqEmpty 183 184 /** 185 * Load queue exception buffer 186 */ 187 exceptionBuffer.io.redirect <> io.redirect 188 for ((buff, w) <- exceptionBuffer.io.req.zipWithIndex) { 189 buff.valid := io.ldu.ldin(w).valid // from load_s3 190 buff.bits := io.ldu.ldin(w).bits 191 } 192 io.exceptionAddr <> exceptionBuffer.io.exceptionAddr 193 194 /** 195 * Load uncache buffer 196 */ 197 uncacheBuffer.io.redirect <> io.redirect 198 uncacheBuffer.io.ldout <> io.ldout 199 uncacheBuffer.io.ld_raw_data <> io.ld_raw_data 200 uncacheBuffer.io.rob <> io.rob 201 uncacheBuffer.io.uncache <> io.uncache 202 for ((buff, w) <- uncacheBuffer.io.req.zipWithIndex) { 203 buff.valid := io.ldu.ldin(w).valid // from load_s3 204 buff.bits := io.ldu.ldin(w).bits // from load_s3 205 } 206 207 208 io.nuke_rollback := loadQueueRAW.io.rollback 209 io.nack_rollback := uncacheBuffer.io.rollback 210 211 /* <------- DANGEROUS: Don't change sequence here ! -------> */ 212 213 /** 214 * LoadQueueReplay 215 */ 216 loadQueueReplay.io.redirect <> io.redirect 217 loadQueueReplay.io.enq <> io.ldu.ldin // from load_s3 218 loadQueueReplay.io.storeAddrIn <> io.sta.storeAddrIn // from store_s1 219 loadQueueReplay.io.storeDataIn <> io.std.storeDataIn // from store_s0 220 loadQueueReplay.io.replay <> io.replay 221 loadQueueReplay.io.refill <> io.refill 222 loadQueueReplay.io.tl_d_channel <> io.tl_d_channel 223 loadQueueReplay.io.stAddrReadySqPtr <> io.sq.stAddrReadySqPtr 224 loadQueueReplay.io.stAddrReadyVec <> io.sq.stAddrReadyVec 225 loadQueueReplay.io.stDataReadySqPtr <> io.sq.stDataReadySqPtr 226 loadQueueReplay.io.stDataReadyVec <> io.sq.stDataReadyVec 227 loadQueueReplay.io.sqEmpty <> io.sq.sqEmpty 228 loadQueueReplay.io.lqFull <> io.lq_rep_full 229 loadQueueReplay.io.ldWbPtr <> virtualLoadQueue.io.ldWbPtr 230 loadQueueReplay.io.rarFull <> loadQueueRAR.io.lqFull 231 loadQueueReplay.io.rawFull <> loadQueueRAW.io.lqFull 232 loadQueueReplay.io.l2_hint <> io.l2_hint 233 loadQueueReplay.io.tlb_hint <> io.tlb_hint 234 loadQueueReplay.io.tlbReplayDelayCycleCtrl <> io.tlbReplayDelayCycleCtrl 235 236 loadQueueReplay.io.debugTopDown <> io.debugTopDown 237 238 val full_mask = Cat(loadQueueRAR.io.lqFull, loadQueueRAW.io.lqFull, loadQueueReplay.io.lqFull) 239 XSPerfAccumulate("full_mask_000", full_mask === 0.U) 240 XSPerfAccumulate("full_mask_001", full_mask === 1.U) 241 XSPerfAccumulate("full_mask_010", full_mask === 2.U) 242 XSPerfAccumulate("full_mask_011", full_mask === 3.U) 243 XSPerfAccumulate("full_mask_100", full_mask === 4.U) 244 XSPerfAccumulate("full_mask_101", full_mask === 5.U) 245 XSPerfAccumulate("full_mask_110", full_mask === 6.U) 246 XSPerfAccumulate("full_mask_111", full_mask === 7.U) 247 XSPerfAccumulate("nuke_rollback", io.nuke_rollback.valid) 248 XSPerfAccumulate("nack_rollabck", io.nack_rollback.valid) 249 250 // perf cnt 251 val perfEvents = Seq(virtualLoadQueue, loadQueueRAR, loadQueueRAW, loadQueueReplay).flatMap(_.getPerfEvents) ++ 252 Seq( 253 ("full_mask_000", full_mask === 0.U), 254 ("full_mask_001", full_mask === 1.U), 255 ("full_mask_010", full_mask === 2.U), 256 ("full_mask_011", full_mask === 3.U), 257 ("full_mask_100", full_mask === 4.U), 258 ("full_mask_101", full_mask === 5.U), 259 ("full_mask_110", full_mask === 6.U), 260 ("full_mask_111", full_mask === 7.U), 261 ("nuke_rollback", io.nuke_rollback.valid), 262 ("nack_rollback", io.nack_rollback.valid) 263 ) 264 generatePerfEvent() 265 // end 266}