1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.mem 18 19import org.chipsalliance.cde.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import utils._ 23import utility._ 24import xiangshan._ 25import xiangshan.backend._ 26import xiangshan.backend.fu.fpu._ 27import xiangshan.backend.rob.RobLsqIO 28import xiangshan.cache._ 29import xiangshan.cache.mmu._ 30import xiangshan.frontend.FtqPtr 31import xiangshan.ExceptionNO._ 32import xiangshan.mem.mdp._ 33import xiangshan.backend.Bundles.{DynInst, MemExuOutput, MemMicroOpRbExt} 34import xiangshan.backend.rob.RobPtr 35 36class LqPtr(implicit p: Parameters) extends CircularQueuePtr[LqPtr]( 37 p => p(XSCoreParamsKey).VirtualLoadQueueSize 38){ 39} 40 41object LqPtr { 42 def apply(f: Bool, v: UInt)(implicit p: Parameters): LqPtr = { 43 val ptr = Wire(new LqPtr) 44 ptr.flag := f 45 ptr.value := v 46 ptr 47 } 48} 49 50trait HasLoadHelper { this: XSModule => 51 def rdataHelper(uop: DynInst, rdata: UInt): UInt = { 52 val fpWen = uop.fpWen 53 LookupTree(uop.fuOpType, List( 54 LSUOpType.lb -> SignExt(rdata(7, 0) , XLEN), 55 LSUOpType.lh -> SignExt(rdata(15, 0), XLEN), 56 /* 57 riscv-spec-20191213: 12.2 NaN Boxing of Narrower Values 58 Any operation that writes a narrower result to an f register must write 59 all 1s to the uppermost FLEN−n bits to yield a legal NaN-boxed value. 60 */ 61 LSUOpType.lw -> Mux(fpWen, FPU.box(rdata, FPU.S), SignExt(rdata(31, 0), XLEN)), 62 LSUOpType.ld -> Mux(fpWen, FPU.box(rdata, FPU.D), SignExt(rdata(63, 0), XLEN)), 63 LSUOpType.lbu -> ZeroExt(rdata(7, 0) , XLEN), 64 LSUOpType.lhu -> ZeroExt(rdata(15, 0), XLEN), 65 LSUOpType.lwu -> ZeroExt(rdata(31, 0), XLEN), 66 )) 67 } 68 69 def rdataVecHelper(alignedType: UInt, rdata: UInt): UInt = { 70 LookupTree(alignedType, List( 71 "b00".U -> ZeroExt(rdata(7, 0), VLEN), 72 "b01".U -> ZeroExt(rdata(15, 0), VLEN), 73 "b10".U -> ZeroExt(rdata(31, 0), VLEN), 74 "b11".U -> ZeroExt(rdata(63, 0), VLEN) 75 )) 76 } 77} 78 79class LqEnqIO(implicit p: Parameters) extends MemBlockBundle { 80 val canAccept = Output(Bool()) 81 val sqCanAccept = Input(Bool()) 82 val needAlloc = Vec(LSQEnqWidth, Input(Bool())) 83 val req = Vec(LSQEnqWidth, Flipped(ValidIO(new DynInst))) 84 val resp = Vec(LSQEnqWidth, Output(new LqPtr)) 85} 86 87class LqTriggerIO(implicit p: Parameters) extends XSBundle { 88 val hitLoadAddrTriggerHitVec = Input(Vec(TriggerNum, Bool())) 89 val lqLoadAddrTriggerHitVec = Output(Vec(TriggerNum, Bool())) 90} 91 92class LoadQueueTopDownIO(implicit p: Parameters) extends XSBundle { 93 val robHeadVaddr = Flipped(Valid(UInt(VAddrBits.W))) 94 val robHeadTlbReplay = Output(Bool()) 95 val robHeadTlbMiss = Output(Bool()) 96 val robHeadLoadVio = Output(Bool()) 97 val robHeadLoadMSHR = Output(Bool()) 98 val robHeadMissInDTlb = Input(Bool()) 99 val robHeadOtherReplay = Output(Bool()) 100} 101 102class LoadQueue(implicit p: Parameters) extends XSModule 103 with HasDCacheParameters 104 with HasCircularQueuePtrHelper 105 with HasLoadHelper 106 with HasPerfEvents 107{ 108 val io = IO(new Bundle() { 109 val redirect = Flipped(Valid(new Redirect)) 110 val enq = new LqEnqIO 111 val ldu = new Bundle() { 112 val stld_nuke_query = Vec(LoadPipelineWidth, Flipped(new LoadNukeQueryIO)) // from load_s2 113 val ldld_nuke_query = Vec(LoadPipelineWidth, Flipped(new LoadNukeQueryIO)) // from load_s2 114 val ldin = Vec(LoadPipelineWidth, Flipped(Decoupled(new LqWriteBundle))) // from load_s3 115 } 116 val sta = new Bundle() { 117 val storeAddrIn = Vec(StorePipelineWidth, Flipped(Valid(new LsPipelineBundle))) // from store_s1 118 val vecStoreAddrIn = Vec(StorePipelineWidth, Flipped(Valid(new LsPipelineBundle))) // from store_s1 119 } 120 val std = new Bundle() { 121 val storeDataIn = Vec(StorePipelineWidth, Flipped(Valid(new MemExuOutput))) // from store_s0, store data, send to sq from rs 122 } 123 val sq = new Bundle() { 124 val stAddrReadySqPtr = Input(new SqPtr) 125 val stAddrReadyVec = Input(Vec(StoreQueueSize, Bool())) 126 val stDataReadySqPtr = Input(new SqPtr) 127 val stDataReadyVec = Input(Vec(StoreQueueSize, Bool())) 128 val stIssuePtr = Input(new SqPtr) 129 val sqEmpty = Input(Bool()) 130 } 131 val ldout = Vec(LoadPipelineWidth, DecoupledIO(new MemExuOutput)) 132 val ld_raw_data = Vec(LoadPipelineWidth, Output(new LoadDataFromLQBundle)) 133 val replay = Vec(LoadPipelineWidth, Decoupled(new LsPipelineBundle)) 134 val refill = Flipped(ValidIO(new Refill)) 135 val tl_d_channel = Input(new DcacheToLduForwardIO) 136 val release = Flipped(Valid(new Release)) 137 val nuke_rollback = Output(Valid(new Redirect)) 138 val nack_rollback = Output(Valid(new Redirect)) 139 val rob = Flipped(new RobLsqIO) 140 val uncache = new UncacheWordIO 141 val exceptionAddr = new ExceptionAddrIO 142 val lqFull = Output(Bool()) 143 val lqDeq = Output(UInt(log2Up(CommitWidth + 1).W)) 144 val lqCancelCnt = Output(UInt(log2Up(VirtualLoadQueueSize+1).W)) 145 val lq_rep_full = Output(Bool()) 146 val tlbReplayDelayCycleCtrl = Vec(4, Input(UInt(ReSelectLen.W))) 147 val l2_hint = Input(Valid(new L2ToL1Hint())) 148 val tlb_hint = Flipped(new TlbHintIO) 149 val lqEmpty = Output(Bool()) 150 151 val vecWriteback = Flipped(ValidIO(new MemExuOutput(isVector = true))) 152 val lqDeqPtr = Output(new LqPtr) 153 val vecMMIOReplay = Vec(VecLoadPipelineWidth, DecoupledIO(new LsPipelineBundle())) 154 155 val trigger = Vec(LoadPipelineWidth, new LqTriggerIO) 156 157 val debugTopDown = new LoadQueueTopDownIO 158 }) 159 160 val loadQueueRAR = Module(new LoadQueueRAR) // read-after-read violation 161 val loadQueueRAW = Module(new LoadQueueRAW) // read-after-write violation 162 val loadQueueReplay = Module(new LoadQueueReplay) // enqueue if need replay 163 val virtualLoadQueue = Module(new VirtualLoadQueue) // control state 164 val exceptionBuffer = Module(new LqExceptionBuffer) // exception buffer 165 val uncacheBuffer = Module(new UncacheBuffer) // uncache buffer 166 167 /** 168 * LoadQueueRAR 169 */ 170 loadQueueRAR.io.redirect <> io.redirect 171 loadQueueRAR.io.release <> io.release 172 loadQueueRAR.io.ldWbPtr <> virtualLoadQueue.io.ldWbPtr 173 for (w <- 0 until LoadPipelineWidth) { 174 loadQueueRAR.io.query(w).req <> io.ldu.ldld_nuke_query(w).req // from load_s1 175 loadQueueRAR.io.query(w).resp <> io.ldu.ldld_nuke_query(w).resp // to load_s2 176 loadQueueRAR.io.query(w).revoke := io.ldu.ldld_nuke_query(w).revoke // from load_s3 177 } 178 179 /** 180 * LoadQueueRAW 181 */ 182 loadQueueRAW.io.redirect <> io.redirect 183 loadQueueRAW.io.storeIn <> io.sta.storeAddrIn 184 loadQueueRAW.io.vecStoreIn <> io.sta.vecStoreAddrIn 185 loadQueueRAW.io.stAddrReadySqPtr <> io.sq.stAddrReadySqPtr 186 loadQueueRAW.io.stIssuePtr <> io.sq.stIssuePtr 187 for (w <- 0 until LoadPipelineWidth) { 188 loadQueueRAW.io.query(w).req <> io.ldu.stld_nuke_query(w).req // from load_s1 189 loadQueueRAW.io.query(w).resp <> io.ldu.stld_nuke_query(w).resp // to load_s2 190 loadQueueRAW.io.query(w).revoke := io.ldu.stld_nuke_query(w).revoke // from load_s3 191 } 192 193 /** 194 * VirtualLoadQueue 195 */ 196 virtualLoadQueue.io.redirect <> io.redirect 197 virtualLoadQueue.io.enq <> io.enq 198 virtualLoadQueue.io.ldin <> io.ldu.ldin // from load_s3 199 virtualLoadQueue.io.lqFull <> io.lqFull 200 virtualLoadQueue.io.lqDeq <> io.lqDeq 201 virtualLoadQueue.io.lqCancelCnt <> io.lqCancelCnt 202 virtualLoadQueue.io.lqEmpty <> io.lqEmpty 203 virtualLoadQueue.io.vecWriteback <> io.vecWriteback 204 virtualLoadQueue.io.ldWbPtr <> io.lqDeqPtr 205 206 /** 207 * Load queue exception buffer 208 */ 209 exceptionBuffer.io.redirect <> io.redirect 210 for ((buff, w) <- exceptionBuffer.io.req.zipWithIndex) { 211 buff.valid := io.ldu.ldin(w).valid // from load_s3 212 buff.bits := io.ldu.ldin(w).bits 213 } 214 io.exceptionAddr <> exceptionBuffer.io.exceptionAddr 215 216 /** 217 * Load uncache buffer 218 */ 219 uncacheBuffer.io.redirect <> io.redirect 220 uncacheBuffer.io.ldout <> io.ldout 221 uncacheBuffer.io.ld_raw_data <> io.ld_raw_data 222 uncacheBuffer.io.rob <> io.rob 223 uncacheBuffer.io.uncache <> io.uncache 224 uncacheBuffer.io.trigger <> io.trigger 225 uncacheBuffer.io.vecReplay <> io.vecMMIOReplay 226 for ((buff, w) <- uncacheBuffer.io.req.zipWithIndex) { 227 buff.valid := io.ldu.ldin(w).valid // from load_s3 228 buff.bits := io.ldu.ldin(w).bits // from load_s3 229 } 230 231 232 io.nuke_rollback := loadQueueRAW.io.rollback 233 io.nack_rollback := uncacheBuffer.io.rollback 234 235 /* <------- DANGEROUS: Don't change sequence here ! -------> */ 236 237 /** 238 * LoadQueueReplay 239 */ 240 loadQueueReplay.io.redirect <> io.redirect 241 loadQueueReplay.io.enq <> io.ldu.ldin // from load_s3 242 loadQueueReplay.io.enq.zip(io.ldu.ldin).foreach { case (sink, source) => 243 sink.valid := source.valid && !source.bits.isvec 244 source.ready := sink.ready && !source.bits.isvec 245 } 246 loadQueueReplay.io.storeAddrIn <> io.sta.storeAddrIn // from store_s1 247 loadQueueReplay.io.storeDataIn <> io.std.storeDataIn // from store_s0 248 loadQueueReplay.io.replay <> io.replay 249 loadQueueReplay.io.refill <> io.refill 250 loadQueueReplay.io.tl_d_channel <> io.tl_d_channel 251 loadQueueReplay.io.stAddrReadySqPtr <> io.sq.stAddrReadySqPtr 252 loadQueueReplay.io.stAddrReadyVec <> io.sq.stAddrReadyVec 253 loadQueueReplay.io.stDataReadySqPtr <> io.sq.stDataReadySqPtr 254 loadQueueReplay.io.stDataReadyVec <> io.sq.stDataReadyVec 255 loadQueueReplay.io.sqEmpty <> io.sq.sqEmpty 256 loadQueueReplay.io.lqFull <> io.lq_rep_full 257 loadQueueReplay.io.ldWbPtr <> virtualLoadQueue.io.ldWbPtr 258 loadQueueReplay.io.rarFull <> loadQueueRAR.io.lqFull 259 loadQueueReplay.io.rawFull <> loadQueueRAW.io.lqFull 260 loadQueueReplay.io.l2_hint <> io.l2_hint 261 loadQueueReplay.io.tlb_hint <> io.tlb_hint 262 loadQueueReplay.io.tlbReplayDelayCycleCtrl <> io.tlbReplayDelayCycleCtrl 263 264 loadQueueReplay.io.debugTopDown <> io.debugTopDown 265 266 val full_mask = Cat(loadQueueRAR.io.lqFull, loadQueueRAW.io.lqFull, loadQueueReplay.io.lqFull) 267 XSPerfAccumulate("full_mask_000", full_mask === 0.U) 268 XSPerfAccumulate("full_mask_001", full_mask === 1.U) 269 XSPerfAccumulate("full_mask_010", full_mask === 2.U) 270 XSPerfAccumulate("full_mask_011", full_mask === 3.U) 271 XSPerfAccumulate("full_mask_100", full_mask === 4.U) 272 XSPerfAccumulate("full_mask_101", full_mask === 5.U) 273 XSPerfAccumulate("full_mask_110", full_mask === 6.U) 274 XSPerfAccumulate("full_mask_111", full_mask === 7.U) 275 XSPerfAccumulate("nuke_rollback", io.nuke_rollback.valid) 276 XSPerfAccumulate("nack_rollabck", io.nack_rollback.valid) 277 278 // perf cnt 279 val perfEvents = Seq(virtualLoadQueue, loadQueueRAR, loadQueueRAW, loadQueueReplay).flatMap(_.getPerfEvents) ++ 280 Seq( 281 ("full_mask_000", full_mask === 0.U), 282 ("full_mask_001", full_mask === 1.U), 283 ("full_mask_010", full_mask === 2.U), 284 ("full_mask_011", full_mask === 3.U), 285 ("full_mask_100", full_mask === 4.U), 286 ("full_mask_101", full_mask === 5.U), 287 ("full_mask_110", full_mask === 6.U), 288 ("full_mask_111", full_mask === 7.U), 289 ("nuke_rollback", io.nuke_rollback.valid), 290 ("nack_rollback", io.nack_rollback.valid) 291 ) 292 generatePerfEvent() 293 // end 294}