1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.mem 18 19import chisel3._ 20import chisel3.util._ 21import difftest._ 22import difftest.common.DifftestMem 23import org.chipsalliance.cde.config.Parameters 24import utility._ 25import utils._ 26import xiangshan._ 27import xiangshan.backend.rob.RobLsqIO 28import xiangshan.cache._ 29 30class SqPtr(implicit p: Parameters) extends CircularQueuePtr[SqPtr]( 31 p => p(XSCoreParamsKey).StoreQueueSize 32){ 33} 34 35object SqPtr { 36 def apply(f: Bool, v: UInt)(implicit p: Parameters): SqPtr = { 37 val ptr = Wire(new SqPtr) 38 ptr.flag := f 39 ptr.value := v 40 ptr 41 } 42} 43 44class SqEnqIO(implicit p: Parameters) extends XSBundle { 45 val canAccept = Output(Bool()) 46 val lqCanAccept = Input(Bool()) 47 val needAlloc = Vec(exuParameters.LsExuCnt, Input(Bool())) 48 val req = Vec(exuParameters.LsExuCnt, Flipped(ValidIO(new MicroOp))) 49 val resp = Vec(exuParameters.LsExuCnt, Output(new SqPtr)) 50} 51 52class DataBufferEntry (implicit p: Parameters) extends DCacheBundle { 53 val addr = UInt(PAddrBits.W) 54 val vaddr = UInt(VAddrBits.W) 55 val data = UInt(VLEN.W) 56 val mask = UInt((VLEN/8).W) 57 val wline = Bool() 58 val sqPtr = new SqPtr 59 val prefetch = Bool() 60} 61 62// Store Queue 63class StoreQueue(implicit p: Parameters) extends XSModule 64 with HasDCacheParameters with HasCircularQueuePtrHelper with HasPerfEvents { 65 val io = IO(new Bundle() { 66 val hartId = Input(UInt(hartIdLen.W)) 67 val enq = new SqEnqIO 68 val brqRedirect = Flipped(ValidIO(new Redirect)) 69 val storeAddrIn = Vec(StorePipelineWidth, Flipped(Valid(new LsPipelineBundle))) // store addr, data is not included 70 val storeAddrInRe = Vec(StorePipelineWidth, Input(new LsPipelineBundle())) // store more mmio and exception 71 val storeDataIn = Vec(StorePipelineWidth, Flipped(Valid(new ExuOutput))) // store data, send to sq from rs 72 val storeMaskIn = Vec(StorePipelineWidth, Flipped(Valid(new StoreMaskBundle))) // store mask, send to sq from rs 73 val sbuffer = Vec(EnsbufferWidth, Decoupled(new DCacheWordReqWithVaddrAndPfFlag)) // write committed store to sbuffer 74 val uncacheOutstanding = Input(Bool()) 75 val mmioStout = DecoupledIO(new ExuOutput) // writeback uncached store 76 val forward = Vec(LoadPipelineWidth, Flipped(new PipeLoadForwardQueryIO)) 77 val rob = Flipped(new RobLsqIO) 78 val uncache = new UncacheWordIO 79 // val refill = Flipped(Valid(new DCacheLineReq )) 80 val exceptionAddr = new ExceptionAddrIO 81 val sqEmpty = Output(Bool()) 82 val stAddrReadySqPtr = Output(new SqPtr) 83 val stAddrReadyVec = Output(Vec(StoreQueueSize, Bool())) 84 val stDataReadySqPtr = Output(new SqPtr) 85 val stDataReadyVec = Output(Vec(StoreQueueSize, Bool())) 86 val stIssuePtr = Output(new SqPtr) 87 val sqDeqPtr = Output(new SqPtr) 88 val sqFull = Output(Bool()) 89 val sqCancelCnt = Output(UInt(log2Up(StoreQueueSize + 1).W)) 90 val sqDeq = Output(UInt(log2Ceil(EnsbufferWidth + 1).W)) 91 val force_write = Output(Bool()) 92 }) 93 94 println("StoreQueue: size:" + StoreQueueSize) 95 96 // data modules 97 val uop = Reg(Vec(StoreQueueSize, new MicroOp)) 98 // val data = Reg(Vec(StoreQueueSize, new LsqEntry)) 99 val dataModule = Module(new SQDataModule( 100 numEntries = StoreQueueSize, 101 numRead = EnsbufferWidth, 102 numWrite = StorePipelineWidth, 103 numForward = StorePipelineWidth 104 )) 105 dataModule.io := DontCare 106 val paddrModule = Module(new SQAddrModule( 107 dataWidth = PAddrBits, 108 numEntries = StoreQueueSize, 109 numRead = EnsbufferWidth, 110 numWrite = StorePipelineWidth, 111 numForward = StorePipelineWidth 112 )) 113 paddrModule.io := DontCare 114 val vaddrModule = Module(new SQAddrModule( 115 dataWidth = VAddrBits, 116 numEntries = StoreQueueSize, 117 numRead = EnsbufferWidth + 1, // sbuffer + badvaddr 1 (TODO) 118 numWrite = StorePipelineWidth, 119 numForward = StorePipelineWidth 120 )) 121 val gpaddrModule = Module(new SQAddrModule( 122 dataWidth = GPAddrBits, 123 numEntries = StoreQueueSize, 124 numRead = EnsbufferWidth + 1, 125 numWrite = StorePipelineWidth, 126 numForward = StorePipelineWidth 127 )) 128 vaddrModule.io := DontCare 129 gpaddrModule.io := DontCare 130 val dataBuffer = Module(new DatamoduleResultBuffer(new DataBufferEntry)) 131 val debug_paddr = Reg(Vec(StoreQueueSize, UInt((PAddrBits).W))) 132 val debug_vaddr = Reg(Vec(StoreQueueSize, UInt((VAddrBits).W))) 133 val debug_data = Reg(Vec(StoreQueueSize, UInt((XLEN).W))) 134 135 // state & misc 136 val allocated = RegInit(VecInit(List.fill(StoreQueueSize)(false.B))) // sq entry has been allocated 137 val addrvalid = RegInit(VecInit(List.fill(StoreQueueSize)(false.B))) // non-mmio addr is valid 138 val datavalid = RegInit(VecInit(List.fill(StoreQueueSize)(false.B))) // non-mmio data is valid 139 val allvalid = VecInit((0 until StoreQueueSize).map(i => addrvalid(i) && datavalid(i))) // non-mmio data & addr is valid 140 val committed = Reg(Vec(StoreQueueSize, Bool())) // inst has been committed by rob 141 val pending = Reg(Vec(StoreQueueSize, Bool())) // mmio pending: inst is an mmio inst, it will not be executed until it reachs the end of rob 142 val mmio = Reg(Vec(StoreQueueSize, Bool())) // mmio: inst is an mmio inst 143 val atomic = Reg(Vec(StoreQueueSize, Bool())) 144 val prefetch = RegInit(VecInit(List.fill(StoreQueueSize)(false.B))) // need prefetch when committing this store to sbuffer? 145 146 // ptr 147 val enqPtrExt = RegInit(VecInit((0 until io.enq.req.length).map(_.U.asTypeOf(new SqPtr)))) 148 val rdataPtrExt = RegInit(VecInit((0 until EnsbufferWidth).map(_.U.asTypeOf(new SqPtr)))) 149 val deqPtrExt = RegInit(VecInit((0 until EnsbufferWidth).map(_.U.asTypeOf(new SqPtr)))) 150 val cmtPtrExt = RegInit(VecInit((0 until CommitWidth).map(_.U.asTypeOf(new SqPtr)))) 151 val addrReadyPtrExt = RegInit(0.U.asTypeOf(new SqPtr)) 152 val dataReadyPtrExt = RegInit(0.U.asTypeOf(new SqPtr)) 153 val validCounter = RegInit(0.U(log2Ceil(VirtualLoadQueueSize + 1).W)) 154 155 val enqPtr = enqPtrExt(0).value 156 val deqPtr = deqPtrExt(0).value 157 val cmtPtr = cmtPtrExt(0).value 158 159 val validCount = distanceBetween(enqPtrExt(0), deqPtrExt(0)) 160 val allowEnqueue = validCount <= (StoreQueueSize - StorePipelineWidth).U 161 162 val deqMask = UIntToMask(deqPtr, StoreQueueSize) 163 val enqMask = UIntToMask(enqPtr, StoreQueueSize) 164 165 val commitCount = RegNext(io.rob.scommit) 166 167 // store can be committed by ROB 168 io.rob.mmio := DontCare 169 io.rob.uop := DontCare 170 171 // Read dataModule 172 assert(EnsbufferWidth <= 2) 173 // rdataPtrExtNext and rdataPtrExtNext+1 entry will be read from dataModule 174 val rdataPtrExtNext = WireInit(Mux(dataBuffer.io.enq(1).fire, 175 VecInit(rdataPtrExt.map(_ + 2.U)), 176 Mux(dataBuffer.io.enq(0).fire || io.mmioStout.fire, 177 VecInit(rdataPtrExt.map(_ + 1.U)), 178 rdataPtrExt 179 ) 180 )) 181 182 // deqPtrExtNext traces which inst is about to leave store queue 183 // 184 // io.sbuffer(i).fire is RegNexted, as sbuffer data write takes 2 cycles. 185 // Before data write finish, sbuffer is unable to provide store to load 186 // forward data. As an workaround, deqPtrExt and allocated flag update 187 // is delayed so that load can get the right data from store queue. 188 // 189 // Modify deqPtrExtNext and io.sqDeq with care! 190 val deqPtrExtNext = Mux(RegNext(io.sbuffer(1).fire), 191 VecInit(deqPtrExt.map(_ + 2.U)), 192 Mux(RegNext(io.sbuffer(0).fire) || io.mmioStout.fire, 193 VecInit(deqPtrExt.map(_ + 1.U)), 194 deqPtrExt 195 ) 196 ) 197 io.sqDeq := RegNext(Mux(RegNext(io.sbuffer(1).fire), 2.U, 198 Mux(RegNext(io.sbuffer(0).fire) || io.mmioStout.fire, 1.U, 0.U) 199 )) 200 assert(!RegNext(RegNext(io.sbuffer(0).fire) && io.mmioStout.fire)) 201 202 for (i <- 0 until EnsbufferWidth) { 203 dataModule.io.raddr(i) := rdataPtrExtNext(i).value 204 paddrModule.io.raddr(i) := rdataPtrExtNext(i).value 205 vaddrModule.io.raddr(i) := rdataPtrExtNext(i).value 206 gpaddrModule.io.raddr(i) := rdataPtrExtNext(i).value 207 } 208 209 // no inst will be committed 1 cycle before tval update 210 vaddrModule.io.raddr(EnsbufferWidth) := (cmtPtrExt(0) + commitCount).value 211 gpaddrModule.io.raddr(EnsbufferWidth) := (cmtPtrExt(0) + commitCount).value 212 /** 213 * Enqueue at dispatch 214 * 215 * Currently, StoreQueue only allows enqueue when #emptyEntries > EnqWidth 216 */ 217 io.enq.canAccept := allowEnqueue 218 val canEnqueue = io.enq.req.map(_.valid) 219 val enqCancel = io.enq.req.map(_.bits.robIdx.needFlush(io.brqRedirect)) 220 for (i <- 0 until io.enq.req.length) { 221 val offset = if (i == 0) 0.U else PopCount(io.enq.needAlloc.take(i)) 222 val sqIdx = enqPtrExt(offset) 223 val index = io.enq.req(i).bits.sqIdx.value 224 when (canEnqueue(i) && !enqCancel(i)) { 225 uop(index) := io.enq.req(i).bits 226 // NOTE: the index will be used when replay 227 uop(index).sqIdx := sqIdx 228 allocated(index) := true.B 229 datavalid(index) := false.B 230 addrvalid(index) := false.B 231 committed(index) := false.B 232 pending(index) := false.B 233 prefetch(index) := false.B 234 mmio(index) := false.B 235 236 XSError(!io.enq.canAccept || !io.enq.lqCanAccept, s"must accept $i\n") 237 XSError(index =/= sqIdx.value, s"must be the same entry $i\n") 238 } 239 io.enq.resp(i) := sqIdx 240 } 241 XSDebug(p"(ready, valid): ${io.enq.canAccept}, ${Binary(Cat(io.enq.req.map(_.valid)))}\n") 242 243 /** 244 * Update addr/dataReadyPtr when issue from rs 245 */ 246 // update issuePtr 247 val IssuePtrMoveStride = 4 248 require(IssuePtrMoveStride >= 2) 249 250 val addrReadyLookupVec = (0 until IssuePtrMoveStride).map(addrReadyPtrExt + _.U) 251 val addrReadyLookup = addrReadyLookupVec.map(ptr => allocated(ptr.value) && (mmio(ptr.value) || addrvalid(ptr.value)) && ptr =/= enqPtrExt(0)) 252 val nextAddrReadyPtr = addrReadyPtrExt + PriorityEncoder(VecInit(addrReadyLookup.map(!_) :+ true.B)) 253 addrReadyPtrExt := nextAddrReadyPtr 254 255 (0 until StoreQueueSize).map(i => { 256 io.stAddrReadyVec(i) := RegNext(allocated(i) && (mmio(i) || addrvalid(i))) 257 }) 258 259 when (io.brqRedirect.valid) { 260 addrReadyPtrExt := Mux( 261 isAfter(cmtPtrExt(0), deqPtrExt(0)), 262 cmtPtrExt(0), 263 deqPtrExtNext(0) // for mmio insts, deqPtr may be ahead of cmtPtr 264 ) 265 } 266 267 io.stAddrReadySqPtr := addrReadyPtrExt 268 269 // update 270 val dataReadyLookupVec = (0 until IssuePtrMoveStride).map(dataReadyPtrExt + _.U) 271 val dataReadyLookup = dataReadyLookupVec.map(ptr => allocated(ptr.value) && (mmio(ptr.value) || datavalid(ptr.value)) && ptr =/= enqPtrExt(0)) 272 val nextDataReadyPtr = dataReadyPtrExt + PriorityEncoder(VecInit(dataReadyLookup.map(!_) :+ true.B)) 273 dataReadyPtrExt := nextDataReadyPtr 274 275 (0 until StoreQueueSize).map(i => { 276 io.stDataReadyVec(i) := RegNext(allocated(i) && (mmio(i) || datavalid(i))) 277 }) 278 279 when (io.brqRedirect.valid) { 280 dataReadyPtrExt := Mux( 281 isAfter(cmtPtrExt(0), deqPtrExt(0)), 282 cmtPtrExt(0), 283 deqPtrExtNext(0) // for mmio insts, deqPtr may be ahead of cmtPtr 284 ) 285 } 286 287 io.stDataReadySqPtr := dataReadyPtrExt 288 io.stIssuePtr := enqPtrExt(0) 289 io.sqDeqPtr := deqPtrExt(0) 290 291 /** 292 * Writeback store from store units 293 * 294 * Most store instructions writeback to regfile in the previous cycle. 295 * However, 296 * (1) For an mmio instruction with exceptions, we need to mark it as addrvalid 297 * (in this way it will trigger an exception when it reaches ROB's head) 298 * instead of pending to avoid sending them to lower level. 299 * (2) For an mmio instruction without exceptions, we mark it as pending. 300 * When the instruction reaches ROB's head, StoreQueue sends it to uncache channel. 301 * Upon receiving the response, StoreQueue writes back the instruction 302 * through arbiter with store units. It will later commit as normal. 303 */ 304 305 // Write addr to sq 306 for (i <- 0 until StorePipelineWidth) { 307 paddrModule.io.wen(i) := false.B 308 vaddrModule.io.wen(i) := false.B 309 gpaddrModule.io.wen(i) := false.B 310 dataModule.io.mask.wen(i) := false.B 311 val stWbIndex = io.storeAddrIn(i).bits.uop.sqIdx.value 312 when (io.storeAddrIn(i).fire) { 313 val addr_valid = !io.storeAddrIn(i).bits.miss 314 addrvalid(stWbIndex) := addr_valid //!io.storeAddrIn(i).bits.mmio 315 // pending(stWbIndex) := io.storeAddrIn(i).bits.mmio 316 317 paddrModule.io.waddr(i) := stWbIndex 318 paddrModule.io.wdata(i) := io.storeAddrIn(i).bits.paddr 319 paddrModule.io.wmask(i) := io.storeAddrIn(i).bits.mask 320 paddrModule.io.wlineflag(i) := io.storeAddrIn(i).bits.wlineflag 321 paddrModule.io.wen(i) := true.B 322 323 vaddrModule.io.waddr(i) := stWbIndex 324 vaddrModule.io.wdata(i) := io.storeAddrIn(i).bits.vaddr 325 vaddrModule.io.wmask(i) := io.storeAddrIn(i).bits.mask 326 vaddrModule.io.wlineflag(i) := io.storeAddrIn(i).bits.wlineflag 327 vaddrModule.io.wen(i) := true.B 328 329 gpaddrModule.io.waddr(i) := stWbIndex 330 gpaddrModule.io.wdata(i) := io.storeAddrIn(i).bits.gpaddr 331 gpaddrModule.io.wmask(i) := io.storeAddrIn(i).bits.mask 332 gpaddrModule.io.wlineflag(i) := io.storeAddrIn(i).bits.wlineflag 333 gpaddrModule.io.wen(i) := true.B 334 335 debug_paddr(paddrModule.io.waddr(i)) := paddrModule.io.wdata(i) 336 337 // mmio(stWbIndex) := io.storeAddrIn(i).bits.mmio 338 339 uop(stWbIndex).ctrl := io.storeAddrIn(i).bits.uop.ctrl 340 uop(stWbIndex).debugInfo := io.storeAddrIn(i).bits.uop.debugInfo 341 XSInfo("store addr write to sq idx %d pc 0x%x miss:%d vaddr %x paddr %x mmio %x\n", 342 io.storeAddrIn(i).bits.uop.sqIdx.value, 343 io.storeAddrIn(i).bits.uop.cf.pc, 344 io.storeAddrIn(i).bits.miss, 345 io.storeAddrIn(i).bits.vaddr, 346 io.storeAddrIn(i).bits.paddr, 347 io.storeAddrIn(i).bits.mmio 348 ) 349 } 350 351 // re-replinish mmio, for pma/pmp will get mmio one cycle later 352 val storeAddrInFireReg = RegNext(io.storeAddrIn(i).fire && !io.storeAddrIn(i).bits.miss) 353 val stWbIndexReg = RegNext(stWbIndex) 354 when (storeAddrInFireReg) { 355 pending(stWbIndexReg) := io.storeAddrInRe(i).mmio 356 mmio(stWbIndexReg) := io.storeAddrInRe(i).mmio 357 atomic(stWbIndexReg) := io.storeAddrInRe(i).atomic 358 } 359 // dcache miss info (one cycle later than storeIn) 360 // if dcache report a miss in sta pipeline, this store will trigger a prefetch when committing to sbuffer (if EnableAtCommitMissTrigger) 361 when (storeAddrInFireReg) { 362 prefetch(stWbIndexReg) := io.storeAddrInRe(i).miss 363 } 364 365 when(vaddrModule.io.wen(i)){ 366 debug_vaddr(vaddrModule.io.waddr(i)) := vaddrModule.io.wdata(i) 367 } 368 } 369 370 // Write data to sq 371 // Now store data pipeline is actually 2 stages 372 for (i <- 0 until StorePipelineWidth) { 373 dataModule.io.data.wen(i) := false.B 374 val stWbIndex = io.storeDataIn(i).bits.uop.sqIdx.value 375 // sq data write takes 2 cycles: 376 // sq data write s0 377 when (io.storeDataIn(i).fire) { 378 // send data write req to data module 379 dataModule.io.data.waddr(i) := stWbIndex 380 dataModule.io.data.wdata(i) := Mux(io.storeDataIn(i).bits.uop.ctrl.fuOpType === LSUOpType.cbo_zero, 381 0.U, 382 genWdata(io.storeDataIn(i).bits.data, io.storeDataIn(i).bits.uop.ctrl.fuOpType(1,0)) 383 ) 384 dataModule.io.data.wen(i) := true.B 385 386 debug_data(dataModule.io.data.waddr(i)) := dataModule.io.data.wdata(i) 387 388 XSInfo("store data write to sq idx %d pc 0x%x data %x -> %x\n", 389 io.storeDataIn(i).bits.uop.sqIdx.value, 390 io.storeDataIn(i).bits.uop.cf.pc, 391 io.storeDataIn(i).bits.data, 392 dataModule.io.data.wdata(i) 393 ) 394 } 395 // sq data write s1 396 when ( 397 RegNext(io.storeDataIn(i).fire) 398 // && !RegNext(io.storeDataIn(i).bits.uop).robIdx.needFlush(io.brqRedirect) 399 ) { 400 datavalid(RegNext(stWbIndex)) := true.B 401 } 402 } 403 404 // Write mask to sq 405 for (i <- 0 until StorePipelineWidth) { 406 // sq mask write s0 407 when (io.storeMaskIn(i).fire) { 408 // send data write req to data module 409 dataModule.io.mask.waddr(i) := io.storeMaskIn(i).bits.sqIdx.value 410 dataModule.io.mask.wdata(i) := io.storeMaskIn(i).bits.mask 411 dataModule.io.mask.wen(i) := true.B 412 } 413 } 414 415 /** 416 * load forward query 417 * 418 * Check store queue for instructions that is older than the load. 419 * The response will be valid at the next cycle after req. 420 */ 421 // check over all lq entries and forward data from the first matched store 422 for (i <- 0 until LoadPipelineWidth) { 423 // Compare deqPtr (deqPtr) and forward.sqIdx, we have two cases: 424 // (1) if they have the same flag, we need to check range(tail, sqIdx) 425 // (2) if they have different flags, we need to check range(tail, VirtualLoadQueueSize) and range(0, sqIdx) 426 // Forward1: Mux(same_flag, range(tail, sqIdx), range(tail, VirtualLoadQueueSize)) 427 // Forward2: Mux(same_flag, 0.U, range(0, sqIdx) ) 428 // i.e. forward1 is the target entries with the same flag bits and forward2 otherwise 429 val differentFlag = deqPtrExt(0).flag =/= io.forward(i).sqIdx.flag 430 val forwardMask = io.forward(i).sqIdxMask 431 // all addrvalid terms need to be checked 432 val addrValidVec = WireInit(VecInit((0 until StoreQueueSize).map(i => addrvalid(i) && allocated(i)))) 433 val dataValidVec = WireInit(VecInit((0 until StoreQueueSize).map(i => datavalid(i)))) 434 val allValidVec = WireInit(VecInit((0 until StoreQueueSize).map(i => addrvalid(i) && datavalid(i) && allocated(i)))) 435 436 val storeSetHitVec = 437 if (LFSTEnable) { 438 WireInit(VecInit((0 until StoreQueueSize).map(j => io.forward(i).uop.cf.loadWaitBit && uop(j).robIdx === io.forward(i).uop.cf.waitForRobIdx))) 439 } else { 440 WireInit(VecInit((0 until StoreQueueSize).map(j => uop(j).cf.storeSetHit && uop(j).cf.ssid === io.forward(i).uop.cf.ssid))) 441 } 442 443 val forwardMask1 = Mux(differentFlag, ~deqMask, deqMask ^ forwardMask) 444 val forwardMask2 = Mux(differentFlag, forwardMask, 0.U(StoreQueueSize.W)) 445 val canForward1 = forwardMask1 & allValidVec.asUInt 446 val canForward2 = forwardMask2 & allValidVec.asUInt 447 val needForward = Mux(differentFlag, ~deqMask | forwardMask, deqMask ^ forwardMask) 448 449 XSDebug(p"$i f1 ${Binary(canForward1)} f2 ${Binary(canForward2)} " + 450 p"sqIdx ${io.forward(i).sqIdx} pa ${Hexadecimal(io.forward(i).paddr)}\n" 451 ) 452 453 // do real fwd query (cam lookup in load_s1) 454 dataModule.io.needForward(i)(0) := canForward1 & vaddrModule.io.forwardMmask(i).asUInt 455 dataModule.io.needForward(i)(1) := canForward2 & vaddrModule.io.forwardMmask(i).asUInt 456 457 vaddrModule.io.forwardMdata(i) := io.forward(i).vaddr 458 vaddrModule.io.forwardDataMask(i) := io.forward(i).mask 459 paddrModule.io.forwardMdata(i) := io.forward(i).paddr 460 paddrModule.io.forwardDataMask(i) := io.forward(i).mask 461 gpaddrModule.io.forwardMdata(i) := io.forward(i).gpaddr 462 gpaddrModule.io.forwardDataMask(i) := io.forward(i).mask 463 464 465 // vaddr cam result does not equal to paddr cam result 466 // replay needed 467 // val vpmaskNotEqual = ((paddrModule.io.forwardMmask(i).asUInt ^ vaddrModule.io.forwardMmask(i).asUInt) & needForward) =/= 0.U 468 // val vaddrMatchFailed = vpmaskNotEqual && io.forward(i).valid 469 val vpmaskNotEqual = ( 470 (RegNext(paddrModule.io.forwardMmask(i).asUInt) ^ RegNext(vaddrModule.io.forwardMmask(i).asUInt)) & 471 RegNext(needForward) & 472 RegNext(addrValidVec.asUInt) 473 ) =/= 0.U 474 val vaddrMatchFailed = vpmaskNotEqual && RegNext(io.forward(i).valid) 475 when (vaddrMatchFailed) { 476 XSInfo("vaddrMatchFailed: pc %x pmask %x vmask %x\n", 477 RegNext(io.forward(i).uop.cf.pc), 478 RegNext(needForward & paddrModule.io.forwardMmask(i).asUInt), 479 RegNext(needForward & vaddrModule.io.forwardMmask(i).asUInt) 480 ); 481 } 482 XSPerfAccumulate("vaddr_match_failed", vpmaskNotEqual) 483 XSPerfAccumulate("vaddr_match_really_failed", vaddrMatchFailed) 484 485 // Fast forward mask will be generated immediately (load_s1) 486 io.forward(i).forwardMaskFast := dataModule.io.forwardMaskFast(i) 487 488 // Forward result will be generated 1 cycle later (load_s2) 489 io.forward(i).forwardMask := dataModule.io.forwardMask(i) 490 io.forward(i).forwardData := dataModule.io.forwardData(i) 491 // If addr match, data not ready, mark it as dataInvalid 492 // load_s1: generate dataInvalid in load_s1 to set fastUop 493 val dataInvalidMask1 = (addrValidVec.asUInt & ~dataValidVec.asUInt & vaddrModule.io.forwardMmask(i).asUInt & forwardMask1.asUInt) 494 val dataInvalidMask2 = (addrValidVec.asUInt & ~dataValidVec.asUInt & vaddrModule.io.forwardMmask(i).asUInt & forwardMask2.asUInt) 495 val dataInvalidMask = dataInvalidMask1 | dataInvalidMask2 496 io.forward(i).dataInvalidFast := dataInvalidMask.orR 497 498 // make chisel happy 499 val dataInvalidMask1Reg = Wire(UInt(StoreQueueSize.W)) 500 dataInvalidMask1Reg := RegNext(dataInvalidMask1) 501 // make chisel happy 502 val dataInvalidMask2Reg = Wire(UInt(StoreQueueSize.W)) 503 dataInvalidMask2Reg := RegNext(dataInvalidMask2) 504 val dataInvalidMaskReg = dataInvalidMask1Reg | dataInvalidMask2Reg 505 506 // If SSID match, address not ready, mark it as addrInvalid 507 // load_s2: generate addrInvalid 508 val addrInvalidMask1 = (~addrValidVec.asUInt & storeSetHitVec.asUInt & forwardMask1.asUInt) 509 val addrInvalidMask2 = (~addrValidVec.asUInt & storeSetHitVec.asUInt & forwardMask2.asUInt) 510 // make chisel happy 511 val addrInvalidMask1Reg = Wire(UInt(StoreQueueSize.W)) 512 addrInvalidMask1Reg := RegNext(addrInvalidMask1) 513 // make chisel happy 514 val addrInvalidMask2Reg = Wire(UInt(StoreQueueSize.W)) 515 addrInvalidMask2Reg := RegNext(addrInvalidMask2) 516 val addrInvalidMaskReg = addrInvalidMask1Reg | addrInvalidMask2Reg 517 518 // load_s2 519 io.forward(i).dataInvalid := RegNext(io.forward(i).dataInvalidFast) 520 // check if vaddr forward mismatched 521 io.forward(i).matchInvalid := vaddrMatchFailed 522 523 // data invalid sq index 524 // check whether false fail 525 // check flag 526 val s2_differentFlag = RegNext(differentFlag) 527 val s2_enqPtrExt = RegNext(enqPtrExt(0)) 528 val s2_deqPtrExt = RegNext(deqPtrExt(0)) 529 530 // addr invalid sq index 531 // make chisel happy 532 val addrInvalidMaskRegWire = Wire(UInt(StoreQueueSize.W)) 533 addrInvalidMaskRegWire := addrInvalidMaskReg 534 val addrInvalidFlag = addrInvalidMaskRegWire.orR 535 val hasInvalidAddr = (~addrValidVec.asUInt & needForward).orR 536 537 val addrInvalidSqIdx1 = OHToUInt(Reverse(PriorityEncoderOH(Reverse(addrInvalidMask1Reg)))) 538 val addrInvalidSqIdx2 = OHToUInt(Reverse(PriorityEncoderOH(Reverse(addrInvalidMask2Reg)))) 539 val addrInvalidSqIdx = Mux(addrInvalidMask2Reg.orR, addrInvalidSqIdx2, addrInvalidSqIdx1) 540 541 // store-set content management 542 // +-----------------------+ 543 // | Search a SSID for the | 544 // | load operation | 545 // +-----------------------+ 546 // | 547 // V 548 // +-------------------+ 549 // | load wait strict? | 550 // +-------------------+ 551 // | 552 // V 553 // +----------------------+ 554 // Set| |Clean 555 // V V 556 // +------------------------+ +------------------------------+ 557 // | Waiting for all older | | Wait until the corresponding | 558 // | stores operations | | older store operations | 559 // +------------------------+ +------------------------------+ 560 561 562 563 when (RegNext(io.forward(i).uop.cf.loadWaitStrict)) { 564 io.forward(i).addrInvalidSqIdx := RegNext(io.forward(i).uop.sqIdx - 1.U) 565 } .elsewhen (addrInvalidFlag) { 566 io.forward(i).addrInvalidSqIdx.flag := Mux(!s2_differentFlag || addrInvalidSqIdx >= s2_deqPtrExt.value, s2_deqPtrExt.flag, s2_enqPtrExt.flag) 567 io.forward(i).addrInvalidSqIdx.value := addrInvalidSqIdx 568 } .otherwise { 569 // may be store inst has been written to sbuffer already. 570 io.forward(i).addrInvalidSqIdx := RegNext(io.forward(i).uop.sqIdx) 571 } 572 io.forward(i).addrInvalid := Mux(RegNext(io.forward(i).uop.cf.loadWaitStrict), RegNext(hasInvalidAddr), addrInvalidFlag) 573 574 // data invalid sq index 575 // make chisel happy 576 val dataInvalidMaskRegWire = Wire(UInt(StoreQueueSize.W)) 577 dataInvalidMaskRegWire := dataInvalidMaskReg 578 val dataInvalidFlag = dataInvalidMaskRegWire.orR 579 580 val dataInvalidSqIdx1 = OHToUInt(Reverse(PriorityEncoderOH(Reverse(dataInvalidMask1Reg)))) 581 val dataInvalidSqIdx2 = OHToUInt(Reverse(PriorityEncoderOH(Reverse(dataInvalidMask2Reg)))) 582 val dataInvalidSqIdx = Mux(dataInvalidMask2Reg.orR, dataInvalidSqIdx2, dataInvalidSqIdx1) 583 584 when (dataInvalidFlag) { 585 io.forward(i).dataInvalidSqIdx.flag := Mux(!s2_differentFlag || dataInvalidSqIdx >= s2_deqPtrExt.value, s2_deqPtrExt.flag, s2_enqPtrExt.flag) 586 io.forward(i).dataInvalidSqIdx.value := dataInvalidSqIdx 587 } .otherwise { 588 // may be store inst has been written to sbuffer already. 589 io.forward(i).dataInvalidSqIdx := RegNext(io.forward(i).uop.sqIdx) 590 } 591 } 592 593 /** 594 * Memory mapped IO / other uncached operations 595 * 596 * States: 597 * (1) writeback from store units: mark as pending 598 * (2) when they reach ROB's head, they can be sent to uncache channel 599 * (3) response from uncache channel: mark as datavalidmask.wen 600 * (4) writeback to ROB (and other units): mark as writebacked 601 * (5) ROB commits the instruction: same as normal instructions 602 */ 603 //(2) when they reach ROB's head, they can be sent to uncache channel 604 val s_idle :: s_req :: s_resp :: s_wb :: s_wait :: Nil = Enum(5) 605 val uncacheState = RegInit(s_idle) 606 switch(uncacheState) { 607 is(s_idle) { 608 when(RegNext(io.rob.pendingst && pending(deqPtr) && allocated(deqPtr) && datavalid(deqPtr) && addrvalid(deqPtr))) { 609 uncacheState := s_req 610 } 611 } 612 is(s_req) { 613 when (io.uncache.req.fire) { 614 when (io.uncacheOutstanding) { 615 uncacheState := s_wb 616 } .otherwise { 617 uncacheState := s_resp 618 } 619 } 620 } 621 is(s_resp) { 622 when(io.uncache.resp.fire) { 623 uncacheState := s_wb 624 } 625 } 626 is(s_wb) { 627 when (io.mmioStout.fire) { 628 uncacheState := s_wait 629 } 630 } 631 is(s_wait) { 632 when(commitCount > 0.U) { 633 uncacheState := s_idle // ready for next mmio 634 } 635 } 636 } 637 io.uncache.req.valid := uncacheState === s_req 638 639 io.uncache.req.bits := DontCare 640 io.uncache.req.bits.cmd := MemoryOpConstants.M_XWR 641 io.uncache.req.bits.addr := paddrModule.io.rdata(0) // data(deqPtr) -> rdata(0) 642 io.uncache.req.bits.data := shiftDataToLow(paddrModule.io.rdata(0), dataModule.io.rdata(0).data) 643 io.uncache.req.bits.mask := shiftMaskToLow(paddrModule.io.rdata(0), dataModule.io.rdata(0).mask) 644 645 // CBO op type check can be delayed for 1 cycle, 646 // as uncache op will not start in s_idle 647 val cbo_mmio_addr = paddrModule.io.rdata(0) >> 2 << 2 // clear lowest 2 bits for op 648 val cbo_mmio_op = 0.U //TODO 649 val cbo_mmio_data = cbo_mmio_addr | cbo_mmio_op 650 when(RegNext(LSUOpType.isCbo(uop(deqPtr).ctrl.fuOpType))){ 651 io.uncache.req.bits.addr := DontCare // TODO 652 io.uncache.req.bits.data := paddrModule.io.rdata(0) 653 io.uncache.req.bits.mask := DontCare // TODO 654 } 655 656 io.uncache.req.bits.atomic := atomic(RegNext(rdataPtrExtNext(0)).value) 657 658 when(io.uncache.req.fire){ 659 // mmio store should not be committed until uncache req is sent 660 pending(deqPtr) := false.B 661 662 XSDebug( 663 p"uncache req: pc ${Hexadecimal(uop(deqPtr).cf.pc)} " + 664 p"addr ${Hexadecimal(io.uncache.req.bits.addr)} " + 665 p"data ${Hexadecimal(io.uncache.req.bits.data)} " + 666 p"op ${Hexadecimal(io.uncache.req.bits.cmd)} " + 667 p"mask ${Hexadecimal(io.uncache.req.bits.mask)}\n" 668 ) 669 } 670 671 // (3) response from uncache channel: mark as datavalid 672 io.uncache.resp.ready := true.B 673 674 // (4) writeback to ROB (and other units): mark as writebacked 675 io.mmioStout.valid := uncacheState === s_wb 676 io.mmioStout.bits.uop := uop(deqPtr) 677 io.mmioStout.bits.uop.sqIdx := deqPtrExt(0) 678 io.mmioStout.bits.data := shiftDataToLow(paddrModule.io.rdata(0), dataModule.io.rdata(0).data) // dataModule.io.rdata.read(deqPtr) 679 io.mmioStout.bits.redirectValid := false.B 680 io.mmioStout.bits.redirect := DontCare 681 io.mmioStout.bits.debug.isMMIO := true.B 682 io.mmioStout.bits.debug.paddr := DontCare 683 io.mmioStout.bits.debug.isPerfCnt := false.B 684 io.mmioStout.bits.fflags := DontCare 685 io.mmioStout.bits.debug.vaddr := DontCare 686 // Remove MMIO inst from store queue after MMIO request is being sent 687 // That inst will be traced by uncache state machine 688 when (io.mmioStout.fire) { 689 allocated(deqPtr) := false.B 690 } 691 692 /** 693 * ROB commits store instructions (mark them as committed) 694 * 695 * (1) When store commits, mark it as committed. 696 * (2) They will not be cancelled and can be sent to lower level. 697 */ 698 XSError(uncacheState =/= s_idle && uncacheState =/= s_wait && commitCount > 0.U, 699 "should not commit instruction when MMIO has not been finished\n") 700 for (i <- 0 until CommitWidth) { 701 when (commitCount > i.U) { // MMIO inst is not in progress 702 if(i == 0){ 703 // MMIO inst should not update committed flag 704 // Note that commit count has been delayed for 1 cycle 705 when(uncacheState === s_idle){ 706 committed(cmtPtrExt(0).value) := true.B 707 } 708 } else { 709 committed(cmtPtrExt(i).value) := true.B 710 } 711 } 712 } 713 cmtPtrExt := cmtPtrExt.map(_ + commitCount) 714 715 // committed stores will not be cancelled and can be sent to lower level. 716 // remove retired insts from sq, add retired store to sbuffer 717 718 // Read data from data module 719 // As store queue grows larger and larger, time needed to read data from data 720 // module keeps growing higher. Now we give data read a whole cycle. 721 722 val mmioStall = mmio(rdataPtrExt(0).value) 723 for (i <- 0 until EnsbufferWidth) { 724 val ptr = rdataPtrExt(i).value 725 dataBuffer.io.enq(i).valid := allocated(ptr) && committed(ptr) && !mmioStall 726 // Note that store data/addr should both be valid after store's commit 727 assert(!dataBuffer.io.enq(i).valid || allvalid(ptr)) 728 dataBuffer.io.enq(i).bits.addr := paddrModule.io.rdata(i) 729 dataBuffer.io.enq(i).bits.vaddr := vaddrModule.io.rdata(i) 730 dataBuffer.io.enq(i).bits.data := dataModule.io.rdata(i).data 731 dataBuffer.io.enq(i).bits.mask := dataModule.io.rdata(i).mask 732 dataBuffer.io.enq(i).bits.wline := paddrModule.io.rlineflag(i) 733 dataBuffer.io.enq(i).bits.sqPtr := rdataPtrExt(i) 734 dataBuffer.io.enq(i).bits.prefetch := prefetch(ptr) 735 } 736 737 // Send data stored in sbufferReqBitsReg to sbuffer 738 for (i <- 0 until EnsbufferWidth) { 739 io.sbuffer(i).valid := dataBuffer.io.deq(i).valid 740 dataBuffer.io.deq(i).ready := io.sbuffer(i).ready 741 // Write line request should have all 1 mask 742 assert(!(io.sbuffer(i).valid && io.sbuffer(i).bits.wline && !io.sbuffer(i).bits.mask.andR)) 743 io.sbuffer(i).bits := DontCare 744 io.sbuffer(i).bits.cmd := MemoryOpConstants.M_XWR 745 io.sbuffer(i).bits.addr := dataBuffer.io.deq(i).bits.addr 746 io.sbuffer(i).bits.vaddr := dataBuffer.io.deq(i).bits.vaddr 747 io.sbuffer(i).bits.data := dataBuffer.io.deq(i).bits.data 748 io.sbuffer(i).bits.mask := dataBuffer.io.deq(i).bits.mask 749 io.sbuffer(i).bits.wline := dataBuffer.io.deq(i).bits.wline 750 io.sbuffer(i).bits.prefetch := dataBuffer.io.deq(i).bits.prefetch 751 752 // io.sbuffer(i).fire is RegNexted, as sbuffer data write takes 2 cycles. 753 // Before data write finish, sbuffer is unable to provide store to load 754 // forward data. As an workaround, deqPtrExt and allocated flag update 755 // is delayed so that load can get the right data from store queue. 756 val ptr = dataBuffer.io.deq(i).bits.sqPtr.value 757 when (RegNext(io.sbuffer(i).fire)) { 758 allocated(RegEnable(ptr, io.sbuffer(i).fire)) := false.B 759 XSDebug("sbuffer "+i+" fire: ptr %d\n", ptr) 760 } 761 } 762 (1 until EnsbufferWidth).foreach(i => when(io.sbuffer(i).fire) { assert(io.sbuffer(i - 1).fire) }) 763 if (coreParams.dcacheParametersOpt.isEmpty) { 764 for (i <- 0 until EnsbufferWidth) { 765 val ptr = deqPtrExt(i).value 766 val ram = DifftestMem(64L * 1024 * 1024 * 1024, 8) 767 val wen = allocated(ptr) && committed(ptr) && !mmio(ptr) 768 val waddr = ((paddrModule.io.rdata(i) - "h80000000".U) >> 3).asUInt 769 val wdata = Mux(paddrModule.io.rdata(i)(3), dataModule.io.rdata(i).data(127, 64), dataModule.io.rdata(i).data(63, 0)) 770 val wmask = Mux(paddrModule.io.rdata(i)(3), dataModule.io.rdata(i).mask(15, 8), dataModule.io.rdata(i).mask(7, 0)) 771 when (wen) { 772 ram.write(waddr, wdata.asTypeOf(Vec(8, UInt(8.W))), wmask.asBools) 773 } 774 } 775 } 776 777 if (env.EnableDifftest) { 778 for (i <- 0 until EnsbufferWidth) { 779 val storeCommit = io.sbuffer(i).fire 780 val waddr = ZeroExt(Cat(io.sbuffer(i).bits.addr(PAddrBits - 1, 3), 0.U(3.W)), 64) 781 val sbufferMask = shiftMaskToLow(io.sbuffer(i).bits.addr, io.sbuffer(i).bits.mask) 782 val sbufferData = shiftDataToLow(io.sbuffer(i).bits.addr, io.sbuffer(i).bits.data) 783 val wmask = sbufferMask 784 val wdata = sbufferData & MaskExpand(sbufferMask) 785 786 val difftest = DifftestModule(new DiffStoreEvent, delay = 2) 787 difftest.coreid := io.hartId 788 difftest.index := i.U 789 difftest.valid := storeCommit 790 difftest.addr := waddr 791 difftest.data := wdata 792 difftest.mask := wmask 793 } 794 } 795 796 // Read vaddr for mem exception 797 io.exceptionAddr.vaddr := vaddrModule.io.rdata(EnsbufferWidth) 798 io.exceptionAddr.gpaddr := gpaddrModule.io.rdata(EnsbufferWidth) 799 // misprediction recovery / exception redirect 800 // invalidate sq term using robIdx 801 val needCancel = Wire(Vec(StoreQueueSize, Bool())) 802 for (i <- 0 until StoreQueueSize) { 803 needCancel(i) := uop(i).robIdx.needFlush(io.brqRedirect) && allocated(i) && !committed(i) 804 when (needCancel(i)) { 805 allocated(i) := false.B 806 } 807 } 808 809 /** 810* update pointers 811**/ 812 val lastEnqCancel = PopCount(RegNext(VecInit(canEnqueue.zip(enqCancel).map(x => x._1 && x._2)))) // 1 cycle after redirect 813 val lastCycleCancelCount = PopCount(RegNext(needCancel)) // 1 cycle after redirect 814 val lastCycleRedirect = RegNext(io.brqRedirect.valid) // 1 cycle after redirect 815 val enqNumber = Mux(!lastCycleRedirect&&io.enq.canAccept && io.enq.lqCanAccept, PopCount(io.enq.req.map(_.valid)), 0.U) // 1 cycle after redirect 816 817 val lastlastCycleRedirect=RegNext(lastCycleRedirect)// 2 cycle after redirect 818 val redirectCancelCount = RegEnable(lastCycleCancelCount + lastEnqCancel, lastCycleRedirect) // 2 cycle after redirect 819 820 when (lastlastCycleRedirect) { 821 // we recover the pointers in 2 cycle after redirect for better timing 822 enqPtrExt := VecInit(enqPtrExt.map(_ - redirectCancelCount)) 823 }.otherwise { 824 // lastCycleRedirect.valid or nornal case 825 // when lastCycleRedirect.valid, enqNumber === 0.U, enqPtrExt will not change 826 enqPtrExt := VecInit(enqPtrExt.map(_ + enqNumber)) 827 } 828 assert(!(lastCycleRedirect && enqNumber =/= 0.U)) 829 830 deqPtrExt := deqPtrExtNext 831 rdataPtrExt := rdataPtrExtNext 832 833 // val dequeueCount = Mux(io.sbuffer(1).fire, 2.U, Mux(io.sbuffer(0).fire || io.mmioStout.fire, 1.U, 0.U)) 834 835 // If redirect at T0, sqCancelCnt is at T2 836 io.sqCancelCnt := redirectCancelCount 837 val ForceWriteUpper = Wire(UInt(log2Up(StoreQueueSize + 1).W)) 838 ForceWriteUpper := Constantin.createRecord("ForceWriteUpper_"+p(XSCoreParamsKey).HartId.toString(), initValue = 60.U) 839 val ForceWriteLower = Wire(UInt(log2Up(StoreQueueSize + 1).W)) 840 ForceWriteLower := Constantin.createRecord("ForceWriteLower_"+p(XSCoreParamsKey).HartId.toString(), initValue = 55.U) 841 842 val valid_cnt = PopCount(allocated) 843 io.force_write := RegNext(Mux(valid_cnt >= ForceWriteUpper, true.B, valid_cnt >= ForceWriteLower && io.force_write), init = false.B) 844 845 // io.sqempty will be used by sbuffer 846 // We delay it for 1 cycle for better timing 847 // When sbuffer need to check if it is empty, the pipeline is blocked, which means delay io.sqempty 848 // for 1 cycle will also promise that sq is empty in that cycle 849 io.sqEmpty := RegNext( 850 enqPtrExt(0).value === deqPtrExt(0).value && 851 enqPtrExt(0).flag === deqPtrExt(0).flag 852 ) 853 // perf counter 854 QueuePerf(StoreQueueSize, validCount, !allowEnqueue) 855 io.sqFull := !allowEnqueue 856 XSPerfAccumulate("mmioCycle", uncacheState =/= s_idle) // lq is busy dealing with uncache req 857 XSPerfAccumulate("mmioCnt", io.uncache.req.fire) 858 XSPerfAccumulate("mmio_wb_success", io.mmioStout.fire) 859 XSPerfAccumulate("mmio_wb_blocked", io.mmioStout.valid && !io.mmioStout.ready) 860 XSPerfAccumulate("validEntryCnt", distanceBetween(enqPtrExt(0), deqPtrExt(0))) 861 XSPerfAccumulate("cmtEntryCnt", distanceBetween(cmtPtrExt(0), deqPtrExt(0))) 862 XSPerfAccumulate("nCmtEntryCnt", distanceBetween(enqPtrExt(0), cmtPtrExt(0))) 863 864 val perfValidCount = distanceBetween(enqPtrExt(0), deqPtrExt(0)) 865 val perfEvents = Seq( 866 ("mmioCycle ", uncacheState =/= s_idle), 867 ("mmioCnt ", io.uncache.req.fire), 868 ("mmio_wb_success", io.mmioStout.fire), 869 ("mmio_wb_blocked", io.mmioStout.valid && !io.mmioStout.ready), 870 ("stq_1_4_valid ", (perfValidCount < (StoreQueueSize.U/4.U))), 871 ("stq_2_4_valid ", (perfValidCount > (StoreQueueSize.U/4.U)) & (perfValidCount <= (StoreQueueSize.U/2.U))), 872 ("stq_3_4_valid ", (perfValidCount > (StoreQueueSize.U/2.U)) & (perfValidCount <= (StoreQueueSize.U*3.U/4.U))), 873 ("stq_4_4_valid ", (perfValidCount > (StoreQueueSize.U*3.U/4.U))), 874 ) 875 generatePerfEvent() 876 877 // debug info 878 XSDebug("enqPtrExt %d:%d deqPtrExt %d:%d\n", enqPtrExt(0).flag, enqPtr, deqPtrExt(0).flag, deqPtr) 879 880 def PrintFlag(flag: Bool, name: String): Unit = { 881 when(flag) { 882 XSDebug(false, true.B, name) 883 }.otherwise { 884 XSDebug(false, true.B, " ") 885 } 886 } 887 888 for (i <- 0 until StoreQueueSize) { 889 XSDebug(i + ": pc %x va %x pa %x data %x ", 890 uop(i).cf.pc, 891 debug_vaddr(i), 892 debug_paddr(i), 893 debug_data(i) 894 ) 895 PrintFlag(allocated(i), "a") 896 PrintFlag(allocated(i) && addrvalid(i), "a") 897 PrintFlag(allocated(i) && datavalid(i), "d") 898 PrintFlag(allocated(i) && committed(i), "c") 899 PrintFlag(allocated(i) && pending(i), "p") 900 PrintFlag(allocated(i) && mmio(i), "m") 901 XSDebug(false, true.B, "\n") 902 } 903 904} 905