1package xiangshan.mem 2 3import chisel3._ 4import chisel3.util._ 5import freechips.rocketchip.tile.HasFPUParameters 6import utils._ 7import xiangshan._ 8import xiangshan.cache._ 9import xiangshan.cache.{DCacheLineIO, DCacheWordIO, MemoryOpConstants, TlbRequestIO} 10import xiangshan.backend.LSUOpType 11import xiangshan.mem._ 12import xiangshan.backend.roq.RoqLsqIO 13import xiangshan.backend.fu.HasExceptionNO 14 15 16class LqPtr extends CircularQueuePtr[LqPtr](LqPtr.LoadQueueSize) { } 17 18object LqPtr extends HasXSParameter { 19 def apply(f: Bool, v: UInt): LqPtr = { 20 val ptr = Wire(new LqPtr) 21 ptr.flag := f 22 ptr.value := v 23 ptr 24 } 25} 26 27trait HasFpLoadHelper { this: HasFPUParameters => 28 def fpRdataHelper(uop: MicroOp, rdata: UInt): UInt = { 29 LookupTree(uop.ctrl.fuOpType, List( 30 LSUOpType.lw -> recode(rdata(31, 0), S), 31 LSUOpType.ld -> recode(rdata(63, 0), D) 32 )) 33 } 34} 35trait HasLoadHelper { this: XSModule => 36 def rdataHelper(uop: MicroOp, rdata: UInt): UInt = { 37 val fpWen = uop.ctrl.fpWen 38 LookupTree(uop.ctrl.fuOpType, List( 39 LSUOpType.lb -> SignExt(rdata(7, 0) , XLEN), 40 LSUOpType.lh -> SignExt(rdata(15, 0), XLEN), 41 LSUOpType.lw -> Mux(fpWen, Cat(Fill(32, 1.U(1.W)), rdata(31, 0)), SignExt(rdata(31, 0), XLEN)), 42 LSUOpType.ld -> Mux(fpWen, rdata, SignExt(rdata(63, 0), XLEN)), 43 LSUOpType.lbu -> ZeroExt(rdata(7, 0) , XLEN), 44 LSUOpType.lhu -> ZeroExt(rdata(15, 0), XLEN), 45 LSUOpType.lwu -> ZeroExt(rdata(31, 0), XLEN), 46 )) 47 } 48} 49 50class LqEnqIO extends XSBundle { 51 val canAccept = Output(Bool()) 52 val sqCanAccept = Input(Bool()) 53 val needAlloc = Vec(RenameWidth, Input(Bool())) 54 val req = Vec(RenameWidth, Flipped(ValidIO(new MicroOp))) 55 val resp = Vec(RenameWidth, Output(new LqPtr)) 56} 57 58// Load Queue 59class LoadQueue extends XSModule 60 with HasDCacheParameters 61 with HasCircularQueuePtrHelper 62 with HasLoadHelper 63 with HasExceptionNO 64{ 65 val io = IO(new Bundle() { 66 val enq = new LqEnqIO 67 val brqRedirect = Flipped(ValidIO(new Redirect)) 68 val flush = Input(Bool()) 69 val loadIn = Vec(LoadPipelineWidth, Flipped(Valid(new LsPipelineBundle))) 70 val storeIn = Vec(StorePipelineWidth, Flipped(Valid(new LsPipelineBundle))) 71 val loadDataForwarded = Vec(LoadPipelineWidth, Input(Bool())) 72 val needReplayFromRS = Vec(LoadPipelineWidth, Input(Bool())) 73 val ldout = Vec(2, DecoupledIO(new ExuOutput)) // writeback int load 74 val load_s1 = Vec(LoadPipelineWidth, Flipped(new MaskedLoadForwardQueryIO)) 75 val roq = Flipped(new RoqLsqIO) 76 val rollback = Output(Valid(new Redirect)) // replay now starts from load instead of store 77 val dcache = Flipped(ValidIO(new Refill)) 78 val uncache = new DCacheWordIO 79 val exceptionAddr = new ExceptionAddrIO 80 val lqFull = Output(Bool()) 81 }) 82 83 val uop = Reg(Vec(LoadQueueSize, new MicroOp)) 84 // val data = Reg(Vec(LoadQueueSize, new LsRoqEntry)) 85 val dataModule = Module(new LoadQueueData(LoadQueueSize, wbNumRead = LoadPipelineWidth, wbNumWrite = LoadPipelineWidth)) 86 dataModule.io := DontCare 87 val vaddrModule = Module(new SyncDataModuleTemplate(UInt(VAddrBits.W), LoadQueueSize, numRead = 1, numWrite = LoadPipelineWidth)) 88 vaddrModule.io := DontCare 89 val allocated = RegInit(VecInit(List.fill(LoadQueueSize)(false.B))) // lq entry has been allocated 90 val datavalid = RegInit(VecInit(List.fill(LoadQueueSize)(false.B))) // data is valid 91 val writebacked = RegInit(VecInit(List.fill(LoadQueueSize)(false.B))) // inst has been writebacked to CDB 92 val miss = Reg(Vec(LoadQueueSize, Bool())) // load inst missed, waiting for miss queue to accept miss request 93 // val listening = Reg(Vec(LoadQueueSize, Bool())) // waiting for refill result 94 val pending = Reg(Vec(LoadQueueSize, Bool())) // mmio pending: inst is an mmio inst, it will not be executed until it reachs the end of roq 95 96 val debug_mmio = Reg(Vec(LoadQueueSize, Bool())) // mmio: inst is an mmio inst 97 val debug_paddr = Reg(Vec(LoadQueueSize, UInt(PAddrBits.W))) // mmio: inst is an mmio inst 98 99 val enqPtrExt = RegInit(VecInit((0 until RenameWidth).map(_.U.asTypeOf(new LqPtr)))) 100 val deqPtrExt = RegInit(0.U.asTypeOf(new LqPtr)) 101 val deqPtrExtNext = Wire(new LqPtr) 102 val allowEnqueue = RegInit(true.B) 103 104 val enqPtr = enqPtrExt(0).value 105 val deqPtr = deqPtrExt.value 106 107 val deqMask = UIntToMask(deqPtr, LoadQueueSize) 108 val enqMask = UIntToMask(enqPtr, LoadQueueSize) 109 110 val commitCount = RegNext(io.roq.lcommit) 111 112 /** 113 * Enqueue at dispatch 114 * 115 * Currently, LoadQueue only allows enqueue when #emptyEntries > RenameWidth(EnqWidth) 116 */ 117 io.enq.canAccept := allowEnqueue 118 119 for (i <- 0 until RenameWidth) { 120 val offset = if (i == 0) 0.U else PopCount(io.enq.needAlloc.take(i)) 121 val lqIdx = enqPtrExt(offset) 122 val index = lqIdx.value 123 when (io.enq.req(i).valid && io.enq.canAccept && io.enq.sqCanAccept && !(io.brqRedirect.valid || io.flush)) { 124 uop(index) := io.enq.req(i).bits 125 allocated(index) := true.B 126 datavalid(index) := false.B 127 writebacked(index) := false.B 128 miss(index) := false.B 129 // listening(index) := false.B 130 pending(index) := false.B 131 } 132 io.enq.resp(i) := lqIdx 133 } 134 XSDebug(p"(ready, valid): ${io.enq.canAccept}, ${Binary(Cat(io.enq.req.map(_.valid)))}\n") 135 136 /** 137 * Writeback load from load units 138 * 139 * Most load instructions writeback to regfile at the same time. 140 * However, 141 * (1) For an mmio instruction with exceptions, it writes back to ROB immediately. 142 * (2) For an mmio instruction without exceptions, it does not write back. 143 * The mmio instruction will be sent to lower level when it reaches ROB's head. 144 * After uncache response, it will write back through arbiter with loadUnit. 145 * (3) For cache misses, it is marked miss and sent to dcache later. 146 * After cache refills, it will write back through arbiter with loadUnit. 147 */ 148 for (i <- 0 until LoadPipelineWidth) { 149 dataModule.io.wb.wen(i) := false.B 150 val loadWbIndex = io.loadIn(i).bits.uop.lqIdx.value 151 when(io.loadIn(i).fire()) { 152 when(io.loadIn(i).bits.miss) { 153 XSInfo(io.loadIn(i).valid, "load miss write to lq idx %d pc 0x%x vaddr %x paddr %x data %x mask %x forwardData %x forwardMask: %x mmio %x\n", 154 io.loadIn(i).bits.uop.lqIdx.asUInt, 155 io.loadIn(i).bits.uop.cf.pc, 156 io.loadIn(i).bits.vaddr, 157 io.loadIn(i).bits.paddr, 158 io.loadIn(i).bits.data, 159 io.loadIn(i).bits.mask, 160 io.loadIn(i).bits.forwardData.asUInt, 161 io.loadIn(i).bits.forwardMask.asUInt, 162 io.loadIn(i).bits.mmio 163 ) 164 }.otherwise { 165 XSInfo(io.loadIn(i).valid, "load hit write to cbd lqidx %d pc 0x%x vaddr %x paddr %x data %x mask %x forwardData %x forwardMask: %x mmio %x\n", 166 io.loadIn(i).bits.uop.lqIdx.asUInt, 167 io.loadIn(i).bits.uop.cf.pc, 168 io.loadIn(i).bits.vaddr, 169 io.loadIn(i).bits.paddr, 170 io.loadIn(i).bits.data, 171 io.loadIn(i).bits.mask, 172 io.loadIn(i).bits.forwardData.asUInt, 173 io.loadIn(i).bits.forwardMask.asUInt, 174 io.loadIn(i).bits.mmio 175 )} 176 datavalid(loadWbIndex) := (!io.loadIn(i).bits.miss || io.loadDataForwarded(i)) && 177 !io.loadIn(i).bits.mmio && // mmio data is not valid until we finished uncache access 178 !io.needReplayFromRS(i) // do not writeback if that inst will be resend from rs 179 writebacked(loadWbIndex) := !io.loadIn(i).bits.miss && !io.loadIn(i).bits.mmio 180 181 val loadWbData = Wire(new LQDataEntry) 182 loadWbData.paddr := io.loadIn(i).bits.paddr 183 loadWbData.mask := io.loadIn(i).bits.mask 184 loadWbData.data := io.loadIn(i).bits.forwardData.asUInt // fwd data 185 loadWbData.fwdMask := io.loadIn(i).bits.forwardMask 186 dataModule.io.wbWrite(i, loadWbIndex, loadWbData) 187 dataModule.io.wb.wen(i) := true.B 188 189 190 debug_mmio(loadWbIndex) := io.loadIn(i).bits.mmio 191 debug_paddr(loadWbIndex) := io.loadIn(i).bits.paddr 192 193 val dcacheMissed = io.loadIn(i).bits.miss && !io.loadIn(i).bits.mmio 194 miss(loadWbIndex) := dcacheMissed && !io.loadDataForwarded(i) && !io.needReplayFromRS(i) 195 pending(loadWbIndex) := io.loadIn(i).bits.mmio 196 uop(loadWbIndex).debugInfo.issueTime := io.loadIn(i).bits.uop.debugInfo.issueTime 197 } 198 // vaddrModule write is delayed, as vaddrModule will not be read right after write 199 vaddrModule.io.waddr(i) := RegNext(loadWbIndex) 200 vaddrModule.io.wdata(i) := RegNext(io.loadIn(i).bits.vaddr) 201 vaddrModule.io.wen(i) := RegNext(io.loadIn(i).fire()) 202 } 203 204 when(io.dcache.valid) { 205 XSDebug("miss resp: paddr:0x%x data %x\n", io.dcache.bits.addr, io.dcache.bits.data) 206 } 207 208 // Refill 64 bit in a cycle 209 // Refill data comes back from io.dcache.resp 210 dataModule.io.refill.valid := io.dcache.valid 211 dataModule.io.refill.paddr := io.dcache.bits.addr 212 dataModule.io.refill.data := io.dcache.bits.data 213 214 (0 until LoadQueueSize).map(i => { 215 dataModule.io.refill.refillMask(i) := allocated(i) && miss(i) 216 when(dataModule.io.refill.valid && dataModule.io.refill.refillMask(i) && dataModule.io.refill.matchMask(i)) { 217 datavalid(i) := true.B 218 miss(i) := false.B 219 } 220 }) 221 222 // Writeback up to 2 missed load insts to CDB 223 // 224 // Pick 2 missed load (data refilled), write them back to cdb 225 // 2 refilled load will be selected from even/odd entry, separately 226 227 // Stage 0 228 // Generate writeback indexes 229 230 def getEvenBits(input: UInt): UInt = { 231 require(input.getWidth == LoadQueueSize) 232 VecInit((0 until LoadQueueSize/2).map(i => {input(2*i)})).asUInt 233 } 234 def getOddBits(input: UInt): UInt = { 235 require(input.getWidth == LoadQueueSize) 236 VecInit((0 until LoadQueueSize/2).map(i => {input(2*i+1)})).asUInt 237 } 238 239 val loadWbSel = Wire(Vec(LoadPipelineWidth, UInt(log2Up(LoadQueueSize).W))) // index selected last cycle 240 val loadWbSelV = Wire(Vec(LoadPipelineWidth, Bool())) // index selected in last cycle is valid 241 242 val loadWbSelVec = VecInit((0 until LoadQueueSize).map(i => { 243 allocated(i) && !writebacked(i) && datavalid(i) 244 })).asUInt() // use uint instead vec to reduce verilog lines 245 val evenDeqMask = getEvenBits(deqMask) 246 val oddDeqMask = getOddBits(deqMask) 247 // generate lastCycleSelect mask 248 val evenSelectMask = Mux(io.ldout(0).fire(), getEvenBits(UIntToOH(loadWbSel(0))), 0.U) 249 val oddSelectMask = Mux(io.ldout(1).fire(), getOddBits(UIntToOH(loadWbSel(1))), 0.U) 250 // generate real select vec 251 val loadEvenSelVec = getEvenBits(loadWbSelVec) & ~evenSelectMask 252 val loadOddSelVec = getOddBits(loadWbSelVec) & ~oddSelectMask 253 254 def toVec(a: UInt): Vec[Bool] = { 255 VecInit(a.asBools) 256 } 257 258 val loadWbSelGen = Wire(Vec(LoadPipelineWidth, UInt(log2Up(LoadQueueSize).W))) 259 val loadWbSelVGen = Wire(Vec(LoadPipelineWidth, Bool())) 260 loadWbSelGen(0) := Cat(getFirstOne(toVec(loadEvenSelVec), evenDeqMask), 0.U(1.W)) 261 loadWbSelVGen(0):= loadEvenSelVec.asUInt.orR 262 loadWbSelGen(1) := Cat(getFirstOne(toVec(loadOddSelVec), oddDeqMask), 1.U(1.W)) 263 loadWbSelVGen(1) := loadOddSelVec.asUInt.orR 264 265 (0 until LoadPipelineWidth).map(i => { 266 loadWbSel(i) := RegNext(loadWbSelGen(i)) 267 loadWbSelV(i) := RegNext(loadWbSelVGen(i), init = false.B) 268 when(io.ldout(i).fire()){ 269 // Mark them as writebacked, so they will not be selected in the next cycle 270 writebacked(loadWbSel(i)) := true.B 271 } 272 }) 273 274 // Stage 1 275 // Use indexes generated in cycle 0 to read data 276 // writeback data to cdb 277 (0 until LoadPipelineWidth).map(i => { 278 // data select 279 dataModule.io.wb.raddr(i) := loadWbSelGen(i) 280 val rdata = dataModule.io.wb.rdata(i).data 281 val seluop = uop(loadWbSel(i)) 282 val func = seluop.ctrl.fuOpType 283 val raddr = dataModule.io.wb.rdata(i).paddr 284 val rdataSel = LookupTree(raddr(2, 0), List( 285 "b000".U -> rdata(63, 0), 286 "b001".U -> rdata(63, 8), 287 "b010".U -> rdata(63, 16), 288 "b011".U -> rdata(63, 24), 289 "b100".U -> rdata(63, 32), 290 "b101".U -> rdata(63, 40), 291 "b110".U -> rdata(63, 48), 292 "b111".U -> rdata(63, 56) 293 )) 294 val rdataPartialLoad = rdataHelper(seluop, rdataSel) 295 296 // writeback missed int/fp load 297 // 298 // Int load writeback will finish (if not blocked) in one cycle 299 io.ldout(i).bits.uop := seluop 300 io.ldout(i).bits.uop.lqIdx := loadWbSel(i).asTypeOf(new LqPtr) 301 io.ldout(i).bits.data := rdataPartialLoad 302 io.ldout(i).bits.redirectValid := false.B 303 io.ldout(i).bits.redirect := DontCare 304 io.ldout(i).bits.debug.isMMIO := debug_mmio(loadWbSel(i)) 305 io.ldout(i).bits.debug.isPerfCnt := false.B 306 io.ldout(i).bits.debug.paddr := debug_paddr(loadWbSel(i)) 307 io.ldout(i).bits.fflags := DontCare 308 io.ldout(i).valid := loadWbSelV(i) 309 310 when(io.ldout(i).fire()) { 311 XSInfo("int load miss write to cbd roqidx %d lqidx %d pc 0x%x mmio %x\n", 312 io.ldout(i).bits.uop.roqIdx.asUInt, 313 io.ldout(i).bits.uop.lqIdx.asUInt, 314 io.ldout(i).bits.uop.cf.pc, 315 debug_mmio(loadWbSel(i)) 316 ) 317 } 318 319 }) 320 321 /** 322 * Load commits 323 * 324 * When load commited, mark it as !allocated and move deqPtrExt forward. 325 */ 326 (0 until CommitWidth).map(i => { 327 when(commitCount > i.U){ 328 allocated(deqPtr+i.U) := false.B 329 } 330 }) 331 332 def getFirstOne(mask: Vec[Bool], startMask: UInt) = { 333 val length = mask.length 334 val highBits = (0 until length).map(i => mask(i) & ~startMask(i)) 335 val highBitsUint = Cat(highBits.reverse) 336 PriorityEncoder(Mux(highBitsUint.orR(), highBitsUint, mask.asUInt)) 337 } 338 339 def getOldestInTwo(valid: Seq[Bool], uop: Seq[MicroOp]) = { 340 assert(valid.length == uop.length) 341 assert(valid.length == 2) 342 Mux(valid(0) && valid(1), 343 Mux(isAfter(uop(0).roqIdx, uop(1).roqIdx), uop(1), uop(0)), 344 Mux(valid(0) && !valid(1), uop(0), uop(1))) 345 } 346 347 def getAfterMask(valid: Seq[Bool], uop: Seq[MicroOp]) = { 348 assert(valid.length == uop.length) 349 val length = valid.length 350 (0 until length).map(i => { 351 (0 until length).map(j => { 352 Mux(valid(i) && valid(j), 353 isAfter(uop(i).roqIdx, uop(j).roqIdx), 354 Mux(!valid(i), true.B, false.B)) 355 }) 356 }) 357 } 358 359 /** 360 * Memory violation detection 361 * 362 * When store writes back, it searches LoadQueue for younger load instructions 363 * with the same load physical address. They loaded wrong data and need re-execution. 364 * 365 * Cycle 0: Store Writeback 366 * Generate match vector for store address with rangeMask(stPtr, enqPtr). 367 * Besides, load instructions in LoadUnit_S1 and S2 are also checked. 368 * Cycle 1: Redirect Generation 369 * There're three possible types of violations, up to 6 possible redirect requests. 370 * Choose the oldest load (part 1). (4 + 2) -> (1 + 2) 371 * Cycle 2: Redirect Fire 372 * Choose the oldest load (part 2). (3 -> 1) 373 * Prepare redirect request according to the detected violation. 374 * Fire redirect request (if valid) 375 */ 376 377 // stage 0: lq l1 wb l1 wb lq 378 // | | | | | | (paddr match) 379 // stage 1: lq l1 wb l1 wb lq 380 // | | | | | | 381 // | |------------| | 382 // | | | 383 // stage 2: lq l1wb lq 384 // | | | 385 // -------------------- 386 // | 387 // rollback req 388 io.load_s1 := DontCare 389 def detectRollback(i: Int) = { 390 val startIndex = io.storeIn(i).bits.uop.lqIdx.value 391 val lqIdxMask = UIntToMask(startIndex, LoadQueueSize) 392 val xorMask = lqIdxMask ^ enqMask 393 val sameFlag = io.storeIn(i).bits.uop.lqIdx.flag === enqPtrExt(0).flag 394 val toEnqPtrMask = Mux(sameFlag, xorMask, ~xorMask) 395 396 // check if load already in lq needs to be rolledback 397 dataModule.io.violation(i).paddr := io.storeIn(i).bits.paddr 398 dataModule.io.violation(i).mask := io.storeIn(i).bits.mask 399 val addrMaskMatch = RegNext(dataModule.io.violation(i).violationMask) 400 val entryNeedCheck = RegNext(VecInit((0 until LoadQueueSize).map(j => { 401 allocated(j) && toEnqPtrMask(j) && (datavalid(j) || miss(j)) 402 }))) 403 val lqViolationVec = VecInit((0 until LoadQueueSize).map(j => { 404 addrMaskMatch(j) && entryNeedCheck(j) 405 })) 406 val lqViolation = lqViolationVec.asUInt().orR() 407 val lqViolationIndex = getFirstOne(lqViolationVec, RegNext(lqIdxMask)) 408 val lqViolationUop = uop(lqViolationIndex) 409 // lqViolationUop.lqIdx.flag := deqMask(lqViolationIndex) ^ deqPtrExt.flag 410 // lqViolationUop.lqIdx.value := lqViolationIndex 411 XSDebug(lqViolation, p"${Binary(Cat(lqViolationVec))}, $startIndex, $lqViolationIndex\n") 412 413 // when l/s writeback to roq together, check if rollback is needed 414 val wbViolationVec = RegNext(VecInit((0 until LoadPipelineWidth).map(j => { 415 io.loadIn(j).valid && 416 isAfter(io.loadIn(j).bits.uop.roqIdx, io.storeIn(i).bits.uop.roqIdx) && 417 io.storeIn(i).bits.paddr(PAddrBits - 1, 3) === io.loadIn(j).bits.paddr(PAddrBits - 1, 3) && 418 (io.storeIn(i).bits.mask & io.loadIn(j).bits.mask).orR 419 }))) 420 val wbViolation = wbViolationVec.asUInt().orR() 421 val wbViolationUop = getOldestInTwo(wbViolationVec, RegNext(VecInit(io.loadIn.map(_.bits.uop)))) 422 XSDebug(wbViolation, p"${Binary(Cat(wbViolationVec))}, $wbViolationUop\n") 423 424 // check if rollback is needed for load in l1 425 val l1ViolationVec = RegNext(VecInit((0 until LoadPipelineWidth).map(j => { 426 io.load_s1(j).valid && // L1 valid 427 isAfter(io.load_s1(j).uop.roqIdx, io.storeIn(i).bits.uop.roqIdx) && 428 io.storeIn(i).bits.paddr(PAddrBits - 1, 3) === io.load_s1(j).paddr(PAddrBits - 1, 3) && 429 (io.storeIn(i).bits.mask & io.load_s1(j).mask).orR 430 }))) 431 val l1Violation = l1ViolationVec.asUInt().orR() 432 val l1ViolationUop = getOldestInTwo(l1ViolationVec, RegNext(VecInit(io.load_s1.map(_.uop)))) 433 XSDebug(l1Violation, p"${Binary(Cat(l1ViolationVec))}, $l1ViolationUop\n") 434 435 XSDebug( 436 l1Violation, 437 "need rollback (l1 load) pc %x roqidx %d target %x\n", 438 io.storeIn(i).bits.uop.cf.pc, io.storeIn(i).bits.uop.roqIdx.asUInt, l1ViolationUop.roqIdx.asUInt 439 ) 440 XSDebug( 441 lqViolation, 442 "need rollback (ld wb before store) pc %x roqidx %d target %x\n", 443 io.storeIn(i).bits.uop.cf.pc, io.storeIn(i).bits.uop.roqIdx.asUInt, lqViolationUop.roqIdx.asUInt 444 ) 445 XSDebug( 446 wbViolation, 447 "need rollback (ld/st wb together) pc %x roqidx %d target %x\n", 448 io.storeIn(i).bits.uop.cf.pc, io.storeIn(i).bits.uop.roqIdx.asUInt, wbViolationUop.roqIdx.asUInt 449 ) 450 451 ((lqViolation, lqViolationUop), (wbViolation, wbViolationUop), (l1Violation, l1ViolationUop)) 452 } 453 454 def rollbackSel(a: Valid[MicroOp], b: Valid[MicroOp]): ValidIO[MicroOp] = { 455 Mux( 456 a.valid, 457 Mux( 458 b.valid, 459 Mux(isAfter(a.bits.roqIdx, b.bits.roqIdx), b, a), // a,b both valid, sel oldest 460 a // sel a 461 ), 462 b // sel b 463 ) 464 } 465 val lastCycleRedirect = RegNext(io.brqRedirect) 466 val lastlastCycleRedirect = RegNext(lastCycleRedirect) 467 val lastCycleFlush = RegNext(io.flush) 468 val lastlastCycleFlush = RegNext(lastCycleFlush) 469 470 // S2: select rollback (part1) and generate rollback request 471 // rollback check 472 // Wb/L1 rollback seq check is done in s2 473 val rollbackWb = Wire(Vec(StorePipelineWidth, Valid(new MicroOp))) 474 val rollbackL1 = Wire(Vec(StorePipelineWidth, Valid(new MicroOp))) 475 val rollbackL1Wb = Wire(Vec(StorePipelineWidth*2, Valid(new MicroOp))) 476 // Lq rollback seq check is done in s3 (next stage), as getting rollbackLq MicroOp is slow 477 val rollbackLq = Wire(Vec(StorePipelineWidth, Valid(new MicroOp))) 478 for (i <- 0 until StorePipelineWidth) { 479 val detectedRollback = detectRollback(i) 480 rollbackLq(i).valid := detectedRollback._1._1 && RegNext(io.storeIn(i).valid) 481 rollbackLq(i).bits := detectedRollback._1._2 482 rollbackWb(i).valid := detectedRollback._2._1 && RegNext(io.storeIn(i).valid) 483 rollbackWb(i).bits := detectedRollback._2._2 484 rollbackL1(i).valid := detectedRollback._3._1 && RegNext(io.storeIn(i).valid) 485 rollbackL1(i).bits := detectedRollback._3._2 486 rollbackL1Wb(2*i) := rollbackL1(i) 487 rollbackL1Wb(2*i+1) := rollbackWb(i) 488 } 489 490 val rollbackL1WbSelected = ParallelOperation(rollbackL1Wb, rollbackSel) 491 val rollbackL1WbVReg = RegNext(rollbackL1WbSelected.valid) 492 val rollbackL1WbReg = RegEnable(rollbackL1WbSelected.bits, rollbackL1WbSelected.valid) 493 val rollbackLq0VReg = RegNext(rollbackLq(0).valid) 494 val rollbackLq0Reg = RegEnable(rollbackLq(0).bits, rollbackLq(0).valid) 495 val rollbackLq1VReg = RegNext(rollbackLq(1).valid) 496 val rollbackLq1Reg = RegEnable(rollbackLq(1).bits, rollbackLq(1).valid) 497 498 // S3: select rollback (part2), generate rollback request, then fire rollback request 499 // Note that we use roqIdx - 1.U to flush the load instruction itself. 500 // Thus, here if last cycle's roqIdx equals to this cycle's roqIdx, it still triggers the redirect. 501 502 // FIXME: this is ugly 503 val rollbackValidVec = Seq(rollbackL1WbVReg, rollbackLq0VReg, rollbackLq1VReg) 504 val rollbackUopVec = Seq(rollbackL1WbReg, rollbackLq0Reg, rollbackLq1Reg) 505 506 // select uop in parallel 507 val mask = getAfterMask(rollbackValidVec, rollbackUopVec) 508 val oneAfterZero = mask(1)(0) 509 val rollbackUop = Mux(oneAfterZero && mask(2)(0), 510 rollbackUopVec(0), 511 Mux(!oneAfterZero && mask(2)(1), rollbackUopVec(1), rollbackUopVec(2))) 512 513 // check if rollback request is still valid in parallel 514 val rollbackValidVecChecked = Wire(Vec(3, Bool())) 515 for(((v, uop), idx) <- rollbackValidVec.zip(rollbackUopVec).zipWithIndex) { 516 rollbackValidVecChecked(idx) := v && 517 (!lastCycleRedirect.valid || isBefore(uop.roqIdx, lastCycleRedirect.bits.roqIdx)) && 518 (!lastlastCycleRedirect.valid || isBefore(uop.roqIdx, lastlastCycleRedirect.bits.roqIdx)) 519 } 520 521 io.rollback.bits.roqIdx := rollbackUop.roqIdx 522 io.rollback.bits.ftqIdx := rollbackUop.cf.ftqPtr 523 io.rollback.bits.ftqOffset := rollbackUop.cf.ftqOffset 524 io.rollback.bits.level := RedirectLevel.flush 525 io.rollback.bits.interrupt := DontCare 526 io.rollback.bits.cfiUpdate := DontCare 527 io.rollback.bits.cfiUpdate.target := rollbackUop.cf.pc 528 // io.rollback.bits.pc := DontCare 529 530 io.rollback.valid := rollbackValidVecChecked.asUInt.orR && !lastCycleFlush && !lastlastCycleFlush 531 532 when(io.rollback.valid) { 533 // XSDebug("Mem rollback: pc %x roqidx %d\n", io.rollback.bits.cfi, io.rollback.bits.roqIdx.asUInt) 534 } 535 536 /** 537 * Memory mapped IO / other uncached operations 538 * 539 * States: 540 * (1) writeback from store units: mark as pending 541 * (2) when they reach ROB's head, they can be sent to uncache channel 542 * (3) response from uncache channel: mark as datavalid 543 * (4) writeback to ROB (and other units): mark as writebacked 544 * (5) ROB commits the instruction: same as normal instructions 545 */ 546 //(2) when they reach ROB's head, they can be sent to uncache channel 547 val lqTailMmioPending = WireInit(pending(deqPtr)) 548 val lqTailAllocated = WireInit(allocated(deqPtr)) 549 val s_idle :: s_req :: s_resp :: s_wait :: Nil = Enum(4) 550 val uncacheState = RegInit(s_idle) 551 switch(uncacheState) { 552 is(s_idle) { 553 when(io.roq.pendingld && lqTailMmioPending && lqTailAllocated) { 554 uncacheState := s_req 555 } 556 } 557 is(s_req) { 558 when(io.uncache.req.fire()) { 559 uncacheState := s_resp 560 } 561 } 562 is(s_resp) { 563 when(io.uncache.resp.fire()) { 564 uncacheState := s_wait 565 } 566 } 567 is(s_wait) { 568 when(io.roq.commit) { 569 uncacheState := s_idle // ready for next mmio 570 } 571 } 572 } 573 io.uncache.req.valid := uncacheState === s_req 574 575 dataModule.io.uncache.raddr := deqPtrExtNext.value 576 577 io.uncache.req.bits.cmd := MemoryOpConstants.M_XRD 578 io.uncache.req.bits.addr := dataModule.io.uncache.rdata.paddr 579 io.uncache.req.bits.data := dataModule.io.uncache.rdata.data 580 io.uncache.req.bits.mask := dataModule.io.uncache.rdata.mask 581 582 io.uncache.req.bits.id := DontCare 583 584 io.uncache.resp.ready := true.B 585 586 when (io.uncache.req.fire()) { 587 pending(deqPtr) := false.B 588 589 XSDebug("uncache req: pc %x addr %x data %x op %x mask %x\n", 590 uop(deqPtr).cf.pc, 591 io.uncache.req.bits.addr, 592 io.uncache.req.bits.data, 593 io.uncache.req.bits.cmd, 594 io.uncache.req.bits.mask 595 ) 596 } 597 598 // (3) response from uncache channel: mark as datavalid 599 dataModule.io.uncache.wen := false.B 600 when(io.uncache.resp.fire()){ 601 datavalid(deqPtr) := true.B 602 dataModule.io.uncacheWrite(deqPtr, io.uncache.resp.bits.data(XLEN-1, 0)) 603 dataModule.io.uncache.wen := true.B 604 605 XSDebug("uncache resp: data %x\n", io.dcache.bits.data) 606 } 607 608 // Read vaddr for mem exception 609 // no inst will be commited 1 cycle before tval update 610 vaddrModule.io.raddr(0) := (deqPtrExt + commitCount).value 611 io.exceptionAddr.vaddr := vaddrModule.io.rdata(0) 612 613 // misprediction recovery / exception redirect 614 // invalidate lq term using robIdx 615 val needCancel = Wire(Vec(LoadQueueSize, Bool())) 616 for (i <- 0 until LoadQueueSize) { 617 needCancel(i) := uop(i).roqIdx.needFlush(io.brqRedirect, io.flush) && allocated(i) 618 when (needCancel(i)) { 619 allocated(i) := false.B 620 } 621 } 622 623 /** 624 * update pointers 625 */ 626 val lastCycleCancelCount = PopCount(RegNext(needCancel)) 627 // when io.brqRedirect.valid, we don't allow eneuque even though it may fire. 628 val enqNumber = Mux(io.enq.canAccept && io.enq.sqCanAccept && !(io.brqRedirect.valid || io.flush), PopCount(io.enq.req.map(_.valid)), 0.U) 629 when (lastCycleRedirect.valid || lastCycleFlush) { 630 // we recover the pointers in the next cycle after redirect 631 enqPtrExt := VecInit(enqPtrExt.map(_ - lastCycleCancelCount)) 632 }.otherwise { 633 enqPtrExt := VecInit(enqPtrExt.map(_ + enqNumber)) 634 } 635 636 deqPtrExtNext := deqPtrExt + commitCount 637 deqPtrExt := deqPtrExtNext 638 639 val validCount = distanceBetween(enqPtrExt(0), deqPtrExt) 640 641 allowEnqueue := validCount + enqNumber <= (LoadQueueSize - RenameWidth).U 642 643 // perf counter 644 QueuePerf(LoadQueueSize, validCount, !allowEnqueue) 645 io.lqFull := !allowEnqueue 646 XSPerfAccumulate("rollback", io.rollback.valid) // rollback redirect generated 647 XSPerfAccumulate("mmioCycle", uncacheState =/= s_idle) // lq is busy dealing with uncache req 648 XSPerfAccumulate("mmioCnt", io.uncache.req.fire()) 649 XSPerfAccumulate("refill", io.dcache.valid) 650 XSPerfAccumulate("writeback_success", PopCount(VecInit(io.ldout.map(i => i.fire())))) 651 XSPerfAccumulate("writeback_blocked", PopCount(VecInit(io.ldout.map(i => i.valid && !i.ready)))) 652 XSPerfAccumulate("utilization_miss", PopCount((0 until LoadQueueSize).map(i => allocated(i) && miss(i)))) 653 654 // debug info 655 XSDebug("enqPtrExt %d:%d deqPtrExt %d:%d\n", enqPtrExt(0).flag, enqPtr, deqPtrExt.flag, deqPtr) 656 657 def PrintFlag(flag: Bool, name: String): Unit = { 658 when(flag) { 659 XSDebug(false, true.B, name) 660 }.otherwise { 661 XSDebug(false, true.B, " ") 662 } 663 } 664 665 for (i <- 0 until LoadQueueSize) { 666 if (i % 4 == 0) XSDebug("") 667 XSDebug(false, true.B, "%x [%x] ", uop(i).cf.pc, dataModule.io.debug(i).paddr) 668 PrintFlag(allocated(i), "a") 669 PrintFlag(allocated(i) && datavalid(i), "v") 670 PrintFlag(allocated(i) && writebacked(i), "w") 671 PrintFlag(allocated(i) && miss(i), "m") 672 // PrintFlag(allocated(i) && listening(i), "l") 673 PrintFlag(allocated(i) && pending(i), "p") 674 XSDebug(false, true.B, " ") 675 if (i % 4 == 3 || i == LoadQueueSize - 1) XSDebug(false, true.B, "\n") 676 } 677 678} 679