1package xiangshan.mem 2 3import chisel3._ 4import chisel3.util._ 5import xiangshan._ 6import utils._ 7import xiangshan.cache._ 8import utils.ParallelAND 9import utils.TrueLRU 10 11class SbufferUserBundle extends XSBundle { 12 val pc = UInt(VAddrBits.W) //for debug 13 val lsroqId = UInt(log2Up(LsroqSize).W) 14} 15 16trait HasSBufferConst extends HasXSParameter { 17 val sBufferIndexWidth: Int = log2Up(StoreBufferSize) // a.k.a. index of cache line 18 19 // paddr = tag + offset 20 val tagWidth: Int = PAddrBits - log2Up(CacheLineSize / 8) 21 val offsetWidth: Int = log2Up(CacheLineSize / 8) 22 23 val cacheMaskWidth: Int = CacheLineSize / 8 24 val instMaskWidth: Int = XLEN / 8 25} 26 27class SBufferCacheLine extends XSBundle with HasSBufferConst { 28 val valid = Bool() 29 val tag = UInt(tagWidth.W) 30 val data = Vec(cacheMaskWidth, UInt(8.W))// UInt(CacheLineSize.W) 31 val mask = Vec(cacheMaskWidth, Bool()) 32} 33 34class UpdateInfo extends XSBundle with HasSBufferConst { 35 val idx: UInt = UInt(sBufferIndexWidth.W) // cache index effected by this store req 36 val newTag: UInt = UInt(tagWidth.W) 37 val newMask: Vec[Bool] = Vec(cacheMaskWidth, Bool()) 38 val newData: Vec[UInt] = Vec(cacheMaskWidth, UInt(8.W)) 39 40 val isForward: Bool = Bool() // this req has same tag as some former req 41 val isUpdated: Bool = Bool() 42 val isInserted: Bool = Bool() 43 val isIgnored: Bool = Bool() 44} 45 46class SbufferFlushBundle extends Bundle { 47 val valid = Output(Bool()) 48 val empty = Input(Bool()) 49} 50 51// Store buffer for XiangShan Out of Order LSU 52class Sbuffer extends XSModule with HasSBufferConst { 53 val io = IO(new Bundle() { 54 val in = Vec(StorePipelineWidth, Flipped(Decoupled(new DCacheWordReq))) 55 val dcache = new DCacheLineIO 56 val forward = Vec(LoadPipelineWidth, Flipped(new LoadForwardQueryIO)) 57 val flush = new Bundle { 58 val valid = Input(Bool()) 59 val empty = Output(Bool()) 60 } // sbuffer flush 61 }) 62 63 val cache: Vec[SBufferCacheLine] = RegInit(VecInit(Seq.fill(StoreBufferSize)(0.U.asTypeOf(new SBufferCacheLine)))) 64 65 val updateInfo = WireInit(VecInit(Seq.fill(StorePipelineWidth)(0.U.asTypeOf(new UpdateInfo)))) 66 updateInfo := DontCare 67 68 val lru = new TrueLRU(StoreBufferSize) 69 70 def getTag(pa: UInt): UInt = 71 pa(PAddrBits - 1, PAddrBits - tagWidth) 72 73 def getAddr(tag: UInt): UInt = 74 Cat(tag, 0.U((PAddrBits - tagWidth).W)) 75 76 def getByteOffset(pa: UInt): UInt = 77 Cat(pa(offsetWidth - 1, log2Up(8)), Fill(3, 0.U)) 78 79 // check if cacheIdx is modified by former request in this cycle 80 def busy(cacheIdx: UInt, max: Int): Bool = { 81 if (max == 0) 82 false.B 83 else 84 ParallelOR((0 until max).map(i => updateInfo(i).idx === cacheIdx && io.in(i).valid)).asBool() 85 } 86 87 88 val lru_accessed = WireInit(VecInit(Seq.fill(StorePipelineWidth)(false.B))) 89 90 // Get retired store from lsroq 91 //-------------------------------------------------------------------------------------------------------------------- 92 for (storeIdx <- 0 until StorePipelineWidth) { 93 io.in(storeIdx).ready := false.B // when there is empty line or target address already in this buffer, assert true 94 // otherwise, assert false 95 // when d-cache write port is valid, write back the oldest line to d-cache 96 97 updateInfo(storeIdx).isForward := false.B 98 updateInfo(storeIdx).isUpdated := false.B 99 updateInfo(storeIdx).isInserted := false.B 100 updateInfo(storeIdx).isIgnored := false.B 101 102 // 0. compare with former requests 103 for (formerIdx <- 0 until storeIdx) { 104 // i: former request 105 when ((getTag(io.in(storeIdx).bits.addr) === updateInfo(formerIdx).newTag) && 106 (updateInfo(formerIdx).isUpdated || updateInfo(formerIdx).isInserted) && io.in(storeIdx).valid && io.in(formerIdx).valid) { 107 updateInfo(storeIdx).isForward := true.B 108 updateInfo(formerIdx).isIgnored := true.B 109 updateInfo(storeIdx).idx := updateInfo(formerIdx).idx 110 XSDebug("req#%d writes same line with req#%d\n", storeIdx.U, formerIdx.U) 111 112 updateInfo(storeIdx).isInserted := updateInfo(formerIdx).isInserted 113 updateInfo(storeIdx).isUpdated := updateInfo(formerIdx).isUpdated 114 115 116 updateInfo(storeIdx).newTag := updateInfo(formerIdx).newTag 117 // update mask and data 118 (0 until cacheMaskWidth).foreach(i => { 119 when (i.U < getByteOffset(io.in(storeIdx).bits.addr).asUInt() || 120 i.U > (getByteOffset(io.in(storeIdx).bits.addr) | 7.U)) { 121 updateInfo(storeIdx).newMask(i) := updateInfo(formerIdx).newMask(i) 122 updateInfo(storeIdx).newData(i) := updateInfo(formerIdx).newData(i) 123 } otherwise { 124 when (io.in(storeIdx).bits.mask.asBools()(i % 8)) { 125 updateInfo(storeIdx).newMask(i) := true.B 126 updateInfo(storeIdx).newData(i) := io.in(storeIdx).bits.data(8 * (i % 8 + 1) - 1, 8 * (i % 8)) 127 } .otherwise { 128 updateInfo(storeIdx).newMask(i) := updateInfo(formerIdx).newMask(i) 129 updateInfo(storeIdx).newData(i) := updateInfo(formerIdx).newData(i) 130 } 131 } 132 }) 133 134 } 135 } 136 137 138 // 1. search for existing lines 139 for (bufIdx <- 0 until StoreBufferSize) { 140 when (!updateInfo(storeIdx).isForward && (getTag(io.in(storeIdx).bits.addr) === cache(bufIdx).tag) && cache(bufIdx).valid && io.in(storeIdx).valid) { 141 // mark this line as UPDATE 142 updateInfo(storeIdx).isUpdated := true.B 143 updateInfo(storeIdx).idx := bufIdx.U 144 updateInfo(storeIdx).newTag := getTag(io.in(storeIdx).bits.addr) 145 146 // update mask and data 147 (0 until cacheMaskWidth).foreach(i => { 148 when (i.U < getByteOffset(io.in(storeIdx).bits.addr).asUInt() || 149 i.U > (getByteOffset(io.in(storeIdx).bits.addr) | 7.U)) { 150 updateInfo(storeIdx).newMask(i) := cache(bufIdx).mask(i) 151 updateInfo(storeIdx).newData(i) := cache(bufIdx).data(i) 152 } otherwise { 153 when (io.in(storeIdx).bits.mask.asBools()(i % 8)) { 154 updateInfo(storeIdx).newMask(i) := true.B 155 updateInfo(storeIdx).newData(i) := io.in(storeIdx).bits.data(8 * (i % 8 + 1) - 1, 8 * (i % 8)) 156 } .otherwise { 157 updateInfo(storeIdx).newMask(i) := cache(bufIdx).mask(i) 158 updateInfo(storeIdx).newData(i) := cache(bufIdx).data(i) 159 } 160 } 161 }) 162 163 164 } 165 } 166 167 168 // 2. not found target address in existing lines, try to insert a new line 169 val freeVec = WireInit(VecInit((0 until StoreBufferSize).map(i => cache(i).valid || busy(i.U, storeIdx)))) 170 val hasFree = !ParallelAND(freeVec) 171 val nextFree = PriorityEncoder(freeVec.map(i => !i)) 172 // XSInfo("hasFree: %d, nextFreeIdx: %d\n", hasFree, nextFree) 173 174 when (!updateInfo(storeIdx).isForward && !updateInfo(storeIdx).isUpdated && hasFree && io.in(storeIdx).valid) { 175 updateInfo(storeIdx).isInserted := true.B 176 updateInfo(storeIdx).idx := nextFree 177 updateInfo(storeIdx).newTag := getTag(io.in(storeIdx).bits.addr) 178 179 // set mask and data 180 (0 until cacheMaskWidth).foreach(i => { 181 when (i.U < getByteOffset(io.in(storeIdx).bits.addr).asUInt() || 182 i.U > (getByteOffset(io.in(storeIdx).bits.addr) | 7.U)) { 183 updateInfo(storeIdx).newMask(i) := false.B 184 updateInfo(storeIdx).newData(i) := 0.U 185 } otherwise { 186 when (io.in(storeIdx).bits.mask.asBools()(i % 8)) { 187 updateInfo(storeIdx).newMask(i) := true.B 188 updateInfo(storeIdx).newData(i) := io.in(storeIdx).bits.data(8 * (i % 8 + 1) - 1, 8 * (i % 8)) 189// XSInfo("[%d] write data %x\n", i.U, io.in(storeIdx).bits.data(8 * (i % 8 + 1) - 1, 8 * (i % 8))) 190 } .otherwise { 191 updateInfo(storeIdx).newMask(i) := false.B 192 updateInfo(storeIdx).newData(i) := 0.U 193 } 194 } 195 }) 196 197 198 } 199 200 // 3. not enough space for this query 201 when (!updateInfo(storeIdx).isForward && !updateInfo(storeIdx).isUpdated && !updateInfo(storeIdx).isInserted) { 202 updateInfo(storeIdx).isIgnored := true.B 203 } 204 205 XSInfo(updateInfo(storeIdx).isUpdated && updateInfo(storeIdx).isInserted, "Error: one line is both updated and inserted!\n") 206 207 if (storeIdx > 0) 208 io.in(storeIdx).ready := io.in(storeIdx - 1).ready && (updateInfo(storeIdx).isUpdated || updateInfo(storeIdx).isInserted) 209 else 210 io.in(storeIdx).ready := updateInfo(storeIdx).isUpdated || updateInfo(storeIdx).isInserted 211 212 when(io.in(storeIdx).fire()){ 213 214 215 when(updateInfo(storeIdx).isIgnored) { 216 XSInfo("Ignore req#%d with paddr %x, mask %x, data %x\n", storeIdx.U, io.in(storeIdx).bits.addr, io.in(storeIdx).bits.mask, io.in(storeIdx).bits.data) 217 218 219 // Update 220 // ---------------------------------------- 221 } .elsewhen(updateInfo(storeIdx).isUpdated) { 222 // clear lruCnt 223// cache(updateInfo(storeIdx).idx).lruCnt := 0.U 224 lru.access(updateInfo(storeIdx).idx) 225 lru_accessed(storeIdx) := true.B 226 // update mask and data 227// cache(updateInfo(storeIdx).idx).data := updateInfo(storeIdx).newData 228 cache(updateInfo(storeIdx).idx).data.zipWithIndex.foreach { case (int, i) => 229 int := updateInfo(storeIdx).newData(i) 230 } 231// cache(updateInfo(storeIdx).idx).mask := updateInfo(storeIdx).newMask 232 cache(updateInfo(storeIdx).idx).mask.zipWithIndex.foreach { case (int, i) => 233 int := updateInfo(storeIdx).newMask(i) 234 } 235 236 XSInfo("Update line#%d with tag %x, mask %x, data %x\n", updateInfo(storeIdx).idx, cache(updateInfo(storeIdx).idx).tag, 237 io.in(storeIdx).bits.mask, io.in(storeIdx).bits.data) 238 239 240 // Insert 241 // ---------------------------------------- 242 } .elsewhen(updateInfo(storeIdx).isInserted) { 243 // clear lruCnt 244// cache(updateInfo(storeIdx).idx).lruCnt := 0.U 245 lru.access(updateInfo(storeIdx).idx) 246 lru_accessed(storeIdx) := true.B 247 // set valid 248 cache(updateInfo(storeIdx).idx).valid := true.B 249 // set tag 250 cache(updateInfo(storeIdx).idx).tag := updateInfo(storeIdx).newTag 251 // update mask and data 252// cache(updateInfo(storeIdx).idx).data := updateInfo(storeIdx).newData 253// cache(updateInfo(storeIdx).idx).mask := updateInfo(storeIdx).newMask 254 cache(updateInfo(storeIdx).idx).data.zipWithIndex.foreach { case (int, i) => 255 int := updateInfo(storeIdx).newData(i) 256 } 257 cache(updateInfo(storeIdx).idx).mask.zipWithIndex.foreach { case (int, i) => 258 int := updateInfo(storeIdx).newMask(i) 259 } 260 261 XSInfo("Insert into line#%d with tag %x, mask: %x, data: %x, pa: %x\n", updateInfo(storeIdx).idx, getTag(io.in(storeIdx).bits.addr), 262 io.in(storeIdx).bits.mask, io.in(storeIdx).bits.data, io.in(storeIdx).bits.addr) 263 } // ignore UNCHANGED & EVICTED state 264 } 265 } 266 267 268 // Write back to d-cache 269 //-------------------------------------------------------------------------------------------------------------------- 270 271 val WriteBackPortCount = 2 272 val FlushPort = 0 // flush has higher priority 273 val EvictionPort = 1 274 275 val wb_arb = Module(new Arbiter(UInt(), WriteBackPortCount)) 276 val wb_resp = WireInit(false.B) 277 278 val waitingCacheLine: SBufferCacheLine = RegInit(0.U.asTypeOf(new SBufferCacheLine)) 279 280 281 // LRU eviction 282 //------------------------------------------------- 283 val validCnt: UInt = Wire(UInt((sBufferIndexWidth + 1).W)) 284 validCnt := PopCount((0 until StoreBufferSize).map(i => cache(i).valid)) 285 XSInfo("[ %d ] lines valid this cycle\n", validCnt) 286 287 val oldestLineIdx: UInt = Wire(UInt(sBufferIndexWidth.W)) 288 oldestLineIdx := lru.way 289 XSInfo("Least recently used #[ %d ] line\n", oldestLineIdx) 290 291 292 // eviction state machine 293 val e_wb_req :: e_wb_resp :: Nil = Enum(2) 294 val eviction_state = RegInit(e_wb_req) 295 296 wb_arb.io.in(EvictionPort).valid := false.B 297 wb_arb.io.in(EvictionPort).bits := DontCare 298 299 when (eviction_state === e_wb_req) { 300 wb_arb.io.in(EvictionPort).valid := validCnt === StoreBufferSize.U && !waitingCacheLine.valid 301 wb_arb.io.in(EvictionPort).bits := oldestLineIdx 302 when (wb_arb.io.in(EvictionPort).fire()) { 303 eviction_state := e_wb_resp 304 } 305 } 306 307 val lru_miss = WireInit(false.B) 308 when (eviction_state === e_wb_resp) { 309 when (wb_resp) { 310 lru.miss 311 lru_miss := true.B 312 eviction_state := e_wb_req 313 } 314 } 315 316 317 // Sbuffer flush 318 //------------------------------------------------- 319 // flush state machine 320 val f_idle :: f_req :: f_wait_resp :: Nil = Enum(3) 321 val f_state = RegInit(f_idle) 322 val flush = io.flush 323 // empty means there are no valid cache line in sbuffer 324 // but there may exist cache line being flushed to dcache and not finished 325 val empty = validCnt === 0.U 326 327 // sbuffer is flushed empty only when: 328 // 1. there no valid line in sbuffer and 329 // 2. cache line waiting to be flushed are flushed out 330 flush.empty := empty && !waitingCacheLine.valid 331 332 wb_arb.io.in(FlushPort).valid := f_state === f_req 333 wb_arb.io.in(FlushPort).bits := PriorityEncoder((0 until StoreBufferSize).map(i => cache(i).valid)) 334 335 // we only expect flush signal in f_idle state 336 assert(!(flush.valid && f_state =/= f_idle)) 337 338 switch (f_state) { 339 is (f_idle) { 340 when (flush.valid && !empty) { f_state := f_req } 341 } 342 is (f_req) { 343 assert(!empty, "when flush, should not be empty") 344 when (wb_arb.io.in(FlushPort).fire()) { f_state := f_wait_resp } 345 } 346 is (f_wait_resp) { when (wb_resp) { 347 when (empty) { f_state := f_idle } 348 .otherwise { f_state := f_req } } 349 } 350 } 351 352 XSDebug(flush.valid, p"Reveive flush. f_state:${f_state}\n") 353 XSDebug(f_state =/= f_idle || flush.valid, p"f_state:${f_state} idx:${wb_arb.io.in(FlushPort).bits} In(${wb_arb.io.in(FlushPort).valid} ${wb_arb.io.in(FlushPort).ready}) wb_resp:${wb_resp}\n") 354 355 // write back unit 356 // --------------------------------------------------------------- 357 val s_invalid :: s_dcache_req :: s_dcache_resp :: Nil = Enum(3) 358 val state = RegInit(s_invalid) 359 360 val wb_idx = Reg(UInt()) 361 362 val dcacheData = Wire(UInt(io.dcache.req.bits.data.getWidth.W)) 363 val dcacheMask = Wire(UInt(io.dcache.req.bits.mask.getWidth.W)) 364 dcacheData := DontCare 365 dcacheMask := DontCare 366 367 io.dcache.req.valid := false.B //needWriteToCache 368 io.dcache.req.bits.addr := DontCare 369 io.dcache.req.bits.data := dcacheData 370 io.dcache.req.bits.mask := dcacheMask 371 io.dcache.req.bits.cmd := MemoryOpConstants.M_XWR 372 io.dcache.req.bits.meta := DontCare // NOT USED 373 io.dcache.resp.ready := false.B 374 375 wb_arb.io.out.ready := false.B 376 377 // wbu state machine 378 when (state === s_invalid) { 379 wb_arb.io.out.ready := true.B 380 when (wb_arb.io.out.fire()) { 381 assert(cache(wb_arb.io.out.bits).valid) 382 wb_idx := wb_arb.io.out.bits 383 state := s_dcache_req 384 } 385 } 386 387 when (state === s_dcache_req) { 388 // assert valid and send data + mask + addr(ends with 000b) to d-cache 389 io.dcache.req.valid := true.B 390 io.dcache.req.bits.addr := getAddr(cache(wb_idx).tag) 391 392 // prepare write data and write mask 393 // first, we get data from cache 394 dcacheData := cache(wb_idx).data.asUInt() 395 dcacheMask := cache(wb_idx).mask.asUInt() 396 397 // then, we tried to merge any updates 398 for (i <- 0 until StorePipelineWidth) { 399 // get data from updateInfo 400 when (updateInfo(i).idx === wb_idx && updateInfo(i).isUpdated && io.in(i).valid) { 401 dcacheData := updateInfo(i).newData.asUInt() 402 dcacheMask := updateInfo(i).newMask.asUInt() 403 } 404 } 405 406 when(io.dcache.req.fire()) { 407 // save current req 408 waitingCacheLine := cache(wb_idx) 409 waitingCacheLine.data := dcacheData.asTypeOf(Vec(cacheMaskWidth, UInt(8.W))) 410 waitingCacheLine.mask := dcacheMask.asTypeOf(Vec(cacheMaskWidth, Bool())) 411 waitingCacheLine.valid := true.B 412 413 cache(wb_idx).valid := false.B 414 415 state := s_dcache_resp 416 417 assert(cache(wb_idx).valid, "sbuffer cache line not valid\n") 418 XSInfo("send req to dcache %x\n", wb_idx) 419 XSDebug("[New D-Cache Req] idx: %d, addr: %x, mask: %x, data: %x\n", 420 wb_idx, io.dcache.req.bits.addr, dcacheMask.asUInt(), dcacheData.asUInt()) 421 } 422 } 423 424 when (state === s_dcache_resp) { 425 io.dcache.resp.ready := true.B 426 when(io.dcache.resp.fire()) { 427 waitingCacheLine.valid := false.B 428 wb_resp := true.B 429 state := s_invalid 430 XSInfo("recv resp from dcache. wb tag %x mask %x data %x\n", waitingCacheLine.tag, waitingCacheLine.mask.asUInt(), waitingCacheLine.data.asUInt()) 431 } 432 433 // the inflight req 434 XSDebug("[Pending Write Back] tag: %x, mask: %x, data: %x\n", 435 waitingCacheLine.tag, waitingCacheLine.mask.asUInt(), waitingCacheLine.data.asUInt()) 436 } 437 438 439 // loadForwardQuery 440 //-------------------------------------------------------------------------------------------------------------------- 441 (0 until LoadPipelineWidth).map(loadIdx => { 442 io.forward(loadIdx).forwardMask := VecInit(List.fill(instMaskWidth)(false.B)) 443 io.forward(loadIdx).forwardData := DontCare 444 445 when(getTag(io.forward(loadIdx).paddr) === waitingCacheLine.tag && waitingCacheLine.valid) { 446 (0 until XLEN / 8).foreach(i => { 447 when (waitingCacheLine.mask(i.U + getByteOffset(io.forward(loadIdx).paddr)) && io.forward(loadIdx).mask(i)) { 448 io.forward(loadIdx).forwardData(i) := waitingCacheLine.data(i.U + getByteOffset(io.forward(loadIdx).paddr)) 449 io.forward(loadIdx).forwardMask(i) := true.B 450 } 451 }) 452 } 453 454 // data in StoreBuffer should have higer priority than waitingCacheLine 455 for (sBufIdx <- 0 until StoreBufferSize) { 456 when(getTag(io.forward(loadIdx).paddr) === cache(sBufIdx).tag && cache(sBufIdx).valid) { 457 // send data with mask in this line 458 // this mask is not 'mask for cache line' and we need to check low bits of paddr 459 // to get certain part of one line 460 // P.S. data in io.in will be manipulated by lsroq 461 (0 until XLEN / 8).foreach(i => { 462 when (cache(sBufIdx).mask(i.U + getByteOffset(io.forward(loadIdx).paddr)) && io.forward(loadIdx).mask(i)) { 463 io.forward(loadIdx).forwardData(i) := cache(sBufIdx).data(i.U + getByteOffset(io.forward(loadIdx).paddr)) 464 io.forward(loadIdx).forwardMask(i) := true.B 465 } 466 }) 467 468 when (io.forward(loadIdx).valid) { 469 XSDebug("[ForwardReq] paddr: %x mask: %x pc: %x\n", io.forward(loadIdx).paddr, io.forward(loadIdx).mask, io.forward(loadIdx).pc) 470 XSDebug("[Forwarding] forward-data: %x forward-mask: %x\n", io.forward(loadIdx).forwardData.asUInt(), 471 io.forward(loadIdx).forwardMask.asUInt()) 472 } 473 } 474 } 475 476 }) 477 478 // additional logs 479 XSInfo(io.in(0).fire(), "ensbuffer addr 0x%x wdata 0x%x\n", io.in(0).bits.addr, io.in(0).bits.data) 480 XSInfo(io.in(1).fire(), "ensbuffer addr 0x%x wdata 0x%x\n", io.in(1).bits.addr, io.in(1).bits.data) 481 XSInfo(io.dcache.req.fire(), "desbuffer addr 0x%x wdata 0x%x\n", io.dcache.req.bits.addr, io.dcache.req.bits.data) 482 483 // output cache line 484 cache.zipWithIndex.foreach { case (line, i) => { 485 XSDebug(line.valid, "[#%d line] Tag: %x, data: %x, mask: %x\n", i.U, line.tag, line.data.asUInt(), line.mask.asUInt()) 486 }} 487} 488