1package xiangshan.mem 2 3import chisel3._ 4import chisel3.util._ 5import xiangshan._ 6import utils._ 7import xiangshan.cache._ 8import utils.ParallelAND 9import utils.TrueLRU 10 11class SbufferUserBundle extends XSBundle { 12 val pc = UInt(VAddrBits.W) //for debug 13} 14 15trait HasSBufferConst extends HasXSParameter { 16 val sBufferIndexWidth: Int = log2Up(StoreBufferSize) // a.k.a. index of cache line 17 18 // paddr = tag + offset 19 val tagWidth: Int = PAddrBits - log2Up(CacheLineSize / 8) 20 val offsetWidth: Int = log2Up(CacheLineSize / 8) 21 22 val cacheMaskWidth: Int = CacheLineSize / 8 23 val instMaskWidth: Int = XLEN / 8 24} 25 26class SBufferCacheLine extends XSBundle with HasSBufferConst { 27 val valid = Bool() 28 val tag = UInt(tagWidth.W) 29 val data = Vec(cacheMaskWidth, UInt(8.W))// UInt(CacheLineSize.W) 30 val mask = Vec(cacheMaskWidth, Bool()) 31} 32 33class UpdateInfo extends XSBundle with HasSBufferConst { 34 val idx: UInt = UInt(sBufferIndexWidth.W) // cache index effected by this store req 35 val newTag: UInt = UInt(tagWidth.W) 36 val newMask: Vec[Bool] = Vec(cacheMaskWidth, Bool()) 37 val newData: Vec[UInt] = Vec(cacheMaskWidth, UInt(8.W)) 38 39 val isForward: Bool = Bool() // this req has same tag as some former req 40 val isUpdated: Bool = Bool() 41 val isInserted: Bool = Bool() 42 val isIgnored: Bool = Bool() 43} 44 45class SbufferFlushBundle extends Bundle { 46 val valid = Output(Bool()) 47 val empty = Input(Bool()) 48} 49 50// Store buffer for XiangShan Out of Order LSU 51class Sbuffer extends XSModule with HasSBufferConst { 52 val io = IO(new Bundle() { 53 val in = Vec(StorePipelineWidth, Flipped(Decoupled(new DCacheWordReq))) 54 val dcache = new DCacheLineIO 55 val forward = Vec(LoadPipelineWidth, Flipped(new LoadForwardQueryIO)) 56 val flush = new Bundle { 57 val valid = Input(Bool()) 58 val empty = Output(Bool()) 59 } // sbuffer flush 60 }) 61 62 val cache: Vec[SBufferCacheLine] = RegInit(VecInit(Seq.fill(StoreBufferSize)(0.U.asTypeOf(new SBufferCacheLine)))) 63 64 val updateInfo = WireInit(VecInit(Seq.fill(StorePipelineWidth)(0.U.asTypeOf(new UpdateInfo)))) 65 updateInfo := DontCare 66 67 val lru = new TrueLRU(StoreBufferSize) 68 69 def getTag(pa: UInt): UInt = 70 pa(PAddrBits - 1, PAddrBits - tagWidth) 71 72 def getAddr(tag: UInt): UInt = 73 Cat(tag, 0.U((PAddrBits - tagWidth).W)) 74 75 def getByteOffset(pa: UInt): UInt = 76 Cat(pa(offsetWidth - 1, log2Up(8)), Fill(3, 0.U)) 77 78 // check if cacheIdx is modified by former request in this cycle 79 def busy(cacheIdx: UInt, max: Int): Bool = { 80 if (max == 0) 81 false.B 82 else 83 ParallelOR((0 until max).map(i => updateInfo(i).idx === cacheIdx && io.in(i).valid)).asBool() 84 } 85 86 87 val lru_accessed = WireInit(VecInit(Seq.fill(StorePipelineWidth)(false.B))) 88 89 // Get retired store from lsq 90 //-------------------------------------------------------------------------------------------------------------------- 91 for (storeIdx <- 0 until StorePipelineWidth) { 92 io.in(storeIdx).ready := false.B // when there is empty line or target address already in this buffer, assert true 93 // otherwise, assert false 94 // when d-cache write port is valid, write back the oldest line to d-cache 95 96 updateInfo(storeIdx).isForward := false.B 97 updateInfo(storeIdx).isUpdated := false.B 98 updateInfo(storeIdx).isInserted := false.B 99 updateInfo(storeIdx).isIgnored := false.B 100 101 // 0. compare with former requests 102 for (formerIdx <- 0 until storeIdx) { 103 // i: former request 104 when ((getTag(io.in(storeIdx).bits.addr) === updateInfo(formerIdx).newTag) && 105 (updateInfo(formerIdx).isUpdated || updateInfo(formerIdx).isInserted) && io.in(storeIdx).valid && io.in(formerIdx).valid) { 106 updateInfo(storeIdx).isForward := true.B 107 updateInfo(formerIdx).isIgnored := true.B 108 updateInfo(storeIdx).idx := updateInfo(formerIdx).idx 109 XSDebug("req#%d writes same line with req#%d\n", storeIdx.U, formerIdx.U) 110 111 updateInfo(storeIdx).isInserted := updateInfo(formerIdx).isInserted 112 updateInfo(storeIdx).isUpdated := updateInfo(formerIdx).isUpdated 113 114 115 updateInfo(storeIdx).newTag := updateInfo(formerIdx).newTag 116 // update mask and data 117 (0 until cacheMaskWidth).foreach(i => { 118 when (i.U < getByteOffset(io.in(storeIdx).bits.addr).asUInt() || 119 i.U > (getByteOffset(io.in(storeIdx).bits.addr) | 7.U)) { 120 updateInfo(storeIdx).newMask(i) := updateInfo(formerIdx).newMask(i) 121 updateInfo(storeIdx).newData(i) := updateInfo(formerIdx).newData(i) 122 } otherwise { 123 when (io.in(storeIdx).bits.mask.asBools()(i % 8)) { 124 updateInfo(storeIdx).newMask(i) := true.B 125 updateInfo(storeIdx).newData(i) := io.in(storeIdx).bits.data(8 * (i % 8 + 1) - 1, 8 * (i % 8)) 126 } .otherwise { 127 updateInfo(storeIdx).newMask(i) := updateInfo(formerIdx).newMask(i) 128 updateInfo(storeIdx).newData(i) := updateInfo(formerIdx).newData(i) 129 } 130 } 131 }) 132 133 } 134 } 135 136 137 // 1. search for existing lines 138 for (bufIdx <- 0 until StoreBufferSize) { 139 when (!updateInfo(storeIdx).isForward && (getTag(io.in(storeIdx).bits.addr) === cache(bufIdx).tag) && cache(bufIdx).valid && io.in(storeIdx).valid) { 140 // mark this line as UPDATE 141 updateInfo(storeIdx).isUpdated := true.B 142 updateInfo(storeIdx).idx := bufIdx.U 143 updateInfo(storeIdx).newTag := getTag(io.in(storeIdx).bits.addr) 144 145 // update mask and data 146 (0 until cacheMaskWidth).foreach(i => { 147 when (i.U < getByteOffset(io.in(storeIdx).bits.addr).asUInt() || 148 i.U > (getByteOffset(io.in(storeIdx).bits.addr) | 7.U)) { 149 updateInfo(storeIdx).newMask(i) := cache(bufIdx).mask(i) 150 updateInfo(storeIdx).newData(i) := cache(bufIdx).data(i) 151 } otherwise { 152 when (io.in(storeIdx).bits.mask.asBools()(i % 8)) { 153 updateInfo(storeIdx).newMask(i) := true.B 154 updateInfo(storeIdx).newData(i) := io.in(storeIdx).bits.data(8 * (i % 8 + 1) - 1, 8 * (i % 8)) 155 } .otherwise { 156 updateInfo(storeIdx).newMask(i) := cache(bufIdx).mask(i) 157 updateInfo(storeIdx).newData(i) := cache(bufIdx).data(i) 158 } 159 } 160 }) 161 162 163 } 164 } 165 166 167 // 2. not found target address in existing lines, try to insert a new line 168 val freeVec = WireInit(VecInit((0 until StoreBufferSize).map(i => cache(i).valid || busy(i.U, storeIdx)))) 169 val hasFree = !ParallelAND(freeVec) 170 val nextFree = PriorityEncoder(freeVec.map(i => !i)) 171 // XSInfo("hasFree: %d, nextFreeIdx: %d\n", hasFree, nextFree) 172 173 when (!updateInfo(storeIdx).isForward && !updateInfo(storeIdx).isUpdated && hasFree && io.in(storeIdx).valid) { 174 updateInfo(storeIdx).isInserted := true.B 175 updateInfo(storeIdx).idx := nextFree 176 updateInfo(storeIdx).newTag := getTag(io.in(storeIdx).bits.addr) 177 178 // set mask and data 179 (0 until cacheMaskWidth).foreach(i => { 180 when (i.U < getByteOffset(io.in(storeIdx).bits.addr).asUInt() || 181 i.U > (getByteOffset(io.in(storeIdx).bits.addr) | 7.U)) { 182 updateInfo(storeIdx).newMask(i) := false.B 183 updateInfo(storeIdx).newData(i) := 0.U 184 } otherwise { 185 when (io.in(storeIdx).bits.mask.asBools()(i % 8)) { 186 updateInfo(storeIdx).newMask(i) := true.B 187 updateInfo(storeIdx).newData(i) := io.in(storeIdx).bits.data(8 * (i % 8 + 1) - 1, 8 * (i % 8)) 188// XSInfo("[%d] write data %x\n", i.U, io.in(storeIdx).bits.data(8 * (i % 8 + 1) - 1, 8 * (i % 8))) 189 } .otherwise { 190 updateInfo(storeIdx).newMask(i) := false.B 191 updateInfo(storeIdx).newData(i) := 0.U 192 } 193 } 194 }) 195 196 197 } 198 199 // 3. not enough space for this query 200 when (!updateInfo(storeIdx).isForward && !updateInfo(storeIdx).isUpdated && !updateInfo(storeIdx).isInserted) { 201 updateInfo(storeIdx).isIgnored := true.B 202 } 203 204 XSInfo(updateInfo(storeIdx).isUpdated && updateInfo(storeIdx).isInserted, "Error: one line is both updated and inserted!\n") 205 206 if (storeIdx > 0) 207 io.in(storeIdx).ready := io.in(storeIdx - 1).ready && (updateInfo(storeIdx).isUpdated || updateInfo(storeIdx).isInserted) 208 else 209 io.in(storeIdx).ready := updateInfo(storeIdx).isUpdated || updateInfo(storeIdx).isInserted 210 211 when(io.in(storeIdx).fire()){ 212 213 214 when(updateInfo(storeIdx).isIgnored) { 215 XSInfo("Ignore req#%d with paddr %x, mask %x, data %x\n", storeIdx.U, io.in(storeIdx).bits.addr, io.in(storeIdx).bits.mask, io.in(storeIdx).bits.data) 216 217 218 // Update 219 // ---------------------------------------- 220 } .elsewhen(updateInfo(storeIdx).isUpdated) { 221 // clear lruCnt 222// cache(updateInfo(storeIdx).idx).lruCnt := 0.U 223 lru.access(updateInfo(storeIdx).idx) 224 lru_accessed(storeIdx) := true.B 225 // update mask and data 226// cache(updateInfo(storeIdx).idx).data := updateInfo(storeIdx).newData 227 cache(updateInfo(storeIdx).idx).data.zipWithIndex.foreach { case (int, i) => 228 int := updateInfo(storeIdx).newData(i) 229 } 230// cache(updateInfo(storeIdx).idx).mask := updateInfo(storeIdx).newMask 231 cache(updateInfo(storeIdx).idx).mask.zipWithIndex.foreach { case (int, i) => 232 int := updateInfo(storeIdx).newMask(i) 233 } 234 235 XSInfo("Update line#%d with tag %x, mask %x, data %x\n", updateInfo(storeIdx).idx, cache(updateInfo(storeIdx).idx).tag, 236 io.in(storeIdx).bits.mask, io.in(storeIdx).bits.data) 237 238 239 // Insert 240 // ---------------------------------------- 241 } .elsewhen(updateInfo(storeIdx).isInserted) { 242 // clear lruCnt 243// cache(updateInfo(storeIdx).idx).lruCnt := 0.U 244 lru.access(updateInfo(storeIdx).idx) 245 lru_accessed(storeIdx) := true.B 246 // set valid 247 cache(updateInfo(storeIdx).idx).valid := true.B 248 // set tag 249 cache(updateInfo(storeIdx).idx).tag := updateInfo(storeIdx).newTag 250 // update mask and data 251// cache(updateInfo(storeIdx).idx).data := updateInfo(storeIdx).newData 252// cache(updateInfo(storeIdx).idx).mask := updateInfo(storeIdx).newMask 253 cache(updateInfo(storeIdx).idx).data.zipWithIndex.foreach { case (int, i) => 254 int := updateInfo(storeIdx).newData(i) 255 } 256 cache(updateInfo(storeIdx).idx).mask.zipWithIndex.foreach { case (int, i) => 257 int := updateInfo(storeIdx).newMask(i) 258 } 259 260 XSInfo("Insert into line#%d with tag %x, mask: %x, data: %x, pa: %x\n", updateInfo(storeIdx).idx, getTag(io.in(storeIdx).bits.addr), 261 io.in(storeIdx).bits.mask, io.in(storeIdx).bits.data, io.in(storeIdx).bits.addr) 262 } // ignore UNCHANGED & EVICTED state 263 } 264 } 265 266 267 // Write back to d-cache 268 //-------------------------------------------------------------------------------------------------------------------- 269 270 val WriteBackPortCount = 2 271 val FlushPort = 0 // flush has higher priority 272 val EvictionPort = 1 273 274 val wb_arb = Module(new Arbiter(UInt(), WriteBackPortCount)) 275 val wb_resp = WireInit(false.B) 276 277 val waitingCacheLine: SBufferCacheLine = RegInit(0.U.asTypeOf(new SBufferCacheLine)) 278 279 280 // LRU eviction 281 //------------------------------------------------- 282 val validCnt: UInt = Wire(UInt((sBufferIndexWidth + 1).W)) 283 validCnt := PopCount((0 until StoreBufferSize).map(i => cache(i).valid)) 284 XSInfo("[ %d ] lines valid this cycle\n", validCnt) 285 286 val oldestLineIdx: UInt = Wire(UInt(sBufferIndexWidth.W)) 287 oldestLineIdx := lru.way 288 XSInfo("Least recently used #[ %d ] line\n", oldestLineIdx) 289 290 291 // eviction state machine 292 val e_wb_req :: e_wb_resp :: Nil = Enum(2) 293 val eviction_state = RegInit(e_wb_req) 294 295 wb_arb.io.in(EvictionPort).valid := false.B 296 wb_arb.io.in(EvictionPort).bits := DontCare 297 298 when (eviction_state === e_wb_req) { 299 wb_arb.io.in(EvictionPort).valid := validCnt === StoreBufferSize.U && !waitingCacheLine.valid 300 wb_arb.io.in(EvictionPort).bits := oldestLineIdx 301 when (wb_arb.io.in(EvictionPort).fire()) { 302 eviction_state := e_wb_resp 303 } 304 } 305 306 val lru_miss = WireInit(false.B) 307 when (eviction_state === e_wb_resp) { 308 when (wb_resp) { 309 lru.miss 310 lru_miss := true.B 311 eviction_state := e_wb_req 312 } 313 } 314 315 316 // Sbuffer flush 317 //------------------------------------------------- 318 // flush state machine 319 val f_idle :: f_req :: f_wait_resp :: Nil = Enum(3) 320 val f_state = RegInit(f_idle) 321 val flush = io.flush 322 // empty means there are no valid cache line in sbuffer 323 // but there may exist cache line being flushed to dcache and not finished 324 val empty = validCnt === 0.U 325 326 // sbuffer is flushed empty only when: 327 // 1. there no valid line in sbuffer and 328 // 2. cache line waiting to be flushed are flushed out 329 flush.empty := empty && !waitingCacheLine.valid 330 331 wb_arb.io.in(FlushPort).valid := f_state === f_req 332 wb_arb.io.in(FlushPort).bits := PriorityEncoder((0 until StoreBufferSize).map(i => cache(i).valid)) 333 334 // we only expect flush signal in f_idle state 335 assert(!(flush.valid && f_state =/= f_idle)) 336 337 switch (f_state) { 338 is (f_idle) { 339 when (flush.valid && !empty) { f_state := f_req } 340 } 341 is (f_req) { 342 assert(!empty, "when flush, should not be empty") 343 when (wb_arb.io.in(FlushPort).fire()) { f_state := f_wait_resp } 344 } 345 is (f_wait_resp) { when (wb_resp) { 346 when (empty) { f_state := f_idle } 347 .otherwise { f_state := f_req } } 348 } 349 } 350 351 XSDebug(flush.valid, p"Reveive flush. f_state:${f_state}\n") 352 XSDebug(f_state =/= f_idle || flush.valid, p"f_state:${f_state} idx:${wb_arb.io.in(FlushPort).bits} In(${wb_arb.io.in(FlushPort).valid} ${wb_arb.io.in(FlushPort).ready}) wb_resp:${wb_resp}\n") 353 354 // write back unit 355 // --------------------------------------------------------------- 356 val s_invalid :: s_dcache_req :: s_dcache_resp :: Nil = Enum(3) 357 val state = RegInit(s_invalid) 358 359 val wb_idx = Reg(UInt()) 360 361 val dcacheData = Wire(UInt(io.dcache.req.bits.data.getWidth.W)) 362 val dcacheMask = Wire(UInt(io.dcache.req.bits.mask.getWidth.W)) 363 dcacheData := DontCare 364 dcacheMask := DontCare 365 366 io.dcache.req.valid := false.B //needWriteToCache 367 io.dcache.req.bits.addr := DontCare 368 io.dcache.req.bits.data := dcacheData 369 io.dcache.req.bits.mask := dcacheMask 370 io.dcache.req.bits.cmd := MemoryOpConstants.M_XWR 371 io.dcache.req.bits.meta := DontCare // NOT USED 372 io.dcache.resp.ready := false.B 373 374 wb_arb.io.out.ready := false.B 375 376 // wbu state machine 377 when (state === s_invalid) { 378 wb_arb.io.out.ready := true.B 379 when (wb_arb.io.out.fire()) { 380 assert(cache(wb_arb.io.out.bits).valid) 381 wb_idx := wb_arb.io.out.bits 382 state := s_dcache_req 383 } 384 } 385 386 when (state === s_dcache_req) { 387 // assert valid and send data + mask + addr(ends with 000b) to d-cache 388 io.dcache.req.valid := true.B 389 io.dcache.req.bits.addr := getAddr(cache(wb_idx).tag) 390 391 // prepare write data and write mask 392 // first, we get data from cache 393 dcacheData := cache(wb_idx).data.asUInt() 394 dcacheMask := cache(wb_idx).mask.asUInt() 395 396 // then, we tried to merge any updates 397 for (i <- 0 until StorePipelineWidth) { 398 // get data from updateInfo 399 when (updateInfo(i).idx === wb_idx && updateInfo(i).isUpdated && io.in(i).valid) { 400 dcacheData := updateInfo(i).newData.asUInt() 401 dcacheMask := updateInfo(i).newMask.asUInt() 402 } 403 } 404 405 when(io.dcache.req.fire()) { 406 // save current req 407 waitingCacheLine := cache(wb_idx) 408 waitingCacheLine.data := dcacheData.asTypeOf(Vec(cacheMaskWidth, UInt(8.W))) 409 waitingCacheLine.mask := dcacheMask.asTypeOf(Vec(cacheMaskWidth, Bool())) 410 waitingCacheLine.valid := true.B 411 412 cache(wb_idx).valid := false.B 413 414 state := s_dcache_resp 415 416 assert(cache(wb_idx).valid, "sbuffer cache line not valid\n") 417 XSInfo("send req to dcache %x\n", wb_idx) 418 XSDebug("[New D-Cache Req] idx: %d, addr: %x, mask: %x, data: %x\n", 419 wb_idx, io.dcache.req.bits.addr, dcacheMask.asUInt(), dcacheData.asUInt()) 420 } 421 } 422 423 when (state === s_dcache_resp) { 424 io.dcache.resp.ready := true.B 425 when(io.dcache.resp.fire()) { 426 waitingCacheLine.valid := false.B 427 wb_resp := true.B 428 state := s_invalid 429 XSInfo("recv resp from dcache. wb tag %x mask %x data %x\n", waitingCacheLine.tag, waitingCacheLine.mask.asUInt(), waitingCacheLine.data.asUInt()) 430 } 431 432 // the inflight req 433 XSDebug("[Pending Write Back] tag: %x, mask: %x, data: %x\n", 434 waitingCacheLine.tag, waitingCacheLine.mask.asUInt(), waitingCacheLine.data.asUInt()) 435 } 436 437 438 // loadForwardQuery 439 //-------------------------------------------------------------------------------------------------------------------- 440 (0 until LoadPipelineWidth).map(loadIdx => { 441 io.forward(loadIdx).forwardMask := VecInit(List.fill(instMaskWidth)(false.B)) 442 io.forward(loadIdx).forwardData := DontCare 443 444 when(getTag(io.forward(loadIdx).paddr) === waitingCacheLine.tag && waitingCacheLine.valid) { 445 (0 until XLEN / 8).foreach(i => { 446 when (waitingCacheLine.mask(i.U + getByteOffset(io.forward(loadIdx).paddr)) && io.forward(loadIdx).mask(i)) { 447 io.forward(loadIdx).forwardData(i) := waitingCacheLine.data(i.U + getByteOffset(io.forward(loadIdx).paddr)) 448 io.forward(loadIdx).forwardMask(i) := true.B 449 } 450 }) 451 } 452 453 // data in StoreBuffer should have higer priority than waitingCacheLine 454 for (sBufIdx <- 0 until StoreBufferSize) { 455 when(getTag(io.forward(loadIdx).paddr) === cache(sBufIdx).tag && cache(sBufIdx).valid) { 456 // send data with mask in this line 457 // this mask is not 'mask for cache line' and we need to check low bits of paddr 458 // to get certain part of one line 459 // P.S. data in io.in will be manipulated by lsq 460 (0 until XLEN / 8).foreach(i => { 461 when (cache(sBufIdx).mask(i.U + getByteOffset(io.forward(loadIdx).paddr)) && io.forward(loadIdx).mask(i)) { 462 io.forward(loadIdx).forwardData(i) := cache(sBufIdx).data(i.U + getByteOffset(io.forward(loadIdx).paddr)) 463 io.forward(loadIdx).forwardMask(i) := true.B 464 } 465 }) 466 467 when (io.forward(loadIdx).valid) { 468 XSDebug("[ForwardReq] paddr: %x mask: %x pc: %x\n", io.forward(loadIdx).paddr, io.forward(loadIdx).mask, io.forward(loadIdx).pc) 469 XSDebug("[Forwarding] forward-data: %x forward-mask: %x\n", io.forward(loadIdx).forwardData.asUInt(), 470 io.forward(loadIdx).forwardMask.asUInt()) 471 } 472 } 473 } 474 475 }) 476 477 // additional logs 478 XSInfo(io.in(0).fire(), "ensbuffer addr 0x%x wdata 0x%x\n", io.in(0).bits.addr, io.in(0).bits.data) 479 XSInfo(io.in(1).fire(), "ensbuffer addr 0x%x wdata 0x%x\n", io.in(1).bits.addr, io.in(1).bits.data) 480 XSInfo(io.dcache.req.fire(), "desbuffer addr 0x%x wdata 0x%x\n", io.dcache.req.bits.addr, io.dcache.req.bits.data) 481 482 // output cache line 483 cache.zipWithIndex.foreach { case (line, i) => { 484 XSDebug(line.valid, "[#%d line] Tag: %x, data: %x, mask: %x\n", i.U, line.tag, line.data.asUInt(), line.mask.asUInt()) 485 }} 486} 487