1/*************************************************************************************** 2* Copyright (c) 2024 Beijing Institute of Open Source Chip (BOSC) 3* Copyright (c) 2020-2024 Institute of Computing Technology, Chinese Academy of Sciences 4* Copyright (c) 2020-2021 Peng Cheng Laboratory 5* 6* XiangShan is licensed under Mulan PSL v2. 7* You can use this software according to the terms and conditions of the Mulan PSL v2. 8* You may obtain a copy of Mulan PSL v2 at: 9* http://license.coscl.org.cn/MulanPSL2 10* 11* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 12* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 13* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 14* 15* See the Mulan PSL v2 for more details. 16* 17* 18* Acknowledgement 19* 20* This implementation is inspired by several key papers: 21* [1] Glenn Reinman, Brad Calder, and Todd Austin. "[Fetch directed instruction prefetching.] 22* (https://doi.org/10.1109/MICRO.1999.809439)" 32nd Annual ACM/IEEE International Symposium on Microarchitecture 23* (MICRO). 1999. 24***************************************************************************************/ 25 26package xiangshan.frontend.icache 27 28import chisel3._ 29import chisel3.util._ 30import freechips.rocketchip.diplomacy.AddressSet 31import freechips.rocketchip.diplomacy.IdRange 32import freechips.rocketchip.diplomacy.LazyModule 33import freechips.rocketchip.diplomacy.LazyModuleImp 34import freechips.rocketchip.tilelink._ 35import freechips.rocketchip.util.BundleFieldBase 36import huancun.AliasField 37import huancun.PrefetchField 38import org.chipsalliance.cde.config.Parameters 39import utility._ 40import utility.mbist.MbistPipeline 41import utility.sram.SRAMReadBus 42import utility.sram.SRAMTemplate 43import utility.sram.SRAMWriteBus 44import utils._ 45import xiangshan._ 46import xiangshan.cache._ 47import xiangshan.cache.mmu.TlbRequestIO 48import xiangshan.frontend._ 49 50case class ICacheParameters( 51 nSets: Int = 256, 52 nWays: Int = 4, 53 rowBits: Int = 64, 54 nTLBEntries: Int = 32, 55 tagECC: Option[String] = None, 56 dataECC: Option[String] = None, 57 replacer: Option[String] = Some("random"), 58 PortNumber: Int = 2, 59 nFetchMshr: Int = 4, 60 nPrefetchMshr: Int = 10, 61 nWayLookupSize: Int = 32, 62 DataCodeUnit: Int = 64, 63 ICacheDataBanks: Int = 8, 64 ICacheDataSRAMWidth: Int = 66, 65 // TODO: hard code, need delete 66 partWayNum: Int = 4, 67 nMMIOs: Int = 1, 68 blockBytes: Int = 64, 69 cacheCtrlAddressOpt: Option[AddressSet] = None 70) extends L1CacheParameters { 71 72 val setBytes: Int = nSets * blockBytes 73 val aliasBitsOpt: Option[Int] = Option.when(setBytes > pageSize)(log2Ceil(setBytes / pageSize)) 74 val reqFields: Seq[BundleFieldBase] = Seq( 75 PrefetchField(), 76 ReqSourceField() 77 ) ++ aliasBitsOpt.map(AliasField) 78 val echoFields: Seq[BundleFieldBase] = Nil 79 def tagCode: Code = Code.fromString(tagECC) 80 def dataCode: Code = Code.fromString(dataECC) 81 def replacement = ReplacementPolicy.fromString(replacer, nWays, nSets) 82} 83 84trait HasICacheParameters extends HasL1CacheParameters with HasInstrMMIOConst with HasIFUConst { 85 val cacheParams: ICacheParameters = icacheParameters 86 87 def ctrlUnitParamsOpt: Option[L1ICacheCtrlParams] = OptionWrapper( 88 cacheParams.cacheCtrlAddressOpt.nonEmpty, 89 L1ICacheCtrlParams( 90 address = cacheParams.cacheCtrlAddressOpt.get, 91 regWidth = XLEN 92 ) 93 ) 94 95 def ICacheSets: Int = cacheParams.nSets 96 def ICacheWays: Int = cacheParams.nWays 97 def PortNumber: Int = cacheParams.PortNumber 98 def nFetchMshr: Int = cacheParams.nFetchMshr 99 def nPrefetchMshr: Int = cacheParams.nPrefetchMshr 100 def nWayLookupSize: Int = cacheParams.nWayLookupSize 101 def DataCodeUnit: Int = cacheParams.DataCodeUnit 102 def ICacheDataBanks: Int = cacheParams.ICacheDataBanks 103 def ICacheDataSRAMWidth: Int = cacheParams.ICacheDataSRAMWidth 104 def partWayNum: Int = cacheParams.partWayNum 105 106 def ICacheMetaBits: Int = tagBits // FIXME: unportable: maybe use somemethod to get width 107 def ICacheMetaCodeBits: Int = 1 // FIXME: unportable: maybe use cacheParams.tagCode.somemethod to get width 108 def ICacheMetaEntryBits: Int = ICacheMetaBits + ICacheMetaCodeBits 109 110 def ICacheDataBits: Int = blockBits / ICacheDataBanks 111 def ICacheDataCodeSegs: Int = 112 math.ceil(ICacheDataBits / DataCodeUnit).toInt // split data to segments for ECC checking 113 def ICacheDataCodeBits: Int = 114 ICacheDataCodeSegs * 1 // FIXME: unportable: maybe use cacheParams.dataCode.somemethod to get width 115 def ICacheDataEntryBits: Int = ICacheDataBits + ICacheDataCodeBits 116 def ICacheBankVisitNum: Int = 32 * 8 / ICacheDataBits + 1 117 def highestIdxBit: Int = log2Ceil(nSets) - 1 118 119 require((ICacheDataBanks >= 2) && isPow2(ICacheDataBanks)) 120 require(ICacheDataSRAMWidth >= ICacheDataEntryBits) 121 require(isPow2(ICacheSets), s"nSets($ICacheSets) must be pow2") 122 require(isPow2(ICacheWays), s"nWays($ICacheWays) must be pow2") 123 124 def generatePipeControl(lastFire: Bool, thisFire: Bool, thisFlush: Bool, lastFlush: Bool): Bool = { 125 val valid = RegInit(false.B) 126 when(thisFlush)(valid := false.B) 127 .elsewhen(lastFire && !lastFlush)(valid := true.B) 128 .elsewhen(thisFire)(valid := false.B) 129 valid 130 } 131 132 def ResultHoldBypass[T <: Data](data: T, valid: Bool): T = 133 Mux(valid, data, RegEnable(data, valid)) 134 135 def ResultHoldBypass[T <: Data](data: T, init: T, valid: Bool): T = 136 Mux(valid, data, RegEnable(data, init, valid)) 137 138 def holdReleaseLatch(valid: Bool, release: Bool, flush: Bool): Bool = { 139 val bit = RegInit(false.B) 140 when(flush)(bit := false.B) 141 .elsewhen(valid && !release)(bit := true.B) 142 .elsewhen(release)(bit := false.B) 143 bit || valid 144 } 145 146 def blockCounter(block: Bool, flush: Bool, threshold: Int): Bool = { 147 val counter = RegInit(0.U(log2Up(threshold + 1).W)) 148 when(block)(counter := counter + 1.U) 149 when(flush)(counter := 0.U) 150 counter > threshold.U 151 } 152 153 def InitQueue[T <: Data](entry: T, size: Int): Vec[T] = 154 RegInit(VecInit(Seq.fill(size)(0.U.asTypeOf(entry.cloneType)))) 155 156 def getBankSel(blkOffset: UInt, valid: Bool = true.B): Vec[UInt] = { 157 val bankIdxLow = (Cat(0.U(1.W), blkOffset) >> log2Ceil(blockBytes / ICacheDataBanks)).asUInt 158 val bankIdxHigh = ((Cat(0.U(1.W), blkOffset) + 32.U) >> log2Ceil(blockBytes / ICacheDataBanks)).asUInt 159 val bankSel = VecInit((0 until ICacheDataBanks * 2).map(i => (i.U >= bankIdxLow) && (i.U <= bankIdxHigh))) 160 assert( 161 !valid || PopCount(bankSel) === ICacheBankVisitNum.U, 162 "The number of bank visits must be %d, but bankSel=0x%x", 163 ICacheBankVisitNum.U, 164 bankSel.asUInt 165 ) 166 bankSel.asTypeOf(UInt((ICacheDataBanks * 2).W)).asTypeOf(Vec(2, UInt(ICacheDataBanks.W))) 167 } 168 169 def getLineSel(blkOffset: UInt): Vec[Bool] = { 170 val bankIdxLow = (blkOffset >> log2Ceil(blockBytes / ICacheDataBanks)).asUInt 171 val lineSel = VecInit((0 until ICacheDataBanks).map(i => i.U < bankIdxLow)) 172 lineSel 173 } 174 175 def getBlkAddr(addr: UInt): UInt = (addr >> blockOffBits).asUInt 176 def getPhyTagFromBlk(addr: UInt): UInt = (addr >> (pgUntagBits - blockOffBits)).asUInt 177 def getIdxFromBlk(addr: UInt): UInt = addr(idxBits - 1, 0) 178 def getPaddrFromPtag(vaddr: UInt, ptag: UInt): UInt = Cat(ptag, vaddr(pgUntagBits - 1, 0)) 179 def getPaddrFromPtag(vaddrVec: Vec[UInt], ptagVec: Vec[UInt]): Vec[UInt] = 180 VecInit((vaddrVec zip ptagVec).map { case (vaddr, ptag) => getPaddrFromPtag(vaddr, ptag) }) 181} 182 183trait HasICacheECCHelper extends HasICacheParameters { 184 def encodeMetaECC(meta: UInt, poison: Bool = false.B): UInt = { 185 require(meta.getWidth == ICacheMetaBits) 186 val code = cacheParams.tagCode.encode(meta, poison) >> ICacheMetaBits 187 code.asTypeOf(UInt(ICacheMetaCodeBits.W)) 188 } 189 190 def encodeDataECC(data: UInt, poison: Bool = false.B): UInt = { 191 require(data.getWidth == ICacheDataBits) 192 val datas = data.asTypeOf(Vec(ICacheDataCodeSegs, UInt((ICacheDataBits / ICacheDataCodeSegs).W))) 193 val codes = VecInit(datas.map(cacheParams.dataCode.encode(_, poison) >> (ICacheDataBits / ICacheDataCodeSegs))) 194 codes.asTypeOf(UInt(ICacheDataCodeBits.W)) 195 } 196} 197 198abstract class ICacheBundle(implicit p: Parameters) extends XSBundle 199 with HasICacheParameters 200 201abstract class ICacheModule(implicit p: Parameters) extends XSModule 202 with HasICacheParameters 203 204abstract class ICacheArray(implicit p: Parameters) extends XSModule 205 with HasICacheParameters 206 207class ICacheMetadata(implicit p: Parameters) extends ICacheBundle { 208 val tag: UInt = UInt(tagBits.W) 209} 210 211object ICacheMetadata { 212 def apply(tag: Bits)(implicit p: Parameters): ICacheMetadata = { 213 val meta = Wire(new ICacheMetadata) 214 meta.tag := tag 215 meta 216 } 217} 218 219class ICacheMetaArrayIO(implicit p: Parameters) extends ICacheBundle { 220 val write: DecoupledIO[ICacheMetaWriteBundle] = Flipped(DecoupledIO(new ICacheMetaWriteBundle)) 221 val read: DecoupledIO[ICacheReadBundle] = Flipped(DecoupledIO(new ICacheReadBundle)) 222 val readResp: ICacheMetaRespBundle = Output(new ICacheMetaRespBundle) 223 val flush: Vec[Valid[ICacheMetaFlushBundle]] = Vec(PortNumber, Flipped(ValidIO(new ICacheMetaFlushBundle))) 224 val flushAll: Bool = Input(Bool()) 225} 226 227class ICacheMetaArray(implicit p: Parameters) extends ICacheArray with HasICacheECCHelper { 228 class ICacheMetaEntry(implicit p: Parameters) extends ICacheBundle { 229 val meta: ICacheMetadata = new ICacheMetadata 230 val code: UInt = UInt(ICacheMetaCodeBits.W) 231 } 232 233 private object ICacheMetaEntry { 234 def apply(meta: ICacheMetadata, poison: Bool)(implicit p: Parameters): ICacheMetaEntry = { 235 val entry = Wire(new ICacheMetaEntry) 236 entry.meta := meta 237 entry.code := encodeMetaECC(meta.asUInt, poison) 238 entry 239 } 240 } 241 242 // sanity check 243 require(ICacheMetaEntryBits == (new ICacheMetaEntry).getWidth) 244 245 val io: ICacheMetaArrayIO = IO(new ICacheMetaArrayIO) 246 247 private val port_0_read_0 = io.read.valid && !io.read.bits.vSetIdx(0)(0) 248 private val port_0_read_1 = io.read.valid && io.read.bits.vSetIdx(0)(0) 249 private val port_1_read_1 = io.read.valid && io.read.bits.vSetIdx(1)(0) && io.read.bits.isDoubleLine 250 private val port_1_read_0 = io.read.valid && !io.read.bits.vSetIdx(1)(0) && io.read.bits.isDoubleLine 251 252 private val port_0_read_0_reg = RegEnable(port_0_read_0, 0.U.asTypeOf(port_0_read_0), io.read.fire) 253 private val port_0_read_1_reg = RegEnable(port_0_read_1, 0.U.asTypeOf(port_0_read_1), io.read.fire) 254 private val port_1_read_1_reg = RegEnable(port_1_read_1, 0.U.asTypeOf(port_1_read_1), io.read.fire) 255 private val port_1_read_0_reg = RegEnable(port_1_read_0, 0.U.asTypeOf(port_1_read_0), io.read.fire) 256 257 private val bank_0_idx = Mux(port_0_read_0, io.read.bits.vSetIdx(0), io.read.bits.vSetIdx(1)) 258 private val bank_1_idx = Mux(port_0_read_1, io.read.bits.vSetIdx(0), io.read.bits.vSetIdx(1)) 259 260 private val write_bank_0 = io.write.valid && !io.write.bits.bankIdx 261 private val write_bank_1 = io.write.valid && io.write.bits.bankIdx 262 263 private val write_meta_bits = ICacheMetaEntry( 264 meta = ICacheMetadata( 265 tag = io.write.bits.phyTag 266 ), 267 poison = io.write.bits.poison 268 ) 269 270 private val tagArrays = (0 until PortNumber) map { bank => 271 val tagArray = Module(new SRAMTemplate( 272 new ICacheMetaEntry(), 273 set = nSets / PortNumber, 274 way = nWays, 275 shouldReset = true, 276 holdRead = true, 277 singlePort = true, 278 withClockGate = true, 279 hasMbist = hasMbist 280 )) 281 282 // meta connection 283 if (bank == 0) { 284 tagArray.io.r.req.valid := port_0_read_0 || port_1_read_0 285 tagArray.io.r.req.bits.apply(setIdx = bank_0_idx(highestIdxBit, 1)) 286 tagArray.io.w.req.valid := write_bank_0 287 tagArray.io.w.req.bits.apply( 288 data = write_meta_bits, 289 setIdx = io.write.bits.virIdx(highestIdxBit, 1), 290 waymask = io.write.bits.waymask 291 ) 292 } else { 293 tagArray.io.r.req.valid := port_0_read_1 || port_1_read_1 294 tagArray.io.r.req.bits.apply(setIdx = bank_1_idx(highestIdxBit, 1)) 295 tagArray.io.w.req.valid := write_bank_1 296 tagArray.io.w.req.bits.apply( 297 data = write_meta_bits, 298 setIdx = io.write.bits.virIdx(highestIdxBit, 1), 299 waymask = io.write.bits.waymask 300 ) 301 } 302 303 tagArray 304 } 305 private val mbistPl = MbistPipeline.PlaceMbistPipeline(1, "MbistPipeIcacheTag", hasMbist) 306 307 private val read_set_idx_next = RegEnable(io.read.bits.vSetIdx, 0.U.asTypeOf(io.read.bits.vSetIdx), io.read.fire) 308 private val valid_array = RegInit(VecInit(Seq.fill(nWays)(0.U(nSets.W)))) 309 private val valid_metas = Wire(Vec(PortNumber, Vec(nWays, Bool()))) 310 // valid read 311 (0 until PortNumber).foreach(i => 312 (0 until nWays).foreach(way => 313 valid_metas(i)(way) := valid_array(way)(read_set_idx_next(i)) 314 ) 315 ) 316 io.readResp.entryValid := valid_metas 317 318 io.read.ready := !io.write.valid && !io.flush.map(_.valid).reduce(_ || _) && !io.flushAll && 319 tagArrays.map(_.io.r.req.ready).reduce(_ && _) 320 321 // valid write 322 private val way_num = OHToUInt(io.write.bits.waymask) 323 when(io.write.valid) { 324 valid_array(way_num) := valid_array(way_num).bitSet(io.write.bits.virIdx, true.B) 325 } 326 327 XSPerfAccumulate("meta_refill_num", io.write.valid) 328 329 io.readResp.metas <> DontCare 330 io.readResp.codes <> DontCare 331 private val readMetaEntries = tagArrays.map(port => port.io.r.resp.asTypeOf(Vec(nWays, new ICacheMetaEntry()))) 332 private val readMetas = readMetaEntries.map(_.map(_.meta)) 333 private val readCodes = readMetaEntries.map(_.map(_.code)) 334 335 // TEST: force ECC to fail by setting readCodes to 0 336 if (ICacheForceMetaECCError) { 337 readCodes.foreach(_.foreach(_ := 0.U)) 338 } 339 340 when(port_0_read_0_reg) { 341 io.readResp.metas(0) := readMetas(0) 342 io.readResp.codes(0) := readCodes(0) 343 }.elsewhen(port_0_read_1_reg) { 344 io.readResp.metas(0) := readMetas(1) 345 io.readResp.codes(0) := readCodes(1) 346 } 347 348 when(port_1_read_0_reg) { 349 io.readResp.metas(1) := readMetas(0) 350 io.readResp.codes(1) := readCodes(0) 351 }.elsewhen(port_1_read_1_reg) { 352 io.readResp.metas(1) := readMetas(1) 353 io.readResp.codes(1) := readCodes(1) 354 } 355 356 io.write.ready := true.B // TODO : has bug ? should be !io.cacheOp.req.valid 357 358 /* 359 * flush logic 360 */ 361 // flush standalone set (e.g. flushed by mainPipe before doing re-fetch) 362 when(io.flush.map(_.valid).reduce(_ || _)) { 363 (0 until nWays).foreach { w => 364 valid_array(w) := (0 until PortNumber).map { i => 365 Mux( 366 // check if set `virIdx` in way `w` is requested to be flushed by port `i` 367 io.flush(i).valid && io.flush(i).bits.waymask(w), 368 valid_array(w).bitSet(io.flush(i).bits.virIdx, false.B), 369 valid_array(w) 370 ) 371 }.reduce(_ & _) 372 } 373 } 374 375 // flush all (e.g. fence.i) 376 when(io.flushAll) { 377 (0 until nWays).foreach(w => valid_array(w) := 0.U) 378 } 379 380 // PERF: flush counter 381 XSPerfAccumulate("flush", io.flush.map(_.valid).reduce(_ || _)) 382 XSPerfAccumulate("flush_all", io.flushAll) 383} 384 385class ICacheDataArrayIO(implicit p: Parameters) extends ICacheBundle { 386 val write: DecoupledIO[ICacheDataWriteBundle] = Flipped(DecoupledIO(new ICacheDataWriteBundle)) 387 val read: Vec[DecoupledIO[ICacheReadBundle]] = Flipped(Vec(partWayNum, DecoupledIO(new ICacheReadBundle))) 388 val readResp: ICacheDataRespBundle = Output(new ICacheDataRespBundle) 389} 390 391class ICacheDataArray(implicit p: Parameters) extends ICacheArray with HasICacheECCHelper { 392 class ICacheDataEntry(implicit p: Parameters) extends ICacheBundle { 393 val data: UInt = UInt(ICacheDataBits.W) 394 val code: UInt = UInt(ICacheDataCodeBits.W) 395 } 396 397 private object ICacheDataEntry { 398 def apply(data: UInt, poison: Bool)(implicit p: Parameters): ICacheDataEntry = { 399 val entry = Wire(new ICacheDataEntry) 400 entry.data := data 401 entry.code := encodeDataECC(data, poison) 402 entry 403 } 404 } 405 406 val io: ICacheDataArrayIO = IO(new ICacheDataArrayIO) 407 408 /** 409 ****************************************************************************** 410 * data array 411 ****************************************************************************** 412 */ 413 private val writeDatas = io.write.bits.data.asTypeOf(Vec(ICacheDataBanks, UInt(ICacheDataBits.W))) 414 private val writeEntries = writeDatas.map(ICacheDataEntry(_, io.write.bits.poison).asUInt) 415 416 // io.read() are copies to control fan-out, we can simply use .head here 417 private val bankSel = getBankSel(io.read.head.bits.blkOffset, io.read.head.valid) 418 private val lineSel = getLineSel(io.read.head.bits.blkOffset) 419 private val waymasks = io.read.head.bits.waymask 420 private val masks = Wire(Vec(nWays, Vec(ICacheDataBanks, Bool()))) 421 (0 until nWays).foreach { way => 422 (0 until ICacheDataBanks).foreach { bank => 423 masks(way)(bank) := Mux( 424 lineSel(bank), 425 waymasks(1)(way) && bankSel(1)(bank).asBool, 426 waymasks(0)(way) && bankSel(0)(bank).asBool 427 ) 428 } 429 } 430 431 private val dataArrays = (0 until nWays).map { way => 432 val banks = (0 until ICacheDataBanks).map { bank => 433 val sramBank = Module(new SRAMTemplateWithFixedWidth( 434 UInt(ICacheDataEntryBits.W), 435 set = nSets, 436 width = ICacheDataSRAMWidth, 437 shouldReset = true, 438 holdRead = true, 439 singlePort = true, 440 withClockGate = false, // enable signal timing is bad, no gating here 441 hasMbist = hasMbist 442 )) 443 444 // read 445 sramBank.io.r.req.valid := io.read(bank % 4).valid && masks(way)(bank) 446 sramBank.io.r.req.bits.apply(setIdx = 447 Mux(lineSel(bank), io.read(bank % 4).bits.vSetIdx(1), io.read(bank % 4).bits.vSetIdx(0)) 448 ) 449 // write 450 sramBank.io.w.req.valid := io.write.valid && io.write.bits.waymask(way).asBool 451 sramBank.io.w.req.bits.apply( 452 data = writeEntries(bank), 453 setIdx = io.write.bits.virIdx, 454 // waymask is invalid when way of SRAMTemplate <= 1 455 waymask = 0.U 456 ) 457 sramBank 458 } 459 MbistPipeline.PlaceMbistPipeline(1, s"MbistPipeIcacheDataWay${way}", hasMbist) 460 banks 461 } 462 463 /** 464 ****************************************************************************** 465 * read logic 466 ****************************************************************************** 467 */ 468 private val masksReg = RegEnable(masks, 0.U.asTypeOf(masks), io.read(0).valid) 469 private val readDataWithCode = (0 until ICacheDataBanks).map { bank => 470 Mux1H(VecInit(masksReg.map(_(bank))).asTypeOf(UInt(nWays.W)), dataArrays.map(_(bank).io.r.resp.asUInt)) 471 } 472 private val readEntries = readDataWithCode.map(_.asTypeOf(new ICacheDataEntry())) 473 private val readDatas = VecInit(readEntries.map(_.data)) 474 private val readCodes = VecInit(readEntries.map(_.code)) 475 476 // TEST: force ECC to fail by setting readCodes to 0 477 if (ICacheForceDataECCError) { 478 readCodes.foreach(_ := 0.U) 479 } 480 481 /** 482 ****************************************************************************** 483 * IO 484 ****************************************************************************** 485 */ 486 io.readResp.datas := readDatas 487 io.readResp.codes := readCodes 488 io.write.ready := true.B 489 io.read.foreach(_.ready := !io.write.valid) 490} 491 492class ICacheReplacerIO(implicit p: Parameters) extends ICacheBundle { 493 val touch: Vec[Valid[ReplacerTouch]] = Vec(PortNumber, Flipped(ValidIO(new ReplacerTouch))) 494 val victim: ReplacerVictim = Flipped(new ReplacerVictim) 495} 496 497class ICacheReplacer(implicit p: Parameters) extends ICacheModule { 498 val io: ICacheReplacerIO = IO(new ICacheReplacerIO) 499 500 private val replacers = 501 Seq.fill(PortNumber)(ReplacementPolicy.fromString(cacheParams.replacer, nWays, nSets / PortNumber)) 502 503 // touch 504 private val touch_sets = Seq.fill(PortNumber)(Wire(Vec(PortNumber, UInt(log2Ceil(nSets / PortNumber).W)))) 505 private val touch_ways = Seq.fill(PortNumber)(Wire(Vec(PortNumber, Valid(UInt(wayBits.W))))) 506 (0 until PortNumber).foreach { i => 507 touch_sets(i)(0) := Mux( 508 io.touch(i).bits.vSetIdx(0), 509 io.touch(1).bits.vSetIdx(highestIdxBit, 1), 510 io.touch(0).bits.vSetIdx(highestIdxBit, 1) 511 ) 512 touch_ways(i)(0).bits := Mux(io.touch(i).bits.vSetIdx(0), io.touch(1).bits.way, io.touch(0).bits.way) 513 touch_ways(i)(0).valid := Mux(io.touch(i).bits.vSetIdx(0), io.touch(1).valid, io.touch(0).valid) 514 } 515 516 // victim 517 io.victim.way := Mux( 518 io.victim.vSetIdx.bits(0), 519 replacers(1).way(io.victim.vSetIdx.bits(highestIdxBit, 1)), 520 replacers(0).way(io.victim.vSetIdx.bits(highestIdxBit, 1)) 521 ) 522 523 // touch the victim in next cycle 524 private val victim_vSetIdx_reg = 525 RegEnable(io.victim.vSetIdx.bits, 0.U.asTypeOf(io.victim.vSetIdx.bits), io.victim.vSetIdx.valid) 526 private val victim_way_reg = RegEnable(io.victim.way, 0.U.asTypeOf(io.victim.way), io.victim.vSetIdx.valid) 527 (0 until PortNumber).foreach { i => 528 touch_sets(i)(1) := victim_vSetIdx_reg(highestIdxBit, 1) 529 touch_ways(i)(1).bits := victim_way_reg 530 touch_ways(i)(1).valid := RegNext(io.victim.vSetIdx.valid) && (victim_vSetIdx_reg(0) === i.U) 531 } 532 533 ((replacers zip touch_sets) zip touch_ways).foreach { case ((r, s), w) => r.access(s, w) } 534} 535 536class ICacheIO(implicit p: Parameters) extends ICacheBundle { 537 val hartId: UInt = Input(UInt(hartIdLen.W)) 538 // FTQ 539 val fetch: ICacheMainPipeBundle = new ICacheMainPipeBundle 540 val ftqPrefetch: FtqToPrefetchIO = Flipped(new FtqToPrefetchIO) 541 // memblock 542 val softPrefetch: Vec[Valid[SoftIfetchPrefetchBundle]] = 543 Vec(backendParams.LduCnt, Flipped(Valid(new SoftIfetchPrefetchBundle))) 544 // IFU 545 val stop: Bool = Input(Bool()) 546 val toIFU: Bool = Output(Bool()) 547 // PMP: mainPipe & prefetchPipe need PortNumber each 548 val pmp: Vec[ICachePMPBundle] = Vec(2 * PortNumber, new ICachePMPBundle) 549 // iTLB 550 val itlb: Vec[TlbRequestIO] = Vec(PortNumber, new TlbRequestIO) 551 val itlbFlushPipe: Bool = Bool() 552 // backend/BEU 553 val error: Valid[L1CacheErrorInfo] = ValidIO(new L1CacheErrorInfo) 554 // backend/CSR 555 val csr_pf_enable: Bool = Input(Bool()) 556 // flush 557 val fencei: Bool = Input(Bool()) 558 val flush: Bool = Input(Bool()) 559 560 // perf 561 val perfInfo: ICachePerfInfo = Output(new ICachePerfInfo) 562} 563 564class ICache()(implicit p: Parameters) extends LazyModule with HasICacheParameters { 565 override def shouldBeInlined: Boolean = false 566 567 val clientParameters: TLMasterPortParameters = TLMasterPortParameters.v1( 568 Seq(TLMasterParameters.v1( 569 name = "icache", 570 sourceId = IdRange(0, cacheParams.nFetchMshr + cacheParams.nPrefetchMshr + 1) 571 )), 572 requestFields = cacheParams.reqFields, 573 echoFields = cacheParams.echoFields 574 ) 575 576 val clientNode: TLClientNode = TLClientNode(Seq(clientParameters)) 577 578 val ctrlUnitOpt: Option[ICacheCtrlUnit] = ctrlUnitParamsOpt.map(params => LazyModule(new ICacheCtrlUnit(params))) 579 580 lazy val module: ICacheImp = new ICacheImp(this) 581} 582 583class ICacheImp(outer: ICache) extends LazyModuleImp(outer) with HasICacheParameters with HasPerfEvents { 584 val io: ICacheIO = IO(new ICacheIO) 585 586 println("ICache:") 587 println(" TagECC: " + cacheParams.tagECC) 588 println(" DataECC: " + cacheParams.dataECC) 589 println(" ICacheSets: " + cacheParams.nSets) 590 println(" ICacheWays: " + cacheParams.nWays) 591 println(" PortNumber: " + cacheParams.PortNumber) 592 println(" nFetchMshr: " + cacheParams.nFetchMshr) 593 println(" nPrefetchMshr: " + cacheParams.nPrefetchMshr) 594 println(" nWayLookupSize: " + cacheParams.nWayLookupSize) 595 println(" DataCodeUnit: " + cacheParams.DataCodeUnit) 596 println(" ICacheDataBanks: " + cacheParams.ICacheDataBanks) 597 println(" ICacheDataSRAMWidth: " + cacheParams.ICacheDataSRAMWidth) 598 599 val (bus, edge) = outer.clientNode.out.head 600 601 private val metaArray = Module(new ICacheMetaArray) 602 private val dataArray = Module(new ICacheDataArray) 603 private val mainPipe = Module(new ICacheMainPipe) 604 private val missUnit = Module(new ICacheMissUnit(edge)) 605 private val replacer = Module(new ICacheReplacer) 606 private val prefetcher = Module(new IPrefetchPipe) 607 private val wayLookup = Module(new WayLookup) 608 609 private val ecc_enable = if (outer.ctrlUnitOpt.nonEmpty) outer.ctrlUnitOpt.get.module.io.ecc_enable else true.B 610 611 // dataArray io 612 if (outer.ctrlUnitOpt.nonEmpty) { 613 val ctrlUnit = outer.ctrlUnitOpt.get.module 614 when(ctrlUnit.io.injecting) { 615 dataArray.io.write <> ctrlUnit.io.dataWrite 616 missUnit.io.data_write.ready := false.B 617 }.otherwise { 618 ctrlUnit.io.dataWrite.ready := false.B 619 dataArray.io.write <> missUnit.io.data_write 620 } 621 } else { 622 dataArray.io.write <> missUnit.io.data_write 623 } 624 dataArray.io.read <> mainPipe.io.dataArray.toIData 625 mainPipe.io.dataArray.fromIData := dataArray.io.readResp 626 627 // metaArray io 628 metaArray.io.flushAll := io.fencei 629 metaArray.io.flush <> mainPipe.io.metaArrayFlush 630 if (outer.ctrlUnitOpt.nonEmpty) { 631 val ctrlUnit = outer.ctrlUnitOpt.get.module 632 when(ctrlUnit.io.injecting) { 633 metaArray.io.write <> ctrlUnit.io.metaWrite 634 metaArray.io.read <> ctrlUnit.io.metaRead 635 missUnit.io.meta_write.ready := false.B 636 prefetcher.io.metaRead.toIMeta.ready := false.B 637 }.otherwise { 638 ctrlUnit.io.metaWrite.ready := false.B 639 ctrlUnit.io.metaRead.ready := false.B 640 metaArray.io.write <> missUnit.io.meta_write 641 metaArray.io.read <> prefetcher.io.metaRead.toIMeta 642 } 643 ctrlUnit.io.metaReadResp := metaArray.io.readResp 644 } else { 645 metaArray.io.write <> missUnit.io.meta_write 646 metaArray.io.read <> prefetcher.io.metaRead.toIMeta 647 } 648 prefetcher.io.metaRead.fromIMeta := metaArray.io.readResp 649 650 prefetcher.io.flush := io.flush 651 prefetcher.io.csr_pf_enable := io.csr_pf_enable 652 prefetcher.io.ecc_enable := ecc_enable 653 prefetcher.io.MSHRResp := missUnit.io.fetch_resp 654 prefetcher.io.flushFromBpu := io.ftqPrefetch.flushFromBpu 655 // cache softPrefetch 656 private val softPrefetchValid = RegInit(false.B) 657 private val softPrefetch = RegInit(0.U.asTypeOf(new IPrefetchReq)) 658 /* FIXME: 659 * If there is already a pending softPrefetch request, it will be overwritten. 660 * Also, if there are multiple softPrefetch requests in the same cycle, only the first one will be accepted. 661 * We should implement a softPrefetchQueue (like ibuffer, multi-in, single-out) to solve this. 662 * However, the impact on performance still needs to be assessed. 663 * Considering that the frequency of prefetch.i may not be high, let's start with a temporary dummy solution. 664 */ 665 when(io.softPrefetch.map(_.valid).reduce(_ || _)) { 666 softPrefetchValid := true.B 667 softPrefetch.fromSoftPrefetch(MuxCase( 668 0.U.asTypeOf(new SoftIfetchPrefetchBundle), 669 io.softPrefetch.map(req => req.valid -> req.bits) 670 )) 671 }.elsewhen(prefetcher.io.req.fire) { 672 softPrefetchValid := false.B 673 } 674 // pass ftqPrefetch 675 private val ftqPrefetch = WireInit(0.U.asTypeOf(new IPrefetchReq)) 676 ftqPrefetch.fromFtqICacheInfo(io.ftqPrefetch.req.bits) 677 // software prefetch has higher priority 678 prefetcher.io.req.valid := softPrefetchValid || io.ftqPrefetch.req.valid 679 prefetcher.io.req.bits := Mux(softPrefetchValid, softPrefetch, ftqPrefetch) 680 prefetcher.io.req.bits.backendException := io.ftqPrefetch.backendException 681 io.ftqPrefetch.req.ready := prefetcher.io.req.ready && !softPrefetchValid 682 683 missUnit.io.hartId := io.hartId 684 missUnit.io.fencei := io.fencei 685 missUnit.io.flush := io.flush 686 missUnit.io.fetch_req <> mainPipe.io.mshr.req 687 missUnit.io.prefetch_req <> prefetcher.io.MSHRReq 688 missUnit.io.mem_grant.valid := false.B 689 missUnit.io.mem_grant.bits := DontCare 690 missUnit.io.mem_grant <> bus.d 691 692 mainPipe.io.flush := io.flush 693 mainPipe.io.respStall := io.stop 694 mainPipe.io.ecc_enable := ecc_enable 695 mainPipe.io.hartId := io.hartId 696 mainPipe.io.mshr.resp := missUnit.io.fetch_resp 697 mainPipe.io.fetch.req <> io.fetch.req 698 mainPipe.io.wayLookupRead <> wayLookup.io.read 699 700 wayLookup.io.flush := io.flush 701 wayLookup.io.write <> prefetcher.io.wayLookupWrite 702 wayLookup.io.update := missUnit.io.fetch_resp 703 704 replacer.io.touch <> mainPipe.io.touch 705 replacer.io.victim <> missUnit.io.victim 706 707 io.pmp(0) <> mainPipe.io.pmp(0) 708 io.pmp(1) <> mainPipe.io.pmp(1) 709 io.pmp(2) <> prefetcher.io.pmp(0) 710 io.pmp(3) <> prefetcher.io.pmp(1) 711 712 io.itlb(0) <> prefetcher.io.itlb(0) 713 io.itlb(1) <> prefetcher.io.itlb(1) 714 io.itlbFlushPipe := prefetcher.io.itlbFlushPipe 715 716 // notify IFU that Icache pipeline is available 717 io.toIFU := mainPipe.io.fetch.req.ready 718 io.perfInfo := mainPipe.io.perfInfo 719 720 io.fetch.resp <> mainPipe.io.fetch.resp 721 io.fetch.topdownIcacheMiss := mainPipe.io.fetch.topdownIcacheMiss 722 io.fetch.topdownItlbMiss := mainPipe.io.fetch.topdownItlbMiss 723 724 bus.b.ready := false.B 725 bus.c.valid := false.B 726 bus.c.bits := DontCare 727 bus.e.valid := false.B 728 bus.e.bits := DontCare 729 730 bus.a <> missUnit.io.mem_acquire 731 732 // Parity error port 733 private val errors = mainPipe.io.errors 734 private val errors_valid = errors.map(e => e.valid).reduce(_ | _) 735 io.error.bits <> RegEnable( 736 PriorityMux(errors.map(e => e.valid -> e.bits)), 737 0.U.asTypeOf(errors(0).bits), 738 errors_valid 739 ) 740 io.error.valid := RegNext(errors_valid, false.B) 741 742 XSPerfAccumulate( 743 "softPrefetch_drop_not_ready", 744 io.softPrefetch.map(_.valid).reduce(_ || _) && softPrefetchValid && !prefetcher.io.req.fire 745 ) 746 XSPerfAccumulate("softPrefetch_drop_multi_req", PopCount(io.softPrefetch.map(_.valid)) > 1.U) 747 XSPerfAccumulate("softPrefetch_block_ftq", softPrefetchValid && io.ftqPrefetch.req.valid) 748 749 val perfEvents: Seq[(String, Bool)] = Seq( 750 ("icache_miss_cnt ", false.B), 751 ("icache_miss_penalty", BoolStopWatch(start = false.B, stop = false.B || false.B, startHighPriority = true)) 752 ) 753 generatePerfEvent() 754} 755 756//class ICachePartWayReadBundle[T <: Data](gen: T, pWay: Int)(implicit p: Parameters) 757// extends ICacheBundle { 758// val req = Flipped(Vec( 759// PortNumber, 760// Decoupled(new Bundle { 761// val ridx = UInt((log2Ceil(nSets) - 1).W) 762// }) 763// )) 764// val resp = Output(new Bundle { 765// val rdata = Vec(PortNumber, Vec(pWay, gen)) 766// }) 767//} 768 769//class ICacheWriteBundle[T <: Data](gen: T, pWay: Int)(implicit p: Parameters) 770// extends ICacheBundle { 771// val wdata = gen 772// val widx = UInt((log2Ceil(nSets) - 1).W) 773// val wbankidx = Bool() 774// val wmask = Vec(pWay, Bool()) 775//} 776 777//class ICachePartWayArray[T <: Data](gen: T, pWay: Int)(implicit p: Parameters) extends ICacheArray { 778// 779// // including part way data 780// val io = IO { 781// new Bundle { 782// val read = new ICachePartWayReadBundle(gen, pWay) 783// val write = Flipped(ValidIO(new ICacheWriteBundle(gen, pWay))) 784// } 785// } 786// 787// io.read.req.map(_.ready := !io.write.valid) 788// 789// val srams = (0 until PortNumber) map { bank => 790// val sramBank = Module(new SRAMTemplate( 791// gen, 792// set = nSets / 2, 793// way = pWay, 794// shouldReset = true, 795// holdRead = true, 796// singlePort = true, 797// withClockGate = true 798// )) 799// 800// sramBank.io.r.req.valid := io.read.req(bank).valid 801// sramBank.io.r.req.bits.apply(setIdx = io.read.req(bank).bits.ridx) 802// 803// if (bank == 0) sramBank.io.w.req.valid := io.write.valid && !io.write.bits.wbankidx 804// else sramBank.io.w.req.valid := io.write.valid && io.write.bits.wbankidx 805// sramBank.io.w.req.bits.apply( 806// data = io.write.bits.wdata, 807// setIdx = io.write.bits.widx, 808// waymask = io.write.bits.wmask.asUInt 809// ) 810// 811// sramBank 812// } 813// 814// io.read.req.map(_.ready := !io.write.valid && srams.map(_.io.r.req.ready).reduce(_ && _)) 815// 816// io.read.resp.rdata := VecInit(srams.map(bank => bank.io.r.resp.asTypeOf(Vec(pWay, gen)))) 817// 818//} 819 820class SRAMTemplateWithFixedWidthIO[T <: Data](gen: T, set: Int, way: Int) extends Bundle { 821 val r: SRAMReadBus[T] = Flipped(new SRAMReadBus(gen, set, way)) 822 val w: SRAMWriteBus[T] = Flipped(new SRAMWriteBus(gen, set, way)) 823} 824 825// Automatically partition the SRAM based on the width of the data and the desired width. 826// final SRAM width = width * way 827class SRAMTemplateWithFixedWidth[T <: Data]( 828 gen: T, 829 set: Int, 830 width: Int, 831 way: Int = 1, 832 shouldReset: Boolean = false, 833 holdRead: Boolean = false, 834 singlePort: Boolean = false, 835 bypassWrite: Boolean = false, 836 withClockGate: Boolean = false, 837 hasMbist: Boolean = false 838) extends Module { 839 840 private val dataBits = gen.getWidth 841 private val bankNum = math.ceil(dataBits.toDouble / width.toDouble).toInt 842 private val totalBits = bankNum * width 843 844 val io: SRAMTemplateWithFixedWidthIO[T] = IO(new SRAMTemplateWithFixedWidthIO(gen, set, way)) 845 846 private val wordType = UInt(width.W) 847 private val writeDatas = (0 until bankNum).map { bank => 848 VecInit((0 until way).map { i => 849 io.w.req.bits.data(i).asTypeOf(UInt(totalBits.W)).asTypeOf(Vec(bankNum, wordType))(bank) 850 }) 851 } 852 853 private val srams = (0 until bankNum) map { bank => 854 val sramBank = Module(new SRAMTemplate( 855 wordType, 856 set = set, 857 way = way, 858 shouldReset = shouldReset, 859 holdRead = holdRead, 860 singlePort = singlePort, 861 bypassWrite = bypassWrite, 862 withClockGate = withClockGate, 863 hasMbist = hasMbist 864 )) 865 // read req 866 sramBank.io.r.req.valid := io.r.req.valid 867 sramBank.io.r.req.bits.setIdx := io.r.req.bits.setIdx 868 869 // write req 870 sramBank.io.w.req.valid := io.w.req.valid 871 sramBank.io.w.req.bits.setIdx := io.w.req.bits.setIdx 872 sramBank.io.w.req.bits.data := writeDatas(bank) 873 sramBank.io.w.req.bits.waymask.foreach(_ := io.w.req.bits.waymask.get) 874 875 sramBank 876 } 877 878 io.r.req.ready := !io.w.req.valid 879 (0 until way).foreach { i => 880 io.r.resp.data(i) := VecInit((0 until bankNum).map(bank => 881 srams(bank).io.r.resp.data(i) 882 )).asTypeOf(UInt(totalBits.W))(dataBits - 1, 0).asTypeOf(gen.cloneType) 883 } 884 885 io.r.req.ready := srams.head.io.r.req.ready 886 io.w.req.ready := srams.head.io.w.req.ready 887} 888