1/*************************************************************************************** 2* Copyright (c) 2021-2025 Beijing Institute of Open Source Chip (BOSC) 3* Copyright (c) 2020-2024 Institute of Computing Technology, Chinese Academy of Sciences 4* Copyright (c) 2020-2021 Peng Cheng Laboratory 5* Copyright (c) 2024-2025 Institute of Information Engineering, Chinese Academy of Sciences 6* 7* XiangShan is licensed under Mulan PSL v2. 8* You can use this software according to the terms and conditions of the Mulan PSL v2. 9* You may obtain a copy of Mulan PSL v2 at: 10* http://license.coscl.org.cn/MulanPSL2 11* 12* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 13* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 14* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 15* 16* See the Mulan PSL v2 for more details. 17***************************************************************************************/ 18 19package xiangshan.cache.mmu 20 21import org.chipsalliance.cde.config.Parameters 22import chisel3._ 23import chisel3.experimental.ExtModule 24import chisel3.util._ 25import xiangshan._ 26import xiangshan.cache.{HasDCacheParameters, MemoryOpConstants} 27import utils._ 28import utility._ 29import freechips.rocketchip.diplomacy.{IdRange, LazyModule, LazyModuleImp} 30import freechips.rocketchip.tilelink._ 31import xiangshan.backend.fu.{PMP, PMPChecker, PMPReqBundle, PMPRespBundle} 32import xiangshan.backend.fu.util.HasCSRConst 33import difftest._ 34 35class L2TLB()(implicit p: Parameters) extends LazyModule with HasPtwConst { 36 override def shouldBeInlined: Boolean = false 37 38 val node = TLClientNode(Seq(TLMasterPortParameters.v1( 39 clients = Seq(TLMasterParameters.v1( 40 "ptw", 41 sourceId = IdRange(0, MemReqWidth) 42 )), 43 requestFields = Seq(ReqSourceField()) 44 ))) 45 46 lazy val module = new L2TLBImp(this) 47} 48 49class L2TLBImp(outer: L2TLB)(implicit p: Parameters) extends PtwModule(outer) with HasCSRConst with HasPerfEvents { 50 51 val (mem, edge) = outer.node.out.head 52 53 val io = IO(new L2TLBIO) 54 val difftestIO = IO(new Bundle() { 55 val ptwResp = Output(Bool()) 56 val ptwAddr = Output(UInt(64.W)) 57 val ptwData = Output(Vec(4, UInt(64.W))) 58 }) 59 60 /* Ptw processes multiple requests 61 * Divide Ptw procedure into two stages: cache access ; mem access if cache miss 62 * miss queue itlb dtlb 63 * | | | 64 * ------arbiter------ 65 * | 66 * l1 - l2 - l3 - sp 67 * | 68 * ------------------------------------------- 69 * miss | queue | hit 70 * [][][][][][] | 71 * | | 72 * state machine accessing mem | 73 * | | 74 * ---------------arbiter--------------------- 75 * | | 76 * itlb dtlb 77 */ 78 79 difftestIO <> DontCare 80 81 val sfence_tmp = DelayN(io.sfence, 1) 82 val csr_tmp = DelayN(io.csr.tlb, 1) 83 val sfence_dup = Seq.fill(if (HasBitmapCheck) 11 else 9)(RegNext(sfence_tmp)) 84 val csr_dup = Seq.fill(if (HasBitmapCheck) 10 else 8)(RegNext(csr_tmp)) // TODO: add csr_modified? 85 val satp = csr_dup(0).satp 86 val vsatp = csr_dup(0).vsatp 87 val hgatp = csr_dup(0).hgatp 88 val priv = csr_dup(0).priv 89 val mPBMTE = csr_dup(0).mPBMTE 90 val hPBMTE = csr_dup(0).hPBMTE 91 val flush = sfence_dup(0).valid || satp.changed || vsatp.changed || hgatp.changed 92 93 val pmp = Module(new PMP()) 94 val pmp_check = VecInit(Seq.fill(if (HasBitmapCheck) 4 else 3)(Module(new PMPChecker(lgMaxSize = 3, sameCycle = true)).io)) 95 pmp.io.distribute_csr := io.csr.distribute_csr 96 if (HasBitmapCheck) { 97 pmp_check.foreach(_.check_env.apply(csr_dup(0).mbmc.CMODE.asBool, ModeS, pmp.io.pmp, pmp.io.pma)) 98 } else { 99 pmp_check.foreach(_.check_env.apply(ModeS, pmp.io.pmp, pmp.io.pma)) 100 } 101 102 // add bitmapcheck 103 val bitmap = Option.when(HasBitmapCheck)(Module(new Bitmap)) 104 val bitmapcache = Option.when(HasBitmapCheck)(Module(new BitmapCache)) 105 106 if (HasBitmapCheck) { 107 bitmap.foreach { Bitmap => 108 Bitmap.io.csr := csr_dup(8) 109 Bitmap.io.sfence := sfence_dup(9) 110 bitmapcache.foreach { BitmapCache => 111 // connect bitmap and bitmapcache 112 BitmapCache.io.req <> Bitmap.io.cache.req 113 Bitmap.io.cache.resp <> BitmapCache.io.resp 114 BitmapCache.io.refill <> Bitmap.io.refill 115 BitmapCache.io.csr := csr_dup(9) 116 BitmapCache.io.sfence := sfence_dup(10) 117 } 118 } 119 } 120 121 val missQueue = Module(new L2TlbMissQueue) 122 val cache = Module(new PtwCache) 123 val ptw = Module(new PTW) 124 val hptw = Module(new HPTW) 125 val llptw = Module(new LLPTW) 126 val blockmq = Module(new BlockHelper(3)) 127 val arb1 = Module(new Arbiter(new PtwReq, PtwWidth)) 128 val arb2 = Module(new Arbiter(new L2TlbWithHptwIdBundle, ((if (l2tlbParams.enablePrefetch) 4 else 3) + (if(HasHExtension) 1 else 0)))) 129 val hptw_req_arb = Module(new Arbiter(new Bundle { 130 val id = UInt(log2Up(l2tlbParams.llptwsize).W) 131 val source = UInt(bSourceWidth.W) 132 val gvpn = UInt(gvpnLen.W) 133 }, 2)) 134 val hptw_resp_arb = Module(new Arbiter(new Bundle { 135 val resp = new HptwResp() 136 val id = UInt(log2Up(l2tlbParams.llptwsize).W) 137 }, 2)) 138 val outArb = (0 until PtwWidth).map(i => Module(new Arbiter(new Bundle { 139 val s2xlate = UInt(2.W) 140 val s1 = new PtwSectorResp () 141 val s2 = new HptwResp() 142 }, 1)).io) 143 val mergeArb = (0 until PtwWidth).map(i => Module(new Arbiter(new Bundle { 144 val s2xlate = UInt(2.W) 145 val s1 = new PtwMergeResp() 146 val s2 = new HptwResp() 147 }, 3)).io) 148 val outArbCachePort = 0 149 val outArbFsmPort = 1 150 val outArbMqPort = 2 151 152 if (HasBitmapCheck) { 153 // connect ptwcache and bitmap sleep-wakeup port 154 cache.io.bitmap_wakeup.get <> bitmap.get.io.wakeup 155 } 156 157 // hptw arb input port 158 val InHptwArbPTWPort = 0 159 val InHptwArbLLPTWPort = 1 160 hptw_req_arb.io.in(InHptwArbPTWPort).valid := ptw.io.hptw.req.valid 161 hptw_req_arb.io.in(InHptwArbPTWPort).bits.gvpn := ptw.io.hptw.req.bits.gvpn 162 hptw_req_arb.io.in(InHptwArbPTWPort).bits.id := ptw.io.hptw.req.bits.id 163 hptw_req_arb.io.in(InHptwArbPTWPort).bits.source := ptw.io.hptw.req.bits.source 164 ptw.io.hptw.req.ready := hptw_req_arb.io.in(InHptwArbPTWPort).ready 165 166 hptw_req_arb.io.in(InHptwArbLLPTWPort).valid := llptw.io.hptw.req.valid 167 hptw_req_arb.io.in(InHptwArbLLPTWPort).bits.gvpn := llptw.io.hptw.req.bits.gvpn 168 hptw_req_arb.io.in(InHptwArbLLPTWPort).bits.id := llptw.io.hptw.req.bits.id 169 hptw_req_arb.io.in(InHptwArbLLPTWPort).bits.source := llptw.io.hptw.req.bits.source 170 llptw.io.hptw.req.ready := hptw_req_arb.io.in(InHptwArbLLPTWPort).ready 171 172 // arb2 input port 173 val InArbHPTWPort = 0 174 val InArbPTWPort = 1 175 val InArbMissQueuePort = 2 176 val InArbTlbPort = 3 177 val InArbPrefetchPort = 4 178 // NOTE: when cache out but miss and ptw doesnt accept, 179 arb1.io.in <> VecInit(io.tlb.map(_.req(0))) 180 181 val tlbCounter = RegInit(0.U(log2Ceil(MissQueueSize + 1).W)) 182 val reqVec = WireInit(VecInit(Seq.fill(PtwWidth)(false.B))) 183 val respVec = WireInit(VecInit(Seq.fill(PtwWidth)(false.B))) 184 185 for (i <- 0 until PtwWidth) { 186 when (io.tlb(i).req(0).fire) { 187 reqVec(i) := true.B 188 } 189 when (io.tlb(i).resp.fire) { 190 respVec(i) := true.B 191 } 192 } 193 194 when (flush) { 195 tlbCounter := 0.U 196 } .otherwise { 197 tlbCounter := tlbCounter + PopCount(reqVec) - PopCount(respVec) 198 } 199 XSError(!(tlbCounter >= 0.U && tlbCounter <= MissQueueSize.U), s"l2tlb full!") 200 201 arb2.io.in(InArbPTWPort).valid := ptw.io.llptw.valid 202 arb2.io.in(InArbPTWPort).bits.req_info := ptw.io.llptw.bits.req_info 203 arb2.io.in(InArbPTWPort).bits.isHptwReq := false.B 204 arb2.io.in(InArbPTWPort).bits.isLLptw := false.B 205 arb2.io.in(InArbPTWPort).bits.hptwId := DontCare 206 ptw.io.llptw.ready := arb2.io.in(InArbPTWPort).ready 207 block_decoupled(missQueue.io.out, arb2.io.in(InArbMissQueuePort), Mux(missQueue.io.out.bits.isLLptw, !llptw.io.in.ready, !ptw.io.req.ready)) 208 209 arb2.io.in(InArbTlbPort).valid := arb1.io.out.fire 210 arb2.io.in(InArbTlbPort).bits.req_info.vpn := arb1.io.out.bits.vpn 211 arb2.io.in(InArbTlbPort).bits.req_info.s2xlate := arb1.io.out.bits.s2xlate 212 arb2.io.in(InArbTlbPort).bits.req_info.source := arb1.io.chosen 213 arb2.io.in(InArbTlbPort).bits.isHptwReq := false.B 214 arb2.io.in(InArbTlbPort).bits.isLLptw := false.B 215 arb2.io.in(InArbTlbPort).bits.hptwId := DontCare 216 // 1. arb1 and arb2 are both comb logic, so ready can work just the same cycle 217 // 2. arb1 can send one req at most in a cycle, so do not need to write 218 // "tlbCounter <= (MissQueueSize - 2).U" 219 arb1.io.out.ready := arb2.io.in(InArbTlbPort).ready && tlbCounter < MissQueueSize.U 220 221 arb2.io.in(InArbHPTWPort).valid := hptw_req_arb.io.out.valid 222 arb2.io.in(InArbHPTWPort).bits.req_info.vpn := hptw_req_arb.io.out.bits.gvpn 223 arb2.io.in(InArbHPTWPort).bits.req_info.s2xlate := onlyStage2 224 arb2.io.in(InArbHPTWPort).bits.req_info.source := hptw_req_arb.io.out.bits.source 225 arb2.io.in(InArbHPTWPort).bits.isHptwReq := true.B 226 arb2.io.in(InArbHPTWPort).bits.isLLptw := false.B 227 arb2.io.in(InArbHPTWPort).bits.hptwId := hptw_req_arb.io.out.bits.id 228 hptw_req_arb.io.out.ready := arb2.io.in(InArbHPTWPort).ready 229 val hartId = p(XSCoreParamsKey).HartId 230 if (l2tlbParams.enablePrefetch) { 231 val prefetch = Module(new L2TlbPrefetch()) 232 val recv = cache.io.resp 233 // NOTE: 1. prefetch doesn't gen prefetch 2. req from mq doesn't gen prefetch 234 // NOTE: 1. miss req gen prefetch 2. hit but prefetched gen prefetch 235 prefetch.io.in.valid := recv.fire && !from_pre(recv.bits.req_info.source) && (!recv.bits.hit || 236 recv.bits.prefetch) && recv.bits.isFirst 237 prefetch.io.in.bits.vpn := recv.bits.req_info.vpn 238 prefetch.io.sfence := sfence_dup(0) 239 prefetch.io.csr := csr_dup(0) 240 arb2.io.in(InArbPrefetchPort) <> prefetch.io.out 241 242 val isWriteL2TlbPrefetchTable = Constantin.createRecord(s"isWriteL2TlbPrefetchTable$hartId") 243 val L2TlbPrefetchTable = ChiselDB.createTable(s"L2TlbPrefetch_hart$hartId", new L2TlbPrefetchDB) 244 val L2TlbPrefetchDB = Wire(new L2TlbPrefetchDB) 245 L2TlbPrefetchDB.vpn := prefetch.io.out.bits.req_info.vpn 246 L2TlbPrefetchTable.log(L2TlbPrefetchDB, isWriteL2TlbPrefetchTable.orR && prefetch.io.out.fire, "L2TlbPrefetch", clock, reset) 247 } 248 arb2.io.out.ready := cache.io.req.ready 249 250 // Instructs requests from cache need go to LLPTW for processing 251 val toFsm_toLLPTW = if (HasBitmapCheck) cache.io.resp.bits.toFsm.bitmapCheck.get.toLLPTW else false.B 252 253 val mq_arb = Module(new Arbiter(new L2TlbWithHptwIdBundle, 2)) 254 mq_arb.io.in(0).valid := cache.io.resp.valid && !cache.io.resp.bits.hit && 255 !from_pre(cache.io.resp.bits.req_info.source) && !cache.io.resp.bits.isHptwReq && // hptw reqs are not sent to missqueue 256 (cache.io.resp.bits.bypassed || ( 257 (((!cache.io.resp.bits.toFsm.l1Hit && !toFsm_toLLPTW) || cache.io.resp.bits.toFsm.stage1Hit) && !cache.io.resp.bits.isHptwReq && (cache.io.resp.bits.isFirst || !ptw.io.req.ready)) // send to ptw, is first or ptw is busy; 258 || ((cache.io.resp.bits.toFsm.l1Hit || toFsm_toLLPTW) && !llptw.io.in.ready) // send to llptw, llptw is full 259 )) 260 261 mq_arb.io.in(0).bits.req_info := cache.io.resp.bits.req_info 262 mq_arb.io.in(0).bits.isHptwReq := false.B 263 mq_arb.io.in(0).bits.hptwId := DontCare 264 mq_arb.io.in(0).bits.isLLptw := cache.io.resp.bits.toFsm.l1Hit || toFsm_toLLPTW 265 mq_arb.io.in(1).bits.req_info := llptw.io.cache.bits 266 mq_arb.io.in(1).bits.isHptwReq := false.B 267 mq_arb.io.in(1).bits.hptwId := DontCare 268 mq_arb.io.in(1).bits.isLLptw := false.B 269 mq_arb.io.in(1).valid := llptw.io.cache.valid 270 llptw.io.cache.ready := mq_arb.io.in(1).ready 271 missQueue.io.in <> mq_arb.io.out 272 missQueue.io.sfence := sfence_dup(6) 273 missQueue.io.csr := csr_dup(5) 274 275 blockmq.io.start := missQueue.io.out.fire 276 blockmq.io.enable := ptw.io.req.fire 277 278 llptw.io.in.valid := cache.io.resp.valid && 279 !cache.io.resp.bits.hit && 280 (toFsm_toLLPTW || cache.io.resp.bits.toFsm.l1Hit) && 281 !cache.io.resp.bits.bypassed && 282 !cache.io.resp.bits.isHptwReq 283 llptw.io.in.bits.req_info := cache.io.resp.bits.req_info 284 llptw.io.in.bits.ppn := cache.io.resp.bits.toFsm.ppn 285 if (HasBitmapCheck) { 286 llptw.io.in.bits.bitmapCheck.get.jmp_bitmap_check := cache.io.resp.bits.toFsm.bitmapCheck.get.jmp_bitmap_check 287 llptw.io.in.bits.bitmapCheck.get.ptes := cache.io.resp.bits.toFsm.bitmapCheck.get.ptes 288 llptw.io.in.bits.bitmapCheck.get.cfs := cache.io.resp.bits.toFsm.bitmapCheck.get.cfs 289 llptw.io.in.bits.bitmapCheck.get.hitway := cache.io.resp.bits.toFsm.bitmapCheck.get.hitway 290 } 291 llptw.io.sfence := sfence_dup(1) 292 llptw.io.csr := csr_dup(1) 293 val llptw_stage1 = Reg(Vec(l2tlbParams.llptwsize, new PtwMergeResp())) 294 when(llptw.io.in.fire){ 295 llptw_stage1(llptw.io.mem.enq_ptr) := cache.io.resp.bits.stage1 296 } 297 298 cache.io.req.valid := arb2.io.out.fire 299 cache.io.req.bits.req_info := arb2.io.out.bits.req_info 300 cache.io.req.bits.isFirst := (arb2.io.chosen =/= InArbMissQueuePort.U && !arb2.io.out.bits.isHptwReq) 301 cache.io.req.bits.isHptwReq := arb2.io.out.bits.isHptwReq 302 cache.io.req.bits.hptwId := arb2.io.out.bits.hptwId 303 cache.io.req.bits.bypassed.map(_ := false.B) 304 cache.io.sfence := sfence_dup(2) 305 cache.io.csr := csr_dup(2) 306 cache.io.sfence_dup.zip(sfence_dup.drop(2).take(4)).map(s => s._1 := s._2) 307 cache.io.csr_dup.zip(csr_dup.drop(2).take(3)).map(c => c._1 := c._2) 308 cache.io.resp.ready := MuxCase(mq_arb.io.in(0).ready || ptw.io.req.ready, Seq( 309 (!cache.io.resp.bits.hit && cache.io.resp.bits.isHptwReq) -> hptw.io.req.ready, 310 (cache.io.resp.bits.hit && cache.io.resp.bits.isHptwReq) -> hptw_resp_arb.io.in(HptwRespArbCachePort).ready, 311 cache.io.resp.bits.hit -> outReady(cache.io.resp.bits.req_info.source, outArbCachePort), 312 ((toFsm_toLLPTW || cache.io.resp.bits.toFsm.l1Hit) && !cache.io.resp.bits.bypassed && llptw.io.in.ready) -> llptw.io.in.ready, 313 (cache.io.resp.bits.bypassed || cache.io.resp.bits.isFirst) -> mq_arb.io.in(0).ready 314 )) 315 316 // NOTE: missQueue req has higher priority 317 ptw.io.req.valid := cache.io.resp.valid && !cache.io.resp.bits.hit && !cache.io.resp.bits.toFsm.l1Hit && 318 !cache.io.resp.bits.bypassed && 319 !cache.io.resp.bits.isFirst && 320 !cache.io.resp.bits.isHptwReq && 321 !toFsm_toLLPTW 322 ptw.io.req.bits.req_info := cache.io.resp.bits.req_info 323 if (EnableSv48) { 324 ptw.io.req.bits.l3Hit.get := cache.io.resp.bits.toFsm.l3Hit.get 325 } 326 ptw.io.req.bits.l2Hit := cache.io.resp.bits.toFsm.l2Hit 327 ptw.io.req.bits.ppn := cache.io.resp.bits.toFsm.ppn 328 ptw.io.req.bits.stage1Hit := cache.io.resp.bits.toFsm.stage1Hit 329 ptw.io.req.bits.stage1 := cache.io.resp.bits.stage1 330 if (HasBitmapCheck) { 331 ptw.io.req.bits.bitmapCheck.get.jmp_bitmap_check := cache.io.resp.bits.toFsm.bitmapCheck.get.jmp_bitmap_check 332 ptw.io.req.bits.bitmapCheck.get.pte := cache.io.resp.bits.toFsm.bitmapCheck.get.pte 333 ptw.io.req.bits.bitmapCheck.get.cfs := cache.io.resp.bits.toFsm.bitmapCheck.get.cfs 334 ptw.io.req.bits.bitmapCheck.get.SPlevel := cache.io.resp.bits.toFsm.bitmapCheck.get.SPlevel 335 } 336 ptw.io.sfence := sfence_dup(7) 337 ptw.io.csr := csr_dup(6) 338 ptw.io.resp.ready := outReady(ptw.io.resp.bits.source, outArbFsmPort) 339 340 hptw.io.req.valid := cache.io.resp.valid && !cache.io.resp.bits.hit && cache.io.resp.bits.isHptwReq 341 hptw.io.req.bits.gvpn := cache.io.resp.bits.req_info.vpn 342 hptw.io.req.bits.id := cache.io.resp.bits.toHptw.id 343 hptw.io.req.bits.source := cache.io.resp.bits.req_info.source 344 if (EnableSv48) { 345 hptw.io.req.bits.l3Hit.get := cache.io.resp.bits.toHptw.l3Hit.get 346 } 347 hptw.io.req.bits.l2Hit := cache.io.resp.bits.toHptw.l2Hit 348 hptw.io.req.bits.l1Hit := cache.io.resp.bits.toHptw.l1Hit 349 hptw.io.req.bits.ppn := cache.io.resp.bits.toHptw.ppn 350 hptw.io.req.bits.bypassed := cache.io.resp.bits.toHptw.bypassed 351 if (HasBitmapCheck) { 352 hptw.io.req.bits.bitmapCheck.get <> cache.io.resp.bits.toHptw.bitmapCheck.get 353 } 354 hptw.io.sfence := sfence_dup(8) 355 hptw.io.csr := csr_dup(7) 356 // mem req 357 def blockBytes_align(addr: UInt) = { 358 Cat(addr(PAddrBits - 1, log2Up(l2tlbParams.blockBytes)), 0.U(log2Up(l2tlbParams.blockBytes).W)) 359 } 360 def addr_low_from_vpn(vpn: UInt) = { 361 vpn(log2Ceil(l2tlbParams.blockBytes)-log2Ceil(XLEN/8)-1, 0) 362 } 363 def addr_low_from_paddr(paddr: UInt) = { 364 paddr(log2Up(l2tlbParams.blockBytes)-1, log2Up(XLEN/8)) 365 } 366 def from_llptw(id: UInt) = { 367 id < l2tlbParams.llptwsize.U 368 } 369 def from_ptw(id: UInt) = { 370 id === l2tlbParams.llptwsize.U 371 } 372 def from_hptw(id: UInt) = { 373 id === l2tlbParams.llptwsize.U + 1.U 374 } 375 def from_bitmap(id: UInt) = { 376 (id > l2tlbParams.llptwsize.U + 1.U) && (id < MemReqWidth.U) 377 } 378 val waiting_resp = RegInit(VecInit(Seq.fill(MemReqWidth)(false.B))) 379 val flush_latch = RegInit(VecInit(Seq.fill(MemReqWidth)(false.B))) 380 val hptw_bypassed = RegInit(false.B) 381 for (i <- waiting_resp.indices) { 382 assert(!flush_latch(i) || waiting_resp(i)) // when sfence_latch wait for mem resp, waiting_resp should be true 383 } 384 385 val llptw_out = llptw.io.out 386 val llptw_mem = llptw.io.mem 387 llptw_mem.flush_latch := flush_latch.take(l2tlbParams.llptwsize) 388 llptw_mem.req_mask := waiting_resp.take(l2tlbParams.llptwsize) 389 ptw.io.mem.mask := waiting_resp.apply(l2tlbParams.llptwsize) 390 hptw.io.mem.mask := waiting_resp.apply(l2tlbParams.llptwsize + 1) 391 if (HasBitmapCheck) { 392 bitmap.get.io.mem.req_mask := waiting_resp.slice(MemReqWidth - (l2tlbParams.llptwsize + 2), MemReqWidth) 393 } 394 val mem_arb = Module(new Arbiter(new L2TlbMemReqBundle(), if (HasBitmapCheck) 4 else 3)) 395 mem_arb.io.in(0) <> ptw.io.mem.req 396 mem_arb.io.in(1) <> llptw_mem.req 397 mem_arb.io.in(2) <> hptw.io.mem.req 398 if (HasBitmapCheck) { 399 mem_arb.io.in(3) <> bitmap.get.io.mem.req 400 } 401 mem_arb.io.out.ready := mem.a.ready && !flush 402 403 // // assert, should not send mem access at same addr for twice. 404 // val last_resp_vpn = RegEnable(cache.io.refill.bits.req_info_dup(0).vpn, cache.io.refill.valid) 405 // val last_resp_s2xlate = RegEnable(cache.io.refill.bits.req_info_dup(0).s2xlate, cache.io.refill.valid) 406 // val last_resp_level = RegEnable(cache.io.refill.bits.level_dup(0), cache.io.refill.valid) 407 // val last_resp_v = RegInit(false.B) 408 // val last_has_invalid = !Cat(cache.io.refill.bits.ptes.asTypeOf(Vec(blockBits/XLEN, UInt(XLEN.W))).map(a => a(0))).andR || cache.io.refill.bits.sel_pte_dup(0).asTypeOf(new PteBundle).isAf() 409 // when (cache.io.refill.valid) { last_resp_v := !last_has_invalid} 410 // when (flush) { last_resp_v := false.B } 411 // XSError(last_resp_v && cache.io.refill.valid && 412 // (cache.io.refill.bits.req_info_dup(0).vpn === last_resp_vpn) && 413 // (cache.io.refill.bits.level_dup(0) === last_resp_level) && 414 // (cache.io.refill.bits.req_info_dup(0).s2xlate === last_resp_s2xlate), 415 // "l2tlb should not access mem at same addr for twice") 416 // // ATTENTION: this may wrongly assert when: a ptes is l2, last part is valid, 417 // // but the current part is invalid, so one more mem access happened 418 // // If this happened, remove the assert. 419 420 val req_addr_low = Reg(Vec(MemReqWidth, UInt((log2Up(l2tlbParams.blockBytes)-log2Up(XLEN/8)).W))) 421 422 when (llptw.io.in.fire) { 423 // when enq miss queue, set the req_addr_low to receive the mem resp data part 424 req_addr_low(llptw_mem.enq_ptr) := addr_low_from_vpn(llptw.io.in.bits.req_info.vpn) 425 } 426 when (mem_arb.io.out.fire) { 427 req_addr_low(mem_arb.io.out.bits.id) := addr_low_from_paddr(mem_arb.io.out.bits.addr) 428 waiting_resp(mem_arb.io.out.bits.id) := true.B 429 hptw_bypassed := from_hptw(mem_arb.io.out.bits.id) && mem_arb.io.out.bits.hptw_bypassed 430 } 431 // mem read 432 val memRead = edge.Get( 433 fromSource = mem_arb.io.out.bits.id, 434 // toAddress = memAddr(log2Up(CacheLineSize / 2 / 8) - 1, 0), 435 toAddress = blockBytes_align(mem_arb.io.out.bits.addr), 436 lgSize = log2Up(l2tlbParams.blockBytes).U 437 )._2 438 mem.a.bits := memRead 439 mem.a.valid := mem_arb.io.out.valid && !flush 440 mem.a.bits.user.lift(ReqSourceKey).foreach(_ := MemReqSource.PTW.id.U) 441 mem.d.ready := true.B 442 // mem -> data buffer 443 val refill_data = RegInit(VecInit.fill(blockBits / l1BusDataWidth)(0.U(l1BusDataWidth.W))) 444 val refill_helper = edge.firstlastHelper(mem.d.bits, mem.d.fire) 445 val mem_resp_done = refill_helper._3 446 val mem_resp_from_llptw = from_llptw(mem.d.bits.source) 447 val mem_resp_from_ptw = from_ptw(mem.d.bits.source) 448 val mem_resp_from_hptw = from_hptw(mem.d.bits.source) 449 val mem_resp_from_bitmap = from_bitmap(mem.d.bits.source) 450 when (mem.d.valid) { 451 assert(mem.d.bits.source < MemReqWidth.U) 452 refill_data(refill_helper._4) := mem.d.bits.data 453 } 454 // refill_data_tmp is the wire fork of refill_data, but one cycle earlier 455 val refill_data_tmp = WireInit(refill_data) 456 refill_data_tmp(refill_helper._4) := mem.d.bits.data 457 458 // save only one pte for each id 459 // (miss queue may can't resp to tlb with low latency, it should have highest priority, but diffcult to design cache) 460 val resp_pte = VecInit((0 until (if (HasBitmapCheck) MemReqWidth / 2 else MemReqWidth)).map(i => 461 if (i == l2tlbParams.llptwsize + 1) {RegEnable(get_part(refill_data_tmp, req_addr_low(i)), 0.U.asTypeOf(get_part(refill_data_tmp, req_addr_low(i))), mem_resp_done && mem_resp_from_hptw) } 462 else if (i == l2tlbParams.llptwsize) {RegEnable(get_part(refill_data_tmp, req_addr_low(i)), 0.U.asTypeOf(get_part(refill_data_tmp, req_addr_low(i))), mem_resp_done && mem_resp_from_ptw) } 463 else { Mux(llptw_mem.buffer_it(i), get_part(refill_data, req_addr_low(i)), RegEnable(get_part(refill_data, req_addr_low(i)), 0.U.asTypeOf(get_part(refill_data, req_addr_low(i))), llptw_mem.buffer_it(i))) } 464 // llptw could not use refill_data_tmp, because enq bypass's result works at next cycle 465 )) 466 467 // save eight ptes for each id when sector tlb 468 // (miss queue may can't resp to tlb with low latency, it should have highest priority, but diffcult to design cache) 469 val resp_pte_sector = VecInit((0 until (if (HasBitmapCheck) MemReqWidth / 2 else MemReqWidth)).map(i => 470 if (i == l2tlbParams.llptwsize + 1) {RegEnable(refill_data_tmp, 0.U.asTypeOf(refill_data_tmp), mem_resp_done && mem_resp_from_hptw) } 471 else if (i == l2tlbParams.llptwsize) {RegEnable(refill_data_tmp, 0.U.asTypeOf(refill_data_tmp), mem_resp_done && mem_resp_from_ptw) } 472 else { Mux(llptw_mem.buffer_it(i), refill_data, RegEnable(refill_data, 0.U.asTypeOf(refill_data), llptw_mem.buffer_it(i))) } 473 // llptw could not use refill_data_tmp, because enq bypass's result works at next cycle 474 )) 475 476 if (HasBitmapCheck) { 477 // add bitmap arb 478 bitmap.foreach { Bitmap => 479 val bitmap_arb = Module(new Arbiter(new bitmapReqBundle(), 3)) 480 bitmap_arb.io.in(0) <> ptw.io.bitmap.get.req 481 bitmap_arb.io.in(1) <> llptw.io.bitmap.get.req 482 bitmap_arb.io.in(2) <> hptw.io.bitmap.get.req 483 bitmap_arb.io.out.ready := Bitmap.io.req.ready 484 485 Bitmap.io.req <> bitmap_arb.io.out 486 487 // connect bitmap resp to PTW 488 val bitmapresp_to_llptw = from_llptw(Bitmap.io.resp.bits.id) 489 val bitmapresp_to_hptw = from_hptw(Bitmap.io.resp.bits.id) 490 val bitmapresp_to_ptw = from_ptw(Bitmap.io.resp.bits.id) 491 492 Bitmap.io.resp.ready := (llptw.io.bitmap.get.resp.ready && bitmapresp_to_llptw) || (hptw.io.bitmap.get.resp.ready && bitmapresp_to_hptw) || (ptw.io.bitmap.get.resp.ready && bitmapresp_to_ptw) 493 494 // bitmap -> llptw ptw hptw 495 llptw.io.bitmap.get.resp.valid := Bitmap.io.resp.valid && bitmapresp_to_llptw 496 hptw.io.bitmap.get.resp.valid := Bitmap.io.resp.valid && bitmapresp_to_hptw 497 ptw.io.bitmap.get.resp.valid := Bitmap.io.resp.valid && bitmapresp_to_ptw 498 499 // add ptw、hptw、llptw with bitmap resp connect 500 ptw.io.bitmap.get.resp.bits := Bitmap.io.resp.bits 501 hptw.io.bitmap.get.resp.bits := Bitmap.io.resp.bits 502 llptw.io.bitmap.get.resp.bits := Bitmap.io.resp.bits 503 504 // mem -> bitmap 505 Bitmap.io.mem.resp.valid := mem_resp_done && mem_resp_from_bitmap 506 Bitmap.io.mem.resp.bits.id := DataHoldBypass(mem.d.bits.source, mem.d.valid) 507 Bitmap.io.mem.resp.bits.value := DataHoldBypass(refill_data_tmp.asUInt, mem.d.valid) 508 } 509 510 // ptwcache -> hptw llptw 511 hptw.io.l0_way_info.get := cache.io.l0_way_info.get 512 llptw.io.l0_way_info.get := cache.io.l0_way_info.get 513 } 514 515 // mem -> llptw 516 llptw_mem.resp.valid := mem_resp_done && mem_resp_from_llptw 517 llptw_mem.resp.bits.id := DataHoldBypass(mem.d.bits.source, mem.d.valid) 518 llptw_mem.resp.bits.value := DataHoldBypass(refill_data_tmp.asUInt, mem.d.valid) 519 // mem -> ptw 520 ptw.io.mem.resp.valid := mem_resp_done && mem_resp_from_ptw 521 ptw.io.mem.resp.bits := resp_pte.apply(l2tlbParams.llptwsize) 522 // mem -> hptw 523 hptw.io.mem.resp.valid := mem_resp_done && mem_resp_from_hptw 524 hptw.io.mem.resp.bits := resp_pte.apply(l2tlbParams.llptwsize + 1) 525 // mem -> cache 526 val refill_from_llptw = mem_resp_from_llptw 527 val refill_from_ptw = mem_resp_from_ptw 528 val refill_from_hptw = mem_resp_from_hptw 529 val refill_level = Mux(refill_from_llptw, 0.U, Mux(refill_from_ptw, RegEnable(ptw.io.refill.level, 0.U, ptw.io.mem.req.fire), RegEnable(hptw.io.refill.level, 0.U, hptw.io.mem.req.fire))) 530 val refill_valid = mem_resp_done && (if (HasBitmapCheck) !mem_resp_from_bitmap else true.B) && !flush && !flush_latch(mem.d.bits.source) && !(from_hptw(mem.d.bits.source) && hptw_bypassed) 531 532 cache.io.refill.valid := GatedValidRegNext(refill_valid, false.B) 533 cache.io.refill.bits.ptes := refill_data.asUInt 534 cache.io.refill.bits.req_info_dup.map(_ := RegEnable(Mux(refill_from_llptw, llptw_mem.refill, Mux(refill_from_ptw, ptw.io.refill.req_info, hptw.io.refill.req_info)), refill_valid)) 535 cache.io.refill.bits.level_dup.map(_ := RegEnable(refill_level, refill_valid)) 536 cache.io.refill.bits.levelOH(refill_level, refill_valid) 537 cache.io.refill.bits.sel_pte_dup.map(_ := RegEnable(sel_data(refill_data_tmp.asUInt, req_addr_low(mem.d.bits.source)), refill_valid)) 538 539 if (env.EnableDifftest) { 540 val difftest_ptw_addr = RegInit(VecInit(Seq.fill(MemReqWidth)(0.U(PAddrBits.W)))) 541 when (mem.a.valid) { 542 difftest_ptw_addr(mem.a.bits.source) := mem.a.bits.address 543 } 544 545 val difftest = DifftestModule(new DiffRefillEvent, dontCare = true) 546 difftest.coreid := io.hartId 547 difftest.index := 2.U 548 difftest.valid := cache.io.refill.valid 549 difftest.addr := difftest_ptw_addr(RegEnable(mem.d.bits.source, mem.d.valid)) 550 difftest.data := refill_data.asTypeOf(difftest.data) 551 difftest.idtfr := DontCare 552 } 553 554 if (env.EnableDifftest) { 555 for (i <- 0 until PtwWidth) { 556 val difftest = DifftestModule(new DiffL2TLBEvent) 557 difftest.coreid := io.hartId 558 difftest.valid := io.tlb(i).resp.fire && !io.tlb(i).resp.bits.s1.af && !io.tlb(i).resp.bits.s2.gaf 559 difftest.index := i.U 560 difftest.vpn := Cat(io.tlb(i).resp.bits.s1.entry.tag, 0.U(sectortlbwidth.W)) 561 difftest.pbmt := io.tlb(i).resp.bits.s1.entry.pbmt 562 difftest.g_pbmt := io.tlb(i).resp.bits.s2.entry.pbmt 563 for (j <- 0 until tlbcontiguous) { 564 difftest.ppn(j) := Cat(io.tlb(i).resp.bits.s1.entry.ppn, io.tlb(i).resp.bits.s1.ppn_low(j)) 565 difftest.valididx(j) := io.tlb(i).resp.bits.s1.valididx(j) 566 difftest.pteidx(j) := io.tlb(i).resp.bits.s1.pteidx(j) 567 } 568 difftest.perm := io.tlb(i).resp.bits.s1.entry.perm.getOrElse(0.U.asTypeOf(new PtePermBundle)).asUInt 569 difftest.level := io.tlb(i).resp.bits.s1.entry.level.getOrElse(0.U.asUInt) 570 difftest.pf := io.tlb(i).resp.bits.s1.pf 571 difftest.satp := Cat(io.csr.tlb.satp.mode, io.csr.tlb.satp.asid, io.csr.tlb.satp.ppn) 572 difftest.vsatp := Cat(io.csr.tlb.vsatp.mode, io.csr.tlb.vsatp.asid, io.csr.tlb.vsatp.ppn) 573 difftest.hgatp := Cat(io.csr.tlb.hgatp.mode, io.csr.tlb.hgatp.vmid, io.csr.tlb.hgatp.ppn) 574 difftest.gvpn := io.tlb(i).resp.bits.s2.entry.tag 575 difftest.g_perm := io.tlb(i).resp.bits.s2.entry.perm.getOrElse(0.U.asTypeOf(new PtePermBundle)).asUInt 576 difftest.g_level := io.tlb(i).resp.bits.s2.entry.level.getOrElse(0.U.asUInt) 577 difftest.s2ppn := io.tlb(i).resp.bits.s2.entry.ppn 578 difftest.gpf := io.tlb(i).resp.bits.s2.gpf 579 difftest.s2xlate := io.tlb(i).resp.bits.s2xlate 580 } 581 } 582 583 // pmp 584 pmp_check(0).req <> ptw.io.pmp.req 585 ptw.io.pmp.resp <> pmp_check(0).resp 586 pmp_check(1).req <> llptw.io.pmp.req 587 llptw.io.pmp.resp <> pmp_check(1).resp 588 pmp_check(2).req <> hptw.io.pmp.req 589 hptw.io.pmp.resp <> pmp_check(2).resp 590 if (HasBitmapCheck) { 591 pmp_check(3).req <> bitmap.get.io.pmp.req 592 bitmap.get.io.pmp.resp <> pmp_check(3).resp 593 } 594 595 llptw_out.ready := outReady(llptw_out.bits.req_info.source, outArbMqPort) 596 597 // hptw and page cache -> ptw and llptw 598 val HptwRespArbCachePort = 0 599 val HptwRespArbHptw = 1 600 hptw_resp_arb.io.in(HptwRespArbCachePort).valid := cache.io.resp.valid && cache.io.resp.bits.hit && cache.io.resp.bits.isHptwReq 601 hptw_resp_arb.io.in(HptwRespArbCachePort).bits.id := cache.io.resp.bits.toHptw.id 602 hptw_resp_arb.io.in(HptwRespArbCachePort).bits.resp := cache.io.resp.bits.toHptw.resp 603 hptw_resp_arb.io.in(HptwRespArbHptw).valid := hptw.io.resp.valid 604 hptw_resp_arb.io.in(HptwRespArbHptw).bits.id := hptw.io.resp.bits.id 605 hptw_resp_arb.io.in(HptwRespArbHptw).bits.resp := hptw.io.resp.bits.resp 606 hptw.io.resp.ready := hptw_resp_arb.io.in(HptwRespArbHptw).ready 607 608 ptw.io.hptw.resp.valid := hptw_resp_arb.io.out.valid && hptw_resp_arb.io.out.bits.id === FsmReqID.U 609 ptw.io.hptw.resp.bits.h_resp := hptw_resp_arb.io.out.bits.resp 610 llptw.io.hptw.resp.valid := hptw_resp_arb.io.out.valid && hptw_resp_arb.io.out.bits.id =/= FsmReqID.U 611 llptw.io.hptw.resp.bits.id := hptw_resp_arb.io.out.bits.id 612 llptw.io.hptw.resp.bits.h_resp := hptw_resp_arb.io.out.bits.resp 613 hptw_resp_arb.io.out.ready := true.B 614 615 val cfsValue = Option.when(HasBitmapCheck)(llptw_out.bits.bitmapCheck.get.cfs) 616 617 // Timing: Maybe need to do some optimization or even add one more cycle 618 for (i <- 0 until PtwWidth) { 619 mergeArb(i).in(outArbCachePort).valid := cache.io.resp.valid && cache.io.resp.bits.hit && cache.io.resp.bits.req_info.source===i.U && !cache.io.resp.bits.isHptwReq 620 mergeArb(i).in(outArbCachePort).bits.s2xlate := cache.io.resp.bits.req_info.s2xlate 621 mergeArb(i).in(outArbCachePort).bits.s1 := cache.io.resp.bits.stage1 622 mergeArb(i).in(outArbCachePort).bits.s2 := cache.io.resp.bits.toHptw.resp 623 mergeArb(i).in(outArbFsmPort).valid := ptw.io.resp.valid && ptw.io.resp.bits.source===i.U 624 mergeArb(i).in(outArbFsmPort).bits.s2xlate := ptw.io.resp.bits.s2xlate 625 mergeArb(i).in(outArbFsmPort).bits.s1 := ptw.io.resp.bits.resp 626 mergeArb(i).in(outArbFsmPort).bits.s2 := ptw.io.resp.bits.h_resp 627 mergeArb(i).in(outArbMqPort).valid := llptw_out.valid && llptw_out.bits.req_info.source===i.U 628 mergeArb(i).in(outArbMqPort).bits.s2xlate := llptw_out.bits.req_info.s2xlate 629 mergeArb(i).in(outArbMqPort).bits.s1 := Mux( 630 llptw_out.bits.first_s2xlate_fault, llptw_stage1(llptw_out.bits.id), 631 // When G-stage triggers gpf | gaf, `first_s2xlate_fault` will be true 632 // So Here only including pf | af in Stage1, or Stage1Gpf (gpf = llptw_out.bits.h_resp.gpf) 633 contiguous_pte_to_merge_ptwResp( 634 if (HasBitmapCheck) Mux(llptw_out.bits.bitmapCheck.get.jmp_bitmap_check, llptw_out.bits.bitmapCheck.get.ptes.asUInt, resp_pte_sector(llptw_out.bits.id).asUInt) else resp_pte_sector(llptw_out.bits.id).asUInt, llptw_out.bits.req_info.vpn, llptw_out.bits.af, 635 true, s2xlate = llptw_out.bits.req_info.s2xlate, mPBMTE = mPBMTE, hPBMTE = hPBMTE, gpf = llptw_out.bits.h_resp.gpf, 636 cfs = cfsValue.getOrElse(VecInit(Seq.fill(tlbcontiguous)(false.B))) 637 ) 638 ) 639 mergeArb(i).in(outArbMqPort).bits.s2 := llptw_out.bits.h_resp 640 mergeArb(i).out.ready := outArb(i).in(0).ready 641 } 642 643 for (i <- 0 until PtwWidth) { 644 outArb(i).in(0).valid := mergeArb(i).out.valid 645 outArb(i).in(0).bits.s2xlate := mergeArb(i).out.bits.s2xlate 646 outArb(i).in(0).bits.s1 := merge_ptwResp_to_sector_ptwResp(mergeArb(i).out.bits.s1) 647 outArb(i).in(0).bits.s2 := mergeArb(i).out.bits.s2 648 } 649 650 // io.tlb.map(_.resp) <> outArb.map(_.out) 651 io.tlb.map(_.resp).zip(outArb.map(_.out)).map{ 652 case (resp, out) => resp <> out 653 } 654 655 // sfence 656 when (flush) { 657 for (i <- 0 until MemReqWidth) { 658 when (waiting_resp(i)) { 659 flush_latch(i) := true.B 660 } 661 } 662 } 663 // mem -> control signal 664 // waiting_resp and sfence_latch will be reset when mem_resp_done 665 when (mem_resp_done) { 666 waiting_resp(mem.d.bits.source) := false.B 667 flush_latch(mem.d.bits.source) := false.B 668 } 669 670 def block_decoupled[T <: Data](source: DecoupledIO[T], sink: DecoupledIO[T], block_signal: Bool) = { 671 sink.valid := source.valid && !block_signal 672 source.ready := sink.ready && !block_signal 673 sink.bits := source.bits 674 } 675 676 def get_part(data: Vec[UInt], index: UInt): UInt = { 677 val inner_data = data.asTypeOf(Vec(data.getWidth / XLEN, UInt(XLEN.W))) 678 inner_data(index) 679 } 680 681 // not_super means that this is a normal page 682 // valididx(i) will be all true when super page to be convenient for l1 tlb matching 683 def contiguous_pte_to_merge_ptwResp(pte: UInt, vpn: UInt, af: Bool, af_first: Boolean, s2xlate: UInt, mPBMTE: Bool, hPBMTE: Bool, not_super: Boolean = true, gpf: Bool, cfs : Vec[Bool]) : PtwMergeResp = { 684 assert(tlbcontiguous == 8, "Only support tlbcontiguous = 8!") 685 val ptw_merge_resp = Wire(new PtwMergeResp()) 686 val hasS2xlate = s2xlate =/= noS2xlate 687 val pbmte = Mux(s2xlate === onlyStage1 || s2xlate === allStage, hPBMTE, mPBMTE) 688 for (i <- 0 until tlbcontiguous) { 689 val pte_in = pte(64 * i + 63, 64 * i).asTypeOf(new PteBundle()) 690 val ptw_resp = Wire(new PtwMergeEntry(tagLen = sectorvpnLen, hasPerm = true, hasLevel = true, hasNapot = true)) 691 ptw_resp.ppn := pte_in.getPPN()(ptePPNLen - 1, sectortlbwidth) 692 ptw_resp.ppn_low := pte_in.getPPN()(sectortlbwidth - 1, 0) 693 ptw_resp.level.map(_ := 0.U) 694 ptw_resp.pbmt := pte_in.pbmt 695 ptw_resp.n.map(_ := pte_in.n) 696 ptw_resp.perm.map(_ := pte_in.getPerm()) 697 ptw_resp.tag := vpn(vpnLen - 1, sectortlbwidth) 698 // LLPTW will not handle onlyS2 situations 699 // noS2xlate. pf: allStagePf; af: af(pmp_af) || pte_in.isAf(ppn_af); gpf: never 700 // onlyStage1. pf: allStagePf; af: af(pmp_af) || pte_in.isAf(ppn_af); gpf: never 701 // allStage. pf: allStagePf; af: af(pmp_af) && !gpf; gpf: incoming parameter `gpf` 702 // priority: af (pmp check) > pf (pte check) > af (pte check) 703 val isPf = pte_in.isPf(0.U, pbmte) || !pte_in.isLeaf() 704 val isAf = pte_in.isAf() && (s2xlate === noS2xlate || s2xlate === onlyStage1) && !isPf 705 ptw_resp.pf := (if (af_first) !af else true.B) && isPf 706 ptw_resp.af := (if (af_first) true.B else !isPf) && (af || isAf) 707 ptw_resp.cf := cfs(ptw_resp.ppn(sectortlbwidth - 1, 0)) 708 ptw_resp.v := !ptw_resp.pf 709 ptw_resp.prefetch := DontCare 710 ptw_resp.asid := Mux(hasS2xlate, vsatp.asid, satp.asid) 711 ptw_resp.vmid.map(_ := hgatp.vmid) 712 ptw_merge_resp.entry(i) := ptw_resp 713 } 714 ptw_merge_resp.pteidx := UIntToOH(vpn(sectortlbwidth - 1, 0)).asBools 715 val napot = ptw_merge_resp.entry(vpn(sectortlbwidth - 1, 0)).n.getOrElse(0.U) 716 ptw_merge_resp.not_super := not_super.B && !napot 717 ptw_merge_resp.not_merge := hasS2xlate 718 ptw_merge_resp 719 } 720 721 def merge_ptwResp_to_sector_ptwResp(pte: PtwMergeResp) : PtwSectorResp = { 722 assert(tlbcontiguous == 8, "Only support tlbcontiguous = 8!") 723 val ptw_sector_resp = Wire(new PtwSectorResp) 724 ptw_sector_resp.entry.tag := pte.entry(OHToUInt(pte.pteidx)).tag 725 ptw_sector_resp.entry.asid := pte.entry(OHToUInt(pte.pteidx)).asid 726 ptw_sector_resp.entry.vmid.map(_ := pte.entry(OHToUInt(pte.pteidx)).vmid.getOrElse(0.U)) 727 ptw_sector_resp.entry.ppn := pte.entry(OHToUInt(pte.pteidx)).ppn 728 ptw_sector_resp.entry.pbmt := pte.entry(OHToUInt(pte.pteidx)).pbmt 729 ptw_sector_resp.entry.n.map(_ := pte.entry(OHToUInt(pte.pteidx)).n.getOrElse(0.U)) 730 ptw_sector_resp.entry.perm.map(_ := pte.entry(OHToUInt(pte.pteidx)).perm.getOrElse(0.U.asTypeOf(new PtePermBundle))) 731 ptw_sector_resp.entry.level.map(_ := pte.entry(OHToUInt(pte.pteidx)).level.getOrElse(0.U(log2Up(Level + 1).W))) 732 ptw_sector_resp.entry.prefetch := pte.entry(OHToUInt(pte.pteidx)).prefetch 733 ptw_sector_resp.entry.v := pte.entry(OHToUInt(pte.pteidx)).v 734 ptw_sector_resp.af := pte.entry(OHToUInt(pte.pteidx)).af 735 ptw_sector_resp.pf := pte.entry(OHToUInt(pte.pteidx)).pf 736 ptw_sector_resp.addr_low := OHToUInt(pte.pteidx) 737 ptw_sector_resp.pteidx := pte.pteidx 738 for (i <- 0 until tlbcontiguous) { 739 val ppn_equal = pte.entry(i).ppn === pte.entry(OHToUInt(pte.pteidx)).ppn 740 val pbmt_equal = pte.entry(i).pbmt === pte.entry(OHToUInt(pte.pteidx)).pbmt 741 val perm_equal = pte.entry(i).perm.getOrElse(0.U.asTypeOf(new PtePermBundle)).asUInt === pte.entry(OHToUInt(pte.pteidx)).perm.getOrElse(0.U.asTypeOf(new PtePermBundle)).asUInt 742 val v_equal = pte.entry(i).v === pte.entry(OHToUInt(pte.pteidx)).v 743 val af_equal = pte.entry(i).af === pte.entry(OHToUInt(pte.pteidx)).af 744 val pf_equal = pte.entry(i).pf === pte.entry(OHToUInt(pte.pteidx)).pf 745 val cf_equal = if (HasBitmapCheck) pte.entry(i).cf === pte.entry(OHToUInt(pte.pteidx)).cf else true.B 746 ptw_sector_resp.valididx(i) := ((ppn_equal && pbmt_equal && perm_equal && v_equal && af_equal && pf_equal && cf_equal) || !pte.not_super) && !pte.not_merge 747 ptw_sector_resp.ppn_low(i) := pte.entry(i).ppn_low 748 } 749 ptw_sector_resp.valididx(OHToUInt(pte.pteidx)) := true.B 750 ptw_sector_resp 751 } 752 753 def outReady(source: UInt, port: Int): Bool = { 754 MuxLookup(source, true.B)((0 until PtwWidth).map(i => i.U -> mergeArb(i).in(port).ready)) 755 } 756 757 // debug info 758 for (i <- 0 until PtwWidth) { 759 XSDebug(p"[io.tlb(${i.U})] ${io.tlb(i)}\n") 760 } 761 XSDebug(p"[sfence] ${io.sfence}\n") 762 XSDebug(p"[io.csr.tlb] ${io.csr.tlb}\n") 763 764 for (i <- 0 until PtwWidth) { 765 XSPerfAccumulate(s"req_count${i}", io.tlb(i).req(0).fire) 766 XSPerfAccumulate(s"req_blocked_count_${i}", io.tlb(i).req(0).valid && !io.tlb(i).req(0).ready) 767 } 768 XSPerfAccumulate(s"req_blocked_by_mq", arb1.io.out.valid && missQueue.io.out.valid) 769 for (i <- 0 until (MemReqWidth + 1)) { 770 XSPerfAccumulate(s"mem_req_util${i}", PopCount(waiting_resp) === i.U) 771 } 772 XSPerfAccumulate("mem_cycle", PopCount(waiting_resp) =/= 0.U) 773 XSPerfAccumulate("mem_count", mem.a.fire) 774 for (i <- 0 until PtwWidth) { 775 XSPerfAccumulate(s"llptw_ppn_af${i}", mergeArb(i).in(outArbMqPort).valid && mergeArb(i).in(outArbMqPort).bits.s1.entry(OHToUInt(mergeArb(i).in(outArbMqPort).bits.s1.pteidx)).af && !llptw_out.bits.af) 776 XSPerfAccumulate(s"access_fault${i}", io.tlb(i).resp.fire && io.tlb(i).resp.bits.s1.af) 777 } 778 779 // print configs 780 println(s"${l2tlbParams.name}: a ptw, a llptw with size ${l2tlbParams.llptwsize}, miss queue size ${MissQueueSize} l2:${l2tlbParams.l2Size} fa l1: nSets ${l2tlbParams.l1nSets} nWays ${l2tlbParams.l1nWays} l0: ${l2tlbParams.l0nSets} nWays ${l2tlbParams.l0nWays} blockBytes:${l2tlbParams.blockBytes}") 781 782 val perfEvents = Seq(llptw, cache, ptw).flatMap(_.getPerfEvents) 783 generatePerfEvent() 784 785 val isWriteL1TlbTable = Constantin.createRecord(s"isWriteL1TlbTable$hartId") 786 val L1TlbTable = ChiselDB.createTable(s"L1Tlb_hart$hartId", new L1TlbDB) 787 val ITlbReqDB, DTlbReqDB, ITlbRespDB, DTlbRespDB = Wire(new L1TlbDB) 788 ITlbReqDB.vpn := io.tlb(0).req(0).bits.vpn 789 DTlbReqDB.vpn := io.tlb(1).req(0).bits.vpn 790 ITlbRespDB.vpn := Cat(io.tlb(0).resp.bits.s1.entry.tag, OHToUInt(io.tlb(0).resp.bits.s1.pteidx)) 791 DTlbRespDB.vpn := Cat(io.tlb(1).resp.bits.s1.entry.tag, OHToUInt(io.tlb(1).resp.bits.s1.pteidx)) 792 L1TlbTable.log(ITlbReqDB, isWriteL1TlbTable.orR && io.tlb(0).req(0).fire, "ITlbReq", clock, reset) 793 L1TlbTable.log(DTlbReqDB, isWriteL1TlbTable.orR && io.tlb(1).req(0).fire, "DTlbReq", clock, reset) 794 L1TlbTable.log(ITlbRespDB, isWriteL1TlbTable.orR && io.tlb(0).resp.fire, "ITlbResp", clock, reset) 795 L1TlbTable.log(DTlbRespDB, isWriteL1TlbTable.orR && io.tlb(1).resp.fire, "DTlbResp", clock, reset) 796 797 val isWritePageCacheTable = Constantin.createRecord(s"isWritePageCacheTable$hartId") 798 val PageCacheTable = ChiselDB.createTable(s"PageCache_hart$hartId", new PageCacheDB) 799 val PageCacheDB = Wire(new PageCacheDB) 800 PageCacheDB.vpn := Cat(cache.io.resp.bits.stage1.entry(0).tag, OHToUInt(cache.io.resp.bits.stage1.pteidx)) 801 PageCacheDB.source := cache.io.resp.bits.req_info.source 802 PageCacheDB.bypassed := cache.io.resp.bits.bypassed 803 PageCacheDB.is_first := cache.io.resp.bits.isFirst 804 PageCacheDB.prefetched := cache.io.resp.bits.stage1.entry(0).prefetch 805 PageCacheDB.prefetch := cache.io.resp.bits.prefetch 806 PageCacheDB.l2Hit := cache.io.resp.bits.toFsm.l2Hit 807 PageCacheDB.l1Hit := cache.io.resp.bits.toFsm.l1Hit 808 PageCacheDB.hit := cache.io.resp.bits.hit 809 PageCacheTable.log(PageCacheDB, isWritePageCacheTable.orR && cache.io.resp.fire, "PageCache", clock, reset) 810 811 val isWritePTWTable = Constantin.createRecord(s"isWritePTWTable$hartId") 812 val PTWTable = ChiselDB.createTable(s"PTW_hart$hartId", new PTWDB) 813 val PTWReqDB, PTWRespDB, LLPTWReqDB, LLPTWRespDB = Wire(new PTWDB) 814 PTWReqDB.vpn := ptw.io.req.bits.req_info.vpn 815 PTWReqDB.source := ptw.io.req.bits.req_info.source 816 PTWRespDB.vpn := ptw.io.refill.req_info.vpn 817 PTWRespDB.source := ptw.io.refill.req_info.source 818 LLPTWReqDB.vpn := llptw.io.in.bits.req_info.vpn 819 LLPTWReqDB.source := llptw.io.in.bits.req_info.source 820 LLPTWRespDB.vpn := llptw.io.mem.refill.vpn 821 LLPTWRespDB.source := llptw.io.mem.refill.source 822 PTWTable.log(PTWReqDB, isWritePTWTable.orR && ptw.io.req.fire, "PTWReq", clock, reset) 823 PTWTable.log(PTWRespDB, isWritePTWTable.orR && ptw.io.mem.resp.fire, "PTWResp", clock, reset) 824 PTWTable.log(LLPTWReqDB, isWritePTWTable.orR && llptw.io.in.fire, "LLPTWReq", clock, reset) 825 PTWTable.log(LLPTWRespDB, isWritePTWTable.orR && llptw.io.mem.resp.fire, "LLPTWResp", clock, reset) 826 827 val isWriteL2TlbMissQueueTable = Constantin.createRecord(s"isWriteL2TlbMissQueueTable$hartId") 828 val L2TlbMissQueueTable = ChiselDB.createTable(s"L2TlbMissQueue_hart$hartId", new L2TlbMissQueueDB) 829 val L2TlbMissQueueInDB, L2TlbMissQueueOutDB = Wire(new L2TlbMissQueueDB) 830 L2TlbMissQueueInDB.vpn := missQueue.io.in.bits.req_info.vpn 831 L2TlbMissQueueOutDB.vpn := missQueue.io.out.bits.req_info.vpn 832 L2TlbMissQueueTable.log(L2TlbMissQueueInDB, isWriteL2TlbMissQueueTable.orR && missQueue.io.in.fire, "L2TlbMissQueueIn", clock, reset) 833 L2TlbMissQueueTable.log(L2TlbMissQueueOutDB, isWriteL2TlbMissQueueTable.orR && missQueue.io.out.fire, "L2TlbMissQueueOut", clock, reset) 834} 835 836/** BlockHelper, block missqueue, not to send too many req to cache 837 * Parameter: 838 * enable: enable BlockHelper, mq should not send too many reqs 839 * start: when miss queue out fire and need, block miss queue's out 840 * block: block miss queue's out 841 * latency: last missqueue out's cache access latency 842 */ 843class BlockHelper(latency: Int)(implicit p: Parameters) extends XSModule { 844 val io = IO(new Bundle { 845 val enable = Input(Bool()) 846 val start = Input(Bool()) 847 val block = Output(Bool()) 848 }) 849 850 val count = RegInit(0.U(log2Ceil(latency).W)) 851 val valid = RegInit(false.B) 852 val work = RegInit(true.B) 853 854 io.block := valid 855 856 when (io.start && work) { valid := true.B } 857 when (valid) { count := count + 1.U } 858 when (count === (latency.U) || io.enable) { 859 valid := false.B 860 work := io.enable 861 count := 0.U 862 } 863} 864 865class PTEHelper() extends ExtModule { 866 val clock = IO(Input(Clock())) 867 val enable = IO(Input(Bool())) 868 val satp = IO(Input(UInt(64.W))) 869 val vpn = IO(Input(UInt(64.W))) 870 val pte = IO(Output(UInt(64.W))) 871 val level = IO(Output(UInt(8.W))) 872 val pf = IO(Output(UInt(8.W))) 873} 874 875class PTWDelayN[T <: Data](gen: T, n: Int, flush: Bool) extends Module { 876 val io = IO(new Bundle() { 877 val in = Input(gen) 878 val out = Output(gen) 879 val ptwflush = Input(flush.cloneType) 880 }) 881 val out = RegInit(VecInit(Seq.fill(n)(0.U.asTypeOf(gen)))) 882 val t = RegInit(VecInit(Seq.fill(n)(0.U.asTypeOf(gen)))) 883 out(0) := io.in 884 if (n == 1) { 885 io.out := out(0) 886 } else { 887 when (io.ptwflush) { 888 for (i <- 0 until n) { 889 t(i) := 0.U.asTypeOf(gen) 890 out(i) := 0.U.asTypeOf(gen) 891 } 892 io.out := 0.U.asTypeOf(gen) 893 } .otherwise { 894 for (i <- 1 until n) { 895 t(i-1) := out(i-1) 896 out(i) := t(i-1) 897 } 898 io.out := out(n-1) 899 } 900 } 901} 902 903object PTWDelayN { 904 def apply[T <: Data](in: T, n: Int, flush: Bool): T = { 905 val delay = Module(new PTWDelayN(in.cloneType, n, flush)) 906 delay.io.in := in 907 delay.io.ptwflush := flush 908 delay.io.out 909 } 910} 911 912class FakePTW()(implicit p: Parameters) extends XSModule with HasPtwConst { 913 val io = IO(new L2TLBIO) 914 val flush = VecInit(Seq.fill(PtwWidth)(false.B)) 915 flush(0) := DelayN(io.sfence.valid || io.csr.tlb.satp.changed, itlbParams.fenceDelay) 916 flush(1) := DelayN(io.sfence.valid || io.csr.tlb.satp.changed, ldtlbParams.fenceDelay) 917 for (i <- 0 until PtwWidth) { 918 val helper = Module(new PTEHelper()) 919 helper.clock := clock 920 helper.satp := io.csr.tlb.satp.ppn 921 922 if (coreParams.softPTWDelay == 1) { 923 helper.enable := io.tlb(i).req(0).fire 924 helper.vpn := io.tlb(i).req(0).bits.vpn 925 } else { 926 helper.enable := PTWDelayN(io.tlb(i).req(0).fire, coreParams.softPTWDelay - 1, flush(i)) 927 helper.vpn := PTWDelayN(io.tlb(i).req(0).bits.vpn, coreParams.softPTWDelay - 1, flush(i)) 928 } 929 930 val pte = helper.pte.asTypeOf(new PteBundle) 931 val level = helper.level 932 val pf = helper.pf 933 val empty = RegInit(true.B) 934 when (io.tlb(i).req(0).fire) { 935 empty := false.B 936 } .elsewhen (io.tlb(i).resp.fire || flush(i)) { 937 empty := true.B 938 } 939 940 io.tlb(i).req(0).ready := empty || io.tlb(i).resp.fire 941 io.tlb(i).resp.valid := PTWDelayN(io.tlb(i).req(0).fire, coreParams.softPTWDelay, flush(i)) 942 assert(!io.tlb(i).resp.valid || io.tlb(i).resp.ready) 943 io.tlb(i).resp.bits.s1.entry.tag := PTWDelayN(io.tlb(i).req(0).bits.vpn, coreParams.softPTWDelay, flush(i)) 944 io.tlb(i).resp.bits.s1.entry.pbmt := pte.pbmt 945 io.tlb(i).resp.bits.s1.entry.ppn := pte.ppn 946 io.tlb(i).resp.bits.s1.entry.perm.map(_ := pte.getPerm()) 947 io.tlb(i).resp.bits.s1.entry.level.map(_ := level) 948 io.tlb(i).resp.bits.s1.pf := pf 949 io.tlb(i).resp.bits.s1.af := DontCare // TODO: implement it 950 io.tlb(i).resp.bits.s1.entry.v := !pf 951 io.tlb(i).resp.bits.s1.entry.prefetch := DontCare 952 io.tlb(i).resp.bits.s1.entry.asid := io.csr.tlb.satp.asid 953 } 954} 955 956class L2TLBWrapper()(implicit p: Parameters) extends LazyModule with HasXSParameter { 957 override def shouldBeInlined: Boolean = false 958 val useSoftPTW = coreParams.softPTW 959 val node = if (!useSoftPTW) TLIdentityNode() else null 960 val ptw = if (!useSoftPTW) LazyModule(new L2TLB()) else null 961 if (!useSoftPTW) { 962 node := ptw.node 963 } 964 965 class L2TLBWrapperImp(wrapper: LazyModule) extends LazyModuleImp(wrapper) with HasPerfEvents { 966 val io = IO(new L2TLBIO) 967 val perfEvents = if (useSoftPTW) { 968 val fake_ptw = Module(new FakePTW()) 969 io <> fake_ptw.io 970 Seq() 971 } 972 else { 973 io <> ptw.module.io 974 ptw.module.getPerfEvents 975 } 976 generatePerfEvent() 977 } 978 979 lazy val module = new L2TLBWrapperImp(this) 980} 981