1/*************************************************************************************** 2* Copyright (c) 2021-2025 Beijing Institute of Open Source Chip (BOSC) 3* Copyright (c) 2020-2024 Institute of Computing Technology, Chinese Academy of Sciences 4* Copyright (c) 2020-2021 Peng Cheng Laboratory 5* Copyright (c) 2024-2025 Institute of Information Engineering, Chinese Academy of Sciences 6* 7* XiangShan is licensed under Mulan PSL v2. 8* You can use this software according to the terms and conditions of the Mulan PSL v2. 9* You may obtain a copy of Mulan PSL v2 at: 10* http://license.coscl.org.cn/MulanPSL2 11* 12* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 13* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 14* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 15* 16* See the Mulan PSL v2 for more details. 17***************************************************************************************/ 18 19package xiangshan.cache.mmu 20 21import org.chipsalliance.cde.config.Parameters 22import chisel3._ 23import chisel3.util._ 24import xiangshan._ 25import xiangshan.cache.{HasDCacheParameters, MemoryOpConstants} 26import utils._ 27import utility._ 28import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp} 29import freechips.rocketchip.tilelink._ 30import xiangshan.backend.fu.{PMPReqBundle, PMPRespBundle} 31 32/** Page Table Walk is divided into two parts 33 * One, PTW: page walk for pde, except for leaf entries, one by one 34 * Two, LLPTW: page walk for pte, only the leaf entries(4KB), in parallel 35 */ 36 37 38/** PTW : page table walker 39 * a finite state machine 40 * only take 1GB and 2MB page walks 41 * or in other words, except the last level(leaf) 42 **/ 43class PTWIO()(implicit p: Parameters) extends MMUIOBaseBundle with HasPtwConst { 44 val req = Flipped(DecoupledIO(new Bundle { 45 val req_info = new L2TlbInnerBundle() 46 val l3Hit = if (EnableSv48) Some(new Bool()) else None 47 val l2Hit = Bool() 48 val ppn = UInt(ptePPNLen.W) 49 val stage1Hit = Bool() 50 val stage1 = new PtwMergeResp 51 val bitmapCheck = Option.when(HasBitmapCheck)(new Bundle { 52 val jmp_bitmap_check = Bool() // super page in PtwCache ptw hit, but need bitmap check 53 val pte = UInt(XLEN.W) // Page Table Entry 54 val cfs = Vec(tlbcontiguous, Bool()) // Bitmap Check Failed Vector 55 val SPlevel = UInt(log2Up(Level).W) 56 }) 57 })) 58 val resp = DecoupledIO(new Bundle { 59 val source = UInt(bSourceWidth.W) 60 val s2xlate = UInt(2.W) 61 val resp = new PtwMergeResp 62 val h_resp = new HptwResp 63 }) 64 65 val llptw = DecoupledIO(new LLPTWInBundle()) 66 // NOTE: llptw change from "connect to llptw" to "connect to page cache" 67 // to avoid corner case that caused duplicate entries 68 69 val hptw = new Bundle { 70 val req = DecoupledIO(new Bundle { 71 val source = UInt(bSourceWidth.W) 72 val id = UInt(log2Up(l2tlbParams.llptwsize).W) 73 val gvpn = UInt(ptePPNLen.W) 74 }) 75 val resp = Flipped(Valid(new Bundle { 76 val h_resp = Output(new HptwResp) 77 })) 78 } 79 val mem = new Bundle { 80 val req = DecoupledIO(new L2TlbMemReqBundle()) 81 val resp = Flipped(ValidIO(UInt(XLEN.W))) 82 val mask = Input(Bool()) 83 } 84 val pmp = new Bundle { 85 val req = ValidIO(new PMPReqBundle()) 86 val resp = Flipped(new PMPRespBundle()) 87 } 88 89 val refill = Output(new Bundle { 90 val req_info = new L2TlbInnerBundle() 91 val level = UInt(log2Up(Level + 1).W) 92 }) 93 val bitmap = Option.when(HasBitmapCheck)(new Bundle { 94 val req = DecoupledIO(new bitmapReqBundle()) 95 val resp = Flipped(DecoupledIO(new bitmapRespBundle())) 96 }) 97} 98 99class PTW()(implicit p: Parameters) extends XSModule with HasPtwConst with HasPerfEvents { 100 val io = IO(new PTWIO) 101 val sfence = io.sfence 102 val mem = io.mem 103 val req_s2xlate = Reg(UInt(2.W)) 104 val enableS2xlate = req_s2xlate =/= noS2xlate 105 val onlyS1xlate = req_s2xlate === onlyStage1 106 val onlyS2xlate = req_s2xlate === onlyStage2 107 108 // mbmc:bitmap csr 109 val mbmc = io.csr.mbmc 110 val bitmap_enable = (if (HasBitmapCheck) true.B else false.B) && mbmc.BME === 1.U && mbmc.CMODE === 0.U 111 112 val satp = Wire(new TlbSatpBundle()) 113 when (io.req.fire) { 114 satp := Mux(io.req.bits.req_info.s2xlate =/= noS2xlate, io.csr.vsatp, io.csr.satp) 115 } .otherwise { 116 satp := Mux(enableS2xlate, io.csr.vsatp, io.csr.satp) 117 } 118 val s1Pbmte = Mux(req_s2xlate =/= noS2xlate, io.csr.hPBMTE, io.csr.mPBMTE) 119 120 val mode = satp.mode 121 val hgatp = io.csr.hgatp 122 val flush = io.sfence.valid || io.csr.satp.changed || io.csr.vsatp.changed || io.csr.hgatp.changed 123 val s2xlate = enableS2xlate && !onlyS1xlate 124 val level = RegInit(3.U(log2Up(Level + 1).W)) 125 val af_level = RegInit(3.U(log2Up(Level + 1).W)) // access fault return this level 126 val gpf_level = RegInit(3.U(log2Up(Level + 1).W)) 127 val ppn = Reg(UInt(ptePPNLen.W)) 128 val vpn = Reg(UInt(vpnLen.W)) // vpn or gvpn(onlyS2xlate) 129 val levelNext = level - 1.U 130 val l3Hit = Reg(Bool()) 131 val l2Hit = Reg(Bool()) 132 val jmp_bitmap_check_w = if (HasBitmapCheck) { io.req.bits.bitmapCheck.get.jmp_bitmap_check && io.req.bits.req_info.s2xlate =/= onlyStage2 } else { false.B } 133 val jmp_bitmap_check_r = if (HasBitmapCheck) { RegEnable(jmp_bitmap_check_w, io.req.fire) } else { false.B } 134 val cache_pte = Option.when(HasBitmapCheck)(RegEnable(io.req.bits.bitmapCheck.get.pte.asTypeOf(new PteBundle().cloneType), io.req.fire)) 135 val pte = if (HasBitmapCheck) { Mux(jmp_bitmap_check_r, cache_pte.get, io.mem.resp.bits.asTypeOf(new PteBundle().cloneType)) } else { mem.resp.bits.asTypeOf(new PteBundle()) } 136 137 // s/w register 138 val s_pmp_check = RegInit(true.B) 139 val s_mem_req = RegInit(true.B) 140 val s_llptw_req = RegInit(true.B) 141 val w_mem_resp = RegInit(true.B) 142 val s_hptw_req = RegInit(true.B) 143 val w_hptw_resp = RegInit(true.B) 144 val s_last_hptw_req = RegInit(true.B) 145 val w_last_hptw_resp = RegInit(true.B) 146 // for updating "level" 147 val mem_addr_update = RegInit(false.B) 148 149 val s_bitmap_check = RegInit(true.B) 150 val w_bitmap_resp = RegInit(true.B) 151 val whether_need_bitmap_check = RegInit(false.B) 152 val bitmap_checkfailed = RegInit(false.B) 153 154 val idle = RegInit(true.B) 155 val finish = WireInit(false.B) 156 dontTouch(finish) 157 val vs_finish = WireInit(false.B) // need to wait for G-stage translate, should not do pmp check 158 dontTouch(vs_finish) 159 160 val hptw_pageFault = RegInit(false.B) 161 val hptw_accessFault = RegInit(false.B) 162 val need_last_s2xlate = RegInit(false.B) 163 val stage1Hit = RegEnable(io.req.bits.stage1Hit, io.req.fire) 164 val stage1 = RegEnable(io.req.bits.stage1, io.req.fire) 165 val hptw_resp_stage2 = Reg(Bool()) 166 val first_gvpn_check_fail = RegInit(false.B) 167 168 // use accessfault repersent bitmap check failed 169 val pte_isAf = Mux(bitmap_enable, pte.isAf() || bitmap_checkfailed, pte.isAf()) 170 val ppn_af = if (HasBitmapCheck) { 171 Mux(enableS2xlate, Mux(onlyS1xlate, pte_isAf, false.B), pte_isAf) // In two-stage address translation, stage 1 ppn is a vpn for host, so don't need to check ppn_high 172 } else { 173 Mux(enableS2xlate, Mux(onlyS1xlate, pte.isAf(), false.B), pte.isAf()) // In two-stage address translation, stage 1 ppn is a vpn for host, so don't need to check ppn_high 174 } 175 val pte_valid = RegInit(false.B) // avoid l1tlb pf from stage1 when gpf happens in the first s2xlate in PTW 176 177 val pageFault = pte.isPf(level, s1Pbmte) 178 val find_pte = pte.isLeaf() || ppn_af || pageFault 179 val to_find_pte = level === 1.U && find_pte === false.B 180 val source = RegEnable(io.req.bits.req_info.source, io.req.fire) 181 182 val sent_to_pmp = idle === false.B && (s_pmp_check === false.B || mem_addr_update) && !finish && !vs_finish && !first_gvpn_check_fail && !(find_pte && pte_valid) 183 val accessFault = RegEnable(io.pmp.resp.ld || io.pmp.resp.mmio, false.B, sent_to_pmp) 184 185 val l3addr = Wire(UInt(ptePaddrLen.W)) 186 val l2addr = Wire(UInt(ptePaddrLen.W)) 187 val l1addr = Wire(UInt(ptePaddrLen.W)) 188 val hptw_addr = Wire(UInt(ptePaddrLen.W)) 189 val mem_addr = Wire(UInt(PAddrBits.W)) 190 191 l3addr := MakeAddr(satp.ppn, getVpnn(vpn, 3)) 192 if (EnableSv48) { 193 when (mode === Sv48) { 194 l2addr := MakeAddr(Mux(l3Hit, ppn, pte.getPPN()), getVpnn(vpn, 2)) 195 } .otherwise { 196 l2addr := MakeAddr(satp.ppn, getVpnn(vpn, 2)) 197 } 198 } else { 199 l2addr := MakeAddr(satp.ppn, getVpnn(vpn, 2)) 200 } 201 l1addr := MakeAddr(Mux(l2Hit, ppn, pte.getPPN()), getVpnn(vpn, 1)) 202 hptw_addr := Mux(af_level === 3.U, l3addr, Mux(af_level === 2.U, l2addr, l1addr)) 203 mem_addr := hptw_addr(PAddrBits - 1, 0) 204 205 val hptw_resp = Reg(new HptwResp) 206 207 val update_full_gvpn_mem_resp = RegInit(false.B) 208 val full_gvpn_reg = Reg(UInt(ptePPNLen.W)) 209 val full_gvpn_wire = pte.getPPN() 210 val full_gvpn = Mux(update_full_gvpn_mem_resp, full_gvpn_wire, full_gvpn_reg) 211 212 val gpaddr = MuxCase(hptw_addr, Seq( 213 (stage1Hit || onlyS2xlate) -> Cat(full_gvpn, 0.U(offLen.W)), 214 !s_last_hptw_req -> Cat(MuxLookup(level, pte.getPPN())(Seq( 215 3.U -> Cat(pte.getPPN()(ptePPNLen - 1, vpnnLen * 3), vpn(vpnnLen * 3 - 1, 0)), 216 2.U -> Cat(pte.getPPN()(ptePPNLen - 1, vpnnLen * 2), vpn(vpnnLen * 2 - 1, 0)), 217 1.U -> Cat(pte.getPPN()(ptePPNLen - 1, vpnnLen), vpn(vpnnLen - 1, 0) 218 ))), 219 0.U(offLen.W)) 220 )) 221 val gvpn_gpf = 222 (!(hptw_pageFault || hptw_accessFault || ((pageFault || ppn_af) && pte_valid)) && 223 Mux( 224 s2xlate && io.csr.hgatp.mode === Sv39x4, 225 full_gvpn(ptePPNLen - 1, GPAddrBitsSv39x4 - offLen) =/= 0.U, 226 Mux( 227 s2xlate && io.csr.hgatp.mode === Sv48x4, 228 full_gvpn(ptePPNLen - 1, GPAddrBitsSv48x4 - offLen) =/= 0.U, 229 false.B 230 ) 231 )) || first_gvpn_check_fail 232 233 val guestFault = hptw_pageFault || hptw_accessFault || gvpn_gpf 234 val hpaddr = Cat(hptw_resp.genPPNS2(get_pn(gpaddr)), get_off(gpaddr)) 235 val fake_h_resp = WireInit(0.U.asTypeOf(new HptwResp)) 236 fake_h_resp.entry.tag := get_pn(gpaddr) 237 fake_h_resp.entry.vmid.map(_ := io.csr.hgatp.vmid) 238 fake_h_resp.gpf := true.B 239 240 val fake_pte = WireInit(0.U.asTypeOf(new PteBundle())) 241 fake_pte.perm.v := false.B // tell L1TLB this is fake pte 242 fake_pte.ppn := ppn(ppnLen - 1, 0) 243 fake_pte.ppn_high := ppn(ptePPNLen - 1, ppnLen) 244 245 io.req.ready := idle 246 val ptw_resp = Wire(new PtwMergeResp) 247 // pageFault is always valid when pte_valid 248 val resp_pf = pte_valid && pageFault 249 // when (pte_valid && (pageFault || guestFault), should not report accessFault or ppn_af 250 val resp_af = (accessFault || ppn_af) && !((pte_valid && pageFault) || guestFault) 251 // should use af_level when accessFault && !((pte_valid && pageFault) || guestFault) 252 val resp_level = Mux(accessFault && resp_af, af_level, Mux(guestFault, gpf_level, level)) 253 // when ptw do not really send a memory request, should use fake_pte 254 val resp_pte = Mux(pte_valid, pte, fake_pte) 255 ptw_resp.apply(resp_pf, resp_af, resp_level, resp_pte, vpn, satp.asid, hgatp.vmid, vpn(sectortlbwidth - 1, 0), not_super = false, not_merge = false, bitmap_checkfailed.asBool) 256 257 val normal_resp = idle === false.B && mem_addr_update && !need_last_s2xlate && (guestFault || (w_mem_resp && find_pte) || (s_pmp_check && accessFault) || onlyS2xlate ) 258 val stageHit_resp = idle === false.B && hptw_resp_stage2 259 io.resp.valid := Mux(stage1Hit, stageHit_resp, normal_resp) 260 io.resp.bits.source := source 261 io.resp.bits.resp := Mux(stage1Hit || (l3Hit || l2Hit) && guestFault && !pte_valid, stage1, ptw_resp) 262 io.resp.bits.h_resp := Mux(gvpn_gpf, fake_h_resp, hptw_resp) 263 io.resp.bits.s2xlate := req_s2xlate 264 265 io.llptw.valid := s_llptw_req === false.B && to_find_pte && !accessFault && !guestFault 266 io.llptw.bits.req_info.source := source 267 io.llptw.bits.req_info.vpn := vpn 268 io.llptw.bits.req_info.s2xlate := req_s2xlate 269 io.llptw.bits.ppn := DontCare 270 if (HasBitmapCheck) { 271 io.llptw.bits.bitmapCheck.get.jmp_bitmap_check := DontCare 272 io.llptw.bits.bitmapCheck.get.ptes := DontCare 273 io.llptw.bits.bitmapCheck.get.cfs := DontCare 274 io.llptw.bits.bitmapCheck.get.hitway := DontCare 275 } 276 277 io.pmp.req.valid := DontCare // samecycle, do not use valid 278 io.pmp.req.bits.addr := Mux(s2xlate, hpaddr, mem_addr) 279 io.pmp.req.bits.size := 3.U // TODO: fix it 280 io.pmp.req.bits.cmd := TlbCmd.read 281 282 if (HasBitmapCheck) { 283 val cache_level = RegEnable(io.req.bits.bitmapCheck.get.SPlevel, io.req.fire) 284 io.bitmap.get.req.valid := !s_bitmap_check 285 io.bitmap.get.req.bits.bmppn := pte.ppn 286 io.bitmap.get.req.bits.id := FsmReqID.U(bMemID.W) 287 io.bitmap.get.req.bits.vpn := vpn 288 io.bitmap.get.req.bits.level := Mux(jmp_bitmap_check_r, cache_level, level) 289 io.bitmap.get.req.bits.way_info := DontCare 290 io.bitmap.get.req.bits.hptw_bypassed := false.B 291 io.bitmap.get.resp.ready := !w_bitmap_resp 292 } 293 mem.req.valid := s_mem_req === false.B && !mem.mask && !accessFault && s_pmp_check 294 mem.req.bits.addr := Mux(s2xlate, hpaddr, mem_addr) 295 mem.req.bits.id := FsmReqID.U(bMemID.W) 296 mem.req.bits.hptw_bypassed := false.B 297 298 io.refill.req_info.s2xlate := req_s2xlate 299 io.refill.req_info.vpn := vpn 300 io.refill.level := level 301 io.refill.req_info.source := source 302 303 io.hptw.req.valid := !s_hptw_req || !s_last_hptw_req 304 io.hptw.req.bits.id := FsmReqID.U(bMemID.W) 305 io.hptw.req.bits.gvpn := get_pn(gpaddr) 306 io.hptw.req.bits.source := source 307 308 if (HasBitmapCheck) { 309 when (io.req.fire && jmp_bitmap_check_w) { 310 idle := false.B 311 req_s2xlate := io.req.bits.req_info.s2xlate 312 vpn := io.req.bits.req_info.vpn 313 s_bitmap_check := false.B 314 need_last_s2xlate := false.B 315 hptw_pageFault := false.B 316 hptw_accessFault := false.B 317 level := io.req.bits.bitmapCheck.get.SPlevel 318 pte_valid := true.B 319 accessFault := false.B 320 } 321 } 322 323 when (io.req.fire && io.req.bits.stage1Hit && (if (HasBitmapCheck) !jmp_bitmap_check_w else true.B)) { 324 idle := false.B 325 req_s2xlate := io.req.bits.req_info.s2xlate 326 s_last_hptw_req := false.B 327 hptw_resp_stage2 := false.B 328 need_last_s2xlate := false.B 329 hptw_pageFault := false.B 330 hptw_accessFault := false.B 331 full_gvpn_reg := io.req.bits.stage1.genPPN() 332 } 333 334 when (io.resp.fire && stage1Hit){ 335 idle := true.B 336 } 337 338 when (io.req.fire && !io.req.bits.stage1Hit && (if (HasBitmapCheck) !jmp_bitmap_check_w else true.B)) { 339 val req = io.req.bits 340 val gvpn_wire = Wire(UInt(ptePPNLen.W)) 341 if (EnableSv48) { 342 when (mode === Sv48) { 343 level := Mux(req.l2Hit, 1.U, Mux(req.l3Hit.get, 2.U, 3.U)) 344 af_level := Mux(req.l2Hit, 1.U, Mux(req.l3Hit.get, 2.U, 3.U)) 345 gpf_level := Mux(req.l2Hit, 2.U, Mux(req.l3Hit.get, 3.U, 0.U)) 346 ppn := Mux(req.l2Hit || req.l3Hit.get, io.req.bits.ppn, satp.ppn) 347 l3Hit := req.l3Hit.get 348 gvpn_wire := Mux(req.l2Hit || req.l3Hit.get, io.req.bits.ppn, satp.ppn) 349 } .otherwise { 350 level := Mux(req.l2Hit, 1.U, 2.U) 351 af_level := Mux(req.l2Hit, 1.U, 2.U) 352 gpf_level := Mux(req.l2Hit, 2.U, 0.U) 353 ppn := Mux(req.l2Hit, io.req.bits.ppn, satp.ppn) 354 l3Hit := false.B 355 gvpn_wire := Mux(req.l2Hit, io.req.bits.ppn, satp.ppn) 356 } 357 } else { 358 level := Mux(req.l2Hit, 1.U, 2.U) 359 af_level := Mux(req.l2Hit, 1.U, 2.U) 360 gpf_level := Mux(req.l2Hit, 2.U, 0.U) 361 ppn := Mux(req.l2Hit, io.req.bits.ppn, satp.ppn) 362 l3Hit := false.B 363 gvpn_wire := Mux(req.l2Hit, io.req.bits.ppn, satp.ppn) 364 } 365 vpn := io.req.bits.req_info.vpn 366 l2Hit := req.l2Hit 367 accessFault := false.B 368 idle := false.B 369 hptw_pageFault := false.B 370 hptw_accessFault := false.B 371 pte_valid := false.B 372 req_s2xlate := io.req.bits.req_info.s2xlate 373 when(io.req.bits.req_info.s2xlate === onlyStage2){ 374 full_gvpn_reg := io.req.bits.req_info.vpn 375 val onlys2_gpaddr = Cat(io.req.bits.req_info.vpn, 0.U(offLen.W)) // is 50 bits, don't need to check high bits when sv48x4 is enabled 376 val check_gpa_high_fail = Mux(io.req.bits.req_info.s2xlate === onlyStage2 && io.csr.hgatp.mode === Sv39x4, onlys2_gpaddr(onlys2_gpaddr.getWidth - 1, GPAddrBitsSv39x4) =/= 0.U, false.B) 377 need_last_s2xlate := false.B 378 when(check_gpa_high_fail){ 379 mem_addr_update := true.B 380 first_gvpn_check_fail := true.B 381 }.otherwise{ 382 s_last_hptw_req := false.B 383 } 384 }.elsewhen(io.req.bits.req_info.s2xlate === allStage){ 385 full_gvpn_reg := 0.U 386 val allstage_gpaddr = Cat(gvpn_wire, 0.U(offLen.W)) 387 val check_gpa_high_fail = Mux(io.csr.hgatp.mode === Sv39x4, allstage_gpaddr(allstage_gpaddr.getWidth - 1, GPAddrBitsSv39x4) =/= 0.U, Mux(io.csr.hgatp.mode === Sv48x4, allstage_gpaddr(allstage_gpaddr.getWidth - 1, GPAddrBitsSv48x4) =/= 0.U, false.B)) 388 when(check_gpa_high_fail){ 389 mem_addr_update := true.B 390 first_gvpn_check_fail := true.B 391 }.otherwise{ 392 need_last_s2xlate := true.B 393 s_hptw_req := false.B 394 } 395 }.otherwise { 396 full_gvpn_reg := 0.U 397 need_last_s2xlate := false.B 398 s_pmp_check := false.B 399 } 400 } 401 402 when(io.hptw.req.fire && s_hptw_req === false.B){ 403 s_hptw_req := true.B 404 w_hptw_resp := false.B 405 } 406 407 when(io.hptw.resp.fire && w_hptw_resp === false.B) { 408 w_hptw_resp := true.B 409 val g_perm_fail = !io.hptw.resp.bits.h_resp.gaf && (!io.hptw.resp.bits.h_resp.entry.perm.get.r && !(io.csr.priv.mxr && io.hptw.resp.bits.h_resp.entry.perm.get.x)) 410 hptw_pageFault := io.hptw.resp.bits.h_resp.gpf || g_perm_fail 411 hptw_accessFault := io.hptw.resp.bits.h_resp.gaf 412 hptw_resp := io.hptw.resp.bits.h_resp 413 hptw_resp.gpf := io.hptw.resp.bits.h_resp.gpf || g_perm_fail 414 when(!(g_perm_fail || io.hptw.resp.bits.h_resp.gpf || io.hptw.resp.bits.h_resp.gaf)) { 415 s_pmp_check := false.B 416 }.otherwise { 417 mem_addr_update := true.B 418 need_last_s2xlate := false.B 419 } 420 } 421 422 when(io.hptw.req.fire && s_last_hptw_req === false.B) { 423 w_last_hptw_resp := false.B 424 s_last_hptw_req := true.B 425 } 426 427 when (io.hptw.resp.fire && w_last_hptw_resp === false.B && stage1Hit){ 428 w_last_hptw_resp := true.B 429 hptw_resp_stage2 := true.B 430 hptw_resp := io.hptw.resp.bits.h_resp 431 } 432 433 when(io.hptw.resp.fire && w_last_hptw_resp === false.B && !stage1Hit){ 434 hptw_pageFault := io.hptw.resp.bits.h_resp.gpf 435 hptw_accessFault := io.hptw.resp.bits.h_resp.gaf 436 hptw_resp := io.hptw.resp.bits.h_resp 437 w_last_hptw_resp := true.B 438 mem_addr_update := true.B 439 } 440 441 when(sent_to_pmp && mem_addr_update === false.B){ 442 s_mem_req := false.B 443 s_pmp_check := true.B 444 } 445 446 when(accessFault && idle === false.B){ 447 s_pmp_check := true.B 448 s_mem_req := true.B 449 w_mem_resp := true.B 450 s_llptw_req := true.B 451 s_hptw_req := true.B 452 w_hptw_resp := true.B 453 s_last_hptw_req := true.B 454 w_last_hptw_resp := true.B 455 mem_addr_update := true.B 456 need_last_s2xlate := false.B 457 if (HasBitmapCheck) { 458 s_bitmap_check := true.B 459 w_bitmap_resp := true.B 460 whether_need_bitmap_check := false.B 461 bitmap_checkfailed := false.B 462 } 463 } 464 465 when(guestFault && idle === false.B){ 466 s_pmp_check := true.B 467 s_mem_req := true.B 468 w_mem_resp := true.B 469 s_llptw_req := true.B 470 s_hptw_req := true.B 471 w_hptw_resp := true.B 472 s_last_hptw_req := true.B 473 w_last_hptw_resp := true.B 474 mem_addr_update := true.B 475 need_last_s2xlate := false.B 476 if (HasBitmapCheck) { 477 s_bitmap_check := true.B 478 w_bitmap_resp := true.B 479 whether_need_bitmap_check := false.B 480 bitmap_checkfailed := false.B 481 } 482 } 483 484 when (mem.req.fire){ 485 s_mem_req := true.B 486 w_mem_resp := false.B 487 } 488 489 when(mem.resp.fire && w_mem_resp === false.B){ 490 w_mem_resp := true.B 491 af_level := af_level - 1.U 492 gpf_level := Mux(mode === Sv39 && !pte_valid && !l2Hit, gpf_level - 2.U, gpf_level - 1.U) 493 pte_valid := true.B 494 update_full_gvpn_mem_resp := true.B 495 if (HasBitmapCheck) { 496 when (bitmap_enable) { 497 whether_need_bitmap_check := true.B 498 } .otherwise { 499 s_llptw_req := false.B 500 mem_addr_update := true.B 501 whether_need_bitmap_check := false.B 502 } 503 } else { 504 s_llptw_req := false.B 505 mem_addr_update := true.B 506 } 507 } 508 509 when(update_full_gvpn_mem_resp) { 510 update_full_gvpn_mem_resp := false.B 511 full_gvpn_reg := pte.getPPN() 512 } 513 514 if (HasBitmapCheck) { 515 when (whether_need_bitmap_check) { 516 when (bitmap_enable && (!enableS2xlate || onlyS1xlate) && pte.isLeaf()) { 517 s_bitmap_check := false.B 518 whether_need_bitmap_check := false.B 519 } .otherwise { 520 mem_addr_update := true.B 521 s_llptw_req := false.B 522 whether_need_bitmap_check := false.B 523 } 524 } 525 // bitmapcheck 526 when (io.bitmap.get.req.fire) { 527 s_bitmap_check := true.B 528 w_bitmap_resp := false.B 529 } 530 when (io.bitmap.get.resp.fire) { 531 w_bitmap_resp := true.B 532 mem_addr_update := true.B 533 bitmap_checkfailed := io.bitmap.get.resp.bits.cf 534 } 535 } 536 537 when(mem_addr_update){ 538 when(level >= 2.U && !onlyS2xlate && !(guestFault || find_pte || accessFault)) { 539 level := levelNext 540 when(s2xlate){ 541 s_hptw_req := false.B 542 vs_finish := true.B 543 }.otherwise{ 544 s_mem_req := false.B 545 } 546 s_llptw_req := true.B 547 mem_addr_update := false.B 548 }.elsewhen(io.llptw.valid){ 549 when(io.llptw.fire) { 550 idle := true.B 551 s_llptw_req := true.B 552 mem_addr_update := false.B 553 need_last_s2xlate := false.B 554 } 555 finish := true.B 556 }.elsewhen(s2xlate && need_last_s2xlate === true.B) { 557 need_last_s2xlate := false.B 558 when(!(guestFault || accessFault || pageFault || ppn_af)){ 559 s_last_hptw_req := false.B 560 mem_addr_update := false.B 561 } 562 }.elsewhen(io.resp.valid){ 563 when(io.resp.fire) { 564 idle := true.B 565 s_llptw_req := true.B 566 mem_addr_update := false.B 567 accessFault := false.B 568 first_gvpn_check_fail := false.B 569 } 570 finish := true.B 571 } 572 } 573 574 575 when (flush) { 576 idle := true.B 577 s_pmp_check := true.B 578 s_mem_req := true.B 579 s_llptw_req := true.B 580 w_mem_resp := true.B 581 accessFault := false.B 582 mem_addr_update := false.B 583 first_gvpn_check_fail := false.B 584 s_hptw_req := true.B 585 w_hptw_resp := true.B 586 s_last_hptw_req := true.B 587 w_last_hptw_resp := true.B 588 if (HasBitmapCheck) { 589 s_bitmap_check := true.B 590 w_bitmap_resp := true.B 591 whether_need_bitmap_check := false.B 592 bitmap_checkfailed := false.B 593 } 594 } 595 596 597 XSDebug(p"[ptw] level:${level} notFound:${pageFault}\n") 598 599 // perf 600 XSPerfAccumulate("fsm_count", io.req.fire) 601 for (i <- 0 until PtwWidth) { 602 XSPerfAccumulate(s"fsm_count_source${i}", io.req.fire && io.req.bits.req_info.source === i.U) 603 } 604 XSPerfAccumulate("fsm_busy", !idle) 605 XSPerfAccumulate("fsm_idle", idle) 606 XSPerfAccumulate("resp_blocked", io.resp.valid && !io.resp.ready) 607 XSPerfAccumulate("ptw_ppn_af", io.resp.fire && ppn_af) 608 XSPerfAccumulate("mem_count", mem.req.fire) 609 XSPerfAccumulate("mem_cycle", BoolStopWatch(mem.req.fire, mem.resp.fire, true)) 610 XSPerfAccumulate("mem_blocked", mem.req.valid && !mem.req.ready) 611 612 val perfEvents = Seq( 613 ("fsm_count ", io.req.fire ), 614 ("fsm_busy ", !idle ), 615 ("fsm_idle ", idle ), 616 ("resp_blocked ", io.resp.valid && !io.resp.ready ), 617 ("mem_count ", mem.req.fire ), 618 ("mem_cycle ", BoolStopWatch(mem.req.fire, mem.resp.fire, true)), 619 ("mem_blocked ", mem.req.valid && !mem.req.ready ), 620 ) 621 generatePerfEvent() 622} 623 624/*========================= LLPTW ==============================*/ 625 626/** LLPTW : Last Level Page Table Walker 627 * the page walker that only takes 4KB(last level) page walk. 628 **/ 629 630class LLPTWInBundle(implicit p: Parameters) extends XSBundle with HasPtwConst { 631 val req_info = Output(new L2TlbInnerBundle()) 632 val ppn = Output(UInt(ptePPNLen.W)) 633 val bitmapCheck = Option.when(HasBitmapCheck)(new Bundle { 634 val jmp_bitmap_check = Bool() // find pte in l0 or sp, but need bitmap check 635 val ptes = Vec(tlbcontiguous, UInt(XLEN.W)) // Page Table Entry Vector 636 val cfs = Vec(tlbcontiguous, Bool()) // Bitmap Check Failed Vector 637 val hitway = UInt(l2tlbParams.l0nWays.W) 638 }) 639} 640 641class LLPTWIO(implicit p: Parameters) extends MMUIOBaseBundle with HasPtwConst { 642 val in = Flipped(DecoupledIO(new LLPTWInBundle())) 643 val out = DecoupledIO(new Bundle { 644 val req_info = Output(new L2TlbInnerBundle()) 645 val id = Output(UInt(bMemID.W)) 646 val h_resp = Output(new HptwResp) 647 val first_s2xlate_fault = Output(Bool()) // Whether the first stage 2 translation occurs pf/af 648 val af = Output(Bool()) 649 val bitmapCheck = Option.when(HasBitmapCheck)(new Bundle { 650 val jmp_bitmap_check = Bool() // find pte in l0 or sp, but need bitmap check 651 val ptes = Vec(tlbcontiguous, UInt(XLEN.W)) // Page Table Entry Vector 652 val cfs = Vec(tlbcontiguous, Bool()) // Bitmap Check Failed Vector 653 }) 654 }) 655 val mem = new Bundle { 656 val req = DecoupledIO(new L2TlbMemReqBundle()) 657 val resp = Flipped(Valid(new Bundle { 658 val id = Output(UInt(log2Up(l2tlbParams.llptwsize).W)) 659 val value = Output(UInt(blockBits.W)) 660 })) 661 val enq_ptr = Output(UInt(log2Ceil(l2tlbParams.llptwsize).W)) 662 val buffer_it = Output(Vec(l2tlbParams.llptwsize, Bool())) 663 val refill = Output(new L2TlbInnerBundle()) 664 val req_mask = Input(Vec(l2tlbParams.llptwsize, Bool())) 665 val flush_latch = Input(Vec(l2tlbParams.llptwsize, Bool())) 666 } 667 val cache = DecoupledIO(new L2TlbInnerBundle()) 668 val pmp = new Bundle { 669 val req = Valid(new PMPReqBundle()) 670 val resp = Flipped(new PMPRespBundle()) 671 } 672 val hptw = new Bundle { 673 val req = DecoupledIO(new Bundle{ 674 val source = UInt(bSourceWidth.W) 675 val id = UInt(log2Up(l2tlbParams.llptwsize).W) 676 val gvpn = UInt(ptePPNLen.W) 677 }) 678 val resp = Flipped(Valid(new Bundle { 679 val id = Output(UInt(log2Up(l2tlbParams.llptwsize).W)) 680 val h_resp = Output(new HptwResp) 681 })) 682 } 683 val bitmap = Option.when(HasBitmapCheck)(new Bundle { 684 val req = DecoupledIO(new bitmapReqBundle()) 685 val resp = Flipped(DecoupledIO(new bitmapRespBundle())) 686 }) 687 688 val l0_way_info = Option.when(HasBitmapCheck)(Input(UInt(l2tlbParams.l0nWays.W))) 689} 690 691class LLPTWEntry(implicit p: Parameters) extends XSBundle with HasPtwConst { 692 val req_info = new L2TlbInnerBundle() 693 val ppn = UInt(ptePPNLen.W) 694 val wait_id = UInt(log2Up(l2tlbParams.llptwsize).W) 695 val af = Bool() 696 val hptw_resp = new HptwResp() 697 val first_s2xlate_fault = Output(Bool()) 698 val cf = Bool() 699 val from_l0 = Bool() 700 val way_info = UInt(l2tlbParams.l0nWays.W) 701 val jmp_bitmap_check = Bool() 702 val ptes = Vec(tlbcontiguous, UInt(XLEN.W)) 703 val cfs = Vec(tlbcontiguous, Bool()) 704} 705 706 707class LLPTW(implicit p: Parameters) extends XSModule with HasPtwConst with HasPerfEvents { 708 val io = IO(new LLPTWIO()) 709 710 // mbmc:bitmap csr 711 val mbmc = io.csr.mbmc 712 val bitmap_enable = (if (HasBitmapCheck) true.B else false.B) && mbmc.BME === 1.U && mbmc.CMODE === 0.U 713 714 val flush = io.sfence.valid || io.csr.satp.changed || io.csr.vsatp.changed || io.csr.hgatp.changed 715 val entries = RegInit(VecInit(Seq.fill(l2tlbParams.llptwsize)(0.U.asTypeOf(new LLPTWEntry())))) 716 val state_idle :: state_hptw_req :: state_hptw_resp :: state_addr_check :: state_mem_req :: state_mem_waiting :: state_mem_out :: state_last_hptw_req :: state_last_hptw_resp :: state_cache :: state_bitmap_check :: state_bitmap_resp :: Nil = Enum(12) 717 val state = RegInit(VecInit(Seq.fill(l2tlbParams.llptwsize)(state_idle))) 718 719 val is_emptys = state.map(_ === state_idle) 720 val is_mems = state.map(_ === state_mem_req) 721 val is_waiting = state.map(_ === state_mem_waiting) 722 val is_having = state.map(_ === state_mem_out) 723 val is_cache = state.map(_ === state_cache) 724 val is_hptw_req = state.map(_ === state_hptw_req) 725 val is_last_hptw_req = state.map(_ === state_last_hptw_req) 726 val is_hptw_resp = state.map(_ === state_hptw_resp) 727 val is_last_hptw_resp = state.map(_ === state_last_hptw_resp) 728 val is_bitmap_req = state.map(_ === state_bitmap_check) 729 val is_bitmap_resp = state.map(_ === state_bitmap_resp) 730 731 val full = !ParallelOR(is_emptys).asBool 732 val enq_ptr = ParallelPriorityEncoder(is_emptys) 733 734 val mem_ptr = ParallelPriorityEncoder(is_having) // TODO: optimize timing, bad: entries -> ptr -> entry 735 val mem_arb = Module(new RRArbiterInit(new LLPTWEntry(), l2tlbParams.llptwsize)) 736 for (i <- 0 until l2tlbParams.llptwsize) { 737 mem_arb.io.in(i).bits := entries(i) 738 mem_arb.io.in(i).valid := is_mems(i) && !io.mem.req_mask(i) 739 } 740 741 // process hptw requests in serial 742 val hyper_arb1 = Module(new RRArbiterInit(new LLPTWEntry(), l2tlbParams.llptwsize)) 743 for (i <- 0 until l2tlbParams.llptwsize) { 744 hyper_arb1.io.in(i).bits := entries(i) 745 hyper_arb1.io.in(i).valid := is_hptw_req(i) && !(Cat(is_hptw_resp).orR) && !(Cat(is_last_hptw_resp).orR) 746 } 747 val hyper_arb2 = Module(new RRArbiterInit(new LLPTWEntry(), l2tlbParams.llptwsize)) 748 for(i <- 0 until l2tlbParams.llptwsize) { 749 hyper_arb2.io.in(i).bits := entries(i) 750 hyper_arb2.io.in(i).valid := is_last_hptw_req(i) && !(Cat(is_hptw_resp).orR) && !(Cat(is_last_hptw_resp).orR) 751 } 752 753 754 val bitmap_arb = Option.when(HasBitmapCheck)(Module(new RRArbiter(new bitmapReqBundle(), l2tlbParams.llptwsize))) 755 val way_info = Option.when(HasBitmapCheck)(Wire(Vec(l2tlbParams.llptwsize, UInt(l2tlbParams.l0nWays.W)))) 756 if (HasBitmapCheck) { 757 for (i <- 0 until l2tlbParams.llptwsize) { 758 bitmap_arb.get.io.in(i).valid := is_bitmap_req(i) 759 bitmap_arb.get.io.in(i).bits.bmppn := entries(i).ppn 760 bitmap_arb.get.io.in(i).bits.vpn := entries(i).req_info.vpn 761 bitmap_arb.get.io.in(i).bits.id := i.U 762 bitmap_arb.get.io.in(i).bits.level := 0.U // last level 763 bitmap_arb.get.io.in(i).bits.way_info := Mux(entries(i).from_l0, entries(i).way_info, way_info.get(i)) 764 bitmap_arb.get.io.in(i).bits.hptw_bypassed := false.B 765 } 766 } 767 768 val cache_ptr = ParallelMux(is_cache, (0 until l2tlbParams.llptwsize).map(_.U(log2Up(l2tlbParams.llptwsize).W))) 769 770 // duplicate req 771 // to_wait: wait for the last to access mem, set to mem_resp 772 // to_cache: the last is back just right now, set to mem_cache 773 val dup_vec = state.indices.map(i => 774 dup(io.in.bits.req_info.vpn, entries(i).req_info.vpn) && io.in.bits.req_info.s2xlate === entries(i).req_info.s2xlate 775 ) 776 val dup_req_fire = mem_arb.io.out.fire && dup(io.in.bits.req_info.vpn, mem_arb.io.out.bits.req_info.vpn) && io.in.bits.req_info.s2xlate === mem_arb.io.out.bits.req_info.s2xlate // dup with the req fire entry 777 val dup_vec_wait = dup_vec.zip(is_waiting).map{case (d, w) => d && w} // dup with "mem_waiting" entries, sending mem req already 778 val dup_vec_having = dup_vec.zipWithIndex.map{case (d, i) => d && is_having(i)} // dup with the "mem_out" entry recv the data just now 779 val dup_vec_bitmap = dup_vec.zipWithIndex.map{case (d, i) => d && (is_bitmap_req(i) || is_bitmap_resp(i))} 780 val dup_vec_last_hptw = dup_vec.zipWithIndex.map{case (d, i) => d && (is_last_hptw_req(i) || is_last_hptw_resp(i))} 781 val wait_id = Mux(dup_req_fire, mem_arb.io.chosen, ParallelMux(dup_vec_wait zip entries.map(_.wait_id))) 782 val dup_wait_resp = io.mem.resp.fire && VecInit(dup_vec_wait)(io.mem.resp.bits.id) && !io.mem.flush_latch(io.mem.resp.bits.id) // dup with the entry that data coming next cycle 783 val to_wait = Cat(dup_vec_wait).orR || dup_req_fire 784 785 val last_hptw_req_id = io.mem.resp.bits.id 786 val req_paddr = MakeAddr(io.in.bits.ppn(ppnLen-1, 0), getVpnn(io.in.bits.req_info.vpn, 0)) 787 val req_hpaddr = MakeAddr(entries(last_hptw_req_id).hptw_resp.genPPNS2(get_pn(req_paddr)), getVpnn(io.in.bits.req_info.vpn, 0)) 788 val index = Mux(entries(last_hptw_req_id).req_info.s2xlate === allStage, req_hpaddr, req_paddr)(log2Up(l2tlbParams.blockBytes)-1, log2Up(XLEN/8)) 789 val last_hptw_req_pte = io.mem.resp.bits.value.asTypeOf(Vec(blockBits / XLEN, new PteBundle()))(index) 790 val last_hptw_req_ppn = Mux(last_hptw_req_pte.n === 0.U, last_hptw_req_pte.getPPN(), Cat(last_hptw_req_pte.getPPN()(ptePPNLen - 1, pteNapotBits), io.in.bits.req_info.vpn(pteNapotBits - 1, 0))) 791 // in `to_last_hptw_req`, we have already judged whether s2xlate === allStage 792 val last_hptw_vsStagePf = last_hptw_req_pte.isPf(0.U, io.csr.hPBMTE) || !last_hptw_req_pte.isLeaf() 793 val last_hptw_gStagePf = last_hptw_req_pte.isStage1Gpf(io.csr.hgatp.mode) && !last_hptw_vsStagePf 794 795 // noS2xlate || onlyStage1 || allStage but exception; do not need Stage2 translate 796 val noStage2 = ((entries(io.mem.resp.bits.id).req_info.s2xlate === noS2xlate) || (entries(io.mem.resp.bits.id).req_info.s2xlate === onlyStage1)) || 797 (entries(io.mem.resp.bits.id).req_info.s2xlate === allStage && (last_hptw_vsStagePf || last_hptw_gStagePf)) 798 val to_mem_out = dup_wait_resp && noStage2 && !bitmap_enable 799 val to_bitmap_req = (if (HasBitmapCheck) true.B else false.B) && dup_wait_resp && noStage2 && bitmap_enable 800 val to_cache = if (HasBitmapCheck) Cat(dup_vec_bitmap).orR || Cat(dup_vec_having).orR || Cat(dup_vec_last_hptw).orR 801 else Cat(dup_vec_having).orR || Cat(dup_vec_last_hptw).orR 802 val to_hptw_req = io.in.bits.req_info.s2xlate === allStage 803 val to_last_hptw_req = dup_wait_resp && entries(io.mem.resp.bits.id).req_info.s2xlate === allStage && !(last_hptw_vsStagePf || last_hptw_gStagePf) 804 val last_hptw_excp = dup_wait_resp && entries(io.mem.resp.bits.id).req_info.s2xlate === allStage && (last_hptw_vsStagePf || last_hptw_gStagePf) 805 806 XSError(RegNext(dup_req_fire && Cat(dup_vec_wait).orR, init = false.B), "mem req but some entries already waiting, should not happed") 807 808 XSError(io.in.fire && ((to_mem_out && to_cache) || (to_wait && to_cache)), "llptw enq, to cache conflict with to mem") 809 val mem_resp_hit = RegInit(VecInit(Seq.fill(l2tlbParams.llptwsize)(false.B))) 810 val enq_state_normal = MuxCase(state_addr_check, Seq( 811 to_mem_out -> state_mem_out, // same to the blew, but the mem resp now 812 to_bitmap_req -> state_bitmap_check, 813 to_last_hptw_req -> state_last_hptw_req, 814 to_wait -> state_mem_waiting, 815 to_cache -> state_cache, 816 to_hptw_req -> state_hptw_req 817 )) 818 val enq_state = Mux(from_pre(io.in.bits.req_info.source) && enq_state_normal =/= state_addr_check, state_idle, enq_state_normal) 819 when (io.in.fire && (if (HasBitmapCheck) !io.in.bits.bitmapCheck.get.jmp_bitmap_check else true.B)) { 820 // if prefetch req does not need mem access, just give it up. 821 // so there will be at most 1 + FilterSize entries that needs re-access page cache 822 // so 2 + FilterSize is enough to avoid dead-lock 823 state(enq_ptr) := enq_state 824 entries(enq_ptr).req_info := io.in.bits.req_info 825 entries(enq_ptr).ppn := Mux(to_last_hptw_req || last_hptw_excp, last_hptw_req_ppn, io.in.bits.ppn) 826 entries(enq_ptr).wait_id := Mux(to_wait, wait_id, enq_ptr) 827 entries(enq_ptr).af := false.B 828 if (HasBitmapCheck) { 829 entries(enq_ptr).cf := false.B 830 entries(enq_ptr).from_l0 := false.B 831 entries(enq_ptr).way_info := 0.U 832 entries(enq_ptr).jmp_bitmap_check := false.B 833 for (i <- 0 until tlbcontiguous) { 834 entries(enq_ptr).ptes(i) := 0.U 835 } 836 entries(enq_ptr).cfs := io.in.bits.bitmapCheck.get.cfs 837 } 838 entries(enq_ptr).hptw_resp := Mux(to_last_hptw_req, entries(last_hptw_req_id).hptw_resp, Mux(to_wait, entries(wait_id).hptw_resp, entries(enq_ptr).hptw_resp)) 839 entries(enq_ptr).hptw_resp.gpf := Mux(last_hptw_excp, last_hptw_gStagePf, false.B) 840 entries(enq_ptr).first_s2xlate_fault := false.B 841 mem_resp_hit(enq_ptr) := to_bitmap_req || to_mem_out || to_last_hptw_req 842 } 843 844 if (HasBitmapCheck) { 845 when (io.in.bits.bitmapCheck.get.jmp_bitmap_check && io.in.fire) { 846 state(enq_ptr) := state_bitmap_check 847 entries(enq_ptr).req_info := io.in.bits.req_info 848 entries(enq_ptr).ppn := io.in.bits.bitmapCheck.get.ptes(io.in.bits.req_info.vpn(sectortlbwidth - 1, 0)).asTypeOf(new PteBundle().cloneType).ppn 849 entries(enq_ptr).wait_id := enq_ptr 850 entries(enq_ptr).af := false.B 851 entries(enq_ptr).cf := false.B 852 entries(enq_ptr).from_l0 := true.B 853 entries(enq_ptr).way_info := io.in.bits.bitmapCheck.get.hitway 854 entries(enq_ptr).jmp_bitmap_check := io.in.bits.bitmapCheck.get.jmp_bitmap_check 855 entries(enq_ptr).ptes := io.in.bits.bitmapCheck.get.ptes 856 entries(enq_ptr).cfs := io.in.bits.bitmapCheck.get.cfs 857 mem_resp_hit(enq_ptr) := false.B 858 } 859 } 860 861 val enq_ptr_reg = RegNext(enq_ptr) 862 val need_addr_check = GatedValidRegNext(enq_state === state_addr_check && io.in.fire && !flush && (if (HasBitmapCheck) !io.in.bits.bitmapCheck.get.jmp_bitmap_check else true.B)) 863 864 val hasHptwResp = ParallelOR(state.map(_ === state_hptw_resp)).asBool 865 val hptw_resp_ptr_reg = RegNext(io.hptw.resp.bits.id) 866 val hptw_need_addr_check = RegNext(hasHptwResp && io.hptw.resp.fire && !flush) && state(hptw_resp_ptr_reg) === state_addr_check 867 868 val ptes = io.mem.resp.bits.value.asTypeOf(Vec(blockBits / XLEN, new PteBundle())) 869 val gpaddr = MakeGPAddr(entries(hptw_resp_ptr_reg).ppn, getVpnn(entries(hptw_resp_ptr_reg).req_info.vpn, 0)) 870 val hptw_resp = entries(hptw_resp_ptr_reg).hptw_resp 871 val hpaddr = Cat(hptw_resp.genPPNS2(get_pn(gpaddr)), get_off(gpaddr)) 872 val addr = RegEnable(MakeAddr(io.in.bits.ppn(ppnLen - 1, 0), getVpnn(io.in.bits.req_info.vpn, 0)), io.in.fire) 873 io.pmp.req.valid := need_addr_check || hptw_need_addr_check 874 io.pmp.req.bits.addr := Mux(hptw_need_addr_check, hpaddr, addr) 875 io.pmp.req.bits.cmd := TlbCmd.read 876 io.pmp.req.bits.size := 3.U // TODO: fix it 877 val pmp_resp_valid = io.pmp.req.valid // same cycle 878 when (pmp_resp_valid) { 879 // NOTE: when pmp resp but state is not addr check, then the entry is dup with other entry, the state was changed before 880 // when dup with the req-ing entry, set to mem_waiting (above codes), and the ld must be false, so dontcare 881 val ptr = Mux(hptw_need_addr_check, hptw_resp_ptr_reg, enq_ptr_reg); 882 val accessFault = io.pmp.resp.ld || io.pmp.resp.mmio 883 entries(ptr).af := accessFault 884 state(ptr) := Mux(accessFault, state_mem_out, state_mem_req) 885 } 886 887 when (mem_arb.io.out.fire) { 888 for (i <- state.indices) { 889 when (state(i) =/= state_idle && state(i) =/= state_mem_out && state(i) =/= state_last_hptw_req && state(i) =/= state_last_hptw_resp 890 && (if (HasBitmapCheck) state(i) =/= state_bitmap_check && state(i) =/= state_bitmap_resp else true.B) 891 && entries(i).req_info.s2xlate === mem_arb.io.out.bits.req_info.s2xlate 892 && dup(entries(i).req_info.vpn, mem_arb.io.out.bits.req_info.vpn)) { 893 // NOTE: "dup enq set state to mem_wait" -> "sending req set other dup entries to mem_wait" 894 state(i) := state_mem_waiting 895 entries(i).hptw_resp := entries(mem_arb.io.chosen).hptw_resp 896 entries(i).wait_id := mem_arb.io.chosen 897 } 898 } 899 } 900 when (io.mem.resp.fire) { 901 state.indices.map{i => 902 when (state(i) === state_mem_waiting && io.mem.resp.bits.id === entries(i).wait_id) { 903 val req_paddr = MakeAddr(entries(i).ppn, getVpnn(entries(i).req_info.vpn, 0)) 904 val req_hpaddr = MakeAddr(entries(i).hptw_resp.genPPNS2(get_pn(req_paddr)), getVpnn(entries(i).req_info.vpn, 0)) 905 val index = Mux(entries(i).req_info.s2xlate === allStage, req_hpaddr, req_paddr)(log2Up(l2tlbParams.blockBytes)-1, log2Up(XLEN/8)) 906 val enableS2xlate = entries(i).req_info.s2xlate =/= noS2xlate 907 val s1Pbmte = Mux(enableS2xlate, io.csr.hPBMTE, io.csr.mPBMTE) 908 val vsStagePf = ptes(index).isPf(0.U, s1Pbmte) || !ptes(index).isLeaf() // Pagefault in vs-Stage 909 // Pagefault in g-Stage; when vsStagePf valid, should not check gStagepf 910 val gStagePf = ptes(index).isStage1Gpf(io.csr.hgatp.mode) && !vsStagePf 911 state(i) := Mux(entries(i).req_info.s2xlate === allStage && !(vsStagePf || gStagePf), 912 state_last_hptw_req, 913 Mux(bitmap_enable, state_bitmap_check, state_mem_out)) 914 mem_resp_hit(i) := true.B 915 entries(i).ppn := Mux(ptes(index).n === 0.U, ptes(index).getPPN(), Cat(ptes(index).getPPN()(ptePPNLen - 1, pteNapotBits), entries(i).req_info.vpn(pteNapotBits - 1, 0))) // for last stage 2 translation 916 // af will be judged in L2 TLB `contiguous_pte_to_merge_ptwResp` 917 entries(i).hptw_resp.gpf := Mux(entries(i).req_info.s2xlate === allStage, gStagePf, false.B) 918 } 919 } 920 } 921 922 if (HasBitmapCheck) { 923 for (i <- 0 until l2tlbParams.llptwsize) { 924 way_info.get(i) := DataHoldBypass(io.l0_way_info.get, mem_resp_hit(i)) 925 } 926 } 927 928 when (hyper_arb1.io.out.fire) { 929 for (i <- state.indices) { 930 when (state(i) === state_hptw_req && entries(i).ppn === hyper_arb1.io.out.bits.ppn && entries(i).req_info.s2xlate === allStage && hyper_arb1.io.chosen === i.U) { 931 state(i) := state_hptw_resp 932 entries(i).wait_id := hyper_arb1.io.chosen 933 } 934 } 935 } 936 937 when (hyper_arb2.io.out.fire) { 938 for (i <- state.indices) { 939 when (state(i) === state_last_hptw_req && entries(i).ppn === hyper_arb2.io.out.bits.ppn && entries(i).req_info.s2xlate === allStage && hyper_arb2.io.chosen === i.U) { 940 state(i) := state_last_hptw_resp 941 entries(i).wait_id := hyper_arb2.io.chosen 942 } 943 } 944 } 945 946 if (HasBitmapCheck) { 947 when (bitmap_arb.get.io.out.fire) { 948 for (i <- state.indices) { 949 when (is_bitmap_req(i) && bitmap_arb.get.io.out.bits.bmppn === entries(i).ppn(ppnLen - 1, 0)) { 950 state(i) := state_bitmap_resp 951 entries(i).wait_id := bitmap_arb.get.io.chosen 952 } 953 } 954 } 955 956 when (io.bitmap.get.resp.fire) { 957 for (i <- state.indices) { 958 when (is_bitmap_resp(i) && io.bitmap.get.resp.bits.id === entries(i).wait_id) { 959 entries(i).cfs := io.bitmap.get.resp.bits.cfs 960 entries(i).cf := io.bitmap.get.resp.bits.cf 961 state(i) := state_mem_out 962 } 963 } 964 } 965 } 966 967 when (io.hptw.resp.fire) { 968 for (i <- state.indices) { 969 when (state(i) === state_hptw_resp && io.hptw.resp.bits.id === entries(i).wait_id && io.hptw.resp.bits.h_resp.entry.tag === entries(i).ppn) { 970 val check_g_perm_fail = !io.hptw.resp.bits.h_resp.gaf && (!io.hptw.resp.bits.h_resp.entry.perm.get.r && !(io.csr.priv.mxr && io.hptw.resp.bits.h_resp.entry.perm.get.x)) 971 when (check_g_perm_fail || io.hptw.resp.bits.h_resp.gaf || io.hptw.resp.bits.h_resp.gpf) { 972 state(i) := state_mem_out 973 entries(i).hptw_resp := io.hptw.resp.bits.h_resp 974 entries(i).hptw_resp.gpf := io.hptw.resp.bits.h_resp.gpf || check_g_perm_fail 975 entries(i).first_s2xlate_fault := io.hptw.resp.bits.h_resp.gaf || io.hptw.resp.bits.h_resp.gpf 976 }.otherwise{ // change the entry that is waiting hptw resp 977 val need_to_waiting_vec = state.indices.map(i => state(i) === state_mem_waiting && 978 dup(entries(i).req_info.vpn, entries(io.hptw.resp.bits.id).req_info.vpn) && 979 entries(i).req_info.s2xlate === entries(io.hptw.resp.bits.id).req_info.s2xlate) 980 val waiting_index = ParallelMux(need_to_waiting_vec zip entries.map(_.wait_id)) 981 state(i) := Mux(Cat(need_to_waiting_vec).orR, state_mem_waiting, state_addr_check) 982 entries(i).hptw_resp := io.hptw.resp.bits.h_resp 983 entries(i).wait_id := Mux(Cat(need_to_waiting_vec).orR, waiting_index, entries(i).wait_id) 984 //To do: change the entry that is having the same hptw req 985 } 986 } 987 when (state(i) === state_last_hptw_resp && io.hptw.resp.bits.id === entries(i).wait_id && io.hptw.resp.bits.h_resp.entry.tag === entries(i).ppn) { 988 state(i) := state_mem_out 989 entries(i).hptw_resp := io.hptw.resp.bits.h_resp 990 //To do: change the entry that is having the same hptw req 991 } 992 } 993 } 994 when (io.out.fire) { 995 assert(state(mem_ptr) === state_mem_out) 996 state(mem_ptr) := state_idle 997 } 998 mem_resp_hit.map(a => when (a) { a := false.B } ) 999 1000 when (io.cache.fire) { 1001 state(cache_ptr) := state_idle 1002 } 1003 XSError(io.out.fire && io.cache.fire && (mem_ptr === cache_ptr), "mem resp and cache fire at the same time at same entry") 1004 1005 when (flush) { 1006 state.map(_ := state_idle) 1007 } 1008 1009 io.in.ready := !full 1010 1011 io.out.valid := ParallelOR(is_having).asBool 1012 io.out.bits.req_info := entries(mem_ptr).req_info 1013 io.out.bits.id := mem_ptr 1014 if (HasBitmapCheck) { 1015 io.out.bits.af := Mux(bitmap_enable, entries(mem_ptr).af || entries(mem_ptr).cf, entries(mem_ptr).af) 1016 io.out.bits.bitmapCheck.get.jmp_bitmap_check := entries(mem_ptr).jmp_bitmap_check 1017 io.out.bits.bitmapCheck.get.ptes := entries(mem_ptr).ptes 1018 io.out.bits.bitmapCheck.get.cfs := entries(mem_ptr).cfs 1019 } else { 1020 io.out.bits.af := entries(mem_ptr).af 1021 } 1022 1023 io.out.bits.h_resp := entries(mem_ptr).hptw_resp 1024 io.out.bits.first_s2xlate_fault := entries(mem_ptr).first_s2xlate_fault 1025 1026 val hptw_req_arb = Module(new Arbiter(new Bundle{ 1027 val source = UInt(bSourceWidth.W) 1028 val id = UInt(log2Up(l2tlbParams.llptwsize).W) 1029 val ppn = UInt(ptePPNLen.W) 1030 } , 2)) 1031 // first stage 2 translation 1032 hptw_req_arb.io.in(0).valid := hyper_arb1.io.out.valid 1033 hptw_req_arb.io.in(0).bits.source := hyper_arb1.io.out.bits.req_info.source 1034 hptw_req_arb.io.in(0).bits.ppn := hyper_arb1.io.out.bits.ppn 1035 hptw_req_arb.io.in(0).bits.id := hyper_arb1.io.chosen 1036 hyper_arb1.io.out.ready := hptw_req_arb.io.in(0).ready 1037 // last stage 2 translation 1038 hptw_req_arb.io.in(1).valid := hyper_arb2.io.out.valid 1039 hptw_req_arb.io.in(1).bits.source := hyper_arb2.io.out.bits.req_info.source 1040 hptw_req_arb.io.in(1).bits.ppn := hyper_arb2.io.out.bits.ppn 1041 hptw_req_arb.io.in(1).bits.id := hyper_arb2.io.chosen 1042 hyper_arb2.io.out.ready := hptw_req_arb.io.in(1).ready 1043 hptw_req_arb.io.out.ready := io.hptw.req.ready 1044 io.hptw.req.valid := hptw_req_arb.io.out.fire && !flush 1045 io.hptw.req.bits.gvpn := hptw_req_arb.io.out.bits.ppn 1046 io.hptw.req.bits.id := hptw_req_arb.io.out.bits.id 1047 io.hptw.req.bits.source := hptw_req_arb.io.out.bits.source 1048 1049 io.mem.req.valid := mem_arb.io.out.valid && !flush 1050 val mem_paddr = MakeAddr(mem_arb.io.out.bits.ppn, getVpnn(mem_arb.io.out.bits.req_info.vpn, 0)) 1051 val mem_hpaddr = MakeAddr(mem_arb.io.out.bits.hptw_resp.genPPNS2(get_pn(mem_paddr)), getVpnn(mem_arb.io.out.bits.req_info.vpn, 0)) 1052 io.mem.req.bits.addr := Mux(mem_arb.io.out.bits.req_info.s2xlate === allStage, mem_hpaddr, mem_paddr) 1053 io.mem.req.bits.id := mem_arb.io.chosen 1054 io.mem.req.bits.hptw_bypassed := false.B 1055 mem_arb.io.out.ready := io.mem.req.ready 1056 val mem_refill_id = RegNext(io.mem.resp.bits.id(log2Up(l2tlbParams.llptwsize)-1, 0)) 1057 io.mem.refill := entries(mem_refill_id).req_info 1058 io.mem.refill.s2xlate := entries(mem_refill_id).req_info.s2xlate 1059 io.mem.buffer_it := mem_resp_hit 1060 io.mem.enq_ptr := enq_ptr 1061 1062 io.cache.valid := Cat(is_cache).orR 1063 io.cache.bits := ParallelMux(is_cache, entries.map(_.req_info)) 1064 1065 val has_bitmap_resp = ParallelOR(is_bitmap_resp).asBool 1066 if (HasBitmapCheck) { 1067 io.bitmap.get.req.valid := bitmap_arb.get.io.out.valid && !flush 1068 io.bitmap.get.req.bits.bmppn := bitmap_arb.get.io.out.bits.bmppn 1069 io.bitmap.get.req.bits.id := bitmap_arb.get.io.chosen 1070 io.bitmap.get.req.bits.vpn := bitmap_arb.get.io.out.bits.vpn 1071 io.bitmap.get.req.bits.level := 0.U 1072 io.bitmap.get.req.bits.way_info := bitmap_arb.get.io.out.bits.way_info 1073 io.bitmap.get.req.bits.hptw_bypassed := bitmap_arb.get.io.out.bits.hptw_bypassed 1074 bitmap_arb.get.io.out.ready := io.bitmap.get.req.ready 1075 io.bitmap.get.resp.ready := has_bitmap_resp 1076 } 1077 1078 XSPerfAccumulate("llptw_in_count", io.in.fire) 1079 XSPerfAccumulate("llptw_in_block", io.in.valid && !io.in.ready) 1080 for (i <- 0 until 7) { 1081 XSPerfAccumulate(s"enq_state${i}", io.in.fire && enq_state === i.U) 1082 } 1083 for (i <- 0 until (l2tlbParams.llptwsize + 1)) { 1084 XSPerfAccumulate(s"util${i}", PopCount(is_emptys.map(!_)) === i.U) 1085 XSPerfAccumulate(s"mem_util${i}", PopCount(is_mems) === i.U) 1086 XSPerfAccumulate(s"waiting_util${i}", PopCount(is_waiting) === i.U) 1087 } 1088 XSPerfAccumulate("mem_count", io.mem.req.fire) 1089 XSPerfAccumulate("mem_cycle", PopCount(is_waiting) =/= 0.U) 1090 XSPerfAccumulate("blocked_in", io.in.valid && !io.in.ready) 1091 1092 val perfEvents = Seq( 1093 ("tlbllptw_incount ", io.in.fire ), 1094 ("tlbllptw_inblock ", io.in.valid && !io.in.ready), 1095 ("tlbllptw_memcount ", io.mem.req.fire ), 1096 ("tlbllptw_memcycle ", PopCount(is_waiting) ), 1097 ) 1098 generatePerfEvent() 1099} 1100 1101/*========================= HPTW ==============================*/ 1102 1103/** HPTW : Hypervisor Page Table Walker 1104 * the page walker take the virtual machine's page walk. 1105 * guest physical address translation, guest physical address -> host physical address 1106 **/ 1107class HPTWIO()(implicit p: Parameters) extends MMUIOBaseBundle with HasPtwConst { 1108 val req = Flipped(DecoupledIO(new Bundle { 1109 val source = UInt(bSourceWidth.W) 1110 val id = UInt(log2Up(l2tlbParams.llptwsize).W) 1111 val gvpn = UInt(gvpnLen.W) 1112 val ppn = UInt(ppnLen.W) 1113 val l3Hit = if (EnableSv48) Some(new Bool()) else None 1114 val l2Hit = Bool() 1115 val l1Hit = Bool() 1116 val bypassed = Bool() // if bypass, don't refill 1117 val bitmapCheck = Option.when(HasBitmapCheck)(new Bundle { 1118 val jmp_bitmap_check = Bool() // find pte in l0 or sp, but need bitmap check 1119 val pte = UInt(XLEN.W) // Page Table Entry 1120 val ptes = Vec(tlbcontiguous, UInt(XLEN.W)) // Page Table Entry Vector 1121 val cfs = Vec(tlbcontiguous, Bool()) // Bitmap Check Failed Vector 1122 val hitway = UInt(l2tlbParams.l0nWays.W) 1123 val fromSP = Bool() 1124 val SPlevel = UInt(log2Up(Level).W) 1125 }) 1126 })) 1127 val resp = DecoupledIO(new Bundle { 1128 val source = UInt(bSourceWidth.W) 1129 val resp = Output(new HptwResp()) 1130 val id = Output(UInt(bMemID.W)) 1131 }) 1132 1133 val mem = new Bundle { 1134 val req = DecoupledIO(new L2TlbMemReqBundle()) 1135 val resp = Flipped(ValidIO(UInt(XLEN.W))) 1136 val mask = Input(Bool()) 1137 } 1138 val refill = Output(new Bundle { 1139 val req_info = new L2TlbInnerBundle() 1140 val level = UInt(log2Up(Level + 1).W) 1141 }) 1142 val pmp = new Bundle { 1143 val req = ValidIO(new PMPReqBundle()) 1144 val resp = Flipped(new PMPRespBundle()) 1145 } 1146 val bitmap = Option.when(HasBitmapCheck)(new Bundle { 1147 val req = DecoupledIO(new bitmapReqBundle()) 1148 val resp = Flipped(DecoupledIO(new bitmapRespBundle())) 1149 }) 1150 1151 val l0_way_info = Option.when(HasBitmapCheck)(Input(UInt(l2tlbParams.l0nWays.W))) 1152} 1153 1154class HPTW()(implicit p: Parameters) extends XSModule with HasPtwConst { 1155 val io = IO(new HPTWIO) 1156 val hgatp = io.csr.hgatp 1157 val mpbmte = io.csr.mPBMTE 1158 val sfence = io.sfence 1159 val flush = sfence.valid || hgatp.changed || io.csr.satp.changed || io.csr.vsatp.changed 1160 val mode = hgatp.mode 1161 1162 // mbmc:bitmap csr 1163 val mbmc = io.csr.mbmc 1164 val bitmap_enable = (if (HasBitmapCheck) true.B else false.B) && mbmc.BME === 1.U && mbmc.CMODE === 0.U 1165 1166 val level = RegInit(3.U(log2Up(Level + 1).W)) 1167 val af_level = RegInit(3.U(log2Up(Level + 1).W)) // access fault return this level 1168 val gpaddr = Reg(UInt(GPAddrBits.W)) 1169 val req_ppn = Reg(UInt(ppnLen.W)) 1170 val vpn = gpaddr(GPAddrBits-1, offLen) 1171 val levelNext = level - 1.U 1172 val l3Hit = Reg(Bool()) 1173 val l2Hit = Reg(Bool()) 1174 val l1Hit = Reg(Bool()) 1175 val bypassed = Reg(Bool()) 1176// val pte = io.mem.resp.bits.MergeRespToPte() 1177 val jmp_bitmap_check = if (HasBitmapCheck) RegEnable(io.req.bits.bitmapCheck.get.jmp_bitmap_check, io.req.fire) else false.B 1178 val fromSP = if (HasBitmapCheck) RegEnable(io.req.bits.bitmapCheck.get.fromSP, io.req.fire) else false.B 1179 val cache_pte = Option.when(HasBitmapCheck)(RegEnable(Mux(io.req.bits.bitmapCheck.get.fromSP, io.req.bits.bitmapCheck.get.pte.asTypeOf(new PteBundle().cloneType), io.req.bits.bitmapCheck.get.ptes(io.req.bits.gvpn(sectortlbwidth - 1, 0)).asTypeOf(new PteBundle().cloneType)), io.req.fire)) 1180 val pte = if (HasBitmapCheck) Mux(jmp_bitmap_check, cache_pte.get, io.mem.resp.bits.asTypeOf(new PteBundle().cloneType)) else io.mem.resp.bits.asTypeOf(new PteBundle().cloneType) 1181 val ppn_l3 = Mux(l3Hit, req_ppn, pte.ppn) 1182 val ppn_l2 = Mux(l2Hit, req_ppn, pte.ppn) 1183 val ppn_l1 = Mux(l1Hit, req_ppn, pte.ppn) 1184 val ppn = Wire(UInt(PAddrBits.W)) 1185 val p_pte = MakeAddr(ppn, getVpnn(vpn, level)) 1186 val pg_base = Wire(UInt(PAddrBits.W)) 1187 val mem_addr = Wire(UInt(PAddrBits.W)) 1188 if (EnableSv48) { 1189 when (mode === Sv48) { 1190 ppn := Mux(af_level === 2.U, ppn_l3, Mux(af_level === 1.U, ppn_l2, ppn_l1)) // for l2, l1 and l3 1191 pg_base := MakeGPAddr(hgatp.ppn, getGVpnn(vpn, 3.U, mode = Sv48)) // for l3 1192 mem_addr := Mux(af_level === 3.U, pg_base, p_pte) 1193 } .otherwise { 1194 ppn := Mux(af_level === 1.U, ppn_l2, ppn_l1) //for l1 and l2 1195 pg_base := MakeGPAddr(hgatp.ppn, getGVpnn(vpn, 2.U, mode = Sv39)) 1196 mem_addr := Mux(af_level === 2.U, pg_base, p_pte) 1197 } 1198 } else { 1199 ppn := Mux(af_level === 1.U, ppn_l2, ppn_l1) //for l1 and l2 1200 pg_base := MakeGPAddr(hgatp.ppn, getGVpnn(vpn, 2.U, mode = Sv39)) 1201 mem_addr := Mux(af_level === 2.U, pg_base, p_pte) 1202 } 1203 1204 //s/w register 1205 val s_pmp_check = RegInit(true.B) 1206 val s_mem_req = RegInit(true.B) 1207 val w_mem_resp = RegInit(true.B) 1208 val idle = RegInit(true.B) 1209 val mem_addr_update = RegInit(false.B) 1210 val finish = WireInit(false.B) 1211 val s_bitmap_check = RegInit(true.B) 1212 val w_bitmap_resp = RegInit(true.B) 1213 val whether_need_bitmap_check = RegInit(false.B) 1214 val bitmap_checkfailed = RegInit(false.B) 1215 1216 val sent_to_pmp = !idle && (!s_pmp_check || mem_addr_update) && !finish 1217 val pageFault = pte.isGpf(level, mpbmte) || (!pte.isLeaf() && level === 0.U) 1218 val accessFault = RegEnable(io.pmp.resp.ld || io.pmp.resp.mmio, sent_to_pmp) 1219 1220 // use access fault when bitmap check failed 1221 val ppn_af = if (HasBitmapCheck) { 1222 Mux(bitmap_enable, pte.isAf() || bitmap_checkfailed, pte.isAf()) 1223 } else { 1224 pte.isAf() 1225 } 1226 val find_pte = pte.isLeaf() || ppn_af || pageFault 1227 1228 val resp_valid = !idle && mem_addr_update && ((w_mem_resp && find_pte) || (s_pmp_check && accessFault)) 1229 val id = Reg(UInt(log2Up(l2tlbParams.llptwsize).W)) 1230 val source = RegEnable(io.req.bits.source, io.req.fire) 1231 1232 io.req.ready := idle 1233 val resp = Wire(new HptwResp()) 1234 // accessFault > pageFault > ppn_af 1235 resp.apply( 1236 gpf = pageFault && !accessFault, 1237 gaf = accessFault || (ppn_af && !pageFault), 1238 level = Mux(accessFault, af_level, level), 1239 pte = pte, 1240 vpn = vpn, 1241 vmid = hgatp.vmid 1242 ) 1243 io.resp.valid := resp_valid 1244 io.resp.bits.id := id 1245 io.resp.bits.resp := resp 1246 io.resp.bits.source := source 1247 1248 io.pmp.req.valid := DontCare 1249 io.pmp.req.bits.addr := mem_addr 1250 io.pmp.req.bits.size := 3.U 1251 io.pmp.req.bits.cmd := TlbCmd.read 1252 1253 if (HasBitmapCheck) { 1254 val way_info = DataHoldBypass(io.l0_way_info.get, RegNext(io.mem.resp.fire, init=false.B)) 1255 val cache_hitway = RegEnable(io.req.bits.bitmapCheck.get.hitway, io.req.fire) 1256 val cache_level = RegEnable(io.req.bits.bitmapCheck.get.SPlevel, io.req.fire) 1257 io.bitmap.get.req.valid := !s_bitmap_check 1258 io.bitmap.get.req.bits.bmppn := pte.ppn 1259 io.bitmap.get.req.bits.id := HptwReqId.U(bMemID.W) 1260 io.bitmap.get.req.bits.vpn := vpn 1261 io.bitmap.get.req.bits.level := Mux(jmp_bitmap_check, Mux(fromSP,cache_level,0.U), level) 1262 io.bitmap.get.req.bits.way_info := Mux(jmp_bitmap_check, cache_hitway, way_info) 1263 io.bitmap.get.req.bits.hptw_bypassed := bypassed 1264 io.bitmap.get.resp.ready := !w_bitmap_resp 1265 } 1266 1267 io.mem.req.valid := !s_mem_req && !io.mem.mask && !accessFault && s_pmp_check 1268 io.mem.req.bits.addr := mem_addr 1269 io.mem.req.bits.id := HptwReqId.U(bMemID.W) 1270 io.mem.req.bits.hptw_bypassed := bypassed 1271 1272 io.refill.req_info.vpn := vpn 1273 io.refill.level := level 1274 io.refill.req_info.source := source 1275 io.refill.req_info.s2xlate := onlyStage2 1276 1277 when (idle){ 1278 if (HasBitmapCheck) { 1279 when (io.req.bits.bitmapCheck.get.jmp_bitmap_check && io.req.fire) { 1280 idle := false.B 1281 gpaddr := Cat(io.req.bits.gvpn, 0.U(offLen.W)) 1282 s_bitmap_check := false.B 1283 id := io.req.bits.id 1284 level := Mux(io.req.bits.bitmapCheck.get.fromSP, io.req.bits.bitmapCheck.get.SPlevel, 0.U) 1285 } 1286 } 1287 when (io.req.fire && (if (HasBitmapCheck) !io.req.bits.bitmapCheck.get.jmp_bitmap_check else true.B)) { 1288 bypassed := io.req.bits.bypassed 1289 idle := false.B 1290 gpaddr := Cat(io.req.bits.gvpn, 0.U(offLen.W)) 1291 accessFault := false.B 1292 s_pmp_check := false.B 1293 id := io.req.bits.id 1294 req_ppn := io.req.bits.ppn 1295 if (EnableSv48) { 1296 when (mode === Sv48) { 1297 level := Mux(io.req.bits.l1Hit, 0.U, Mux(io.req.bits.l2Hit, 1.U, Mux(io.req.bits.l3Hit.get, 2.U, 3.U))) 1298 af_level := Mux(io.req.bits.l1Hit, 0.U, Mux(io.req.bits.l2Hit, 1.U, Mux(io.req.bits.l3Hit.get, 2.U, 3.U))) 1299 l3Hit := io.req.bits.l3Hit.get 1300 } .otherwise { 1301 level := Mux(io.req.bits.l1Hit, 0.U, Mux(io.req.bits.l2Hit, 1.U, 2.U)) 1302 af_level := Mux(io.req.bits.l1Hit, 0.U, Mux(io.req.bits.l2Hit, 1.U, 2.U)) 1303 l3Hit := false.B 1304 } 1305 } else { 1306 level := Mux(io.req.bits.l1Hit, 0.U, Mux(io.req.bits.l2Hit, 1.U, 2.U)) 1307 af_level := Mux(io.req.bits.l1Hit, 0.U, Mux(io.req.bits.l2Hit, 1.U, 2.U)) 1308 l3Hit := false.B 1309 } 1310 l2Hit := io.req.bits.l2Hit 1311 l1Hit := io.req.bits.l1Hit 1312 } 1313 } 1314 1315 when(sent_to_pmp && !mem_addr_update){ 1316 s_mem_req := false.B 1317 s_pmp_check := true.B 1318 } 1319 1320 when(accessFault && !idle){ 1321 s_pmp_check := true.B 1322 s_mem_req := true.B 1323 w_mem_resp := true.B 1324 mem_addr_update := true.B 1325 if (HasBitmapCheck) { 1326 s_bitmap_check := true.B 1327 w_bitmap_resp := true.B 1328 whether_need_bitmap_check := false.B 1329 bitmap_checkfailed := false.B 1330 } 1331 } 1332 1333 when(io.mem.req.fire){ 1334 s_mem_req := true.B 1335 w_mem_resp := false.B 1336 } 1337 1338 when(io.mem.resp.fire && !w_mem_resp){ 1339 w_mem_resp := true.B 1340 af_level := af_level - 1.U 1341 if (HasBitmapCheck) { 1342 when (bitmap_enable) { 1343 whether_need_bitmap_check := true.B 1344 } .otherwise { 1345 mem_addr_update := true.B 1346 whether_need_bitmap_check := false.B 1347 } 1348 } else { 1349 mem_addr_update := true.B 1350 } 1351 } 1352 1353 if (HasBitmapCheck) { 1354 when (whether_need_bitmap_check) { 1355 when (bitmap_enable && pte.isLeaf()) { 1356 s_bitmap_check := false.B 1357 whether_need_bitmap_check := false.B 1358 } .otherwise { 1359 mem_addr_update := true.B 1360 whether_need_bitmap_check := false.B 1361 } 1362 } 1363 // bitmapcheck 1364 when (io.bitmap.get.req.fire) { 1365 s_bitmap_check := true.B 1366 w_bitmap_resp := false.B 1367 } 1368 when (io.bitmap.get.resp.fire) { 1369 w_bitmap_resp := true.B 1370 mem_addr_update := true.B 1371 bitmap_checkfailed := io.bitmap.get.resp.bits.cf 1372 } 1373 } 1374 1375 when(mem_addr_update){ 1376 when(!(find_pte || accessFault)){ 1377 level := levelNext 1378 s_mem_req := false.B 1379 mem_addr_update := false.B 1380 }.elsewhen(resp_valid){ 1381 when(io.resp.fire){ 1382 idle := true.B 1383 mem_addr_update := false.B 1384 accessFault := false.B 1385 } 1386 finish := true.B 1387 } 1388 } 1389 when (flush) { 1390 idle := true.B 1391 s_pmp_check := true.B 1392 s_mem_req := true.B 1393 w_mem_resp := true.B 1394 accessFault := false.B 1395 mem_addr_update := false.B 1396 if (HasBitmapCheck) { 1397 s_bitmap_check := true.B 1398 w_bitmap_resp := true.B 1399 whether_need_bitmap_check := false.B 1400 bitmap_checkfailed := false.B 1401 } 1402 } 1403} 1404