1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.mem 18 19import chipsalliance.rocketchip.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import utils._ 23import utility._ 24import xiangshan.ExceptionNO._ 25import xiangshan._ 26import xiangshan.backend.fu.PMPRespBundle 27import xiangshan.backend.rob.DebugLsInfoBundle 28import xiangshan.cache.mmu.{TlbCmd, TlbReq, TlbRequestIO, TlbResp} 29 30// Store Pipeline Stage 0 31// Generate addr, use addr to query DCache and DTLB 32class StoreUnit_S0(implicit p: Parameters) extends XSModule { 33 val io = IO(new Bundle() { 34 val in = Flipped(Decoupled(new ExuInput)) 35 val rsIdx = Input(UInt(log2Up(IssQueSize).W)) 36 val isFirstIssue = Input(Bool()) 37 val out = Decoupled(new LsPipelineBundle) 38 val dtlbReq = DecoupledIO(new TlbReq) 39 }) 40 41 // send req to dtlb 42 // val saddr = io.in.bits.src(0) + SignExt(io.in.bits.uop.ctrl.imm(11,0), VAddrBits) 43 val imm12 = WireInit(io.in.bits.uop.ctrl.imm(11,0)) 44 val saddr_lo = io.in.bits.src(0)(11,0) + Cat(0.U(1.W), imm12) 45 val saddr_hi = Mux(saddr_lo(12), 46 Mux(imm12(11), io.in.bits.src(0)(VAddrBits-1, 12), io.in.bits.src(0)(VAddrBits-1, 12)+1.U), 47 Mux(imm12(11), io.in.bits.src(0)(VAddrBits-1, 12)+SignExt(1.U, VAddrBits-12), io.in.bits.src(0)(VAddrBits-1, 12)), 48 ) 49 val saddr = Cat(saddr_hi, saddr_lo(11,0)) 50 51 io.dtlbReq.bits.vaddr := saddr 52 io.dtlbReq.valid := io.in.valid 53 io.dtlbReq.bits.cmd := TlbCmd.write 54 io.dtlbReq.bits.size := LSUOpType.size(io.in.bits.uop.ctrl.fuOpType) 55 io.dtlbReq.bits.kill := DontCare 56 io.dtlbReq.bits.memidx.is_ld := false.B 57 io.dtlbReq.bits.memidx.is_st := true.B 58 io.dtlbReq.bits.memidx.idx := io.in.bits.uop.sqIdx.value 59 io.dtlbReq.bits.debug.robIdx := io.in.bits.uop.robIdx 60 io.dtlbReq.bits.no_translate := false.B 61 io.dtlbReq.bits.debug.pc := io.in.bits.uop.cf.pc 62 io.dtlbReq.bits.debug.isFirstIssue := io.isFirstIssue 63 64 io.out.bits := DontCare 65 io.out.bits.vaddr := saddr 66 67 // Now data use its own io 68 // io.out.bits.data := genWdata(io.in.bits.src(1), io.in.bits.uop.ctrl.fuOpType(1,0)) 69 io.out.bits.data := io.in.bits.src(1) // FIXME: remove data from pipeline 70 io.out.bits.uop := io.in.bits.uop 71 io.out.bits.miss := DontCare 72 io.out.bits.rsIdx := io.rsIdx 73 io.out.bits.mask := genWmask(io.out.bits.vaddr, io.in.bits.uop.ctrl.fuOpType(1,0)) 74 io.out.bits.isFirstIssue := io.isFirstIssue 75 io.out.bits.wlineflag := io.in.bits.uop.ctrl.fuOpType === LSUOpType.cbo_zero 76 io.out.valid := io.in.valid 77 io.in.ready := io.out.ready 78 when(io.in.valid && io.isFirstIssue) { 79 io.out.bits.uop.debugInfo.tlbFirstReqTime := GTimer() 80 } 81 82 // exception check 83 val addrAligned = LookupTree(io.in.bits.uop.ctrl.fuOpType(1,0), List( 84 "b00".U -> true.B, //b 85 "b01".U -> (io.out.bits.vaddr(0) === 0.U), //h 86 "b10".U -> (io.out.bits.vaddr(1,0) === 0.U), //w 87 "b11".U -> (io.out.bits.vaddr(2,0) === 0.U) //d 88 )) 89 90 io.out.bits.uop.cf.exceptionVec(storeAddrMisaligned) := !addrAligned 91 92 XSPerfAccumulate("in_valid", io.in.valid) 93 XSPerfAccumulate("in_fire", io.in.fire) 94 XSPerfAccumulate("in_fire_first_issue", io.in.fire && io.isFirstIssue) 95 XSPerfAccumulate("addr_spec_success", io.out.fire() && saddr(VAddrBits-1, 12) === io.in.bits.src(0)(VAddrBits-1, 12)) 96 XSPerfAccumulate("addr_spec_failed", io.out.fire() && saddr(VAddrBits-1, 12) =/= io.in.bits.src(0)(VAddrBits-1, 12)) 97 XSPerfAccumulate("addr_spec_success_once", io.out.fire() && saddr(VAddrBits-1, 12) === io.in.bits.src(0)(VAddrBits-1, 12) && io.isFirstIssue) 98 XSPerfAccumulate("addr_spec_failed_once", io.out.fire() && saddr(VAddrBits-1, 12) =/= io.in.bits.src(0)(VAddrBits-1, 12) && io.isFirstIssue) 99} 100 101// Store Pipeline Stage 1 102// TLB resp (send paddr to dcache) 103class StoreUnit_S1(implicit p: Parameters) extends XSModule { 104 val io = IO(new Bundle() { 105 val in = Flipped(Decoupled(new LsPipelineBundle)) 106 val out = Decoupled(new LsPipelineBundle) 107 val lsq = ValidIO(new LsPipelineBundle()) 108 val dtlbResp = Flipped(DecoupledIO(new TlbResp())) 109 val rsFeedback = ValidIO(new RSFeedback) 110 val reExecuteQuery = Valid(new LoadReExecuteQueryIO) 111 }) 112 113 // mmio cbo decoder 114 val is_mmio_cbo = io.in.bits.uop.ctrl.fuOpType === LSUOpType.cbo_clean || 115 io.in.bits.uop.ctrl.fuOpType === LSUOpType.cbo_flush || 116 io.in.bits.uop.ctrl.fuOpType === LSUOpType.cbo_inval 117 118 val s1_paddr = io.dtlbResp.bits.paddr(0) 119 val s1_tlb_miss = io.dtlbResp.bits.miss 120 121 val s1_mmio = is_mmio_cbo 122 val s1_exception = ExceptionNO.selectByFu(io.out.bits.uop.cf.exceptionVec, staCfg).asUInt.orR 123 124 io.in.ready := true.B 125 126 io.dtlbResp.ready := true.B // TODO: why dtlbResp needs a ready? 127 128 // st-ld violation dectect request. 129 io.reExecuteQuery.valid := io.in.valid && !s1_tlb_miss 130 io.reExecuteQuery.bits.robIdx := io.in.bits.uop.robIdx 131 io.reExecuteQuery.bits.paddr := s1_paddr 132 io.reExecuteQuery.bits.mask := io.in.bits.mask 133 134 // Send TLB feedback to store issue queue 135 // Store feedback is generated in store_s1, sent to RS in store_s2 136 io.rsFeedback.valid := io.in.valid 137 io.rsFeedback.bits.hit := !s1_tlb_miss 138 io.rsFeedback.bits.flushState := io.dtlbResp.bits.ptwBack 139 io.rsFeedback.bits.rsIdx := io.in.bits.rsIdx 140 io.rsFeedback.bits.sourceType := RSFeedbackType.tlbMiss 141 XSDebug(io.rsFeedback.valid, 142 "S1 Store: tlbHit: %d robIdx: %d\n", 143 io.rsFeedback.bits.hit, 144 io.rsFeedback.bits.rsIdx 145 ) 146 io.rsFeedback.bits.dataInvalidSqIdx := DontCare 147 148 // get paddr from dtlb, check if rollback is needed 149 // writeback store inst to lsq 150 io.out.valid := io.in.valid && !s1_tlb_miss 151 io.out.bits := io.in.bits 152 io.out.bits.paddr := s1_paddr 153 io.out.bits.miss := false.B 154 io.out.bits.mmio := s1_mmio 155 io.out.bits.atomic := s1_mmio 156 io.out.bits.uop.cf.exceptionVec(storePageFault) := io.dtlbResp.bits.excp(0).pf.st 157 io.out.bits.uop.cf.exceptionVec(storeAccessFault) := io.dtlbResp.bits.excp(0).af.st 158 159 io.lsq.valid := io.in.valid 160 io.lsq.bits := io.out.bits 161 io.lsq.bits.miss := s1_tlb_miss 162 163 // mmio inst with exception will be writebacked immediately 164 // io.out.valid := io.in.valid && (!io.out.bits.mmio || s1_exception) && !s1_tlb_miss 165 166 // write below io.out.bits assign sentence to prevent overwriting values 167 val s1_tlb_memidx = io.dtlbResp.bits.memidx 168 when(s1_tlb_memidx.is_st && io.dtlbResp.valid && !s1_tlb_miss && s1_tlb_memidx.idx === io.out.bits.uop.sqIdx.value) { 169 // printf("Store idx = %d\n", s1_tlb_memidx.idx) 170 io.out.bits.uop.debugInfo.tlbRespTime := GTimer() 171 } 172 173 XSPerfAccumulate("in_valid", io.in.valid) 174 XSPerfAccumulate("in_fire", io.in.fire) 175 XSPerfAccumulate("in_fire_first_issue", io.in.fire && io.in.bits.isFirstIssue) 176 XSPerfAccumulate("tlb_miss", io.in.fire && s1_tlb_miss) 177 XSPerfAccumulate("tlb_miss_first_issue", io.in.fire && s1_tlb_miss && io.in.bits.isFirstIssue) 178} 179 180class StoreUnit_S2(implicit p: Parameters) extends XSModule { 181 val io = IO(new Bundle() { 182 val in = Flipped(Decoupled(new LsPipelineBundle)) 183 val pmpResp = Flipped(new PMPRespBundle) 184 val static_pm = Input(Valid(Bool())) 185 val out = Decoupled(new LsPipelineBundle) 186 }) 187 val pmp = WireInit(io.pmpResp) 188 when (io.static_pm.valid) { 189 pmp.ld := false.B 190 pmp.st := false.B 191 pmp.instr := false.B 192 pmp.mmio := io.static_pm.bits 193 } 194 195 val s2_exception = ExceptionNO.selectByFu(io.out.bits.uop.cf.exceptionVec, staCfg).asUInt.orR 196 val is_mmio = io.in.bits.mmio || pmp.mmio 197 198 io.in.ready := true.B 199 io.out.bits := io.in.bits 200 io.out.bits.mmio := is_mmio && !s2_exception 201 io.out.bits.atomic := io.in.bits.atomic || pmp.atomic 202 io.out.bits.uop.cf.exceptionVec(storeAccessFault) := io.in.bits.uop.cf.exceptionVec(storeAccessFault) || pmp.st 203 io.out.valid := io.in.valid && (!is_mmio || s2_exception) 204} 205 206class StoreUnit_WriteBack(implicit p: Parameters) extends XSModule { 207 val io = IO(new Bundle() { 208 val redirect = Flipped(Valid(new Redirect)) 209 val in = Flipped(Decoupled(new LsPipelineBundle)) 210 val stout = DecoupledIO(new ExuOutput) // writeback store 211 }) 212 213 io.in.ready := true.B 214 215 val SelectGroupSize = RollbackGroupSize 216 val lgSelectGroupSize = log2Ceil(SelectGroupSize) 217 val TotalSelectCycles = scala.math.ceil(log2Ceil(LoadQueueRAWSize).toFloat / lgSelectGroupSize).toInt + 1 218 219 val stout = Wire(new ExuOutput) 220 stout := DontCare 221 stout.uop := io.in.bits.uop 222 stout.data := DontCare 223 stout.redirectValid := false.B 224 stout.redirect := DontCare 225 stout.debug.isMMIO := io.in.bits.mmio 226 stout.debug.paddr := io.in.bits.paddr 227 stout.debug.vaddr := io.in.bits.vaddr 228 stout.debug.isPerfCnt := false.B 229 stout.fflags := DontCare 230 231 // delay TotalSelectCycles - 2 cycle(s) 232 var valid = io.in.valid 233 var bits = stout 234 for (i <- 0 until TotalSelectCycles - 2) { 235 valid = RegNext(valid && !bits.uop.robIdx.needFlush(io.redirect)) 236 bits = RegNext(bits) 237 } 238 io.stout.valid := valid && !bits.uop.robIdx.needFlush(io.redirect) 239 io.stout.bits := bits 240} 241 242class StoreUnit(implicit p: Parameters) extends XSModule { 243 val io = IO(new Bundle() { 244 val stin = Flipped(Decoupled(new ExuInput)) 245 val redirect = Flipped(ValidIO(new Redirect)) 246 val feedbackSlow = ValidIO(new RSFeedback) 247 val tlb = new TlbRequestIO() 248 val pmp = Flipped(new PMPRespBundle()) 249 val rsIdx = Input(UInt(log2Up(IssQueSize).W)) 250 val isFirstIssue = Input(Bool()) 251 val lsq = ValidIO(new LsPipelineBundle) 252 val lsq_replenish = Output(new LsPipelineBundle()) 253 val stout = DecoupledIO(new ExuOutput) // writeback store 254 // store mask, send to sq in store_s0 255 val storeMaskOut = Valid(new StoreMaskBundle) 256 val reExecuteQuery = Valid(new LoadReExecuteQueryIO) 257 val issue = Valid(new ExuInput) 258 val debug_ls = Output(new DebugLsInfoBundle) 259 }) 260 261 val store_s0 = Module(new StoreUnit_S0) 262 val store_s1 = Module(new StoreUnit_S1) 263 val store_s2 = Module(new StoreUnit_S2) 264 val store_wb = Module(new StoreUnit_WriteBack) 265 266 store_s0.io.in <> io.stin 267 store_s0.io.dtlbReq <> io.tlb.req 268 io.tlb.req_kill := false.B 269 store_s0.io.rsIdx := io.rsIdx 270 store_s0.io.isFirstIssue := io.isFirstIssue 271 272 io.storeMaskOut.valid := store_s0.io.in.valid 273 io.storeMaskOut.bits.mask := store_s0.io.out.bits.mask 274 io.storeMaskOut.bits.sqIdx := store_s0.io.out.bits.uop.sqIdx 275 276 PipelineConnect(store_s0.io.out, store_s1.io.in, true.B, store_s0.io.out.bits.uop.robIdx.needFlush(io.redirect)) 277 io.issue.valid := store_s1.io.in.valid && !store_s1.io.dtlbResp.bits.miss 278 io.issue.bits := RegEnable(store_s0.io.in.bits, store_s0.io.in.valid) 279 280 store_s1.io.dtlbResp <> io.tlb.resp 281 io.lsq <> store_s1.io.lsq 282 io.reExecuteQuery := store_s1.io.reExecuteQuery 283 284 PipelineConnect(store_s1.io.out, store_s2.io.in, true.B, store_s1.io.out.bits.uop.robIdx.needFlush(io.redirect)) 285 286 // feedback tlb miss to RS in store_s2 287 io.feedbackSlow.bits := RegNext(store_s1.io.rsFeedback.bits) 288 io.feedbackSlow.valid := RegNext(store_s1.io.rsFeedback.valid && !store_s1.io.out.bits.uop.robIdx.needFlush(io.redirect)) 289 290 store_s2.io.pmpResp <> io.pmp 291 store_s2.io.static_pm := RegNext(io.tlb.resp.bits.static_pm) 292 io.lsq_replenish := store_s2.io.out.bits // mmio and exception 293 PipelineConnect(store_s2.io.out, store_wb.io.in, true.B, store_s2.io.out.bits.uop.robIdx.needFlush(io.redirect)) 294 295 store_wb.io.redirect <> io.redirect 296 store_wb.io.stout <> io.stout 297 298 io.debug_ls := DontCare 299 io.debug_ls.s1.isTlbFirstMiss := io.tlb.resp.valid && io.tlb.resp.bits.miss && io.tlb.resp.bits.debug.isFirstIssue 300 io.debug_ls.s1_robIdx := store_s1.io.in.bits.uop.robIdx.value 301 302 private def printPipeLine(pipeline: LsPipelineBundle, cond: Bool, name: String): Unit = { 303 XSDebug(cond, 304 p"$name" + p" pc ${Hexadecimal(pipeline.uop.cf.pc)} " + 305 p"addr ${Hexadecimal(pipeline.vaddr)} -> ${Hexadecimal(pipeline.paddr)} " + 306 p"op ${Binary(pipeline.uop.ctrl.fuOpType)} " + 307 p"data ${Hexadecimal(pipeline.data)} " + 308 p"mask ${Hexadecimal(pipeline.mask)}\n" 309 ) 310 } 311 312 printPipeLine(store_s0.io.out.bits, store_s0.io.out.valid, "S0") 313 printPipeLine(store_s1.io.out.bits, store_s1.io.out.valid, "S1") 314} 315