1/*************************************************************************************** 2* Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences 3* Copyright (c) 2020-2021 Peng Cheng Laboratory 4* 5* XiangShan is licensed under Mulan PSL v2. 6* You can use this software according to the terms and conditions of the Mulan PSL v2. 7* You may obtain a copy of Mulan PSL v2 at: 8* http://license.coscl.org.cn/MulanPSL2 9* 10* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, 11* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, 12* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. 13* 14* See the Mulan PSL v2 for more details. 15***************************************************************************************/ 16 17package xiangshan.mem 18 19import org.chipsalliance.cde.config.Parameters 20import chisel3._ 21import chisel3.util._ 22import utils._ 23import utility._ 24import xiangshan._ 25import xiangshan.ExceptionNO._ 26import xiangshan.backend.Bundles.{MemExuInput, MemExuOutput, connectSamePort} 27import xiangshan.backend.fu.PMPRespBundle 28import xiangshan.backend.fu.FuConfig._ 29import xiangshan.backend.fu.FuType._ 30import xiangshan.backend.ctrlblock.DebugLsInfoBundle 31import xiangshan.backend.fu.NewCSR._ 32import xiangshan.mem.Bundles._ 33import xiangshan.cache.mmu.{Pbmt, TlbCmd, TlbReq, TlbRequestIO, TlbResp} 34import xiangshan.cache.{DCacheStoreIO, DcacheStoreRequestIO, HasDCacheParameters, MemoryOpConstants, StorePrefetchReq} 35 36class StoreUnit(implicit p: Parameters) extends XSModule 37 with HasDCacheParameters 38 with HasVLSUParameters 39 { 40 val io = IO(new Bundle() { 41 val redirect = Flipped(ValidIO(new Redirect)) 42 val csrCtrl = Flipped(new CustomCSRCtrlIO) 43 val stin = Flipped(Decoupled(new MemExuInput)) 44 val issue = Valid(new MemExuInput) 45 // misalignBuffer issue path 46 val misalign_stin = Flipped(Decoupled(new LsPipelineBundle)) 47 val misalign_stout = Valid(new SqWriteBundle) 48 val tlb = new TlbRequestIO() 49 val dcache = new DCacheStoreIO 50 val pmp = Flipped(new PMPRespBundle()) 51 val lsq = ValidIO(new LsPipelineBundle) 52 val lsq_replenish = Output(new LsPipelineBundle()) 53 val feedback_slow = ValidIO(new RSFeedback) 54 val prefetch_req = Flipped(DecoupledIO(new StorePrefetchReq)) 55 // provide prefetch info to sms 56 val prefetch_train = ValidIO(new LsPrefetchTrainBundle()) 57 // speculative for gated control 58 val s1_prefetch_spec = Output(Bool()) 59 val s2_prefetch_spec = Output(Bool()) 60 val stld_nuke_query = Valid(new StoreNukeQueryBundle) 61 val stout = DecoupledIO(new MemExuOutput) // writeback store 62 val vecstout = DecoupledIO(new VecPipelineFeedbackIO(isVStore = true)) 63 // store mask, send to sq in store_s0 64 val st_mask_out = Valid(new StoreMaskBundle) 65 val debug_ls = Output(new DebugLsInfoBundle) 66 // vector 67 val vecstin = Flipped(Decoupled(new VecPipeBundle(isVStore = true))) 68 val vec_isFirstIssue = Input(Bool()) 69 // writeback to misalign buffer 70 val misalign_enq = new MisalignBufferEnqIO 71 // trigger 72 val fromCsrTrigger = Input(new CsrTriggerBundle) 73 74 val s0_s1_valid = Output(Bool()) 75 }) 76 77 PerfCCT.updateInstPos(io.stin.bits.uop.debug_seqNum, PerfCCT.InstPos.AtFU.id.U, io.stin.valid, clock, reset) 78 79 val s1_ready, s2_ready, s3_ready = WireInit(false.B) 80 81 // Pipeline 82 // -------------------------------------------------------------------------------- 83 // stage 0 84 // -------------------------------------------------------------------------------- 85 // generate addr, use addr to query DCache and DTLB 86 val s0_iss_valid = io.stin.valid 87 val s0_prf_valid = io.prefetch_req.valid && io.dcache.req.ready 88 val s0_vec_valid = io.vecstin.valid 89 val s0_ma_st_valid = io.misalign_stin.valid 90 val s0_valid = s0_iss_valid || s0_prf_valid || s0_vec_valid || s0_ma_st_valid 91 val s0_use_flow_ma = s0_ma_st_valid 92 val s0_use_flow_vec = s0_vec_valid && !s0_ma_st_valid 93 val s0_use_flow_rs = s0_iss_valid && !s0_vec_valid && !s0_ma_st_valid 94 val s0_use_flow_prf = s0_prf_valid && !s0_iss_valid && !s0_vec_valid && !s0_ma_st_valid 95 val s0_use_non_prf_flow = s0_use_flow_rs || s0_use_flow_vec || s0_use_flow_ma 96 val s0_stin = Mux(s0_use_flow_rs, io.stin.bits, 0.U.asTypeOf(io.stin.bits)) 97 val s0_vecstin = Mux(s0_use_flow_vec, io.vecstin.bits, 0.U.asTypeOf(io.vecstin.bits)) 98 val s0_uop = Mux( 99 s0_use_flow_ma, 100 io.misalign_stin.bits.uop, 101 Mux( 102 s0_use_flow_rs, 103 s0_stin.uop, 104 s0_vecstin.uop 105 ) 106 ) 107 val s0_isFirstIssue = Mux( 108 s0_use_flow_ma, 109 false.B, 110 s0_use_flow_rs && io.stin.bits.isFirstIssue || s0_use_flow_vec && io.vec_isFirstIssue 111 ) 112 val s0_size = Mux(s0_use_non_prf_flow, s0_uop.fuOpType(2,0), 0.U)// may broken if use it in feature 113 val s0_mem_idx = Mux(s0_use_non_prf_flow, s0_uop.sqIdx.value, 0.U) 114 val s0_rob_idx = Mux(s0_use_non_prf_flow, s0_uop.robIdx, 0.U.asTypeOf(s0_uop.robIdx)) 115 val s0_pc = Mux(s0_use_non_prf_flow, s0_uop.pc, 0.U) 116 val s0_instr_type = Mux(s0_use_non_prf_flow, STORE_SOURCE.U, DCACHE_PREFETCH_SOURCE.U) 117 val s0_wlineflag = Mux(s0_use_flow_rs, LSUOpType.isCboAll(s0_uop.fuOpType), false.B) 118 val s0_out = Wire(new LsPipelineBundle) 119 val s0_kill = s0_uop.robIdx.needFlush(io.redirect) 120 val s0_can_go = s1_ready 121 val s0_fire = s0_valid && !s0_kill && s0_can_go 122 val s0_is128bit = Wire(Bool()) 123 // vector 124 val s0_vecActive = !s0_use_flow_vec || s0_vecstin.vecActive 125 // val s0_flowPtr = s0_vecstin.flowPtr 126 // val s0_isLastElem = s0_vecstin.isLastElem 127 val s0_secondInv = s0_vecstin.usSecondInv 128 val s0_elemIdx = s0_vecstin.elemIdx 129 val s0_alignedType = s0_vecstin.alignedType 130 val s0_mBIndex = s0_vecstin.mBIndex 131 val s0_vecBaseVaddr = s0_vecstin.basevaddr 132 val s0_isFinalSplit = io.misalign_stin.valid && io.misalign_stin.bits.isFinalSplit 133 134 // generate addr 135 val s0_saddr = s0_stin.src(0) + SignExt(s0_stin.uop.imm(11,0), VAddrBits) 136 val s0_fullva = Wire(UInt(XLEN.W)) 137 138 val s0_vaddr = Mux( 139 s0_use_flow_ma, 140 io.misalign_stin.bits.vaddr, 141 Mux( 142 s0_use_flow_rs, 143 s0_saddr, 144 Mux( 145 s0_use_flow_vec, 146 s0_vecstin.vaddr(VAddrBits - 1, 0), 147 io.prefetch_req.bits.vaddr 148 ) 149 ) 150 ) 151 152 val s0_isCbo = s0_use_flow_rs && LSUOpType.isCboAll(s0_stin.uop.fuOpType) 153 // only simulation 154 val cbo_assert_flag = LSUOpType.isCboAll(s0_out.uop.fuOpType) 155 XSError(!s0_use_flow_rs && cbo_assert_flag && s0_valid, "cbo instruction selection error.") 156 157 val s0_alignType = Mux(s0_use_flow_vec, s0_vecstin.alignedType(1,0), s0_uop.fuOpType(1, 0)) 158 // exception check 159 val s0_addr_aligned = LookupTree(s0_alignType, List( 160 "b00".U -> true.B, //b 161 "b01".U -> (s0_vaddr(0) === 0.U), //h 162 "b10".U -> (s0_vaddr(1,0) === 0.U), //w 163 "b11".U -> (s0_vaddr(2,0) === 0.U) //d 164 )) || s0_isCbo 165 // if vector store sends 128-bit requests, its address must be 128-aligned 166 XSError(s0_use_flow_vec && s0_vaddr(3, 0) =/= 0.U && s0_vecstin.alignedType(2), "unit stride 128 bit element is not aligned!") 167 168 val s0_isMisalign = Mux(s0_use_non_prf_flow, (!s0_addr_aligned || s0_vecstin.uop.exceptionVec(storeAddrMisaligned) && s0_vecActive), false.B) 169 val s0_addr_low = s0_vaddr(4, 0) 170 val s0_addr_Up_low = LookupTree(s0_alignType, List( 171 "b00".U -> 0.U, 172 "b01".U -> 1.U, 173 "b10".U -> 3.U, 174 "b11".U -> 7.U 175 )) + s0_addr_low 176 val s0_rs_corss16Bytes = s0_addr_Up_low(4) =/= s0_addr_low(4) 177 val s0_misalignWith16Byte = !s0_rs_corss16Bytes && !s0_addr_aligned && !s0_use_flow_prf 178 s0_is128bit := Mux(s0_use_flow_ma, io.misalign_stin.bits.is128bit, is128Bit(s0_vecstin.alignedType) || s0_misalignWith16Byte) 179 180 s0_fullva := Mux( 181 s0_use_flow_rs, 182 s0_stin.src(0) + SignExt(s0_stin.uop.imm(11,0), XLEN), 183 Mux( 184 s0_use_flow_vec, 185 s0_vecstin.vaddr, 186 s0_vaddr 187 ) 188 ) 189 190 val s0_mask = Mux( 191 s0_use_flow_ma, 192 io.misalign_stin.bits.mask, 193 Mux( 194 s0_use_flow_rs, 195 genVWmask128(s0_saddr, s0_uop.fuOpType(2,0)), 196 Mux( 197 s0_use_flow_vec, 198 s0_vecstin.mask, 199 // -1.asSInt.asUInt 200 Fill(VLEN/8, 1.U(1.W)) 201 ) 202 ) 203 ) 204 205 io.tlb.req.valid := s0_valid 206 io.tlb.req.bits.vaddr := s0_vaddr 207 io.tlb.req.bits.fullva := s0_fullva 208 io.tlb.req.bits.checkfullva := s0_use_flow_rs || s0_use_flow_vec 209 io.tlb.req.bits.cmd := TlbCmd.write 210 io.tlb.req.bits.isPrefetch := s0_use_flow_prf 211 io.tlb.req.bits.size := s0_size 212 io.tlb.req.bits.kill := false.B 213 io.tlb.req.bits.memidx.is_ld := false.B 214 io.tlb.req.bits.memidx.is_st := true.B 215 io.tlb.req.bits.memidx.idx := s0_mem_idx 216 io.tlb.req.bits.debug.robIdx := s0_rob_idx 217 io.tlb.req.bits.no_translate := false.B 218 io.tlb.req.bits.debug.pc := s0_pc 219 io.tlb.req.bits.debug.isFirstIssue := s0_isFirstIssue 220 io.tlb.req_kill := false.B 221 io.tlb.req.bits.hyperinst := LSUOpType.isHsv(s0_uop.fuOpType) 222 io.tlb.req.bits.hlvx := false.B 223 io.tlb.req.bits.pmp_addr := DontCare 224 225 // Dcache access here: not **real** dcache write 226 // just read meta and tag in dcache, to find out the store will hit or miss 227 228 // NOTE: The store request does not wait for the dcache to be ready. 229 // If the dcache is not ready at this time, the dcache is not queried. 230 // But, store prefetch request will always wait for dcache to be ready to make progress. 231 io.dcache.req.valid := s0_fire 232 io.dcache.req.bits.cmd := MemoryOpConstants.M_PFW 233 io.dcache.req.bits.vaddr := s0_vaddr 234 io.dcache.req.bits.instrtype := s0_instr_type 235 236 s0_out := DontCare 237 s0_out.vaddr := s0_vaddr 238 s0_out.fullva := s0_fullva 239 // Now data use its own io 240 s0_out.data := s0_stin.src(1) 241 s0_out.uop := s0_uop 242 s0_out.miss := false.B 243 // For unaligned, we need to generate a base-aligned mask in storeunit and then do a shift split in StoreQueue. 244 s0_out.mask := Mux(s0_rs_corss16Bytes && !s0_addr_aligned, genBasemask(s0_saddr,s0_alignType(1,0)), s0_mask) 245 s0_out.isFirstIssue := s0_isFirstIssue 246 s0_out.isHWPrefetch := s0_use_flow_prf 247 s0_out.wlineflag := s0_wlineflag 248 s0_out.isvec := s0_use_flow_vec 249 s0_out.is128bit := s0_is128bit 250 s0_out.vecActive := s0_vecActive 251 s0_out.usSecondInv := s0_secondInv 252 s0_out.elemIdx := s0_elemIdx 253 s0_out.alignedType := s0_alignedType 254 s0_out.mbIndex := s0_mBIndex 255 s0_out.misalignWith16Byte := s0_misalignWith16Byte 256 s0_out.isMisalign := s0_isMisalign 257 s0_out.vecBaseVaddr := s0_vecBaseVaddr 258 when(s0_valid && s0_isFirstIssue) { 259 s0_out.uop.debugInfo.tlbFirstReqTime := GTimer() 260 } 261 s0_out.isFrmMisAlignBuf := s0_use_flow_ma 262 s0_out.isFinalSplit := s0_isFinalSplit 263// s0_out.uop.exceptionVec(storeAddrMisaligned) := Mux(s0_use_non_prf_flow, (!s0_addr_aligned || s0_vecstin.uop.exceptionVec(storeAddrMisaligned) && s0_vecActive), false.B) && !s0_misalignWith16Byte 264 265 io.st_mask_out.valid := s0_use_flow_rs || s0_use_flow_vec 266 io.st_mask_out.bits.mask := s0_out.mask 267 io.st_mask_out.bits.sqIdx := s0_out.uop.sqIdx 268 269 io.stin.ready := s1_ready && s0_use_flow_rs 270 io.vecstin.ready := s1_ready && s0_use_flow_vec 271 io.prefetch_req.ready := s1_ready && io.dcache.req.ready && !s0_iss_valid && !s0_vec_valid && !s0_ma_st_valid 272 io.misalign_stin.ready := s1_ready && s0_use_flow_ma 273 274 // Pipeline 275 // -------------------------------------------------------------------------------- 276 // stage 1 277 // -------------------------------------------------------------------------------- 278 // TLB resp (send paddr to dcache) 279 val s1_valid = RegInit(false.B) 280 val s1_in = RegEnable(s0_out, s0_fire) 281 val s1_out = Wire(new LsPipelineBundle) 282 val s1_kill = Wire(Bool()) 283 val s1_can_go = s2_ready 284 val s1_fire = s1_valid && !s1_kill && s1_can_go 285 val s1_vecActive = RegEnable(s0_out.vecActive, true.B, s0_fire) 286 val s1_frm_mabuf = s1_in.isFrmMisAlignBuf 287 val s1_is128bit = s1_in.is128bit 288 289 // mmio cbo decoder 290 val s1_isCbo = RegEnable(s0_isCbo, s0_fire) 291 val s1_vaNeedExt = io.tlb.resp.bits.excp(0).vaNeedExt 292 val s1_isHyper = io.tlb.resp.bits.excp(0).isHyper 293 val s1_paddr = io.tlb.resp.bits.paddr(0) 294 val s1_gpaddr = io.tlb.resp.bits.gpaddr(0) 295 val s1_fullva = io.tlb.resp.bits.fullva 296 val s1_isForVSnonLeafPTE = io.tlb.resp.bits.isForVSnonLeafPTE 297 val s1_tlb_miss = io.tlb.resp.bits.miss && io.tlb.resp.valid && s1_valid 298 val s1_pbmt = Mux(!s1_tlb_miss, io.tlb.resp.bits.pbmt.head, 0.U(Pbmt.width.W)) 299 val s1_exception = ExceptionNO.selectByFu(s1_out.uop.exceptionVec, StaCfg).asUInt.orR 300 val s1_isvec = RegEnable(s0_out.isvec, false.B, s0_fire) 301 //We don't want `StoreUnit` to have an additional effect on the Store of vector from a `misalignBuffer,` 302 //But there are places where a marker bit is needed to enable additional processing of vector instructions. 303 //For example: `StoreQueue` is exceptionBuffer 304 val s1_frm_mab_vec = RegEnable(s0_use_flow_ma && io.misalign_stin.bits.isvec, false.B, s0_fire) 305 // val s1_isLastElem = RegEnable(s0_isLastElem, false.B, s0_fire) 306 s1_kill := s1_in.uop.robIdx.needFlush(io.redirect) || (s1_tlb_miss && !s1_isvec && !s1_frm_mabuf) 307 308 s1_ready := !s1_valid || s1_kill || s2_ready 309 io.tlb.resp.ready := true.B // TODO: why dtlbResp needs a ready? 310 when (s0_fire) { s1_valid := true.B } 311 .elsewhen (s1_fire) { s1_valid := false.B } 312 .elsewhen (s1_kill) { s1_valid := false.B } 313 314 // st-ld violation dectect request. 315 io.stld_nuke_query.valid := s1_valid && !s1_tlb_miss && !s1_in.isHWPrefetch 316 io.stld_nuke_query.bits.robIdx := s1_in.uop.robIdx 317 io.stld_nuke_query.bits.paddr := s1_paddr 318 io.stld_nuke_query.bits.mask := s1_in.mask 319 io.stld_nuke_query.bits.matchLine := (s1_in.isvec || s1_in.misalignWith16Byte) && s1_in.is128bit 320 321 // issue 322 io.issue.valid := s1_valid && !s1_tlb_miss && !s1_in.isHWPrefetch && !s1_isvec && !s1_frm_mabuf 323 io.issue.bits := RegEnable(s0_stin, s0_valid) 324 325 // trigger 326 val storeTrigger = Module(new MemTrigger(MemType.STORE)) 327 storeTrigger.io.fromCsrTrigger.tdataVec := io.fromCsrTrigger.tdataVec 328 storeTrigger.io.fromCsrTrigger.tEnableVec := io.fromCsrTrigger.tEnableVec 329 storeTrigger.io.fromCsrTrigger.triggerCanRaiseBpExp := io.fromCsrTrigger.triggerCanRaiseBpExp 330 storeTrigger.io.fromCsrTrigger.debugMode := io.fromCsrTrigger.debugMode 331 storeTrigger.io.fromLoadStore.vaddr := s1_in.vaddr 332 storeTrigger.io.fromLoadStore.isVectorUnitStride := s1_in.isvec && s1_in.is128bit 333 storeTrigger.io.fromLoadStore.mask := s1_in.mask 334 storeTrigger.io.isCbo.get := s1_isCbo 335 336 val s1_trigger_action = storeTrigger.io.toLoadStore.triggerAction 337 val s1_trigger_debug_mode = TriggerAction.isDmode(s1_trigger_action) 338 val s1_trigger_breakpoint = TriggerAction.isExp(s1_trigger_action) 339 340 // for misalign in vsMergeBuffer 341 io.s0_s1_valid := s0_valid || s1_valid 342 343 // Send TLB feedback to store issue queue 344 // Store feedback is generated in store_s1, sent to RS in store_s2 345 val s1_feedback = Wire(Valid(new RSFeedback)) 346 s1_feedback.valid := s1_valid & !s1_in.isHWPrefetch 347 s1_feedback.bits.hit := !s1_tlb_miss 348 s1_feedback.bits.flushState := io.tlb.resp.bits.ptwBack 349 s1_feedback.bits.robIdx := s1_out.uop.robIdx 350 s1_feedback.bits.sourceType := RSFeedbackType.tlbMiss 351 s1_feedback.bits.dataInvalidSqIdx := DontCare 352 s1_feedback.bits.sqIdx := s1_out.uop.sqIdx 353 s1_feedback.bits.lqIdx := s1_out.uop.lqIdx 354 355 XSDebug(s1_feedback.valid, 356 "S1 Store: tlbHit: %d robIdx: %d\n", 357 s1_feedback.bits.hit, 358 s1_feedback.bits.robIdx.value 359 ) 360 361 // io.feedback_slow := s1_feedback 362 363 // get paddr from dtlb, check if rollback is needed 364 // writeback store inst to lsq 365 s1_out := s1_in 366 s1_out.paddr := s1_paddr 367 s1_out.gpaddr := s1_gpaddr 368 s1_out.fullva := s1_fullva 369 s1_out.vaNeedExt := s1_vaNeedExt 370 s1_out.isHyper := s1_isHyper 371 s1_out.miss := false.B 372 s1_out.nc := Pbmt.isNC(s1_pbmt) 373 s1_out.mmio := Pbmt.isIO(s1_pbmt) 374 s1_out.tlbMiss := s1_tlb_miss 375 s1_out.atomic := Pbmt.isIO(s1_pbmt) 376 s1_out.isForVSnonLeafPTE := s1_isForVSnonLeafPTE 377 when (RegNext(io.tlb.req.bits.checkfullva) && 378 (s1_out.uop.exceptionVec(storePageFault) || 379 s1_out.uop.exceptionVec(storeAccessFault) || 380 s1_out.uop.exceptionVec(storeGuestPageFault))) { 381 s1_out.uop.exceptionVec(storeAddrMisaligned) := false.B 382 } 383 s1_out.uop.exceptionVec(storePageFault) := io.tlb.resp.bits.excp(0).pf.st && s1_vecActive 384 s1_out.uop.exceptionVec(storeAccessFault) := io.tlb.resp.bits.excp(0).af.st && s1_vecActive 385 s1_out.uop.exceptionVec(storeGuestPageFault) := io.tlb.resp.bits.excp(0).gpf.st && s1_vecActive 386 387 s1_out.uop.flushPipe := false.B 388 s1_out.uop.trigger := s1_trigger_action 389 s1_out.uop.exceptionVec(breakPoint) := s1_trigger_breakpoint 390 s1_out.uop.exceptionVec(storeAddrMisaligned) := s1_out.mmio && s1_in.isMisalign 391 s1_out.vecVaddrOffset := Mux( 392 s1_trigger_debug_mode || s1_trigger_breakpoint, 393 storeTrigger.io.toLoadStore.triggerVaddr - s1_in.vecBaseVaddr, 394 s1_in.vaddr + genVFirstUnmask(s1_in.mask).asUInt - s1_in.vecBaseVaddr , 395 ) 396 s1_out.vecTriggerMask := Mux(s1_trigger_debug_mode || s1_trigger_breakpoint, storeTrigger.io.toLoadStore.triggerMask, 0.U) 397 398 // scalar store and scalar load nuke check, and also other purposes 399 //A 128-bit aligned unaligned memory access requires changing the unaligned flag bit in sq 400 io.lsq.valid := s1_valid && !s1_in.isHWPrefetch 401 io.lsq.bits := s1_out 402 io.lsq.bits.miss := s1_tlb_miss 403 io.lsq.bits.isvec := s1_out.isvec || s1_frm_mab_vec 404 io.lsq.bits.updateAddrValid := (!s1_in.isMisalign || s1_in.misalignWith16Byte) && (!s1_frm_mabuf || s1_in.isFinalSplit) || s1_exception 405 // kill dcache write intent request when tlb miss or exception 406 io.dcache.s1_kill := (s1_tlb_miss || s1_exception || s1_out.mmio || s1_in.uop.robIdx.needFlush(io.redirect)) 407 io.dcache.s1_paddr := s1_paddr 408 409 // write below io.out.bits assign sentence to prevent overwriting values 410 val s1_tlb_memidx = io.tlb.resp.bits.memidx 411 when(s1_tlb_memidx.is_st && io.tlb.resp.valid && !s1_tlb_miss && s1_tlb_memidx.idx === s1_out.uop.sqIdx.value) { 412 // printf("Store idx = %d\n", s1_tlb_memidx.idx) 413 s1_out.uop.debugInfo.tlbRespTime := GTimer() 414 } 415 val s1_mis_align = s1_valid && !s1_tlb_miss && !s1_in.isHWPrefetch && !s1_isCbo && !s1_out.nc && !s1_out.mmio && 416 GatedValidRegNext(io.csrCtrl.hd_misalign_st_enable) && s1_in.isMisalign && !s1_in.misalignWith16Byte && 417 !s1_trigger_breakpoint && !s1_trigger_debug_mode 418 val s1_toMisalignBufferValid = s1_valid && !s1_tlb_miss && !s1_in.isHWPrefetch && 419 !s1_frm_mabuf && !s1_isCbo && s1_in.isMisalign && !s1_in.misalignWith16Byte && 420 GatedValidRegNext(io.csrCtrl.hd_misalign_st_enable) 421 io.misalign_enq.req.valid := s1_toMisalignBufferValid 422 io.misalign_enq.req.bits.fromLsPipelineBundle(s1_in) 423 424 // Pipeline 425 // -------------------------------------------------------------------------------- 426 // stage 2 427 // -------------------------------------------------------------------------------- 428 // mmio check 429 val s2_valid = RegInit(false.B) 430 val s2_in = RegEnable(s1_out, s1_fire) 431 val s2_out = Wire(new LsPipelineBundle) 432 val s2_kill = Wire(Bool()) 433 val s2_can_go = s3_ready 434 val s2_fire = s2_valid && !s2_kill && s2_can_go 435 val s2_vecActive = RegEnable(s1_out.vecActive, true.B, s1_fire) 436 val s2_frm_mabuf = s2_in.isFrmMisAlignBuf 437 val s2_frm_mab_vec = RegEnable(s1_frm_mab_vec, true.B, s1_fire) 438 val s2_pbmt = RegEnable(s1_pbmt, s1_fire) 439 val s2_trigger_debug_mode = RegEnable(s1_trigger_debug_mode, false.B, s1_fire) 440 441 s2_ready := !s2_valid || s2_kill || s3_ready 442 when (s1_fire) { s2_valid := true.B } 443 .elsewhen (s2_fire) { s2_valid := false.B } 444 .elsewhen (s2_kill) { s2_valid := false.B } 445 446 val s2_pmp = WireInit(io.pmp) 447 448 val s2_exception = RegNext(s1_feedback.bits.hit) && 449 (s2_trigger_debug_mode || ExceptionNO.selectByFu(s2_out.uop.exceptionVec, StaCfg).asUInt.orR) && s2_vecActive 450 val s2_un_misalign_exception = RegNext(s1_feedback.bits.hit) && 451 (s2_trigger_debug_mode || ExceptionNO.selectByFuAndUnSelect(s2_out.uop.exceptionVec, StaCfg, Seq(storeAddrMisaligned)).asUInt.orR) 452 453 val s2_mmio = (s2_in.mmio || (Pbmt.isPMA(s2_pbmt) && s2_pmp.mmio)) && RegNext(s1_feedback.bits.hit) 454 val s2_memBackTypeMM = !s2_pmp.mmio 455 // The response signal of `pmp/pma` is credible only after the physical address is actually generated. 456 // Therefore, the response signals of pmp/pma generated after an address translation has produced an `access fault` or a `page fault` are completely unreliable. 457 val s2_un_access_exception = s2_vecActive && ( 458 s2_in.uop.exceptionVec(storeAccessFault) || 459 s2_in.uop.exceptionVec(storePageFault) || 460 s2_in.uop.exceptionVec(storeGuestPageFault) 461 ) 462 // This real physical address is located in uncache space. 463 val s2_actually_uncache = !s2_in.tlbMiss && !s2_un_access_exception && (Pbmt.isPMA(s2_pbmt) && s2_pmp.mmio || s2_in.nc || s2_in.mmio) && RegNext(s1_feedback.bits.hit) 464 val s2_isCbo = RegEnable(s1_isCbo, s1_fire) // all cbo instr 465 val s2_isCbo_noZero = LSUOpType.isCbo(s2_in.uop.fuOpType) 466 467 s2_kill := ((s2_mmio && !s2_exception) && !s2_in.isvec && !s2_frm_mabuf) || s2_in.uop.robIdx.needFlush(io.redirect) 468 469 s2_out := s2_in 470 s2_out.af := s2_out.uop.exceptionVec(storeAccessFault) 471 s2_out.mmio := s2_mmio && !s2_exception 472 s2_out.atomic := s2_in.atomic || Pbmt.isPMA(s2_pbmt) && s2_pmp.atomic 473 s2_out.memBackTypeMM := s2_memBackTypeMM 474 s2_out.uop.exceptionVec(storeAccessFault) := (s2_in.uop.exceptionVec(storeAccessFault) || 475 s2_pmp.st || 476 ((s2_in.isvec || s2_isCbo) && s2_actually_uncache && RegNext(s1_feedback.bits.hit)) 477 ) && s2_vecActive 478 s2_out.uop.exceptionVec(storeAddrMisaligned) := s2_actually_uncache && s2_in.isMisalign && !s2_un_misalign_exception 479 s2_out.uop.vpu.vstart := s2_in.vecVaddrOffset >> s2_in.uop.vpu.veew 480 481 // kill dcache write intent request when mmio or exception 482 io.dcache.s2_kill := (s2_actually_uncache || s2_exception || s2_in.uop.robIdx.needFlush(io.redirect)) 483 io.dcache.s2_pc := s2_out.uop.pc 484 // TODO: dcache resp 485 io.dcache.resp.ready := true.B 486 487 val s2_mis_align = s2_valid && RegEnable(s1_mis_align, s1_fire) && !s2_exception 488 // goto misalignBuffer 489 io.misalign_enq.revoke := s2_exception 490 val s2_misalignBufferNack = !io.misalign_enq.revoke && 491 RegEnable(s1_toMisalignBufferValid && !io.misalign_enq.req.ready, false.B, s1_fire) 492 493 // feedback tlb miss to RS in store_s2 494 val feedback_slow_valid = WireInit(false.B) 495 496 feedback_slow_valid := s1_feedback.valid && !s1_out.uop.robIdx.needFlush(io.redirect) && !s1_out.isvec && !s1_frm_mabuf 497 io.feedback_slow.valid := GatedValidRegNext(feedback_slow_valid) 498 io.feedback_slow.bits := RegEnable(s1_feedback.bits, feedback_slow_valid) 499 io.feedback_slow.bits.hit := RegEnable(s1_feedback.bits.hit, feedback_slow_valid) && !s2_misalignBufferNack 500 501 val s2_vecFeedback = RegNext(!s1_out.uop.robIdx.needFlush(io.redirect) && s1_feedback.bits.hit && s1_feedback.valid) && 502 !s2_misalignBufferNack && s2_in.isvec && !s2_frm_mabuf 503 504 val s2_misalign_stout = WireInit(0.U.asTypeOf(io.misalign_stout)) 505 s2_misalign_stout.valid := s2_valid && s2_can_go && s2_frm_mabuf 506 connectSamePort(s2_misalign_stout.bits, s2_out) 507 s2_misalign_stout.bits.need_rep := RegEnable(s1_tlb_miss, s1_fire) 508 io.misalign_stout := s2_misalign_stout 509 510 // mmio and exception 511 io.lsq_replenish := s2_out 512 io.lsq_replenish.af := s2_out.af && s2_valid && !s2_kill 513 io.lsq_replenish.mmio := (s2_mmio || s2_isCbo_noZero) && !s2_exception // reuse `mmiostall` logic in sq 514 515 // prefetch related 516 io.lsq_replenish.miss := io.dcache.resp.fire && io.dcache.resp.bits.miss // miss info 517 io.lsq_replenish.updateAddrValid := !s2_mis_align && (!s2_frm_mabuf || s2_out.isFinalSplit) || s2_exception 518 io.lsq_replenish.isvec := s2_out.isvec || s2_frm_mab_vec 519 520 io.lsq_replenish.hasException := (ExceptionNO.selectByFu(s2_out.uop.exceptionVec, StaCfg).asUInt.orR || 521 TriggerAction.isDmode(s2_out.uop.trigger) || s2_out.af) && s2_valid && !s2_kill 522 523 524 // RegNext prefetch train for better timing 525 // ** Now, prefetch train is valid at store s3 ** 526 val s2_prefetch_train_valid = WireInit(false.B) 527 s2_prefetch_train_valid := s2_valid && io.dcache.resp.fire && !s2_out.mmio && !s2_out.nc && !s2_in.tlbMiss && !s2_in.isHWPrefetch 528 if(EnableStorePrefetchSMS) { 529 io.s1_prefetch_spec := s1_fire 530 io.s2_prefetch_spec := s2_prefetch_train_valid 531 io.prefetch_train.valid := RegNext(s2_prefetch_train_valid) 532 io.prefetch_train.bits.fromLsPipelineBundle(s2_in, latch = true, enable = s2_prefetch_train_valid) 533 }else { 534 io.s1_prefetch_spec := false.B 535 io.s2_prefetch_spec := false.B 536 io.prefetch_train.valid := false.B 537 io.prefetch_train.bits.fromLsPipelineBundle(s2_in, latch = true, enable = false.B) 538 } 539 // override miss bit 540 io.prefetch_train.bits.miss := RegEnable(io.dcache.resp.bits.miss, s2_prefetch_train_valid) 541 // TODO: add prefetch and access bit 542 io.prefetch_train.bits.meta_prefetch := false.B 543 io.prefetch_train.bits.meta_access := false.B 544 io.prefetch_train.bits.isFinalSplit := false.B 545 io.prefetch_train.bits.misalignWith16Byte := false.B 546 io.prefetch_train.bits.isMisalign := false.B 547 io.prefetch_train.bits.misalignNeedWakeUp := false.B 548 io.prefetch_train.bits.updateAddrValid := false.B 549 io.prefetch_train.bits.hasException := false.B 550 551 // Pipeline 552 // -------------------------------------------------------------------------------- 553 // stage 3 554 // -------------------------------------------------------------------------------- 555 // store write back 556 val s3_valid = RegInit(false.B) 557 val s3_in = RegEnable(s2_out, s2_fire) 558 val s3_out = Wire(new MemExuOutput(isVector = true)) 559 val s3_kill = s3_in.uop.robIdx.needFlush(io.redirect) 560 val s3_can_go = s3_ready 561 val s3_fire = s3_valid && !s3_kill && s3_can_go 562 val s3_vecFeedback = RegEnable(s2_vecFeedback, s2_fire) 563 val s3_exception = RegEnable(s2_exception, s2_fire) 564 565 // store misalign will not writeback to rob now 566 when (s2_fire) { s3_valid := (!s2_mmio && !s2_isCbo || s2_exception) && !s2_out.isHWPrefetch && !s2_mis_align && !s2_frm_mabuf } 567 .elsewhen (s3_fire) { s3_valid := false.B } 568 .elsewhen (s3_kill) { s3_valid := false.B } 569 570 // wb: writeback 571 572 s3_out := DontCare 573 s3_out.uop := s3_in.uop 574 s3_out.data := DontCare 575 s3_out.debug.isMMIO := s3_in.mmio 576 s3_out.debug.isNC := s3_in.nc 577 s3_out.debug.paddr := s3_in.paddr 578 s3_out.debug.vaddr := s3_in.vaddr 579 s3_out.debug.isPerfCnt := false.B 580 581 XSError(s3_valid && s3_in.isvec && s3_in.vecActive && !s3_in.mask.orR, "In vecActive, mask complement should not be 0") 582 // Pipeline 583 // -------------------------------------------------------------------------------- 584 // stage x 585 // -------------------------------------------------------------------------------- 586 val sx_valid = Wire(Vec(RAWTotalDelayCycles + 1, Bool())) 587 val sx_ready = Wire(Vec(RAWTotalDelayCycles + 1, Bool())) 588 val sx_in = Wire(Vec(RAWTotalDelayCycles + 1, new VecMemExuOutput(isVector = true))) 589 val sx_in_vec = Wire(Vec(RAWTotalDelayCycles +1, Bool())) 590 591 // backward ready signal 592 s3_ready := sx_ready.head 593 for (i <- 0 until RAWTotalDelayCycles + 1) { 594 if (i == 0) { 595 sx_valid(i) := s3_valid 596 sx_in(i).output := s3_out 597 sx_in(i).vecFeedback := s3_vecFeedback 598 sx_in(i).nc := s3_in.nc 599 sx_in(i).mmio := s3_in.mmio 600 sx_in(i).usSecondInv := s3_in.usSecondInv 601 sx_in(i).elemIdx := s3_in.elemIdx 602 sx_in(i).alignedType := s3_in.alignedType 603 sx_in(i).mbIndex := s3_in.mbIndex 604 sx_in(i).mask := s3_in.mask 605 sx_in(i).vaddr := s3_in.fullva 606 sx_in(i).vaNeedExt := s3_in.vaNeedExt 607 sx_in(i).gpaddr := s3_in.gpaddr 608 sx_in(i).isForVSnonLeafPTE := s3_in.isForVSnonLeafPTE 609 sx_in(i).vecTriggerMask := s3_in.vecTriggerMask 610 sx_in(i).hasException := s3_exception 611 sx_in_vec(i) := s3_in.isvec 612 sx_ready(i) := !s3_valid(i) || sx_in(i).output.uop.robIdx.needFlush(io.redirect) || (if (RAWTotalDelayCycles == 0) io.stout.ready else sx_ready(i+1)) 613 } else { 614 val cur_kill = sx_in(i).output.uop.robIdx.needFlush(io.redirect) 615 val cur_can_go = (if (i == RAWTotalDelayCycles) io.stout.ready else sx_ready(i+1)) 616 val cur_fire = sx_valid(i) && !cur_kill && cur_can_go 617 val prev_fire = sx_valid(i-1) && !sx_in(i-1).output.uop.robIdx.needFlush(io.redirect) && sx_ready(i) 618 619 sx_ready(i) := !sx_valid(i) || cur_kill || (if (i == RAWTotalDelayCycles) io.stout.ready else sx_ready(i+1)) 620 val sx_valid_can_go = prev_fire || cur_fire || cur_kill 621 sx_valid(i) := RegEnable(Mux(prev_fire, true.B, false.B), false.B, sx_valid_can_go) 622 sx_in(i) := RegEnable(sx_in(i-1), prev_fire) 623 sx_in_vec(i) := RegEnable(sx_in_vec(i-1), prev_fire) 624 } 625 } 626 val sx_last_valid = sx_valid.takeRight(1).head 627 val sx_last_ready = sx_ready.takeRight(1).head 628 val sx_last_in = sx_in.takeRight(1).head 629 val sx_last_in_vec = sx_in_vec.takeRight(1).head 630 sx_last_ready := !sx_last_valid || sx_last_in.output.uop.robIdx.needFlush(io.redirect) || io.stout.ready 631 632 // write back: normal store, nc store 633 io.stout.valid := sx_last_valid && !sx_last_in_vec //isStore(sx_last_in.output.uop.fuType) 634 io.stout.bits := sx_last_in.output 635 io.stout.bits.uop.exceptionVec := ExceptionNO.selectByFu(sx_last_in.output.uop.exceptionVec, StaCfg) 636 637 io.vecstout.valid := sx_last_valid && sx_last_in_vec //isVStore(sx_last_in.output.uop.fuType) 638 // TODO: implement it! 639 io.vecstout.bits.mBIndex := sx_last_in.mbIndex 640 io.vecstout.bits.hit := sx_last_in.vecFeedback 641 io.vecstout.bits.isvec := true.B 642 io.vecstout.bits.sourceType := RSFeedbackType.tlbMiss 643 io.vecstout.bits.flushState := DontCare 644 io.vecstout.bits.trigger := sx_last_in.output.uop.trigger 645 io.vecstout.bits.nc := sx_last_in.nc 646 io.vecstout.bits.mmio := sx_last_in.mmio 647 io.vecstout.bits.exceptionVec := ExceptionNO.selectByFu(sx_last_in.output.uop.exceptionVec, VstuCfg) 648 io.vecstout.bits.hasException := sx_last_in.hasException 649 io.vecstout.bits.usSecondInv := sx_last_in.usSecondInv 650 io.vecstout.bits.vecFeedback := sx_last_in.vecFeedback 651 io.vecstout.bits.elemIdx := sx_last_in.elemIdx 652 io.vecstout.bits.alignedType := sx_last_in.alignedType 653 io.vecstout.bits.mask := sx_last_in.mask 654 io.vecstout.bits.vaddr := sx_last_in.vaddr 655 io.vecstout.bits.vaNeedExt := sx_last_in.vaNeedExt 656 io.vecstout.bits.gpaddr := sx_last_in.gpaddr 657 io.vecstout.bits.isForVSnonLeafPTE := sx_last_in.isForVSnonLeafPTE 658 io.vecstout.bits.vstart := sx_last_in.output.uop.vpu.vstart 659 io.vecstout.bits.vecTriggerMask := sx_last_in.vecTriggerMask 660 // io.vecstout.bits.reg_offset.map(_ := DontCare) 661 // io.vecstout.bits.elemIdx.map(_ := sx_last_in.elemIdx) 662 // io.vecstout.bits.elemIdxInsideVd.map(_ := DontCare) 663 // io.vecstout.bits.vecdata.map(_ := DontCare) 664 // io.vecstout.bits.mask.map(_ := DontCare) 665 // io.vecstout.bits.alignedType.map(_ := sx_last_in.alignedType) 666 667 io.debug_ls := DontCare 668 io.debug_ls.s1_robIdx := s1_in.uop.robIdx.value 669 io.debug_ls.s1_isTlbFirstMiss := io.tlb.resp.valid && io.tlb.resp.bits.miss && io.tlb.resp.bits.debug.isFirstIssue && !s1_in.isHWPrefetch 670 671 private def printPipeLine(pipeline: LsPipelineBundle, cond: Bool, name: String): Unit = { 672 XSDebug(cond, 673 p"$name" + p" pc ${Hexadecimal(pipeline.uop.pc)} " + 674 p"addr ${Hexadecimal(pipeline.vaddr)} -> ${Hexadecimal(pipeline.paddr)} " + 675 p"op ${Binary(pipeline.uop.fuOpType)} " + 676 p"data ${Hexadecimal(pipeline.data)} " + 677 p"mask ${Hexadecimal(pipeline.mask)}\n" 678 ) 679 } 680 681 printPipeLine(s0_out, s0_valid, "S0") 682 printPipeLine(s1_out, s1_valid, "S1") 683 684 // perf cnt 685 XSPerfAccumulate("s0_in_valid", s0_valid) 686 XSPerfAccumulate("s0_in_fire", s0_fire) 687 XSPerfAccumulate("s0_vecin_fire", s0_fire && s0_use_flow_vec) 688 XSPerfAccumulate("s0_in_fire_first_issue", s0_fire && s0_isFirstIssue) 689 XSPerfAccumulate("s0_addr_spec_success", s0_fire && !s0_use_flow_vec && s0_saddr(VAddrBits-1, 12) === s0_stin.src(0)(VAddrBits-1, 12)) 690 XSPerfAccumulate("s0_addr_spec_failed", s0_fire && !s0_use_flow_vec && s0_saddr(VAddrBits-1, 12) =/= s0_stin.src(0)(VAddrBits-1, 12)) 691 XSPerfAccumulate("s0_addr_spec_success_once", s0_fire && !s0_use_flow_vec && s0_saddr(VAddrBits-1, 12) === s0_stin.src(0)(VAddrBits-1, 12) && s0_isFirstIssue) 692 XSPerfAccumulate("s0_addr_spec_failed_once", s0_fire && !s0_use_flow_vec && s0_saddr(VAddrBits-1, 12) =/= s0_stin.src(0)(VAddrBits-1, 12) && s0_isFirstIssue) 693 694 XSPerfAccumulate("s1_in_valid", s1_valid) 695 XSPerfAccumulate("s1_in_fire", s1_fire) 696 XSPerfAccumulate("s1_in_fire_first_issue", s1_fire && s1_in.isFirstIssue) 697 XSPerfAccumulate("s1_tlb_miss", s1_fire && s1_tlb_miss) 698 XSPerfAccumulate("s1_tlb_miss_first_issue", s1_fire && s1_tlb_miss && s1_in.isFirstIssue) 699 // end 700} 701