Lines Matching +full:adreno +full:- +full:gmu
1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2017-2019 The Linux Foundation. All rights reserved. */
15 #include <linux/soc/qcom/llcc-qcom.h>
24 /* Check that the GMU is idle */ in _a6xx_check_idle()
25 if (!adreno_has_gmu_wrapper(adreno_gpu) && !a6xx_gmu_isidle(&a6xx_gpu->gmu)) in _a6xx_check_idle()
45 gpu->name, __builtin_return_address(0), in a6xx_idle()
62 if (a6xx_gpu->has_whereami && !adreno_gpu->base.hw_apriv) { in update_shadow_rptr()
78 spin_lock_irqsave(&ring->preempt_lock, flags); in a6xx_flush()
81 ring->cur = ring->next; in a6xx_flush()
88 if (a6xx_gpu->cur_ring == ring) in a6xx_flush()
91 ring->restore_wptr = true; in a6xx_flush()
93 ring->restore_wptr = true; in a6xx_flush()
96 spin_unlock_irqrestore(&ring->preempt_lock, flags); in a6xx_flush()
113 bool sysprof = refcount_read(&a6xx_gpu->base.base.sysprof_active) > 1; in a6xx_set_pagetable()
114 struct msm_file_private *ctx = submit->queue->ctx; in a6xx_set_pagetable()
115 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; in a6xx_set_pagetable()
120 if (ctx->seqno == ring->cur_ctx_seqno) in a6xx_set_pagetable()
123 if (msm_iommu_pagetable_params(ctx->aspace->mmu, &ttbr, &asid)) in a6xx_set_pagetable()
126 if (adreno_gpu->info->family >= ADRENO_7XX_GEN1) { in a6xx_set_pagetable()
132 OUT_RING(ring, submit->seqno - 1); in a6xx_set_pagetable()
165 OUT_RING(ring, ctx->seqno); in a6xx_set_pagetable()
172 if (adreno_is_a7xx(&a6xx_gpu->base)) { in a6xx_set_pagetable()
200 /* Re-enable protected mode: */ in a6xx_set_pagetable()
209 unsigned int index = submit->seqno % MSM_GPU_SUBMIT_STATS_COUNT; in a6xx_submit()
212 struct msm_ringbuffer *ring = submit->ring; in a6xx_submit()
221 * For PM4 the GMU register offsets are calculated from the base of the in a6xx_submit()
236 for (i = 0; i < submit->nr_cmds; i++) { in a6xx_submit()
237 switch (submit->cmd[i].type) { in a6xx_submit()
241 if (ring->cur_ctx_seqno == submit->queue->ctx->seqno) in a6xx_submit()
246 OUT_RING(ring, lower_32_bits(submit->cmd[i].iova)); in a6xx_submit()
247 OUT_RING(ring, upper_32_bits(submit->cmd[i].iova)); in a6xx_submit()
248 OUT_RING(ring, A5XX_CP_INDIRECT_BUFFER_2_IB_SIZE(submit->cmd[i].size)); in a6xx_submit()
254 * Periodically update shadow-wptr if needed, so that we in a6xx_submit()
271 OUT_RING(ring, submit->seqno); in a6xx_submit()
282 OUT_RING(ring, submit->seqno); in a6xx_submit()
305 a6xx_gpu->preempt_iova[ring->id])); in a6xx_emit_set_pseudo_reg()
307 a6xx_gpu->preempt_iova[ring->id])); in a6xx_emit_set_pseudo_reg()
320 preempt_postamble = a6xx_gpu->preempt_postamble_iova; in a6xx_emit_set_pseudo_reg()
326 a6xx_gpu->preempt_postamble_len) | in a6xx_emit_set_pseudo_reg()
332 unsigned int index = submit->seqno % MSM_GPU_SUBMIT_STATS_COUNT; in a7xx_submit()
335 struct msm_ringbuffer *ring = submit->ring; in a7xx_submit()
351 if (gpu->nr_rings > 1) in a7xx_submit()
352 a6xx_emit_set_pseudo_reg(ring, a6xx_gpu, submit->queue); in a7xx_submit()
365 if (submit->queue->flags & MSM_SUBMITQUEUE_ALLOW_PREEMPT) { in a7xx_submit()
371 for (i = 0; i < submit->nr_cmds; i++) { in a7xx_submit()
372 switch (submit->cmd[i].type) { in a7xx_submit()
376 if (ring->cur_ctx_seqno == submit->queue->ctx->seqno) in a7xx_submit()
381 OUT_RING(ring, lower_32_bits(submit->cmd[i].iova)); in a7xx_submit()
382 OUT_RING(ring, upper_32_bits(submit->cmd[i].iova)); in a7xx_submit()
383 OUT_RING(ring, A5XX_CP_INDIRECT_BUFFER_2_IB_SIZE(submit->cmd[i].size)); in a7xx_submit()
389 * Periodically update shadow-wptr if needed, so that we in a7xx_submit()
399 if (submit->queue->flags & MSM_SUBMITQUEUE_ALLOW_PREEMPT) { in a7xx_submit()
411 OUT_RING(ring, submit->seqno); in a7xx_submit()
433 OUT_RING(ring, submit->seqno); in a7xx_submit()
447 OUT_RING(ring, submit->seqno); in a7xx_submit()
449 a6xx_gpu->last_seqno[ring->id] = submit->seqno; in a7xx_submit()
456 OUT_RING(ring, submit->seqno); in a7xx_submit()
465 if (gpu->nr_rings > 1) { in a7xx_submit()
476 /* Data value - not used if the address above is 0 */ in a7xx_submit()
496 struct a6xx_gmu *gmu = &a6xx_gpu->gmu; in a6xx_set_hwcg() local
502 if (!(adreno_gpu->info->a6xx->hwcg || adreno_is_a7xx(adreno_gpu))) in a6xx_set_hwcg()
517 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GPU_GMU_AO_GMU_CGC_MODE_CNTL, in a6xx_set_hwcg()
518 state ? adreno_gpu->info->a6xx->gmu_cgc_mode : 0); in a6xx_set_hwcg()
519 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GPU_GMU_AO_GMU_CGC_DELAY_CNTL, in a6xx_set_hwcg()
521 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GPU_GMU_AO_GMU_CGC_HYST_CNTL, in a6xx_set_hwcg()
524 if (!adreno_gpu->info->a6xx->hwcg) { in a6xx_set_hwcg()
533 dev_err(&gpu->pdev->dev, "RBBM_CGC_P2S_STATUS TXDONE Poll failed\n"); in a6xx_set_hwcg()
545 /* Don't re-program the registers if they are already correct */ in a6xx_set_hwcg()
551 gmu_rmw(gmu, REG_A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL, 1, 0); in a6xx_set_hwcg()
553 for (i = 0; (reg = &adreno_gpu->info->a6xx->hwcg[i], reg->offset); i++) in a6xx_set_hwcg()
554 gpu_write(gpu, reg->offset, state ? reg->value : 0); in a6xx_set_hwcg()
558 gmu_rmw(gmu, REG_A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL, 0, 1); in a6xx_set_hwcg()
566 const struct adreno_protect *protect = adreno_gpu->info->a6xx->protect; in a6xx_set_cp_protect()
579 for (i = 0; i < protect->count - 1; i++) { in a6xx_set_cp_protect()
581 if (protect->regs[i]) in a6xx_set_cp_protect()
582 gpu_write(gpu, REG_A6XX_CP_PROTECT(i), protect->regs[i]); in a6xx_set_cp_protect()
585 gpu_write(gpu, REG_A6XX_CP_PROTECT(protect->count_max - 1), protect->regs[i]); in a6xx_set_cp_protect()
590 gpu->ubwc_config.rgb565_predicator = 0; in a6xx_calc_ubwc_config()
591 gpu->ubwc_config.uavflagprd_inv = 0; in a6xx_calc_ubwc_config()
592 gpu->ubwc_config.min_acc_len = 0; in a6xx_calc_ubwc_config()
593 gpu->ubwc_config.ubwc_swizzle = 0x6; in a6xx_calc_ubwc_config()
594 gpu->ubwc_config.macrotile_mode = 0; in a6xx_calc_ubwc_config()
595 gpu->ubwc_config.highest_bank_bit = 15; in a6xx_calc_ubwc_config()
598 gpu->ubwc_config.highest_bank_bit = 13; in a6xx_calc_ubwc_config()
599 gpu->ubwc_config.min_acc_len = 1; in a6xx_calc_ubwc_config()
600 gpu->ubwc_config.ubwc_swizzle = 0x7; in a6xx_calc_ubwc_config()
604 gpu->ubwc_config.highest_bank_bit = 14; in a6xx_calc_ubwc_config()
608 gpu->ubwc_config.highest_bank_bit = 13; in a6xx_calc_ubwc_config()
611 gpu->ubwc_config.highest_bank_bit = 13; in a6xx_calc_ubwc_config()
614 gpu->ubwc_config.highest_bank_bit = 13; in a6xx_calc_ubwc_config()
615 gpu->ubwc_config.amsbc = 1; in a6xx_calc_ubwc_config()
616 gpu->ubwc_config.uavflagprd_inv = 2; in a6xx_calc_ubwc_config()
620 gpu->ubwc_config.amsbc = 1; in a6xx_calc_ubwc_config()
623 gpu->ubwc_config.macrotile_mode = 1; in a6xx_calc_ubwc_config()
631 gpu->ubwc_config.highest_bank_bit = 16; in a6xx_calc_ubwc_config()
632 gpu->ubwc_config.amsbc = 1; in a6xx_calc_ubwc_config()
633 gpu->ubwc_config.rgb565_predicator = 1; in a6xx_calc_ubwc_config()
634 gpu->ubwc_config.uavflagprd_inv = 2; in a6xx_calc_ubwc_config()
635 gpu->ubwc_config.macrotile_mode = 1; in a6xx_calc_ubwc_config()
639 gpu->ubwc_config.highest_bank_bit = 13; in a6xx_calc_ubwc_config()
640 gpu->ubwc_config.amsbc = 1; in a6xx_calc_ubwc_config()
641 gpu->ubwc_config.rgb565_predicator = 1; in a6xx_calc_ubwc_config()
642 gpu->ubwc_config.uavflagprd_inv = 2; in a6xx_calc_ubwc_config()
643 gpu->ubwc_config.macrotile_mode = 1; in a6xx_calc_ubwc_config()
644 gpu->ubwc_config.ubwc_swizzle = 0x4; in a6xx_calc_ubwc_config()
648 gpu->ubwc_config.highest_bank_bit = 14; in a6xx_calc_ubwc_config()
649 gpu->ubwc_config.amsbc = 1; in a6xx_calc_ubwc_config()
650 gpu->ubwc_config.rgb565_predicator = 1; in a6xx_calc_ubwc_config()
651 gpu->ubwc_config.uavflagprd_inv = 2; in a6xx_calc_ubwc_config()
652 gpu->ubwc_config.macrotile_mode = 1; in a6xx_calc_ubwc_config()
656 gpu->ubwc_config.highest_bank_bit = 14; in a6xx_calc_ubwc_config()
657 gpu->ubwc_config.min_acc_len = 1; in a6xx_calc_ubwc_config()
669 BUG_ON(adreno_gpu->ubwc_config.highest_bank_bit < 13); in a6xx_set_ubwc_config()
670 u32 hbb = adreno_gpu->ubwc_config.highest_bank_bit - 13; in a6xx_set_ubwc_config()
673 u32 ubwc_mode = adreno_gpu->ubwc_config.ubwc_swizzle & 1; in a6xx_set_ubwc_config()
674 u32 level2_swizzling_dis = !(adreno_gpu->ubwc_config.ubwc_swizzle & 2); in a6xx_set_ubwc_config()
678 adreno_gpu->ubwc_config.rgb565_predicator << 11 | in a6xx_set_ubwc_config()
679 hbb_hi << 10 | adreno_gpu->ubwc_config.amsbc << 4 | in a6xx_set_ubwc_config()
680 adreno_gpu->ubwc_config.min_acc_len << 3 | in a6xx_set_ubwc_config()
685 adreno_gpu->ubwc_config.min_acc_len << 3 | in a6xx_set_ubwc_config()
690 adreno_gpu->ubwc_config.uavflagprd_inv << 4 | in a6xx_set_ubwc_config()
691 adreno_gpu->ubwc_config.min_acc_len << 3 | in a6xx_set_ubwc_config()
699 adreno_gpu->ubwc_config.min_acc_len << 23 | hbb_lo << 21); in a6xx_set_ubwc_config()
702 adreno_gpu->ubwc_config.macrotile_mode); in a6xx_set_ubwc_config()
710 void *ptr = a6xx_gpu->pwrup_reglist_ptr; in a7xx_patch_pwrup_reglist()
712 u32 *dest = (u32 *)&lock->regs[0]; in a7xx_patch_pwrup_reglist()
715 reglist = adreno_gpu->info->a6xx->pwrup_reglist; in a7xx_patch_pwrup_reglist()
717 lock->gpu_req = lock->cpu_req = lock->turn = 0; in a7xx_patch_pwrup_reglist()
718 lock->ifpc_list_len = 0; in a7xx_patch_pwrup_reglist()
719 lock->preemption_list_len = reglist->count; in a7xx_patch_pwrup_reglist()
725 for (i = 0; i < reglist->count; i++) { in a7xx_patch_pwrup_reglist()
726 *dest++ = reglist->regs[i]; in a7xx_patch_pwrup_reglist()
727 *dest++ = gpu_read(gpu, reglist->regs[i]); in a7xx_patch_pwrup_reglist()
732 * 1. Static IFPC-only registers in a7xx_patch_pwrup_reglist()
745 lock->dynamic_list_len = 0; in a7xx_patch_pwrup_reglist()
752 struct msm_ringbuffer *ring = gpu->rb[0]; in a7xx_preempt_start()
754 if (gpu->nr_rings <= 1) in a7xx_preempt_start()
773 return a6xx_idle(gpu, ring) ? 0 : -EINVAL; in a7xx_preempt_start()
778 struct msm_ringbuffer *ring = gpu->rb[0]; in a6xx_cp_init()
802 return a6xx_idle(gpu, ring) ? 0 : -EINVAL; in a6xx_cp_init()
809 struct msm_ringbuffer *ring = gpu->rb[0]; in a7xx_cp_init()
846 OUT_RING(ring, lower_32_bits(a6xx_gpu->pwrup_reglist_iova)); in a7xx_cp_init()
848 OUT_RING(ring, upper_32_bits(a6xx_gpu->pwrup_reglist_iova)); in a7xx_cp_init()
853 return a6xx_idle(gpu, ring) ? 0 : -EINVAL; in a7xx_cp_init()
863 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; in a6xx_ucode_check_version()
864 struct msm_gpu *gpu = &adreno_gpu->base; in a6xx_ucode_check_version()
865 const char *sqe_name = adreno_gpu->info->fw[ADRENO_FW_SQE]; in a6xx_ucode_check_version()
899 a6xx_gpu->has_whereami = true; in a6xx_ucode_check_version()
904 DRM_DEV_ERROR(&gpu->pdev->dev, in a6xx_ucode_check_version()
913 DRM_DEV_ERROR(&gpu->pdev->dev, in a6xx_ucode_check_version()
919 DRM_DEV_ERROR(&gpu->pdev->dev, in a6xx_ucode_check_version()
932 if (!a6xx_gpu->sqe_bo) { in a6xx_ucode_load()
933 a6xx_gpu->sqe_bo = adreno_fw_create_bo(gpu, in a6xx_ucode_load()
934 adreno_gpu->fw[ADRENO_FW_SQE], &a6xx_gpu->sqe_iova); in a6xx_ucode_load()
936 if (IS_ERR(a6xx_gpu->sqe_bo)) { in a6xx_ucode_load()
937 int ret = PTR_ERR(a6xx_gpu->sqe_bo); in a6xx_ucode_load()
939 a6xx_gpu->sqe_bo = NULL; in a6xx_ucode_load()
940 DRM_DEV_ERROR(&gpu->pdev->dev, in a6xx_ucode_load()
946 msm_gem_object_set_name(a6xx_gpu->sqe_bo, "sqefw"); in a6xx_ucode_load()
947 if (!a6xx_ucode_check_version(a6xx_gpu, a6xx_gpu->sqe_bo)) { in a6xx_ucode_load()
948 msm_gem_unpin_iova(a6xx_gpu->sqe_bo, gpu->aspace); in a6xx_ucode_load()
949 drm_gem_object_put(a6xx_gpu->sqe_bo); in a6xx_ucode_load()
951 a6xx_gpu->sqe_bo = NULL; in a6xx_ucode_load()
952 return -EPERM; in a6xx_ucode_load()
960 if ((adreno_gpu->base.hw_apriv || a6xx_gpu->has_whereami) && in a6xx_ucode_load()
961 !a6xx_gpu->shadow_bo) { in a6xx_ucode_load()
962 a6xx_gpu->shadow = msm_gem_kernel_new(gpu->dev, in a6xx_ucode_load()
963 sizeof(u32) * gpu->nr_rings, in a6xx_ucode_load()
965 gpu->aspace, &a6xx_gpu->shadow_bo, in a6xx_ucode_load()
966 &a6xx_gpu->shadow_iova); in a6xx_ucode_load()
968 if (IS_ERR(a6xx_gpu->shadow)) in a6xx_ucode_load()
969 return PTR_ERR(a6xx_gpu->shadow); in a6xx_ucode_load()
971 msm_gem_object_set_name(a6xx_gpu->shadow_bo, "shadow"); in a6xx_ucode_load()
974 a6xx_gpu->pwrup_reglist_ptr = msm_gem_kernel_new(gpu->dev, PAGE_SIZE, in a6xx_ucode_load()
976 gpu->aspace, &a6xx_gpu->pwrup_reglist_bo, in a6xx_ucode_load()
977 &a6xx_gpu->pwrup_reglist_iova); in a6xx_ucode_load()
979 if (IS_ERR(a6xx_gpu->pwrup_reglist_ptr)) in a6xx_ucode_load()
980 return PTR_ERR(a6xx_gpu->pwrup_reglist_ptr); in a6xx_ucode_load()
982 msm_gem_object_set_name(a6xx_gpu->pwrup_reglist_bo, "pwrup_reglist"); in a6xx_ucode_load()
1041 struct a6xx_gmu *gmu = &a6xx_gpu->gmu; in hw_init() local
1047 /* Make sure the GMU keeps the GPU on while we set it up */ in hw_init()
1048 ret = a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET); in hw_init()
1071 a6xx_sptprac_enable(gmu); in hw_init()
1074 * Disable the trusted memory range - we don't actually supported secure in hw_init()
1126 gpu_write64(gpu, REG_A6XX_UCHE_TRAP_BASE, adreno_gpu->uche_trap_base); in hw_init()
1127 gpu_write64(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE, adreno_gpu->uche_trap_base); in hw_init()
1129 gpu_write64(gpu, REG_A6XX_UCHE_WRITE_RANGE_MAX, adreno_gpu->uche_trap_base + 0xfc0); in hw_init()
1130 gpu_write64(gpu, REG_A6XX_UCHE_TRAP_BASE, adreno_gpu->uche_trap_base); in hw_init()
1131 gpu_write64(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE, adreno_gpu->uche_trap_base); in hw_init()
1139 /* Set the GMEM VA range [0x100000:0x100000 + gpu->gmem - 1] */ in hw_init()
1143 gmem_range_min + adreno_gpu->info->gmem - 1); in hw_init()
1179 if (adreno_gpu->info->a6xx->prim_fifo_threshold) in hw_init()
1181 adreno_gpu->info->a6xx->prim_fifo_threshold); in hw_init()
1191 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_1, in hw_init()
1228 /* Set up the CX GMU counter 0 to count busy ticks */ in hw_init()
1229 gmu_write(gmu, REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_MASK, 0xff000000); in hw_init()
1232 gmu_rmw(gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_0, 0xff, BIT(5)); in hw_init()
1233 gmu_write(gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE, 1); in hw_init()
1263 if (gpu->hw_apriv) { in hw_init()
1283 /* Disable non-ubwc read reqs from passing write reqs */ in hw_init()
1295 gpu_write64(gpu, REG_A6XX_CP_SQE_INSTR_BASE, a6xx_gpu->sqe_iova); in hw_init()
1298 gpu_write64(gpu, REG_A6XX_CP_RB_BASE, gpu->rb[0]->iova); in hw_init()
1304 if (adreno_gpu->base.hw_apriv) in hw_init()
1311 if (a6xx_gpu->shadow_bo) { in hw_init()
1313 shadowptr(a6xx_gpu, gpu->rb[0])); in hw_init()
1314 for (unsigned int i = 0; i < gpu->nr_rings; i++) in hw_init()
1315 a6xx_gpu->shadow[i] = 0; in hw_init()
1321 rbmemptr(gpu->rb[0], bv_rptr)); in hw_init()
1327 a6xx_gpu->cur_ring = gpu->rb[0]; in hw_init()
1329 for (i = 0; i < gpu->nr_rings; i++) in hw_init()
1330 gpu->rb[i]->cur_ctx_seqno = 0; in hw_init()
1335 if (adreno_is_a7xx(adreno_gpu) && !a6xx_gpu->pwrup_reglist_emitted) { in hw_init()
1337 a6xx_gpu->pwrup_reglist_emitted = true; in hw_init()
1353 OUT_PKT7(gpu->rb[0], CP_SET_SECURE_MODE, 1); in hw_init()
1354 OUT_RING(gpu->rb[0], 0x00000000); in hw_init()
1356 a6xx_flush(gpu, gpu->rb[0]); in hw_init()
1357 if (!a6xx_idle(gpu, gpu->rb[0])) in hw_init()
1358 return -EINVAL; in hw_init()
1359 } else if (ret == -ENODEV) { in hw_init()
1366 dev_warn_once(gpu->dev->dev, in hw_init()
1367 "Zap shader not enabled - using SECVID_TRUST_CNTL instead\n"); in hw_init()
1378 /* Last step - yield the ringbuffer */ in hw_init()
1382 * Tell the GMU that we are done touching the GPU and it can start power in hw_init()
1385 a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET); in hw_init()
1387 if (a6xx_gpu->gmu.legacy) { in hw_init()
1388 /* Take the GMU out of its special boot mode */ in hw_init()
1389 a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_BOOT_SLUMBER); in hw_init()
1401 mutex_lock(&a6xx_gpu->gmu.lock); in a6xx_hw_init()
1403 mutex_unlock(&a6xx_gpu->gmu.lock); in a6xx_hw_init()
1410 DRM_DEV_INFO(&gpu->pdev->dev, "status: %08x\n", in a6xx_dump()
1419 struct a6xx_gmu *gmu = &a6xx_gpu->gmu; in a6xx_recover() local
1425 DRM_DEV_INFO(&gpu->pdev->dev, "CP_SCRATCH_REG%d: %u\n", i, in a6xx_recover()
1435 a6xx_gpu->hung = true; in a6xx_recover()
1440 pm_runtime_dont_use_autosuspend(&gpu->pdev->dev); in a6xx_recover()
1443 mutex_lock(&gpu->active_lock); in a6xx_recover()
1444 active_submits = gpu->active_submits; in a6xx_recover()
1450 gpu->active_submits = 0; in a6xx_recover()
1461 reinit_completion(&gmu->pd_gate); in a6xx_recover()
1462 dev_pm_genpd_add_notifier(gmu->cxpd, &gmu->pd_nb); in a6xx_recover()
1463 dev_pm_genpd_synced_poweroff(gmu->cxpd); in a6xx_recover()
1467 pm_runtime_put(&gpu->pdev->dev); in a6xx_recover()
1470 pm_runtime_put_sync(&gpu->pdev->dev); in a6xx_recover()
1472 if (!wait_for_completion_timeout(&gmu->pd_gate, msecs_to_jiffies(1000))) in a6xx_recover()
1473 DRM_DEV_ERROR(&gpu->pdev->dev, "cx gdsc didn't collapse\n"); in a6xx_recover()
1475 dev_pm_genpd_remove_notifier(gmu->cxpd); in a6xx_recover()
1477 pm_runtime_use_autosuspend(&gpu->pdev->dev); in a6xx_recover()
1480 pm_runtime_get(&gpu->pdev->dev); in a6xx_recover()
1482 pm_runtime_get_sync(&gpu->pdev->dev); in a6xx_recover()
1484 gpu->active_submits = active_submits; in a6xx_recover()
1485 mutex_unlock(&gpu->active_lock); in a6xx_recover()
1488 a6xx_gpu->hung = false; in a6xx_recover()
1523 * compute-only some of them do not exist and there are holes in a6xx_uche_fault_block()
1527 "-", "LPAC_SP", "-", "-", in a6xx_uche_fault_block()
1528 "LPAC_HLSQ", "-", "-", "LPAC_TP", in a6xx_uche_fault_block()
1599 return "GMU"; in a6xx_fault_block()
1620 block = a6xx_fault_block(gpu, info->fsynr1 & 0xff); in a6xx_fault_handler()
1634 dev_err_ratelimited(&gpu->pdev->dev, in a6xx_cp_hw_err_irq()
1640 dev_err_ratelimited(&gpu->pdev->dev, in a6xx_cp_hw_err_irq()
1644 dev_err_ratelimited(&gpu->pdev->dev, "CP | HW fault | status=0x%8.8X\n", in a6xx_cp_hw_err_irq()
1650 dev_err_ratelimited(&gpu->pdev->dev, in a6xx_cp_hw_err_irq()
1657 dev_err_ratelimited(&gpu->pdev->dev, "CP AHB error interrupt\n"); in a6xx_cp_hw_err_irq()
1660 dev_err_ratelimited(&gpu->pdev->dev, "CP VSD decoder parity error\n"); in a6xx_cp_hw_err_irq()
1663 dev_err_ratelimited(&gpu->pdev->dev, "CP illegal instruction error\n"); in a6xx_cp_hw_err_irq()
1671 struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu); in a6xx_fault_detect_irq()
1687 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE, 1); in a6xx_fault_detect_irq()
1689 DRM_DEV_ERROR(&gpu->pdev->dev, in a6xx_fault_detect_irq()
1691 ring ? ring->id : -1, ring ? ring->fctx->last_fence : 0, in a6xx_fault_detect_irq()
1701 del_timer(&gpu->hangcheck_timer); in a6xx_fault_detect_irq()
1703 kthread_queue_work(gpu->worker, &gpu->recover_work); in a6xx_fault_detect_irq()
1713 dev_err_ratelimited(&gpu->pdev->dev, "SW fuse violation status=%8.8x\n", status); in a7xx_sw_fuse_violation_irq()
1721 del_timer(&gpu->hangcheck_timer); in a7xx_sw_fuse_violation_irq()
1723 kthread_queue_work(gpu->worker, &gpu->recover_work); in a7xx_sw_fuse_violation_irq()
1729 struct msm_drm_private *priv = gpu->dev->dev_private; in a6xx_irq()
1734 if (priv->disable_err_irq) in a6xx_irq()
1741 dev_err_ratelimited(&gpu->pdev->dev, "CP | AHB bus error\n"); in a6xx_irq()
1747 dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB ASYNC overflow\n"); in a6xx_irq()
1750 dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB bus overflow\n"); in a6xx_irq()
1753 dev_err_ratelimited(&gpu->pdev->dev, "UCHE | Out of bounds access\n"); in a6xx_irq()
1771 llcc_slice_deactivate(a6xx_gpu->llc_slice); in a6xx_llc_deactivate()
1772 llcc_slice_deactivate(a6xx_gpu->htw_llc_slice); in a6xx_llc_deactivate()
1777 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; in a6xx_llc_activate()
1778 struct msm_gpu *gpu = &adreno_gpu->base; in a6xx_llc_activate()
1781 if (IS_ERR(a6xx_gpu->llc_mmio)) in a6xx_llc_activate()
1784 if (!llcc_slice_activate(a6xx_gpu->llc_slice)) { in a6xx_llc_activate()
1785 u32 gpu_scid = llcc_get_slice_id(a6xx_gpu->llc_slice); in a6xx_llc_activate()
1803 if (!llcc_slice_activate(a6xx_gpu->htw_llc_slice)) { in a6xx_llc_activate()
1804 if (!a6xx_gpu->have_mmu500) { in a6xx_llc_activate()
1805 u32 gpuhtw_scid = llcc_get_slice_id(a6xx_gpu->htw_llc_slice); in a6xx_llc_activate()
1819 if (!a6xx_gpu->have_mmu500) { in a6xx_llc_activate()
1837 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; in a7xx_llc_activate()
1838 struct msm_gpu *gpu = &adreno_gpu->base; in a7xx_llc_activate()
1840 if (IS_ERR(a6xx_gpu->llc_mmio)) in a7xx_llc_activate()
1843 if (!llcc_slice_activate(a6xx_gpu->llc_slice)) { in a7xx_llc_activate()
1844 u32 gpu_scid = llcc_get_slice_id(a6xx_gpu->llc_slice); in a7xx_llc_activate()
1861 llcc_slice_activate(a6xx_gpu->htw_llc_slice); in a7xx_llc_activate()
1866 /* No LLCC on non-RPMh (and by extension, non-GMU) SoCs */ in a6xx_llc_slices_destroy()
1867 if (adreno_has_gmu_wrapper(&a6xx_gpu->base)) in a6xx_llc_slices_destroy()
1870 llcc_slice_putd(a6xx_gpu->llc_slice); in a6xx_llc_slices_destroy()
1871 llcc_slice_putd(a6xx_gpu->htw_llc_slice); in a6xx_llc_slices_destroy()
1879 /* No LLCC on non-RPMh (and by extension, non-GMU) SoCs */ in a6xx_llc_slices_init()
1880 if (adreno_has_gmu_wrapper(&a6xx_gpu->base)) in a6xx_llc_slices_init()
1887 phandle = of_parse_phandle(pdev->dev.of_node, "iommus", 0); in a6xx_llc_slices_init()
1888 a6xx_gpu->have_mmu500 = (phandle && in a6xx_llc_slices_init()
1889 of_device_is_compatible(phandle, "arm,mmu-500")); in a6xx_llc_slices_init()
1892 if (is_a7xx || !a6xx_gpu->have_mmu500) in a6xx_llc_slices_init()
1893 a6xx_gpu->llc_mmio = msm_ioremap(pdev, "cx_mem"); in a6xx_llc_slices_init()
1895 a6xx_gpu->llc_mmio = NULL; in a6xx_llc_slices_init()
1897 a6xx_gpu->llc_slice = llcc_slice_getd(LLCC_GPU); in a6xx_llc_slices_init()
1898 a6xx_gpu->htw_llc_slice = llcc_slice_getd(LLCC_GPUHTW); in a6xx_llc_slices_init()
1900 if (IS_ERR_OR_NULL(a6xx_gpu->llc_slice) && IS_ERR_OR_NULL(a6xx_gpu->htw_llc_slice)) in a6xx_llc_slices_init()
1901 a6xx_gpu->llc_mmio = ERR_PTR(-EINVAL); in a6xx_llc_slices_init()
1906 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; in a7xx_cx_mem_init()
1907 struct msm_gpu *gpu = &adreno_gpu->base; in a7xx_cx_mem_init()
1921 dev_warn_once(gpu->dev->dev, in a7xx_cx_mem_init()
1927 adreno_gpu->has_ray_tracing = true; in a7xx_cx_mem_init()
1943 adreno_gpu->has_ray_tracing = in a7xx_cx_mem_init()
1947 adreno_gpu->has_ray_tracing = true; in a7xx_cx_mem_init()
1962 struct msm_gpu *gpu = &adreno_gpu->base; in a6xx_bus_clear_pending_transactions()
2019 gpu->needs_hw_init = true; in a6xx_gmu_pm_resume()
2023 mutex_lock(&a6xx_gpu->gmu.lock); in a6xx_gmu_pm_resume()
2025 mutex_unlock(&a6xx_gpu->gmu.lock); in a6xx_gmu_pm_resume()
2040 struct a6xx_gmu *gmu = &a6xx_gpu->gmu; in a6xx_pm_resume() local
2041 unsigned long freq = gpu->fast_rate; in a6xx_pm_resume()
2045 gpu->needs_hw_init = true; in a6xx_pm_resume()
2049 mutex_lock(&a6xx_gpu->gmu.lock); in a6xx_pm_resume()
2051 opp = dev_pm_opp_find_freq_ceil(&gpu->pdev->dev, &freq); in a6xx_pm_resume()
2059 dev_pm_opp_set_opp(&gpu->pdev->dev, opp); in a6xx_pm_resume()
2061 pm_runtime_resume_and_get(gmu->dev); in a6xx_pm_resume()
2062 pm_runtime_resume_and_get(gmu->gxpd); in a6xx_pm_resume()
2064 ret = clk_bulk_prepare_enable(gpu->nr_clocks, gpu->grp_clks); in a6xx_pm_resume()
2069 a6xx_sptprac_enable(gmu); in a6xx_pm_resume()
2074 pm_runtime_put(gmu->gxpd); in a6xx_pm_resume()
2075 pm_runtime_put(gmu->dev); in a6xx_pm_resume()
2076 dev_pm_opp_set_opp(&gpu->pdev->dev, NULL); in a6xx_pm_resume()
2079 mutex_unlock(&a6xx_gpu->gmu.lock); in a6xx_pm_resume()
2099 mutex_lock(&a6xx_gpu->gmu.lock); in a6xx_gmu_pm_suspend()
2101 mutex_unlock(&a6xx_gpu->gmu.lock); in a6xx_gmu_pm_suspend()
2105 if (a6xx_gpu->shadow_bo) in a6xx_gmu_pm_suspend()
2106 for (i = 0; i < gpu->nr_rings; i++) in a6xx_gmu_pm_suspend()
2107 a6xx_gpu->shadow[i] = 0; in a6xx_gmu_pm_suspend()
2109 gpu->suspend_count++; in a6xx_gmu_pm_suspend()
2118 struct a6xx_gmu *gmu = &a6xx_gpu->gmu; in a6xx_pm_suspend() local
2125 mutex_lock(&a6xx_gpu->gmu.lock); in a6xx_pm_suspend()
2131 a6xx_sptprac_disable(gmu); in a6xx_pm_suspend()
2133 clk_bulk_disable_unprepare(gpu->nr_clocks, gpu->grp_clks); in a6xx_pm_suspend()
2135 pm_runtime_put_sync(gmu->gxpd); in a6xx_pm_suspend()
2136 dev_pm_opp_set_opp(&gpu->pdev->dev, NULL); in a6xx_pm_suspend()
2137 pm_runtime_put_sync(gmu->dev); in a6xx_pm_suspend()
2139 mutex_unlock(&a6xx_gpu->gmu.lock); in a6xx_pm_suspend()
2141 if (a6xx_gpu->shadow_bo) in a6xx_pm_suspend()
2142 for (i = 0; i < gpu->nr_rings; i++) in a6xx_pm_suspend()
2143 a6xx_gpu->shadow[i] = 0; in a6xx_pm_suspend()
2145 gpu->suspend_count++; in a6xx_pm_suspend()
2155 mutex_lock(&a6xx_gpu->gmu.lock); in a6xx_gmu_get_timestamp()
2158 a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET); in a6xx_gmu_get_timestamp()
2162 a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET); in a6xx_gmu_get_timestamp()
2164 mutex_unlock(&a6xx_gpu->gmu.lock); in a6xx_gmu_get_timestamp()
2180 return a6xx_gpu->cur_ring; in a6xx_active_ring()
2188 if (a6xx_gpu->sqe_bo) { in a6xx_destroy()
2189 msm_gem_unpin_iova(a6xx_gpu->sqe_bo, gpu->aspace); in a6xx_destroy()
2190 drm_gem_object_put(a6xx_gpu->sqe_bo); in a6xx_destroy()
2193 if (a6xx_gpu->shadow_bo) { in a6xx_destroy()
2194 msm_gem_unpin_iova(a6xx_gpu->shadow_bo, gpu->aspace); in a6xx_destroy()
2195 drm_gem_object_put(a6xx_gpu->shadow_bo); in a6xx_destroy()
2216 busy_cycles = gmu_read64(&a6xx_gpu->gmu, in a6xx_gpu_busy()
2229 mutex_lock(&a6xx_gpu->gmu.lock); in a6xx_gpu_set_freq()
2231 mutex_unlock(&a6xx_gpu->gmu.lock); in a6xx_gpu_set_freq()
2245 if (!IS_ERR_OR_NULL(a6xx_gpu->htw_llc_slice) && in a6xx_create_address_space()
2246 !device_iommu_capable(&pdev->dev, IOMMU_CAP_CACHE_COHERENCY)) in a6xx_create_address_space()
2257 mmu = msm_iommu_pagetable_create(gpu->aspace->mmu); in a6xx_create_private_address_space()
2272 if (adreno_gpu->base.hw_apriv || a6xx_gpu->has_whereami) in a6xx_get_rptr()
2273 return a6xx_gpu->shadow[ring->id]; in a6xx_get_rptr()
2275 return ring->memptrs->rptr = gpu_read(gpu, REG_A6XX_CP_RB_RPTR); in a6xx_get_rptr()
2304 progress = !!memcmp(&cp_state, &ring->last_cp_state, sizeof(cp_state)); in a6xx_progress()
2306 ring->last_cp_state = cp_state; in a6xx_progress()
2313 if (!info->speedbins) in fuse_to_supp_hw()
2316 for (int i = 0; info->speedbins[i].fuse != SHRT_MAX; i++) in fuse_to_supp_hw()
2317 if (info->speedbins[i].fuse == fuse) in fuse_to_supp_hw()
2318 return BIT(info->speedbins[i].speedbin); in fuse_to_supp_hw()
2331 * -ENOENT means that the platform doesn't support speedbin which is in a6xx_set_supported_hw()
2334 if (ret == -ENOENT) { in a6xx_set_supported_hw()
2338 "failed to read speed-bin. Some OPPs may not be supported by hardware\n"); in a6xx_set_supported_hw()
2346 "missing support for speed-bin: %u. Some OPPs may not be supported by hardware\n", in a6xx_set_supported_hw()
2451 struct msm_drm_private *priv = dev->dev_private; in a6xx_gpu_init()
2452 struct platform_device *pdev = priv->gpu_pdev; in a6xx_gpu_init()
2453 struct adreno_platform_config *config = pdev->dev.platform_data; in a6xx_gpu_init()
2464 return ERR_PTR(-ENOMEM); in a6xx_gpu_init()
2466 adreno_gpu = &a6xx_gpu->base; in a6xx_gpu_init()
2467 gpu = &adreno_gpu->base; in a6xx_gpu_init()
2469 mutex_init(&a6xx_gpu->gmu.lock); in a6xx_gpu_init()
2471 adreno_gpu->registers = NULL; in a6xx_gpu_init()
2473 /* Check if there is a GMU phandle and set it up */ in a6xx_gpu_init()
2474 node = of_parse_phandle(pdev->dev.of_node, "qcom,gmu", 0); in a6xx_gpu_init()
2478 adreno_gpu->gmu_is_wrapper = of_device_is_compatible(node, "qcom,adreno-gmu-wrapper"); in a6xx_gpu_init()
2480 adreno_gpu->base.hw_apriv = in a6xx_gpu_init()
2481 !!(config->info->quirks & ADRENO_QUIRK_HAS_HW_APRIV); in a6xx_gpu_init()
2483 /* gpu->info only gets assigned in adreno_gpu_init() */ in a6xx_gpu_init()
2484 is_a7xx = config->info->family == ADRENO_7XX_GEN1 || in a6xx_gpu_init()
2485 config->info->family == ADRENO_7XX_GEN2 || in a6xx_gpu_init()
2486 config->info->family == ADRENO_7XX_GEN3; in a6xx_gpu_init()
2490 ret = a6xx_set_supported_hw(&pdev->dev, config->info); in a6xx_gpu_init()
2497 if ((enable_preemption == 1) || (enable_preemption == -1 && in a6xx_gpu_init()
2498 (config->info->quirks & ADRENO_QUIRK_PREEMPTION))) in a6xx_gpu_init()
2507 a6xx_destroy(&(a6xx_gpu->base.base)); in a6xx_gpu_init()
2516 priv->gpu_clamp_to_idle = true; in a6xx_gpu_init()
2524 a6xx_destroy(&(a6xx_gpu->base.base)); in a6xx_gpu_init()
2531 a6xx_destroy(&(a6xx_gpu->base.base)); in a6xx_gpu_init()
2536 adreno_gpu->uche_trap_base = 0x1fffffffff000ull; in a6xx_gpu_init()
2538 if (gpu->aspace) in a6xx_gpu_init()
2539 msm_mmu_set_fault_handler(gpu->aspace->mmu, gpu, in a6xx_gpu_init()