1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2016-2017 The Linux Foundation. All rights reserved.
10 #include <linux/nvmem-consumer.h>
26 	if (a5xx_gpu->has_whereami) {  in update_shadow_rptr()
48 	spin_lock_irqsave(&ring->preempt_lock, flags);  in a5xx_flush()
51 	ring->cur = ring->next;  in a5xx_flush()
56 	spin_unlock_irqrestore(&ring->preempt_lock, flags);  in a5xx_flush()
62 	if (a5xx_gpu->cur_ring == ring && !a5xx_in_preempt(a5xx_gpu))  in a5xx_flush()
70 	struct msm_ringbuffer *ring = submit->ring;  in a5xx_submit_in_rb()
75 	for (i = 0; i < submit->nr_cmds; i++) {  in a5xx_submit_in_rb()
76 		switch (submit->cmd[i].type) {  in a5xx_submit_in_rb()
80 			if (ring->cur_ctx_seqno == submit->queue->ctx->seqno)  in a5xx_submit_in_rb()
85 			obj = submit->bos[submit->cmd[i].idx].obj;  in a5xx_submit_in_rb()
86 			dwords = submit->cmd[i].size;  in a5xx_submit_in_rb()
114 	a5xx_gpu->last_seqno[ring->id] = submit->seqno;  in a5xx_submit_in_rb()
123 	ring->memptrs->fence = submit->seqno;  in a5xx_submit_in_rb()
131 	struct msm_ringbuffer *ring = submit->ring;  in a5xx_submit()
134 	if (IS_ENABLED(CONFIG_DRM_MSM_GPU_SUDO) && submit->in_rb) {  in a5xx_submit()
135 		ring->cur_ctx_seqno = 0;  in a5xx_submit()
149 	OUT_RING(ring, lower_32_bits(a5xx_gpu->preempt_iova[submit->ring->id]));  in a5xx_submit()
150 	OUT_RING(ring, upper_32_bits(a5xx_gpu->preempt_iova[submit->ring->id]));  in a5xx_submit()
158 	 * user-space to be aware of it and provide additional handling  in a5xx_submit()
169 	for (i = 0; i < submit->nr_cmds; i++) {  in a5xx_submit()
170 		switch (submit->cmd[i].type) {  in a5xx_submit()
174 			if (ring->cur_ctx_seqno == submit->queue->ctx->seqno)  in a5xx_submit()
179 			OUT_RING(ring, lower_32_bits(submit->cmd[i].iova));  in a5xx_submit()
180 			OUT_RING(ring, upper_32_bits(submit->cmd[i].iova));  in a5xx_submit()
181 			OUT_RING(ring, submit->cmd[i].size);  in a5xx_submit()
187 		 * Periodically update shadow-wptr if needed, so that we  in a5xx_submit()
199 	 * are done rendering - otherwise a lucky preemption would start  in a5xx_submit()
215 	OUT_RING(ring, submit->seqno);  in a5xx_submit()
216 	a5xx_gpu->last_seqno[ring->id] = submit->seqno;  in a5xx_submit()
227 	OUT_RING(ring, submit->seqno);  in a5xx_submit()
238 	/* Data value - not used if the address above is 0 */  in a5xx_submit()
478 	struct msm_ringbuffer *ring = gpu->rb[0];  in a5xx_me_init()
498 		 * Force a WFI after every direct-render 3D mode draw and every  in a5xx_me_init()
514 	return a5xx_idle(gpu, ring) ? 0 : -EINVAL;  in a5xx_me_init()
521 	struct msm_ringbuffer *ring = gpu->rb[0];  in a5xx_preempt_start()
523 	if (gpu->nr_rings == 1)  in a5xx_preempt_start()
532 	OUT_RING(ring, lower_32_bits(a5xx_gpu->preempt_iova[ring->id]));  in a5xx_preempt_start()
533 	OUT_RING(ring, upper_32_bits(a5xx_gpu->preempt_iova[ring->id]));  in a5xx_preempt_start()
558 	return a5xx_idle(gpu, ring) ? 0 : -EINVAL;  in a5xx_preempt_start()
575 		a5xx_gpu->has_whereami = true;  in a5xx_ucode_check_version()
586 	if (!a5xx_gpu->pm4_bo) {  in a5xx_ucode_load()
587 		a5xx_gpu->pm4_bo = adreno_fw_create_bo(gpu,  in a5xx_ucode_load()
588 			adreno_gpu->fw[ADRENO_FW_PM4], &a5xx_gpu->pm4_iova);  in a5xx_ucode_load()
591 		if (IS_ERR(a5xx_gpu->pm4_bo)) {  in a5xx_ucode_load()
592 			ret = PTR_ERR(a5xx_gpu->pm4_bo);  in a5xx_ucode_load()
593 			a5xx_gpu->pm4_bo = NULL;  in a5xx_ucode_load()
594 			DRM_DEV_ERROR(gpu->dev->dev, "could not allocate PM4: %d\n",  in a5xx_ucode_load()
599 		msm_gem_object_set_name(a5xx_gpu->pm4_bo, "pm4fw");  in a5xx_ucode_load()
602 	if (!a5xx_gpu->pfp_bo) {  in a5xx_ucode_load()
603 		a5xx_gpu->pfp_bo = adreno_fw_create_bo(gpu,  in a5xx_ucode_load()
604 			adreno_gpu->fw[ADRENO_FW_PFP], &a5xx_gpu->pfp_iova);  in a5xx_ucode_load()
606 		if (IS_ERR(a5xx_gpu->pfp_bo)) {  in a5xx_ucode_load()
607 			ret = PTR_ERR(a5xx_gpu->pfp_bo);  in a5xx_ucode_load()
608 			a5xx_gpu->pfp_bo = NULL;  in a5xx_ucode_load()
609 			DRM_DEV_ERROR(gpu->dev->dev, "could not allocate PFP: %d\n",  in a5xx_ucode_load()
614 		msm_gem_object_set_name(a5xx_gpu->pfp_bo, "pfpfw");  in a5xx_ucode_load()
615 		a5xx_ucode_check_version(a5xx_gpu, a5xx_gpu->pfp_bo);  in a5xx_ucode_load()
618 	if (a5xx_gpu->has_whereami) {  in a5xx_ucode_load()
619 		if (!a5xx_gpu->shadow_bo) {  in a5xx_ucode_load()
620 			a5xx_gpu->shadow = msm_gem_kernel_new(gpu->dev,  in a5xx_ucode_load()
621 				sizeof(u32) * gpu->nr_rings,  in a5xx_ucode_load()
623 				gpu->aspace, &a5xx_gpu->shadow_bo,  in a5xx_ucode_load()
624 				&a5xx_gpu->shadow_iova);  in a5xx_ucode_load()
626 			if (IS_ERR(a5xx_gpu->shadow))  in a5xx_ucode_load()
627 				return PTR_ERR(a5xx_gpu->shadow);  in a5xx_ucode_load()
629 			msm_gem_object_set_name(a5xx_gpu->shadow_bo, "shadow");  in a5xx_ucode_load()
631 	} else if (gpu->nr_rings > 1) {  in a5xx_ucode_load()
634 		gpu->nr_rings = 1;  in a5xx_ucode_load()
649 	 * to resume zap shader  in a5xx_zap_shader_resume()
656 		DRM_ERROR("%s: zap-shader resume failed: %d\n",  in a5xx_zap_shader_resume()
657 			gpu->name, ret);  in a5xx_zap_shader_resume()
668 	 * If the zap shader is already loaded into memory we just need to kick  in a5xx_zap_shader_init()
712 	if (adreno_gpu->info->quirks & ADRENO_QUIRK_FAULT_DETECT_MASK) {  in a5xx_hw_init()
714 		 * Mask out the activity signals from RB1-3 to avoid false  in a5xx_hw_init()
753 	gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_LO, lower_32_bits(adreno_gpu->uche_trap_base));  in a5xx_hw_init()
754 	gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_HI, upper_32_bits(adreno_gpu->uche_trap_base));  in a5xx_hw_init()
755 	gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_LO, lower_32_bits(adreno_gpu->uche_trap_base));  in a5xx_hw_init()
756 	gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_HI, upper_32_bits(adreno_gpu->uche_trap_base));  in a5xx_hw_init()
758 	/* Set the GMEM VA range (0 to gpu->gmem) */  in a5xx_hw_init()
762 		0x00100000 + adreno_gpu->info->gmem - 1);  in a5xx_hw_init()
797 	if (adreno_gpu->info->quirks & ADRENO_QUIRK_TWO_PASS_USE_WFI)  in a5xx_hw_init()
802 	 * for 1-SP GPUs, as it is enabled by default.  in a5xx_hw_init()
824 	 *  bug is fixed in latest A510 revision. To enable this bug fix -  in a5xx_hw_init()
836 	BUG_ON(adreno_gpu->ubwc_config.highest_bank_bit < 13);  in a5xx_hw_init()
837 	hbb = adreno_gpu->ubwc_config.highest_bank_bit - 13;  in a5xx_hw_init()
890 	 * Disable the trusted memory range - we don't actually supported secure  in a5xx_hw_init()
916 	if (adreno_gpu->info->quirks & ADRENO_QUIRK_LMLOADKILL_DISABLE) {  in a5xx_hw_init()
928 	gpu_write64(gpu, REG_A5XX_CP_ME_INSTR_BASE_LO, a5xx_gpu->pm4_iova);  in a5xx_hw_init()
929 	gpu_write64(gpu, REG_A5XX_CP_PFP_INSTR_BASE_LO, a5xx_gpu->pfp_iova);  in a5xx_hw_init()
932 	gpu_write64(gpu, REG_A5XX_CP_RB_BASE, gpu->rb[0]->iova);  in a5xx_hw_init()
944 	if (a5xx_gpu->shadow_bo) {  in a5xx_hw_init()
946 			    shadowptr(a5xx_gpu, gpu->rb[0]));  in a5xx_hw_init()
969 		OUT_PKT7(gpu->rb[0], CP_EVENT_WRITE, 1);  in a5xx_hw_init()
970 		OUT_RING(gpu->rb[0], CP_EVENT_WRITE_0_EVENT(STAT_EVENT));  in a5xx_hw_init()
972 		a5xx_flush(gpu, gpu->rb[0], true);  in a5xx_hw_init()
973 		if (!a5xx_idle(gpu, gpu->rb[0]))  in a5xx_hw_init()
974 			return -EINVAL;  in a5xx_hw_init()
979 	 * try to load a zap shader into the secure world. If successful  in a5xx_hw_init()
987 		OUT_PKT7(gpu->rb[0], CP_SET_SECURE_MODE, 1);  in a5xx_hw_init()
988 		OUT_RING(gpu->rb[0], 0x00000000);  in a5xx_hw_init()
990 		a5xx_flush(gpu, gpu->rb[0], true);  in a5xx_hw_init()
991 		if (!a5xx_idle(gpu, gpu->rb[0]))  in a5xx_hw_init()
992 			return -EINVAL;  in a5xx_hw_init()
993 	} else if (ret == -ENODEV) {  in a5xx_hw_init()
995 		 * This device does not use zap shader (but print a warning  in a5xx_hw_init()
1000 		dev_warn_once(gpu->dev->dev,  in a5xx_hw_init()
1001 			"Zap shader not enabled - using SECVID_TRUST_CNTL instead\n");  in a5xx_hw_init()
1007 	/* Last step - yield the ringbuffer */  in a5xx_hw_init()
1038 	DBG("%s", gpu->name);  in a5xx_destroy()
1042 	if (a5xx_gpu->pm4_bo) {  in a5xx_destroy()
1043 		msm_gem_unpin_iova(a5xx_gpu->pm4_bo, gpu->aspace);  in a5xx_destroy()
1044 		drm_gem_object_put(a5xx_gpu->pm4_bo);  in a5xx_destroy()
1047 	if (a5xx_gpu->pfp_bo) {  in a5xx_destroy()
1048 		msm_gem_unpin_iova(a5xx_gpu->pfp_bo, gpu->aspace);  in a5xx_destroy()
1049 		drm_gem_object_put(a5xx_gpu->pfp_bo);  in a5xx_destroy()
1052 	if (a5xx_gpu->gpmu_bo) {  in a5xx_destroy()
1053 		msm_gem_unpin_iova(a5xx_gpu->gpmu_bo, gpu->aspace);  in a5xx_destroy()
1054 		drm_gem_object_put(a5xx_gpu->gpmu_bo);  in a5xx_destroy()
1057 	if (a5xx_gpu->shadow_bo) {  in a5xx_destroy()
1058 		msm_gem_unpin_iova(a5xx_gpu->shadow_bo, gpu->aspace);  in a5xx_destroy()
1059 		drm_gem_object_put(a5xx_gpu->shadow_bo);  in a5xx_destroy()
1084 	if (ring != a5xx_gpu->cur_ring) {  in a5xx_idle()
1085 		WARN(1, "Tried to idle a non-current ringbuffer\n");  in a5xx_idle()
1095 			gpu->name, __builtin_return_address(0),  in a5xx_idle()
1119 		snprintf(block, sizeof(block), "%x", info->fsynr1);  in a5xx_fault_handler()
1141 		dev_err_ratelimited(gpu->dev->dev, "CP | opcode error | possible opcode=0x%8.8X\n",  in a5xx_cp_err_irq()
1146 		dev_err_ratelimited(gpu->dev->dev, "CP | HW fault | status=0x%8.8X\n",  in a5xx_cp_err_irq()
1150 		dev_err_ratelimited(gpu->dev->dev, "CP | DMA error\n");  in a5xx_cp_err_irq()
1155 		dev_err_ratelimited(gpu->dev->dev,  in a5xx_cp_err_irq()
1168 		dev_err_ratelimited(gpu->dev->dev,  in a5xx_cp_err_irq()
1180 		dev_err_ratelimited(gpu->dev->dev,  in a5xx_rbbm_err_irq()
1195 		dev_err_ratelimited(gpu->dev->dev, "RBBM | AHB transfer timeout\n");  in a5xx_rbbm_err_irq()
1198 		dev_err_ratelimited(gpu->dev->dev, "RBBM | ME master split | status=0x%X\n",  in a5xx_rbbm_err_irq()
1202 		dev_err_ratelimited(gpu->dev->dev, "RBBM | PFP master split | status=0x%X\n",  in a5xx_rbbm_err_irq()
1206 		dev_err_ratelimited(gpu->dev->dev, "RBBM | ETS master split | status=0x%X\n",  in a5xx_rbbm_err_irq()
1210 		dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB ASYNC overflow\n");  in a5xx_rbbm_err_irq()
1213 		dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB bus overflow\n");  in a5xx_rbbm_err_irq()
1222 	dev_err_ratelimited(gpu->dev->dev, "UCHE | Out of bounds access | addr=0x%llX\n",  in a5xx_uche_err_irq()
1228 	dev_err_ratelimited(gpu->dev->dev, "GPMU | voltage droop\n");  in a5xx_gpmu_err_irq()
1233 	struct drm_device *dev = gpu->dev;  in a5xx_fault_detect_irq()
1234 	struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu);  in a5xx_fault_detect_irq()
1245 …DRM_DEV_ERROR(dev->dev, "gpu fault ring %d fence %x status %8.8X rb %4.4x/%4.4x ib1 %16.16llX/%4.4…  in a5xx_fault_detect_irq()
1246 		ring ? ring->id : -1, ring ? ring->fctx->last_fence : 0,  in a5xx_fault_detect_irq()
1256 	del_timer(&gpu->hangcheck_timer);  in a5xx_fault_detect_irq()
1258 	kthread_queue_work(gpu->worker, &gpu->recover_work);  in a5xx_fault_detect_irq()
1271 	struct msm_drm_private *priv = gpu->dev->dev_private;  in a5xx_irq()
1275 	 * Clear all the interrupts except RBBM_AHB_ERROR - if we clear it  in a5xx_irq()
1281 	if (priv->disable_err_irq) {  in a5xx_irq()
1346 	DRM_DEV_INFO(gpu->dev->dev, "status:   %08x\n",  in a5xx_dump()
1381 			gpu->name,  in a5xx_pm_resume()
1392 			gpu->name);  in a5xx_pm_resume()
1429 	if (a5xx_gpu->has_whereami)  in a5xx_pm_suspend()
1430 		for (i = 0; i < gpu->nr_rings; i++)  in a5xx_pm_suspend()
1431 			a5xx_gpu->shadow[i] = 0;  in a5xx_pm_suspend()
1457 	dumper->ptr = msm_gem_kernel_new(gpu->dev,  in a5xx_crashdumper_init()
1458 		SZ_1M, MSM_BO_WC, gpu->aspace,  in a5xx_crashdumper_init()
1459 		&dumper->bo, &dumper->iova);  in a5xx_crashdumper_init()
1461 	if (!IS_ERR(dumper->ptr))  in a5xx_crashdumper_init()
1462 		msm_gem_object_set_name(dumper->bo, "crashdump");  in a5xx_crashdumper_init()
1464 	return PTR_ERR_OR_ZERO(dumper->ptr);  in a5xx_crashdumper_init()
1472 	if (IS_ERR_OR_NULL(dumper->ptr))  in a5xx_crashdumper_run()
1473 		return -EINVAL;  in a5xx_crashdumper_run()
1475 	gpu_write64(gpu, REG_A5XX_CP_CRASH_SCRIPT_BASE_LO, dumper->iova);  in a5xx_crashdumper_run()
1493 	{ 0x35, 0xe00, 0x32 },   /* HSLQ non-context */
1498 	{ 0x3f, 0x0ec0, 0x40 },  /* SP non-context */
1503 	{ 0x3a, 0x0f00, 0x1c },  /* TP non-context */
1531 	a5xx_state->hlsqregs = kcalloc(count, sizeof(u32), GFP_KERNEL);  in a5xx_gpu_state_get_hlsq_regs()
1532 	if (!a5xx_state->hlsqregs)  in a5xx_gpu_state_get_hlsq_regs()
1557 		kfree(a5xx_state->hlsqregs);  in a5xx_gpu_state_get_hlsq_regs()
1558 		msm_gem_kernel_put(dumper.bo, gpu->aspace);  in a5xx_gpu_state_get_hlsq_regs()
1563 	memcpy(a5xx_state->hlsqregs, dumper.ptr + (256 * SZ_1K),  in a5xx_gpu_state_get_hlsq_regs()
1566 	msm_gem_kernel_put(dumper.bo, gpu->aspace);  in a5xx_gpu_state_get_hlsq_regs()
1576 		return ERR_PTR(-ENOMEM);  in a5xx_gpu_state_get()
1582 	adreno_gpu_state_get(gpu, &(a5xx_state->base));  in a5xx_gpu_state_get()
1584 	a5xx_state->base.rbbm_status = gpu_read(gpu, REG_A5XX_RBBM_STATUS);  in a5xx_gpu_state_get()
1596 	return &a5xx_state->base;  in a5xx_gpu_state_get()
1606 	kfree(a5xx_state->hlsqregs);  in a5xx_gpu_state_destroy()
1617 	return kref_put(&state->ref, a5xx_gpu_state_destroy);  in a5xx_gpu_state_put()
1636 	if (!a5xx_state->hlsqregs)  in a5xx_show()
1639 	drm_printf(p, "registers-hlsq:\n");  in a5xx_show()
1653 			if (a5xx_state->hlsqregs[pos] == 0xdeadbeef)  in a5xx_show()
1656 			drm_printf(p, "  - { offset: 0x%04x, value: 0x%08x }\n",  in a5xx_show()
1657 				o << 2, a5xx_state->hlsqregs[pos]);  in a5xx_show()
1668 	return a5xx_gpu->cur_ring;  in a5xx_active_ring()
1676 	*out_sample_rate = clk_get_rate(gpu->core_clk);  in a5xx_gpu_busy()
1686 	if (a5xx_gpu->has_whereami)  in a5xx_get_rptr()
1687 		return a5xx_gpu->shadow[ring->id];  in a5xx_get_rptr()
1689 	return ring->memptrs->rptr = gpu_read(gpu, REG_A5XX_CP_RB_RPTR);  in a5xx_get_rptr()
1726 	 * If the OPP table specifies a opp-supported-hw property then we have  in check_speed_bin()
1754 	struct msm_drm_private *priv = dev->dev_private;  in a5xx_gpu_init()
1755 	struct platform_device *pdev = priv->gpu_pdev;  in a5xx_gpu_init()
1756 	struct adreno_platform_config *config = pdev->dev.platform_data;  in a5xx_gpu_init()
1765 		return ERR_PTR(-ENOMEM);  in a5xx_gpu_init()
1767 	adreno_gpu = &a5xx_gpu->base;  in a5xx_gpu_init()
1768 	gpu = &adreno_gpu->base;  in a5xx_gpu_init()
1770 	adreno_gpu->registers = a5xx_registers;  in a5xx_gpu_init()
1772 	a5xx_gpu->lm_leakage = 0x4E001A;  in a5xx_gpu_init()
1774 	check_speed_bin(&pdev->dev);  in a5xx_gpu_init()
1778 	if (config->info->revn == 510)  in a5xx_gpu_init()
1783 		a5xx_destroy(&(a5xx_gpu->base.base));  in a5xx_gpu_init()
1787 	if (gpu->aspace)  in a5xx_gpu_init()
1788 		msm_mmu_set_fault_handler(gpu->aspace->mmu, gpu, a5xx_fault_handler);  in a5xx_gpu_init()
1795 		adreno_gpu->ubwc_config.highest_bank_bit = 15;  in a5xx_gpu_init()
1797 		adreno_gpu->ubwc_config.highest_bank_bit = 14;  in a5xx_gpu_init()
1800 	adreno_gpu->ubwc_config.macrotile_mode = 0;  in a5xx_gpu_init()
1801 	adreno_gpu->ubwc_config.ubwc_swizzle = 0x7;  in a5xx_gpu_init()
1803 	adreno_gpu->uche_trap_base = 0x0001ffffffff0000ull;  in a5xx_gpu_init()