1 // SPDX-License-Identifier: GPL-2.0
3  * Copyright (C) 2014-2018 Etnaviv Project
27 	u32 *vaddr = (u32 *)buffer->vaddr;  in OUT()
29 	BUG_ON(buffer->user_size >= buffer->size);  in OUT()
31 	vaddr[buffer->user_size / 4] = data;  in OUT()
32 	buffer->user_size += 4;  in OUT()
40 	buffer->user_size = ALIGN(buffer->user_size, 8);  in CMD_LOAD_STATE()
51 	buffer->user_size = ALIGN(buffer->user_size, 8);  in CMD_END()
59 	buffer->user_size = ALIGN(buffer->user_size, 8);  in CMD_WAIT()
67 	buffer->user_size = ALIGN(buffer->user_size, 8);  in CMD_LINK()
77 	buffer->user_size = ALIGN(buffer->user_size, 8);  in CMD_STALL()
90 static void etnaviv_cmd_select_pipe(struct etnaviv_gpu *gpu,  in etnaviv_cmd_select_pipe()  argument
95 	lockdep_assert_held(&gpu->lock);  in etnaviv_cmd_select_pipe()
103 	if (gpu->exec_state == ETNA_PIPE_2D)  in etnaviv_cmd_select_pipe()
105 	else if (gpu->exec_state == ETNA_PIPE_3D)  in etnaviv_cmd_select_pipe()
116 static void etnaviv_buffer_dump(struct etnaviv_gpu *gpu,  in etnaviv_buffer_dump()  argument
119 	u32 size = buf->size;  in etnaviv_buffer_dump()
120 	u32 *ptr = buf->vaddr + off;  in etnaviv_buffer_dump()
122 	dev_info(gpu->dev, "virt %p phys 0x%08x free 0x%08x\n",  in etnaviv_buffer_dump()
124 			&gpu->mmu_context->cmdbuf_mapping) +  in etnaviv_buffer_dump()
125 			off, size - len * 4 - off);  in etnaviv_buffer_dump()
133  * The GPU may be executing this WAIT while we're modifying it, so we have
134  * to write it in a specific order to avoid the GPU branching to somewhere
140 	u32 *lw = buffer->vaddr + wl_offset;  in etnaviv_buffer_replace_wait()
150  * 'cmd_dwords' 64-bit words into the buffer, wrapping if necessary.
152 static u32 etnaviv_buffer_reserve(struct etnaviv_gpu *gpu,  in etnaviv_buffer_reserve()  argument
155 	if (buffer->user_size + cmd_dwords * sizeof(u64) > buffer->size)  in etnaviv_buffer_reserve()
156 		buffer->user_size = 0;  in etnaviv_buffer_reserve()
159 				     &gpu->mmu_context->cmdbuf_mapping) +  in etnaviv_buffer_reserve()
160 	       buffer->user_size;  in etnaviv_buffer_reserve()
163 u16 etnaviv_buffer_init(struct etnaviv_gpu *gpu)  in etnaviv_buffer_init()  argument
165 	struct etnaviv_cmdbuf *buffer = &gpu->buffer;  in etnaviv_buffer_init()
167 	lockdep_assert_held(&gpu->lock);  in etnaviv_buffer_init()
170 	buffer->user_size = 0;  in etnaviv_buffer_init()
172 	CMD_WAIT(buffer, gpu->fe_waitcycles);  in etnaviv_buffer_init()
174 		 etnaviv_cmdbuf_get_va(buffer, &gpu->mmu_context->cmdbuf_mapping)  in etnaviv_buffer_init()
175 		 + buffer->user_size - 4);  in etnaviv_buffer_init()
177 	return buffer->user_size / 8;  in etnaviv_buffer_init()
180 u16 etnaviv_buffer_config_mmuv2(struct etnaviv_gpu *gpu, u32 mtlb_addr, u32 safe_addr)  in etnaviv_buffer_config_mmuv2()  argument
182 	struct etnaviv_cmdbuf *buffer = &gpu->buffer;  in etnaviv_buffer_config_mmuv2()
184 	lockdep_assert_held(&gpu->lock);  in etnaviv_buffer_config_mmuv2()
186 	buffer->user_size = 0;  in etnaviv_buffer_config_mmuv2()
188 	if (gpu->identity.features & chipFeatures_PIPE_3D) {  in etnaviv_buffer_config_mmuv2()
198 	if (gpu->identity.features & chipFeatures_PIPE_2D) {  in etnaviv_buffer_config_mmuv2()
210 	buffer->user_size = ALIGN(buffer->user_size, 8);  in etnaviv_buffer_config_mmuv2()
212 	return buffer->user_size / 8;  in etnaviv_buffer_config_mmuv2()
215 u16 etnaviv_buffer_config_pta(struct etnaviv_gpu *gpu, unsigned short id)  in etnaviv_buffer_config_pta()  argument
217 	struct etnaviv_cmdbuf *buffer = &gpu->buffer;  in etnaviv_buffer_config_pta()
219 	lockdep_assert_held(&gpu->lock);  in etnaviv_buffer_config_pta()
221 	buffer->user_size = 0;  in etnaviv_buffer_config_pta()
224 		       VIVS_MMUv2_PTA_CONFIG_INDEX(id));  in etnaviv_buffer_config_pta()
228 	buffer->user_size = ALIGN(buffer->user_size, 8);  in etnaviv_buffer_config_pta()
230 	return buffer->user_size / 8;  in etnaviv_buffer_config_pta()
233 void etnaviv_buffer_end(struct etnaviv_gpu *gpu)  in etnaviv_buffer_end()  argument
235 	struct etnaviv_cmdbuf *buffer = &gpu->buffer;  in etnaviv_buffer_end()
236 	unsigned int waitlink_offset = buffer->user_size - 16;  in etnaviv_buffer_end()
238 	bool has_blt = !!(gpu->identity.minor_features5 &  in etnaviv_buffer_end()
241 	lockdep_assert_held(&gpu->lock);  in etnaviv_buffer_end()
243 	if (gpu->exec_state == ETNA_PIPE_2D)  in etnaviv_buffer_end()
245 	else if (gpu->exec_state == ETNA_PIPE_3D)  in etnaviv_buffer_end()
258 		link_target = etnaviv_buffer_reserve(gpu, buffer, dwords);  in etnaviv_buffer_end()
269 		if (gpu->exec_state == ETNA_PIPE_3D) {  in etnaviv_buffer_end()
294 		/* Replace the last link-wait with an "END" command */  in etnaviv_buffer_end()
301 void etnaviv_sync_point_queue(struct etnaviv_gpu *gpu, unsigned int event)  in etnaviv_sync_point_queue()  argument
303 	struct etnaviv_cmdbuf *buffer = &gpu->buffer;  in etnaviv_sync_point_queue()
304 	unsigned int waitlink_offset = buffer->user_size - 16;  in etnaviv_sync_point_queue()
307 	lockdep_assert_held(&gpu->lock);  in etnaviv_sync_point_queue()
314 	target = etnaviv_buffer_reserve(gpu, buffer, dwords);  in etnaviv_sync_point_queue()
320 	/* Stop the FE to 'pause' the GPU */  in etnaviv_sync_point_queue()
324 	CMD_WAIT(buffer, gpu->fe_waitcycles);  in etnaviv_sync_point_queue()
326 		 etnaviv_cmdbuf_get_va(buffer, &gpu->mmu_context->cmdbuf_mapping)  in etnaviv_sync_point_queue()
327 		 + buffer->user_size - 4);  in etnaviv_sync_point_queue()
340 void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, u32 exec_state,  in etnaviv_buffer_queue()  argument
344 	struct etnaviv_cmdbuf *buffer = &gpu->buffer;  in etnaviv_buffer_queue()
345 	unsigned int waitlink_offset = buffer->user_size - 16;  in etnaviv_buffer_queue()
348 	bool switch_context = gpu->exec_state != exec_state;  in etnaviv_buffer_queue()
349 	bool switch_mmu_context = gpu->mmu_context != mmu_context;  in etnaviv_buffer_queue()
350 	unsigned int new_flush_seq = READ_ONCE(gpu->mmu_context->flush_seq);  in etnaviv_buffer_queue()
351 	bool need_flush = switch_mmu_context || gpu->flush_seq != new_flush_seq;  in etnaviv_buffer_queue()
352 	bool has_blt = !!(gpu->identity.minor_features5 &  in etnaviv_buffer_queue()
355 	lockdep_assert_held(&gpu->lock);  in etnaviv_buffer_queue()
358 		etnaviv_buffer_dump(gpu, buffer, 0, 0x50);  in etnaviv_buffer_queue()
361 					    &gpu->mmu_context->cmdbuf_mapping);  in etnaviv_buffer_queue()
362 	link_dwords = cmdbuf->size / 8;  in etnaviv_buffer_queue()
367 	 * link to this buffer - a total of four additional words.  in etnaviv_buffer_queue()
377 			if (gpu->mmu_context->global->version == ETNAVIV_IOMMU_V1)  in etnaviv_buffer_queue()
388 		if (switch_mmu_context && gpu->sec_mode == ETNA_SEC_KERNEL)  in etnaviv_buffer_queue()
391 		target = etnaviv_buffer_reserve(gpu, buffer, extra_dwords);  in etnaviv_buffer_queue()
399 			struct etnaviv_iommu_context *old_context = gpu->mmu_context;  in etnaviv_buffer_queue()
401 			gpu->mmu_context = etnaviv_iommu_context_get(mmu_context);  in etnaviv_buffer_queue()
407 			if (gpu->mmu_context->global->version == ETNAVIV_IOMMU_V1) {  in etnaviv_buffer_queue()
419 				    gpu->sec_mode == ETNA_SEC_KERNEL) {  in etnaviv_buffer_queue()
420 					unsigned short id =  in etnaviv_buffer_queue()  local
421 						etnaviv_iommuv2_get_pta_id(gpu->mmu_context);  in etnaviv_buffer_queue()
424 						VIVS_MMUv2_PTA_CONFIG_INDEX(id));  in etnaviv_buffer_queue()
427 				if (gpu->sec_mode == ETNA_SEC_NONE)  in etnaviv_buffer_queue()
428 					flush |= etnaviv_iommuv2_get_mtlb_addr(gpu->mmu_context);  in etnaviv_buffer_queue()
438 			gpu->flush_seq = new_flush_seq;  in etnaviv_buffer_queue()
442 			etnaviv_cmd_select_pipe(gpu, buffer, exec_state);  in etnaviv_buffer_queue()
443 			gpu->exec_state = exec_state;  in etnaviv_buffer_queue()
448 					&gpu->mmu_context->cmdbuf_mapping);  in etnaviv_buffer_queue()
472 	return_target = etnaviv_buffer_reserve(gpu, buffer, return_dwords);  in etnaviv_buffer_queue()
479 	if (gpu->exec_state == ETNA_PIPE_2D) {  in etnaviv_buffer_queue()
508 	CMD_WAIT(buffer, gpu->fe_waitcycles);  in etnaviv_buffer_queue()
510 		 etnaviv_cmdbuf_get_va(buffer, &gpu->mmu_context->cmdbuf_mapping)  in etnaviv_buffer_queue()
511 		 + buffer->user_size - 4);  in etnaviv_buffer_queue()
516 			etnaviv_cmdbuf_get_va(cmdbuf, &gpu->mmu_context->cmdbuf_mapping),  in etnaviv_buffer_queue()
517 			cmdbuf->vaddr);  in etnaviv_buffer_queue()
521 			       cmdbuf->vaddr, cmdbuf->size, 0);  in etnaviv_buffer_queue()
523 		pr_info("link op: %p\n", buffer->vaddr + waitlink_offset);  in etnaviv_buffer_queue()
539 		etnaviv_buffer_dump(gpu, buffer, 0, 0x50);  in etnaviv_buffer_queue()