xref: /aosp_15_r20/external/swiftshader/src/Pipeline/SpirvShaderControlFlow.cpp (revision 03ce13f70fcc45d86ee91b7ee4cab1936a95046e)
1 // Copyright 2019 The SwiftShader Authors. All Rights Reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 //    http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 
15 #include "SpirvShader.hpp"
16 #include "SpirvShaderDebug.hpp"
17 
18 #include "Reactor/Coroutine.hpp"  // rr::Yield
19 
20 #include "ShaderCore.hpp"
21 
22 #include <spirv/unified1/spirv.hpp>
23 
24 #include <queue>
25 
26 #include <fstream>
27 #include <iostream>
28 
29 namespace sw {
30 
Block(InsnIterator begin,InsnIterator end)31 Spirv::Block::Block(InsnIterator begin, InsnIterator end)
32     : begin_(begin)
33     , end_(end)
34 {
35 	// Default to a Simple, this may change later.
36 	kind = Block::Simple;
37 
38 	// Walk the instructions to find the last two of the block.
39 	InsnIterator insns[2];
40 	for(auto insn : *this)
41 	{
42 		insns[0] = insns[1];
43 		insns[1] = insn;
44 	}
45 
46 	switch(insns[1].opcode())
47 	{
48 	case spv::OpBranch:
49 		branchInstruction = insns[1];
50 		outs.emplace(Block::ID(branchInstruction.word(1)));
51 
52 		switch(insns[0].opcode())
53 		{
54 		case spv::OpLoopMerge:
55 			kind = Loop;
56 			mergeInstruction = insns[0];
57 			mergeBlock = Block::ID(mergeInstruction.word(1));
58 			continueTarget = Block::ID(mergeInstruction.word(2));
59 			break;
60 
61 		default:
62 			kind = Block::Simple;
63 			break;
64 		}
65 		break;
66 
67 	case spv::OpBranchConditional:
68 		branchInstruction = insns[1];
69 		outs.emplace(Block::ID(branchInstruction.word(2)));
70 		outs.emplace(Block::ID(branchInstruction.word(3)));
71 
72 		switch(insns[0].opcode())
73 		{
74 		case spv::OpSelectionMerge:
75 			kind = StructuredBranchConditional;
76 			mergeInstruction = insns[0];
77 			mergeBlock = Block::ID(mergeInstruction.word(1));
78 			break;
79 
80 		case spv::OpLoopMerge:
81 			kind = Loop;
82 			mergeInstruction = insns[0];
83 			mergeBlock = Block::ID(mergeInstruction.word(1));
84 			continueTarget = Block::ID(mergeInstruction.word(2));
85 			break;
86 
87 		default:
88 			kind = UnstructuredBranchConditional;
89 			break;
90 		}
91 		break;
92 
93 	case spv::OpSwitch:
94 		branchInstruction = insns[1];
95 		outs.emplace(Block::ID(branchInstruction.word(2)));
96 		for(uint32_t w = 4; w < branchInstruction.wordCount(); w += 2)
97 		{
98 			outs.emplace(Block::ID(branchInstruction.word(w)));
99 		}
100 
101 		switch(insns[0].opcode())
102 		{
103 		case spv::OpSelectionMerge:
104 			kind = StructuredSwitch;
105 			mergeInstruction = insns[0];
106 			mergeBlock = Block::ID(mergeInstruction.word(1));
107 			break;
108 
109 		default:
110 			kind = UnstructuredSwitch;
111 			break;
112 		}
113 		break;
114 
115 	default:
116 		break;
117 	}
118 }
119 
TraverseReachableBlocks(Block::ID id,Block::Set & reachable) const120 void Spirv::Function::TraverseReachableBlocks(Block::ID id, Block::Set &reachable) const
121 {
122 	if(reachable.count(id) == 0)
123 	{
124 		reachable.emplace(id);
125 		for(auto out : getBlock(id).outs)
126 		{
127 			TraverseReachableBlocks(out, reachable);
128 		}
129 	}
130 }
131 
AssignBlockFields()132 void Spirv::Function::AssignBlockFields()
133 {
134 	Block::Set reachable;
135 	TraverseReachableBlocks(entry, reachable);
136 
137 	for(auto &it : blocks)
138 	{
139 		auto &blockId = it.first;
140 		auto &block = it.second;
141 		if(reachable.count(blockId) > 0)
142 		{
143 			for(auto &outId : it.second.outs)
144 			{
145 				auto outIt = blocks.find(outId);
146 				ASSERT_MSG(outIt != blocks.end(), "Block %d has a non-existent out %d", blockId.value(), outId.value());
147 				auto &out = outIt->second;
148 				out.ins.emplace(blockId);
149 			}
150 			if(block.kind == Block::Loop)
151 			{
152 				auto mergeIt = blocks.find(block.mergeBlock);
153 				ASSERT_MSG(mergeIt != blocks.end(), "Loop block %d has a non-existent merge block %d", blockId.value(), block.mergeBlock.value());
154 				mergeIt->second.isLoopMerge = true;
155 			}
156 		}
157 	}
158 }
159 
ForeachBlockDependency(Block::ID blockId,std::function<void (Block::ID)> f) const160 void Spirv::Function::ForeachBlockDependency(Block::ID blockId, std::function<void(Block::ID)> f) const
161 {
162 	auto block = getBlock(blockId);
163 	for(auto dep : block.ins)
164 	{
165 		if(block.kind != Block::Loop ||                  // if not a loop...
166 		   !ExistsPath(blockId, dep, block.mergeBlock))  // or a loop and not a loop back edge
167 		{
168 			f(dep);
169 		}
170 	}
171 }
172 
ExistsPath(Block::ID from,Block::ID to,Block::ID notPassingThrough) const173 bool Spirv::Function::ExistsPath(Block::ID from, Block::ID to, Block::ID notPassingThrough) const
174 {
175 	// TODO: Optimize: This can be cached on the block.
176 	Block::Set seen;
177 	seen.emplace(notPassingThrough);
178 
179 	std::queue<Block::ID> pending;
180 	pending.emplace(from);
181 
182 	while(pending.size() > 0)
183 	{
184 		auto id = pending.front();
185 		pending.pop();
186 		for(auto out : getBlock(id).outs)
187 		{
188 			if(seen.count(out) != 0) { continue; }
189 			if(out == to) { return true; }
190 			pending.emplace(out);
191 		}
192 		seen.emplace(id);
193 	}
194 
195 	return false;
196 }
197 
addOutputActiveLaneMaskEdge(Block::ID to,RValue<SIMD::Int> mask)198 void SpirvEmitter::addOutputActiveLaneMaskEdge(Block::ID to, RValue<SIMD::Int> mask)
199 {
200 	addActiveLaneMaskEdge(block, to, mask & activeLaneMask());
201 }
202 
addActiveLaneMaskEdge(Block::ID from,Block::ID to,RValue<SIMD::Int> mask)203 void SpirvEmitter::addActiveLaneMaskEdge(Block::ID from, Block::ID to, RValue<SIMD::Int> mask)
204 {
205 	auto edge = Block::Edge{ from, to };
206 	auto it = edgeActiveLaneMasks.find(edge);
207 	if(it == edgeActiveLaneMasks.end())
208 	{
209 		edgeActiveLaneMasks.emplace(edge, mask);
210 	}
211 	else
212 	{
213 		auto combined = it->second | mask;
214 		edgeActiveLaneMasks.erase(edge);
215 		edgeActiveLaneMasks.emplace(edge, combined);
216 	}
217 }
218 
GetActiveLaneMaskEdge(Block::ID from,Block::ID to) const219 RValue<SIMD::Int> SpirvEmitter::GetActiveLaneMaskEdge(Block::ID from, Block::ID to) const
220 {
221 	auto edge = Block::Edge{ from, to };
222 	auto it = edgeActiveLaneMasks.find(edge);
223 	ASSERT_MSG(it != edgeActiveLaneMasks.end(), "Could not find edge %d -> %d", from.value(), to.value());
224 	return it->second;
225 }
226 
EmitBlocks(Block::ID id,Block::ID ignore)227 void SpirvEmitter::EmitBlocks(Block::ID id, Block::ID ignore /* = 0 */)
228 {
229 	auto oldPending = this->pending;
230 	auto &function = shader.getFunction(this->function);
231 
232 	std::deque<Block::ID> pending;
233 	this->pending = &pending;
234 	pending.push_front(id);
235 	while(pending.size() > 0)
236 	{
237 		auto id = pending.front();
238 
239 		const auto &block = function.getBlock(id);
240 		if(id == ignore)
241 		{
242 			pending.pop_front();
243 			continue;
244 		}
245 
246 		// Ensure all dependency blocks have been generated.
247 		auto depsDone = true;
248 		function.ForeachBlockDependency(id, [&](Block::ID dep) {
249 			if(visited.count(dep) == 0)
250 			{
251 				this->pending->push_front(dep);
252 				depsDone = false;
253 			}
254 		});
255 
256 		if(!depsDone)
257 		{
258 			continue;
259 		}
260 
261 		pending.pop_front();
262 
263 		this->block = id;
264 
265 		switch(block.kind)
266 		{
267 		case Block::Simple:
268 		case Block::StructuredBranchConditional:
269 		case Block::UnstructuredBranchConditional:
270 		case Block::StructuredSwitch:
271 		case Block::UnstructuredSwitch:
272 			EmitNonLoop();
273 			break;
274 
275 		case Block::Loop:
276 			EmitLoop();
277 			break;
278 
279 		default:
280 			UNREACHABLE("Unexpected Block Kind: %d", int(block.kind));
281 		}
282 	}
283 
284 	this->pending = oldPending;
285 }
286 
EmitNonLoop()287 void SpirvEmitter::EmitNonLoop()
288 {
289 	auto &function = shader.getFunction(this->function);
290 	auto blockId = block;
291 	auto block = function.getBlock(blockId);
292 
293 	if(!visited.emplace(blockId).second)
294 	{
295 		return;  // Already generated this block.
296 	}
297 
298 	if(blockId != function.entry)
299 	{
300 		// Set the activeLaneMask.
301 		SIMD::Int activeLaneMask(0);
302 		for(auto in : block.ins)
303 		{
304 			auto inMask = GetActiveLaneMaskEdge(in, blockId);
305 			SPIRV_SHADER_DBG("Block {0} -> {1} mask: {2}", in, blockId, inMask);
306 			activeLaneMask |= inMask;
307 		}
308 		SPIRV_SHADER_DBG("Block {0} mask: {1}", blockId, activeLaneMask);
309 		SetActiveLaneMask(activeLaneMask);
310 	}
311 
312 	EmitInstructions(block.begin(), block.end());
313 
314 	for(auto out : block.outs)
315 	{
316 		if(visited.count(out) == 0)
317 		{
318 			pending->push_back(out);
319 		}
320 	}
321 
322 	SPIRV_SHADER_DBG("Block {0} done", blockId);
323 }
324 
EmitLoop()325 void SpirvEmitter::EmitLoop()
326 {
327 	auto &function = shader.getFunction(this->function);
328 	auto blockId = block;
329 	auto &block = function.getBlock(blockId);
330 	auto mergeBlockId = block.mergeBlock;
331 	auto &mergeBlock = function.getBlock(mergeBlockId);
332 
333 	if(!visited.emplace(blockId).second)
334 	{
335 		return;  // Already emitted this loop.
336 	}
337 
338 	SPIRV_SHADER_DBG("*** LOOP HEADER ***");
339 
340 	// Gather all the blocks that make up the loop.
341 	std::unordered_set<Block::ID> loopBlocks;
342 	loopBlocks.emplace(block.mergeBlock);  // Stop traversal at mergeBlock.
343 	function.TraverseReachableBlocks(blockId, loopBlocks);
344 
345 	// incomingBlocks are block ins that are not back-edges.
346 	std::unordered_set<Block::ID> incomingBlocks;
347 	for(auto in : block.ins)
348 	{
349 		if(loopBlocks.count(in) == 0)
350 		{
351 			incomingBlocks.emplace(in);
352 		}
353 	}
354 
355 	// Emit the loop phi instructions, and initialize them with a value from
356 	// the incoming blocks.
357 	for(auto insn = block.begin(); insn != block.mergeInstruction; insn++)
358 	{
359 		if(insn.opcode() == spv::OpPhi)
360 		{
361 			StorePhi(blockId, insn, incomingBlocks);
362 		}
363 	}
364 
365 	// loopActiveLaneMask is the mask of lanes that are continuing to loop.
366 	// This is initialized with the incoming active lane masks.
367 	SIMD::Int loopActiveLaneMask = SIMD::Int(0);
368 	for(auto in : incomingBlocks)
369 	{
370 		loopActiveLaneMask |= GetActiveLaneMaskEdge(in, blockId);
371 	}
372 
373 	// mergeActiveLaneMasks contains edge lane masks for the merge block.
374 	// This is the union of all edge masks across all iterations of the loop.
375 	std::unordered_map<Block::ID, SIMD::Int> mergeActiveLaneMasks;
376 	for(auto in : function.getBlock(mergeBlockId).ins)
377 	{
378 		mergeActiveLaneMasks.emplace(in, SIMD::Int(0));
379 	}
380 
381 	// Create the loop basic blocks
382 	auto headerBasicBlock = Nucleus::createBasicBlock();
383 	auto mergeBasicBlock = Nucleus::createBasicBlock();
384 
385 	// Start emitting code inside the loop.
386 	Nucleus::createBr(headerBasicBlock);
387 	Nucleus::setInsertBlock(headerBasicBlock);
388 
389 	SPIRV_SHADER_DBG("*** LOOP START (mask: {0}) ***", loopActiveLaneMask);
390 
391 	// Load the active lane mask.
392 	SetActiveLaneMask(loopActiveLaneMask);
393 
394 	// Emit the non-phi loop header block's instructions.
395 	for(auto insn = block.begin(); insn != block.end(); insn++)
396 	{
397 		if(insn.opcode() == spv::OpPhi)
398 		{
399 			LoadPhi(insn);
400 		}
401 		else
402 		{
403 			EmitInstruction(insn);
404 		}
405 	}
406 
407 	// Emit all blocks between the loop header and the merge block, but
408 	// don't emit the merge block yet.
409 	for(auto out : block.outs)
410 	{
411 		EmitBlocks(out, mergeBlockId);
412 	}
413 
414 	// Restore current block id after emitting loop blocks.
415 	this->block = blockId;
416 
417 	// Rebuild the loopActiveLaneMask from the loop back edges.
418 	loopActiveLaneMask = SIMD::Int(0);
419 	for(auto in : block.ins)
420 	{
421 		if(function.ExistsPath(blockId, in, mergeBlockId))
422 		{
423 			loopActiveLaneMask |= GetActiveLaneMaskEdge(in, blockId);
424 		}
425 	}
426 
427 	// Add active lanes to the merge lane mask.
428 	for(auto in : function.getBlock(mergeBlockId).ins)
429 	{
430 		auto edge = Block::Edge{ in, mergeBlockId };
431 		auto it = edgeActiveLaneMasks.find(edge);
432 
433 		if(it != edgeActiveLaneMasks.end())
434 		{
435 			mergeActiveLaneMasks[in] |= it->second;
436 		}
437 	}
438 
439 	// Update loop phi values.
440 	for(auto insn = block.begin(); insn != block.mergeInstruction; insn++)
441 	{
442 		if(insn.opcode() == spv::OpPhi)
443 		{
444 			StorePhi(blockId, insn, loopBlocks);
445 		}
446 	}
447 
448 	SPIRV_SHADER_DBG("*** LOOP END (mask: {0}) ***", loopActiveLaneMask);
449 
450 	// Use the [loop -> merge] active lane masks to update the phi values in
451 	// the merge block. We need to do this to handle divergent control flow
452 	// in the loop.
453 	//
454 	// Consider the following:
455 	//
456 	//     int phi_source = 0;
457 	//     for(uint i = 0; i < 4; i++)
458 	//     {
459 	//         phi_source = 0;
460 	//         if(gl_GlobalInvocationID.x % 4 == i) // divergent control flow
461 	//         {
462 	//             phi_source = 42; // single lane assignment.
463 	//             break; // activeLaneMask for [loop->merge] is active for a single lane.
464 	//         }
465 	//         // -- we are here --
466 	//     }
467 	//     // merge block
468 	//     int phi = phi_source; // OpPhi
469 	//
470 	// In this example, with each iteration of the loop, phi_source will
471 	// only have a single lane assigned. However the 'phi' value in the merge
472 	// block needs to be assigned the union of all the per-lane assignments
473 	// of phi_source when that lane exited the loop.
474 	for(auto insn = mergeBlock.begin(); insn != mergeBlock.end(); insn++)
475 	{
476 		if(insn.opcode() == spv::OpPhi)
477 		{
478 			StorePhi(mergeBlockId, insn, loopBlocks);
479 		}
480 	}
481 
482 	// Loop body now done.
483 	// If any lanes are still active, jump back to the loop header,
484 	// otherwise jump to the merge block.
485 	Nucleus::createCondBr(AnyTrue(loopActiveLaneMask).value(), headerBasicBlock, mergeBasicBlock);
486 
487 	// Continue emitting from the merge block.
488 	Nucleus::setInsertBlock(mergeBasicBlock);
489 	pending->push_back(mergeBlockId);
490 
491 	for(const auto &it : mergeActiveLaneMasks)
492 	{
493 		addActiveLaneMaskEdge(it.first, mergeBlockId, it.second);
494 	}
495 }
496 
EmitBranch(InsnIterator insn)497 void SpirvEmitter::EmitBranch(InsnIterator insn)
498 {
499 	auto target = Block::ID(insn.word(1));
500 	addActiveLaneMaskEdge(block, target, activeLaneMask());
501 }
502 
EmitBranchConditional(InsnIterator insn)503 void SpirvEmitter::EmitBranchConditional(InsnIterator insn)
504 {
505 	auto &function = shader.getFunction(this->function);
506 	auto block = function.getBlock(this->block);
507 	ASSERT(block.branchInstruction == insn);
508 
509 	auto condId = Object::ID(block.branchInstruction.word(1));
510 	auto trueBlockId = Block::ID(block.branchInstruction.word(2));
511 	auto falseBlockId = Block::ID(block.branchInstruction.word(3));
512 
513 	auto cond = Operand(shader, *this, condId);
514 	ASSERT_MSG(shader.getObjectType(condId).componentCount == 1, "Condition must be a Boolean type scalar");
515 
516 	// TODO: Optimize for case where all lanes take same path.
517 
518 	addOutputActiveLaneMaskEdge(trueBlockId, cond.Int(0));
519 	addOutputActiveLaneMaskEdge(falseBlockId, ~cond.Int(0));
520 }
521 
EmitSwitch(InsnIterator insn)522 void SpirvEmitter::EmitSwitch(InsnIterator insn)
523 {
524 	auto &function = shader.getFunction(this->function);
525 	auto block = function.getBlock(this->block);
526 	ASSERT(block.branchInstruction == insn);
527 
528 	auto selId = Object::ID(block.branchInstruction.word(1));
529 
530 	auto sel = Operand(shader, *this, selId);
531 	ASSERT_MSG(sel.componentCount == 1, "Selector must be a scalar");
532 	SPIRV_SHADER_DBG("switch({0})", sel);
533 
534 	auto numCases = (block.branchInstruction.wordCount() - 3) / 2;
535 
536 	// TODO: Optimize for case where all lanes take same path.
537 
538 	SIMD::Int defaultLaneMask = activeLaneMask();
539 
540 	// Gather up the case label matches and calculate defaultLaneMask.
541 	std::vector<RValue<SIMD::Int>> caseLabelMatches;
542 	caseLabelMatches.reserve(numCases);
543 
544 	for(uint32_t i = 0; i < numCases; i++)
545 	{
546 		auto label = block.branchInstruction.word(i * 2 + 3);
547 		auto caseBlockId = Block::ID(block.branchInstruction.word(i * 2 + 4));
548 		auto caseLabelMatch = CmpEQ(sel.Int(0), SIMD::Int(label));
549 		SPIRV_SHADER_DBG("case {0}: {1}", label, caseLabelMatch & activeLaneMask());
550 		addOutputActiveLaneMaskEdge(caseBlockId, caseLabelMatch);
551 		defaultLaneMask &= ~caseLabelMatch;
552 	}
553 
554 	auto defaultBlockId = Block::ID(block.branchInstruction.word(2));
555 	SPIRV_SHADER_DBG("default: {0}", defaultLaneMask);
556 	addOutputActiveLaneMaskEdge(defaultBlockId, defaultLaneMask);
557 }
558 
EmitUnreachable(InsnIterator insn)559 void SpirvEmitter::EmitUnreachable(InsnIterator insn)
560 {
561 	// TODO: Log something in this case?
562 	SetActiveLaneMask(SIMD::Int(0));
563 }
564 
EmitReturn(InsnIterator insn)565 void SpirvEmitter::EmitReturn(InsnIterator insn)
566 {
567 	SetActiveLaneMask(SIMD::Int(0));
568 }
569 
EmitTerminateInvocation(InsnIterator insn)570 void SpirvEmitter::EmitTerminateInvocation(InsnIterator insn)
571 {
572 	routine->discardMask |= SignMask(activeLaneMask());
573 	SetActiveLaneMask(SIMD::Int(0));
574 }
575 
EmitDemoteToHelperInvocation(InsnIterator insn)576 void SpirvEmitter::EmitDemoteToHelperInvocation(InsnIterator insn)
577 {
578 	routine->helperInvocation |= activeLaneMask();
579 	routine->discardMask |= SignMask(activeLaneMask());
580 	SetStoresAndAtomicsMask(storesAndAtomicsMask() & ~activeLaneMask());
581 }
582 
EmitIsHelperInvocation(InsnIterator insn)583 void SpirvEmitter::EmitIsHelperInvocation(InsnIterator insn)
584 {
585 	auto &type = shader.getType(insn.resultTypeId());
586 	auto &dst = createIntermediate(insn.resultId(), type.componentCount);
587 	dst.move(0, routine->helperInvocation);
588 }
589 
EmitFunctionCall(InsnIterator insn)590 void SpirvEmitter::EmitFunctionCall(InsnIterator insn)
591 {
592 	auto functionId = Spirv::Function::ID(insn.word(3));
593 	const auto &functionIt = shader.functions.find(functionId);
594 	ASSERT(functionIt != shader.functions.end());
595 	auto &function = functionIt->second;
596 
597 	// TODO(b/141246700): Add full support for spv::OpFunctionCall
598 	// The only supported function is a single OpKill wrapped in a
599 	// function, as a result of the "wrap OpKill" SPIRV-Tools pass
600 	ASSERT(function.blocks.size() == 1);
601 	spv::Op wrapOpKill[] = { spv::OpLabel, spv::OpKill };
602 
603 	for(const auto &block : function.blocks)
604 	{
605 		int insnNumber = 0;
606 		for(auto blockInsn : block.second)
607 		{
608 			if(insnNumber > 1)
609 			{
610 				UNIMPLEMENTED("b/141246700: Function block number of instructions: %d", insnNumber);  // FIXME(b/141246700)
611 			}
612 
613 			if(blockInsn.opcode() != wrapOpKill[insnNumber++])
614 			{
615 				UNIMPLEMENTED("b/141246700: Function block instruction %d : %s", insnNumber - 1, shader.OpcodeName(blockInsn.opcode()));  // FIXME(b/141246700)
616 			}
617 
618 			if(blockInsn.opcode() == spv::OpKill)
619 			{
620 				EmitInstruction(blockInsn);
621 			}
622 		}
623 	}
624 }
625 
EmitControlBarrier(InsnIterator insn)626 void SpirvEmitter::EmitControlBarrier(InsnIterator insn)
627 {
628 	auto executionScope = spv::Scope(shader.GetConstScalarInt(insn.word(1)));
629 	auto semantics = spv::MemorySemanticsMask(shader.GetConstScalarInt(insn.word(3)));
630 	// TODO(b/176819536): We probably want to consider the memory scope here.
631 	// For now, just always emit the full fence.
632 	Fence(semantics);
633 
634 	switch(executionScope)
635 	{
636 	case spv::ScopeWorkgroup:
637 		Yield(YieldResult::ControlBarrier);
638 		break;
639 	case spv::ScopeSubgroup:
640 		break;
641 	default:
642 		// See Vulkan 1.1 spec, Appendix A, Validation Rules within a Module.
643 		UNREACHABLE("Scope for execution must be limited to Workgroup or Subgroup");
644 		break;
645 	}
646 }
647 
EmitPhi(InsnIterator insn)648 void SpirvEmitter::EmitPhi(InsnIterator insn)
649 {
650 	auto &function = shader.getFunction(this->function);
651 	auto currentBlock = function.getBlock(block);
652 
653 	if(!currentBlock.isLoopMerge)
654 	{
655 		// If this is a loop merge block, then don't attempt to update the
656 		// phi values from the ins. EmitLoop() has had to take special care
657 		// of this phi in order to correctly deal with divergent lanes.
658 		StorePhi(block, insn, currentBlock.ins);
659 	}
660 
661 	LoadPhi(insn);
662 }
663 
LoadPhi(InsnIterator insn)664 void SpirvEmitter::LoadPhi(InsnIterator insn)
665 {
666 	auto typeId = Type::ID(insn.word(1));
667 	auto type = shader.getType(typeId);
668 	auto objectId = Object::ID(insn.word(2));
669 
670 	auto storageIt = phis.find(objectId);
671 	ASSERT(storageIt != phis.end());
672 	const auto &storage = storageIt->second;
673 
674 	auto &dst = createIntermediate(objectId, type.componentCount);
675 
676 	for(uint32_t i = 0; i < type.componentCount; i++)
677 	{
678 		dst.move(i, storage[i]);
679 		SPIRV_SHADER_DBG("LoadPhi({0}.{1}): {2}", objectId, i, storage[i]);
680 	}
681 }
682 
StorePhi(Block::ID currentBlock,InsnIterator insn,const std::unordered_set<Block::ID> & filter)683 void SpirvEmitter::StorePhi(Block::ID currentBlock, InsnIterator insn, const std::unordered_set<Block::ID> &filter)
684 {
685 	auto typeId = Type::ID(insn.word(1));
686 	auto type = shader.getType(typeId);
687 	auto objectId = Object::ID(insn.word(2));
688 
689 	auto storageIt = phis.find(objectId);
690 	ASSERT(storageIt != phis.end());
691 	auto &storage = storageIt->second;
692 
693 	for(uint32_t w = 3; w < insn.wordCount(); w += 2)
694 	{
695 		auto varId = Object::ID(insn.word(w + 0));
696 		auto blockId = Block::ID(insn.word(w + 1));
697 
698 		if(filter.count(blockId) == 0)
699 		{
700 			continue;
701 		}
702 
703 		auto mask = GetActiveLaneMaskEdge(blockId, currentBlock);
704 		auto in = Operand(shader, *this, varId);
705 
706 		for(uint32_t i = 0; i < type.componentCount; i++)
707 		{
708 			storage[i] = As<SIMD::Float>((As<SIMD::Int>(storage[i]) & ~mask) | (in.Int(i) & mask));
709 			SPIRV_SHADER_DBG("StorePhi({0}.{1}): [{2} <- {3}] {4}: {5}, mask: {6}",
710 			                 objectId, i, currentBlock, blockId, varId, in.UInt(i), mask);
711 		}
712 	}
713 
714 	for(uint32_t i = 0; i < type.componentCount; i++)
715 	{
716 		SPIRV_SHADER_DBG("StorePhi({0}.{1}): {2}", objectId, i, As<SIMD::UInt>(storage[i]));
717 	}
718 }
719 
Yield(YieldResult res) const720 void SpirvEmitter::Yield(YieldResult res) const
721 {
722 	rr::Yield(RValue<Int>(int(res)));
723 }
724 
SetActiveLaneMask(RValue<SIMD::Int> mask)725 void SpirvEmitter::SetActiveLaneMask(RValue<SIMD::Int> mask)
726 {
727 	activeLaneMaskValue = mask.value();
728 }
729 
SetStoresAndAtomicsMask(RValue<SIMD::Int> mask)730 void SpirvEmitter::SetStoresAndAtomicsMask(RValue<SIMD::Int> mask)
731 {
732 	storesAndAtomicsMaskValue = mask.value();
733 }
734 
WriteCFGGraphVizDotFile(const char * path) const735 void Spirv::WriteCFGGraphVizDotFile(const char *path) const
736 {
737 	std::ofstream file(path);
738 	file << "digraph D {" << std::endl;
739 	for(auto &func : functions)
740 	{
741 		file << "  subgraph cluster_function_" << func.first.value() << " {"
742 		     << std::endl;
743 
744 		file << "    label = \"function<" << func.first.value() << ">"
745 		     << (func.first == entryPoint ? " (entry point)" : "")
746 		     << "\"" << std::endl;
747 
748 		for(auto &block : func.second.blocks)
749 		{
750 			file << "    block_" << block.first.value() << " ["
751 			     << "shape=circle "
752 			     << "label=\"" << block.first.value() << "\""
753 			     << "]" << std::endl;
754 		}
755 		file << std::endl;
756 		for(auto &block : func.second.blocks)
757 		{
758 			file << "    block_" << block.first.value() << " -> {";
759 			bool first = true;
760 			for(auto outs : block.second.outs)
761 			{
762 				if(!first) { file << ", "; }
763 				file << "block_" << outs.value();
764 				first = false;
765 			}
766 			file << "}" << std::endl;
767 		}
768 		file << std::endl;
769 		for(auto &block : func.second.blocks)
770 		{
771 			if(block.second.kind == Block::Loop)
772 			{
773 				if(block.second.mergeBlock != 0)
774 				{
775 					file << "    block_" << block.first.value() << " -> "
776 					     << "block_" << block.second.mergeBlock.value()
777 					     << "[label=\"M\" style=dashed color=blue]"
778 					     << std::endl;
779 				}
780 
781 				if(block.second.continueTarget != 0)
782 				{
783 					file << "    block_" << block.first.value() << " -> "
784 					     << "block_" << block.second.continueTarget.value()
785 					     << "[label=\"C\" style=dashed color=green]"
786 					     << std::endl;
787 				}
788 			}
789 		}
790 
791 		file << "  }" << std::endl;
792 	}
793 
794 	for(auto &func : functions)
795 	{
796 		for(auto &block : func.second.blocks)
797 		{
798 			for(auto insn : block.second)
799 			{
800 				if(insn.opcode() == spv::OpFunctionCall)
801 				{
802 					auto target = getFunction(insn.word(3)).entry;
803 					file << "    block_" << block.first.value() << " -> "
804 					     << "block_" << target.value()
805 					     << "[color=\"#00008050\"]"
806 					     << std::endl;
807 				}
808 			}
809 		}
810 	}
811 
812 	file << "}" << std::endl;
813 }
814 
815 }  // namespace sw
816