1 /* -*- mesa-c++ -*-
2 * Copyright 2021 Collabora LTD
3 * Author: Gert Wollny <[email protected]>
4 * SPDX-License-Identifier: MIT
5 */
6
7 #include "sfn_valuefactory.h"
8
9 #include "gallium/drivers/r600/r600_shader.h"
10 #include "sfn_debug.h"
11 #include "sfn_instr.h"
12
13 #include <algorithm>
14 #include <iostream>
15 #include <queue>
16 #include <sstream>
17
18 namespace r600 {
19
20 using std::istringstream;
21 using std::string;
22
ValueFactory()23 ValueFactory::ValueFactory():
24 m_next_register_index(VirtualValue::virtual_register_base),
25 m_nowrite_idx(0)
26 {
27 }
28
29 void
set_virtual_register_base(int base)30 ValueFactory::set_virtual_register_base(int base)
31 {
32 m_next_register_index = base;
33 }
34
35 bool
allocate_registers(const std::list<nir_intrinsic_instr * > & regs)36 ValueFactory::allocate_registers(const std::list<nir_intrinsic_instr *>& regs)
37 {
38 struct array_entry {
39 unsigned index;
40 unsigned length;
41 int ncomponents;
42
43 bool operator()(const array_entry& a, const array_entry& b) const
44 {
45 return a.ncomponents < b.ncomponents ||
46 (a.ncomponents == b.ncomponents && a.length < b.length);
47 }
48 };
49
50 using array_list =
51 std::priority_queue<array_entry, std::vector<array_entry>, array_entry>;
52
53 std::list<unsigned> non_array;
54 array_list arrays;
55 for(auto intr : regs) {
56 unsigned num_elms = nir_intrinsic_num_array_elems(intr);
57 int num_comp = nir_intrinsic_num_components(intr);
58 int bit_size = nir_intrinsic_bit_size(intr);
59
60 if (num_elms > 0 || num_comp > 1 || bit_size > 32) {
61 array_entry ae = {
62 intr->def.index,
63 num_elms ? num_elms : 1,
64 bit_size / 32 * num_comp};
65 arrays.push(ae);
66 } else {
67 non_array.push_back(intr->def.index);
68 }
69 }
70
71 int free_components = 4;
72 int sel = m_next_register_index;
73 unsigned length = 0;
74
75 while (!arrays.empty()) {
76 auto a = arrays.top();
77 arrays.pop();
78
79 /* This is a bit hackish, return an id that encodes the array merge. To
80 * make sure that the mapping doesn't go wrong we have to make sure the
81 * arrays is longer than the number of instances in this arrays slot */
82 if (a.ncomponents > free_components || a.length > length) {
83 sel = m_next_register_index;
84 free_components = 4;
85 m_next_register_index += a.length;
86 }
87
88 uint32_t frac = free_components - a.ncomponents;
89
90 auto array = new LocalArray(sel, a.ncomponents, a.length, frac);
91
92 for (int i = 0; i < a.ncomponents; ++i) {
93 RegisterKey key(a.index, i, vp_array);
94 m_channel_counts.inc_count(frac + i, a.length);
95 m_registers[key] = array;
96 sfn_log << SfnLog::reg << __func__ << ": Allocate array " << key << ":" << *array
97 << "\n";
98 }
99
100 free_components -= a.ncomponents;
101 length = a.length;
102 }
103
104 m_required_array_registers = m_next_register_index ? m_next_register_index : 0;
105
106 for (auto index : non_array) {
107 RegisterKey key(index, 0, vp_register);
108 auto chan = m_channel_counts.least_used(0xf);
109 m_registers[key] = new Register(m_next_register_index++,
110 chan, pin_free);
111 m_channel_counts.inc_count(chan);
112 }
113
114 return true;
115 }
116
new_register_index()117 int ValueFactory::new_register_index()
118 {
119 return m_next_register_index++;
120 }
121
122 PRegister
allocate_pinned_register(int sel,int chan)123 ValueFactory::allocate_pinned_register(int sel, int chan)
124 {
125 if (m_next_register_index <= sel)
126 m_next_register_index = sel + 1;
127
128 auto reg = new Register(sel, chan, pin_fully);
129 reg->set_flag(Register::pin_start);
130 reg->set_flag(Register::ssa);
131 m_pinned_registers.push_back(reg);
132 return reg;
133 }
134
135 RegisterVec4
allocate_pinned_vec4(int sel,bool is_ssa)136 ValueFactory::allocate_pinned_vec4(int sel, bool is_ssa)
137 {
138 if (m_next_register_index <= sel)
139 m_next_register_index = sel + 1;
140
141 RegisterVec4 retval(sel, is_ssa, {0, 1, 2, 3}, pin_fully);
142 for (int i = 0; i < 4; ++i) {
143 retval[i]->set_flag(Register::pin_start);
144 retval[i]->set_flag(Register::ssa);
145 m_pinned_registers.push_back(retval[i]);
146 }
147 return retval;
148 }
149
150 void
inject_value(const nir_def & def,int chan,PVirtualValue value)151 ValueFactory::inject_value(const nir_def& def, int chan, PVirtualValue value)
152 {
153 RegisterKey key(def.index, chan, vp_ssa);
154 sfn_log << SfnLog::reg << "Inject value with key " << key << "\n";
155 assert(m_values.find(key) == m_values.end());
156 m_values[key] = value;
157 }
158
159 class TranslateRegister : public RegisterVisitor {
160 public:
visit(VirtualValue & value)161 void visit(VirtualValue& value) { (void)value; }
visit(Register & value)162 void visit(Register& value) { (void)value; };
visit(LocalArray & value)163 void visit(LocalArray& value) { m_value = value.element(m_offset, m_addr, m_chan); }
visit(LocalArrayValue & value)164 void visit(LocalArrayValue& value) { (void)value; }
visit(UniformValue & value)165 void visit(UniformValue& value) { (void)value; }
visit(LiteralConstant & value)166 void visit(LiteralConstant& value) { (void)value; }
visit(InlineConstant & value)167 void visit(InlineConstant& value) { (void)value; }
168
TranslateRegister(int offset,PVirtualValue addr,int chan)169 TranslateRegister(int offset, PVirtualValue addr, int chan):
170 m_addr(addr),
171 m_value(nullptr),
172 m_offset(offset),
173 m_chan(chan)
174 {
175 }
176
177 PVirtualValue m_addr;
178 PRegister m_value;
179 int m_offset;
180 int m_chan;
181 };
182
183 void
allocate_const(nir_load_const_instr * load_const)184 ValueFactory::allocate_const(nir_load_const_instr *load_const)
185 {
186 assert(load_const->def.bit_size == 32);
187 for (int i = 0; i < load_const->def.num_components; ++i) {
188 RegisterKey key(load_const->def.index, i, vp_ssa);
189 m_values[key] = literal(load_const->value[i].i32);
190 sfn_log << SfnLog::reg << "Add const with key " << key << " as " << m_values[key]
191 << "\n";
192 }
193 }
194
195 PVirtualValue
uniform(nir_intrinsic_instr * load_uniform,int chan)196 ValueFactory::uniform(nir_intrinsic_instr *load_uniform, int chan)
197 {
198 auto literal = nir_src_as_const_value(load_uniform->src[0]);
199 assert(literal);
200
201 int index = nir_intrinsic_base(load_uniform) + +literal->u32 + 512;
202
203 return uniform(index, chan, 0);
204 }
205
206 PVirtualValue
uniform(uint32_t index,int chan,int kcache)207 ValueFactory::uniform(uint32_t index, int chan, int kcache)
208 {
209 return new UniformValue(index, chan, kcache);
210 }
211
212 PRegister
temp_register(int pinned_channel,bool is_ssa)213 ValueFactory::temp_register(int pinned_channel, bool is_ssa)
214 {
215 int sel = m_next_register_index++;
216 int chan = (pinned_channel >= 0) ? pinned_channel : m_channel_counts.least_used(0xf);
217
218 auto reg = new Register(sel, chan, pinned_channel >= 0 ? pin_chan : pin_free);
219 m_channel_counts.inc_count(chan);
220
221 if (is_ssa)
222 reg->set_flag(Register::ssa);
223
224 m_registers[RegisterKey(sel, chan, vp_temp)] = reg;
225 return reg;
226 }
227
228 RegisterVec4
temp_vec4(Pin pin,const RegisterVec4::Swizzle & swizzle)229 ValueFactory::temp_vec4(Pin pin, const RegisterVec4::Swizzle& swizzle)
230 {
231 int sel = m_next_register_index++;
232
233 if (pin == pin_free)
234 pin = pin_chan;
235
236 PRegister vec4[4];
237
238 for (int i = 0; i < 4; ++i) {
239 vec4[i] = new Register(sel, swizzle[i], pin);
240 vec4[i]->set_flag(Register::ssa);
241 m_registers[RegisterKey(sel, swizzle[i], vp_temp)] = vec4[i];
242 }
243 return RegisterVec4(vec4[0], vec4[1], vec4[2], vec4[3], pin);
244 }
245
246 RegisterVec4
dest_vec4(const nir_def & def,Pin pin)247 ValueFactory::dest_vec4(const nir_def& def, Pin pin)
248 {
249 if (pin != pin_group && pin != pin_chgr)
250 pin = pin_chan;
251 PRegister x = dest(def, 0, pin);
252 PRegister y = dest(def, 1, pin);
253 PRegister z = dest(def, 2, pin);
254 PRegister w = dest(def, 3, pin);
255 return RegisterVec4(x, y, z, w, pin);
256 }
257
addr()258 PRegister ValueFactory::addr()
259 {
260 if (!m_ar)
261 m_ar = new AddressRegister(AddressRegister::addr);
262 return m_ar;
263 }
264
idx_reg(unsigned idx)265 PRegister ValueFactory::idx_reg(unsigned idx)
266 {
267
268 if (idx == 0) {
269 if (!m_idx0)
270 m_idx0 = new AddressRegister(AddressRegister::idx0);
271 return m_idx0;
272 } else {
273 assert(idx == 1);
274 if (!m_idx1)
275 m_idx1 = new AddressRegister(AddressRegister::idx1);
276 return m_idx1;
277 }
278 }
279
280
281 PVirtualValue
src(const nir_alu_src & alu_src,int chan)282 ValueFactory::src(const nir_alu_src& alu_src, int chan)
283 {
284 return src(alu_src.src, alu_src.swizzle[chan]);
285 }
286
287 PVirtualValue
src64(const nir_alu_src & alu_src,int chan,int comp)288 ValueFactory::src64(const nir_alu_src& alu_src, int chan, int comp)
289 {
290 return src(alu_src.src, 2 * alu_src.swizzle[chan] + comp);
291 }
292
293 PVirtualValue
src(const nir_src & src,int chan)294 ValueFactory::src(const nir_src& src, int chan)
295 {
296 sfn_log << SfnLog::reg << "search (ref) " << (void *)&src << "\n";
297
298 sfn_log << SfnLog::reg << "search ssa " << src.ssa->index << " c:" << chan
299 << " got ";
300 auto val = ssa_src(*src.ssa, chan);
301 sfn_log << *val << "\n";
302 return val;
303 }
304
305 PVirtualValue
src(const nir_tex_src & tex_src,int chan)306 ValueFactory::src(const nir_tex_src& tex_src, int chan)
307 {
308 return src(tex_src.src, chan);
309 }
310
311 PRegister
dummy_dest(unsigned chan)312 ValueFactory::dummy_dest(unsigned chan)
313 {
314 assert(chan < 4);
315 return m_dummy_dest_pinned[chan];
316 }
317
318 PRegister
dest(const nir_def & ssa,int chan,Pin pin_channel,uint8_t chan_mask)319 ValueFactory::dest(const nir_def& ssa, int chan, Pin pin_channel, uint8_t chan_mask)
320 {
321 RegisterKey key(ssa.index, chan, vp_ssa);
322
323 /* dirty workaround for Cayman trans ops, because we may request
324 * the same sa reg more than once, but only write to it once. */
325 auto ireg = m_registers.find(key);
326 if (ireg != m_registers.end())
327 return ireg->second;
328
329 auto isel = m_ssa_index_to_sel.find(ssa.index);
330 int sel;
331 if (isel != m_ssa_index_to_sel.end())
332 sel = isel->second;
333 else {
334 sel = m_next_register_index++;
335 sfn_log << SfnLog::reg << "Assign " << sel << " to index " << ssa.index << " in "
336 << &m_ssa_index_to_sel << "\n";
337 m_ssa_index_to_sel[ssa.index] = sel;
338 }
339
340 if (pin_channel == pin_free)
341 chan = m_channel_counts.least_used(chan_mask);
342
343 auto vreg = new Register(sel, chan, pin_channel);
344 m_channel_counts.inc_count(chan);
345 vreg->set_flag(Register::ssa);
346 m_registers[key] = vreg;
347 sfn_log << SfnLog::reg << "allocate Ssa " << key << ":" << *vreg << "\n";
348 return vreg;
349 }
350
351 PVirtualValue
zero()352 ValueFactory::zero()
353 {
354 return inline_const(ALU_SRC_0, 0);
355 }
356
357 PVirtualValue
one()358 ValueFactory::one()
359 {
360 return inline_const(ALU_SRC_1, 0);
361 }
362
363 PVirtualValue
one_i()364 ValueFactory::one_i()
365 {
366 return inline_const(ALU_SRC_1_INT, 0);
367 }
368
369 PRegister
undef(int index,int chan)370 ValueFactory::undef(int index, int chan)
371 {
372 RegisterKey key(index, chan, vp_ssa);
373 PRegister reg = new Register(m_next_register_index++, 0, pin_free);
374 reg->set_flag(Register::ssa);
375 m_registers[key] = reg;
376 return reg;
377 }
378
379 PVirtualValue
ssa_src(const nir_def & ssa,int chan)380 ValueFactory::ssa_src(const nir_def& ssa, int chan)
381 {
382 RegisterKey key(ssa.index, chan, vp_ssa);
383 sfn_log << SfnLog::reg << "search src with key" << key << "\n";
384
385 auto ireg = m_registers.find(key);
386 if (ireg != m_registers.end())
387 return ireg->second;
388
389 auto ival = m_values.find(key);
390 if (ival != m_values.end())
391 return ival->second;
392
393 RegisterKey rkey(ssa.index, chan, vp_register);
394 sfn_log << SfnLog::reg << "search src with key" << rkey << "\n";
395
396 ireg = m_registers.find(rkey);
397 if (ireg != m_registers.end())
398 return ireg->second;
399
400 RegisterKey array_key(ssa.index, chan, vp_array);
401 sfn_log << SfnLog::reg << "search array with key" << array_key << "\n";
402 auto iarray = m_registers.find(array_key);
403 if (iarray != m_registers.end())
404 return iarray->second;
405
406 std::cerr << "Didn't find source with key " << key << "\n";
407 unreachable("Source values should always exist");
408 }
409
410 PVirtualValue
literal(uint32_t value)411 ValueFactory::literal(uint32_t value)
412 {
413 auto iv = m_literal_values.find(value);
414 if (iv != m_literal_values.end())
415 return iv->second;
416
417 auto v = new LiteralConstant(value);
418 m_literal_values[value] = v;
419 return v;
420 }
421
422 PInlineConstant
inline_const(AluInlineConstants sel,int chan)423 ValueFactory::inline_const(AluInlineConstants sel, int chan)
424 {
425 int hash = (sel << 3) | chan;
426 auto iv = m_inline_constants.find(hash);
427 if (iv != m_inline_constants.end())
428 return iv->second;
429 auto v = new InlineConstant(sel, chan);
430 m_inline_constants[hash] = v;
431 return v;
432 }
433
434 std::vector<PVirtualValue, Allocator<PVirtualValue>>
src_vec(const nir_src & source,int components)435 ValueFactory::src_vec(const nir_src& source, int components)
436 {
437 std::vector<PVirtualValue, Allocator<PVirtualValue>> retval;
438 retval.reserve(components);
439 for (int i = 0; i < components; ++i)
440 retval.push_back(src(source, i));
441 return retval;
442 }
443
444 std::vector<PRegister, Allocator<PRegister>>
dest_vec(const nir_def & def,int num_components)445 ValueFactory::dest_vec(const nir_def& def, int num_components)
446 {
447 std::vector<PRegister, Allocator<PRegister>> retval;
448 retval.reserve(num_components);
449 for (int i = 0; i < num_components; ++i)
450 retval.push_back(dest(def, i, num_components > 1 ? pin_none : pin_free));
451 return retval;
452 }
453
454 RegisterVec4
src_vec4(const nir_src & source,Pin pin,const RegisterVec4::Swizzle & swz)455 ValueFactory::src_vec4(const nir_src& source, Pin pin, const RegisterVec4::Swizzle& swz)
456 {
457 auto sx = swz[0] < 4 ? src(source, swz[0])->as_register() : nullptr;
458 auto sy = swz[1] < 4 ? src(source, swz[1])->as_register() : nullptr;
459 auto sz = swz[2] < 4 ? src(source, swz[2])->as_register() : nullptr;
460 auto sw = swz[3] < 4 ? src(source, swz[3])->as_register() : nullptr;
461
462 assert(sx || sy || sz || sw);
463
464 int sel = sx ? sx->sel() : (sy ? sy->sel() : (sz ? sz->sel() : sw ? sw->sel() : -1));
465 if (sel < 0)
466 unreachable("source vector without valid components");
467
468 if (!sx)
469 sx = new Register(sel, 7, pin);
470 if (!sy)
471 sy = new Register(sel, 7, pin);
472 if (!sz)
473 sz = new Register(sel, 7, pin);
474 if (!sw)
475 sw = new Register(sel, 7, pin);
476
477 return RegisterVec4(sx, sy, sz, sw, pin);
478 }
479
480 static Pin
pin_from_string(const std::string & pinstr)481 pin_from_string(const std::string& pinstr)
482 {
483 if (pinstr == "chan")
484 return pin_chan;
485 if (pinstr == "array")
486 return pin_array;
487 if (pinstr == "fully")
488 return pin_fully;
489 if (pinstr == "group")
490 return pin_group;
491 if (pinstr == "chgr")
492 return pin_chgr;
493 if (pinstr == "free")
494 return pin_free;
495 return pin_none;
496 }
497
498 static int
chan_from_char(char chan)499 chan_from_char(char chan)
500 {
501 switch (chan) {
502 case 'x':
503 return 0;
504 case 'y':
505 return 1;
506 case 'z':
507 return 2;
508 case 'w':
509 return 3;
510 case '0':
511 return 4;
512 case '1':
513 return 5;
514 case '_':
515 return 7;
516 }
517 unreachable("Unknown swizzle char");
518 }
519
520 static int
str_to_int(const string & s)521 str_to_int(const string& s)
522 {
523 istringstream ss(s);
524 int retval;
525 ss >> retval;
526 return retval;
527 }
528
529 static bool
split_register_string(const string & s,string & index_str,string & size_str,string & swizzle_str,string & pin_str)530 split_register_string(const string& s,
531 string& index_str,
532 string& size_str,
533 string& swizzle_str,
534 string& pin_str)
535 {
536 int type = 0;
537 for (unsigned i = 1; i < s.length(); ++i) {
538 if (s[i] == '.' && type != 3) {
539 type = 1;
540 continue;
541 } else if (s[i] == '@' && type != 3) {
542 type = 2;
543 continue;
544 } else if (s[i] == '[') {
545 type = 3;
546 continue;
547 } else if (s[i] == ']') {
548 if (type != 3)
549 std::cerr << "s=" << s << ": type=" << type << ": i=" << i << "\n";
550 assert(type == 3);
551
552 type = 4;
553 continue;
554 }
555
556 switch (type) {
557 case 0:
558 index_str.append(1, s[i]);
559 break;
560 case 1:
561 swizzle_str.append(1, s[i]);
562 break;
563 case 2:
564 pin_str.append(1, s[i]);
565 break;
566 case 3:
567 size_str.append(1, s[i]);
568 break;
569 default:
570 unreachable("Malformed Array allocation string");
571 }
572 }
573 return true;
574 }
575
576 PRegister
dest_from_string(const std::string & s)577 ValueFactory::dest_from_string(const std::string& s)
578 {
579 if (s == "AR") {
580 if (!m_ar)
581 m_ar = new AddressRegister(AddressRegister::addr);
582 return m_ar;
583 } else if (s == "IDX0") {
584 if (!m_idx0)
585 m_idx0 = new AddressRegister(AddressRegister::idx0);
586 return m_idx0;
587 } else if (s == "IDX1") {
588 if (!m_idx1)
589 m_idx1 = new AddressRegister(AddressRegister::idx1);
590 return m_idx1;
591 }
592
593 string index_str;
594 string size_str;
595 string swizzle_str;
596 string pin_str;
597
598 assert(s.length() >= 4);
599
600 assert(strchr("ARS_", s[0]));
601
602 split_register_string(s, index_str, size_str, swizzle_str, pin_str);
603
604 int sel = 0;
605 if (s[0] == '_') {
606 /* Since these instructions still may use or switch to a different
607 * channel we have to create a new instance for each occurrence */
608 sel = std::numeric_limits<int>::max() - m_nowrite_idx++;
609 } else {
610 std::istringstream n(index_str);
611 n >> sel;
612 }
613
614 auto p = pin_from_string(pin_str);
615 char chan = chan_from_char(swizzle_str[0]);
616
617 EValuePool pool = vp_temp;
618 switch (s[0]) {
619 case 'A':
620 pool = vp_array;
621 break;
622 case 'R':
623 pool = vp_register;
624 break;
625 case '_':
626 pool = vp_ignore;
627 break;
628 case 'S':
629 pool = vp_ssa;
630 break;
631 default:
632 unreachable("Unknown value type");
633 }
634
635 bool is_ssa = s[0] == 'S';
636
637 RegisterKey key(sel, chan, pool);
638
639 sfn_log << SfnLog::reg << "Search register with key " << key << "\n";
640
641 auto ireg = m_registers.find(key);
642 if (ireg == m_registers.end()) {
643 auto reg = new Register(sel, chan, p);
644 if (s[0] == 'S')
645 reg->set_flag(Register::ssa);
646 if (p == pin_fully)
647 reg->set_flag(Register::pin_start);
648 m_registers[key] = reg;
649 return reg;
650 } else if (pool == vp_ignore) {
651 assert(ireg->second->sel() == std::numeric_limits<int>::max());
652 return ireg->second;
653 } else {
654 assert(!is_ssa || s[0] == '_');
655
656 if (size_str.length()) {
657 auto array = static_cast<LocalArray *>(ireg->second);
658 PVirtualValue addr = nullptr;
659 int offset = 0;
660 if (size_str[0] == 'S' || size_str[0] == 'R' ||
661 size_str == "AR" || size_str.substr(0,3) == "IDX") {
662 addr = src_from_string(size_str);
663 } else {
664 istringstream num_str(size_str);
665 num_str >> offset;
666 }
667
668 return array->element(offset, addr, chan - array->frac());
669 } else
670 return ireg->second;
671 }
672 }
673
674 PVirtualValue
src_from_string(const std::string & s)675 ValueFactory::src_from_string(const std::string& s)
676 {
677 if (s == "AR") {
678 assert(m_ar);
679 return m_ar;
680 } else if (s == "IDX0") {
681 assert(m_idx0);
682 return m_idx0;
683 } else if (s == "IDX1") {
684 assert(m_idx1);
685 return m_idx1;
686 }
687
688 switch (s[0]) {
689 case 'A':
690 case 'S':
691 case 'R':
692 break;
693 case 'L':
694 return LiteralConstant::from_string(s);
695 case 'K':
696 return UniformValue::from_string(s, this);
697 case 'P':
698 return InlineConstant::param_from_string(s);
699 case 'I':
700 return InlineConstant::from_string(s);
701
702 default:
703 std::cerr << "'" << s << "'";
704 unreachable("Unknown register type");
705 }
706
707 assert(strchr("ARS_", s[0]));
708
709 string index_str;
710 string size_str;
711 string swizzle_str;
712 string pin_str;
713
714 split_register_string(s, index_str, size_str, swizzle_str, pin_str);
715
716 int sel = 0;
717 if (s[0] == '_') {
718 sel = std::numeric_limits<int>::max();
719 } else {
720 std::istringstream n(index_str);
721 n >> sel;
722 }
723
724 auto p = pin_from_string(pin_str);
725 char chan = chan_from_char(swizzle_str[0]);
726
727 EValuePool pool = vp_temp;
728 switch (s[0]) {
729 case 'A':
730 pool = vp_array;
731 break;
732 case 'R':
733 pool = vp_register;
734 break;
735 case '_':
736 pool = vp_ignore;
737 break;
738 case 'S':
739 pool = vp_ssa;
740 break;
741 default:
742 unreachable("Unknown value type");
743 }
744
745 RegisterKey key(sel, chan, pool);
746
747 auto ireg = m_registers.find(key);
748 if (ireg != m_registers.end()) {
749 if (pool != vp_ssa && size_str.length()) {
750 auto array = static_cast<LocalArray *>(ireg->second);
751 PVirtualValue addr = nullptr;
752 int offset = 0;
753 if (size_str[0] == 'S' || size_str[0] == 'R' ||
754 size_str == "AR" || size_str.substr(0,3) == "IDX") {
755 addr = src_from_string(size_str);
756 } else {
757 istringstream num_str(size_str);
758 num_str >> offset;
759 }
760 return array->element(offset, addr, chan - array->frac());
761 } else {
762 return ireg->second;
763 }
764 } else {
765 if (sel != std::numeric_limits<int>::max()) {
766 std::cerr << "register " << key << "not found \n";
767 unreachable("Source register should exist");
768 } else {
769 auto reg = new Register(sel, chan, p);
770 m_registers[key] = reg;
771 return reg;
772 }
773 }
774 }
775
776 RegisterVec4
dest_vec4_from_string(const std::string & s,RegisterVec4::Swizzle & swz,Pin pin)777 ValueFactory::dest_vec4_from_string(const std::string& s,
778 RegisterVec4::Swizzle& swz,
779 Pin pin)
780 {
781 bool is_ssa = false;
782 int sel = sel_and_szw_from_string(s, swz, is_ssa);
783
784 PRegister v[4];
785
786 for (int i = 0; i < 4; ++i) {
787 auto pool = is_ssa ? vp_ssa : vp_register;
788 if (swz[i] > 3)
789 pool = vp_ignore;
790
791 RegisterKey key(sel, i, pool);
792 auto ireg = m_registers.find(key);
793 if (ireg != m_registers.end()) {
794 v[i] = ireg->second;
795 assert(!is_ssa || pool == vp_ignore);
796 } else {
797 v[i] = new Register(sel, i, pin);
798 if (is_ssa)
799 v[i]->set_flag(Register::ssa);
800 m_registers[key] = v[i];
801 }
802 }
803 return RegisterVec4(v[0], v[1], v[2], v[3], pin);
804 }
805
806 RegisterVec4
src_vec4_from_string(const std::string & s)807 ValueFactory::src_vec4_from_string(const std::string& s)
808 {
809 RegisterVec4::Swizzle swz;
810 bool is_ssa = false;
811 int sel = sel_and_szw_from_string(s, swz, is_ssa);
812
813 PRegister v[4];
814
815 PRegister used_reg = nullptr;
816 for (int i = 0; i < 4; ++i) {
817 if (swz[i] < 4) {
818 RegisterKey key(sel, swz[i], is_ssa ? vp_ssa : vp_register);
819 auto ireg = m_registers.find(key);
820 if (ireg == m_registers.end()) {
821 std::cerr << s << ": Register with key " << key << " not found\n";
822 assert(0);
823 }
824 used_reg = v[i] = ireg->second;
825 } else {
826 v[i] = nullptr;
827 }
828 }
829 sel = used_reg ? used_reg->sel() : 0;
830 Pin pin = used_reg ? used_reg->pin() : pin_group;
831
832 for (int i = 0; i < 4; ++i) {
833 if (!v[i]) {
834 v[i] = new Register(sel, swz[i], pin);
835 if (is_ssa)
836 v[i]->set_flag(Register::ssa);
837 } else {
838 if (v[i]->pin() == pin_none)
839 v[i]->set_pin(pin_group);
840 }
841 }
842 return RegisterVec4(v[0], v[1], v[2], v[3], pin);
843 }
844
845 LocalArray *
array_from_string(const std::string & s)846 ValueFactory::array_from_string(const std::string& s)
847 {
848 assert(s[0] == 'A');
849 string index_str;
850 string size_str;
851 string swizzle_str;
852 string pin_str;
853
854 int type = 0;
855 for (unsigned i = 1; i < s.length(); ++i) {
856 if (s[i] == '.') {
857 type = 1;
858 continue;
859 } else if (s[i] == '@') {
860 type = 2;
861 continue;
862 } else if (s[i] == '[') {
863 type = 3;
864 continue;
865 } else if (s[i] == ']') {
866 assert(type == 3);
867 type = 4;
868 continue;
869 }
870
871 switch (type) {
872 case 0:
873 index_str.append(1, s[i]);
874 break;
875 case 1:
876 swizzle_str.append(1, s[i]);
877 break;
878 case 2:
879 pin_str.append(1, s[i]);
880 break;
881 case 3:
882 size_str.append(1, s[i]);
883 break;
884 default:
885 unreachable("Malformed Array allocation string");
886 }
887 }
888
889 int sel = str_to_int(index_str);
890 int size = str_to_int(size_str);
891 int ncomp = swizzle_str.length();
892
893 if (ncomp > 4 || ncomp <= 0) {
894 std::cerr << "Error reading array from '" << s << ": ";
895 std::cerr << "index:'" << index_str << "' -> '" << sel << "' size:'" << size_str
896 << "' -> '" << size << " swizzle:'" << swizzle_str << "' -> '" << ncomp
897 << "'\n";
898 assert(0);
899 }
900
901 const char *swz = "xyzw";
902 const char *first_swz = strchr(swz, swizzle_str[0]);
903 long frac = first_swz - swz;
904 assert(frac >= 0 && frac <= 4 - ncomp);
905
906 auto array = new LocalArray(sel, ncomp, size, frac);
907
908 for (int i = 0; i < ncomp; ++i) {
909 RegisterKey key(sel, i + frac, vp_array);
910 m_registers[key] = array;
911 }
912 return array;
913 }
914
915 void
append_register(Register * reg)916 LiveRangeMap::append_register(Register *reg)
917 {
918 sfn_log << SfnLog::merge << __func__ << ": " << *reg << "\n";
919
920 auto chan = reg->chan();
921 auto& ranges = m_life_ranges[chan];
922
923 LiveRangeEntry entry(reg);
924 ranges.emplace_back(entry);
925 }
926
927 std::array<size_t, 4>
sizes() const928 LiveRangeMap::sizes() const
929 {
930 std::array<size_t, 4> result;
931 std::transform(m_life_ranges.begin(),
932 m_life_ranges.end(),
933 result.begin(),
934 [](auto lr) { return lr.size(); });
935 return result;
936 }
937
938 LiveRangeMap
prepare_live_range_map()939 ValueFactory::prepare_live_range_map()
940 {
941 LiveRangeMap result;
942
943 for (auto [key, reg] : m_registers) {
944 if (key.value.pool == vp_ignore)
945 continue;
946
947 if (key.value.pool == vp_array) {
948 auto array = static_cast<LocalArray *>(reg);
949 for (auto& a : *array) {
950 result.append_register(a);
951 }
952 } else {
953 if (reg->chan() < 4)
954 result.append_register(reg);
955 }
956 }
957
958 for (auto r : m_pinned_registers) {
959 result.append_register(r);
960 }
961
962 for (int i = 0; i < 4; ++i) {
963 auto& comp = result.component(i);
964 std::sort(comp.begin(),
965 comp.end(),
966 [](const LiveRangeEntry& lhs, const LiveRangeEntry& rhs) {
967 return lhs.m_register->sel() < rhs.m_register->sel();
968 });
969 for (size_t j = 0; j < comp.size(); ++j)
970 comp[j].m_register->set_index(j);
971 }
972
973 return result;
974 }
975
976 void
clear_pins()977 ValueFactory::clear_pins()
978 {
979 for (auto [key, reg] : m_registers)
980 reg->set_pin(pin_none);
981
982 for (auto reg : m_pinned_registers)
983 reg->set_pin(pin_none);
984 }
985
986 void
clear()987 ValueFactory::clear()
988 {
989 m_registers.clear();
990 m_values.clear();
991 m_literal_values.clear();
992 m_inline_constants.clear();
993 m_ssa_index_to_sel.clear();
994 }
995
996 void
get_shader_info(r600_shader * sh_info)997 ValueFactory::get_shader_info(r600_shader *sh_info)
998 {
999 std::set<LocalArray *> arrays;
1000
1001 for (auto& [key, reg] : m_registers) {
1002 if (key.value.pool == vp_array)
1003 arrays.insert(static_cast<LocalArray *>(reg));
1004 }
1005
1006 if (!arrays.empty()) {
1007
1008 sh_info->num_arrays = arrays.size();
1009 sh_info->arrays =
1010 (r600_shader_array *)malloc(sizeof(struct r600_shader_array) * arrays.size());
1011
1012 for (auto& arr : arrays) {
1013 sh_info->arrays->gpr_start = arr->sel();
1014 sh_info->arrays->gpr_count = arr->size();
1015 sh_info->arrays->comp_mask = ((1 << arr->nchannels()) - 1) << arr->frac();
1016 }
1017 sh_info->indirect_files |= 1 << TGSI_FILE_TEMPORARY;
1018 }
1019 }
1020
1021 } // namespace r600
1022