1 /*
2 * Copyright 2011 Christoph Bumiller
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23 #include "nv50_ir.h"
24 #include "nv50_ir_build_util.h"
25
26 namespace nv50_ir {
27
BuildUtil()28 BuildUtil::BuildUtil()
29 {
30 init(NULL);
31 }
32
BuildUtil(Program * prog)33 BuildUtil::BuildUtil(Program *prog)
34 {
35 init(prog);
36 }
37
38 void
init(Program * prog)39 BuildUtil::init(Program *prog)
40 {
41 this->prog = prog;
42
43 func = NULL;
44 bb = NULL;
45 pos = NULL;
46
47 tail = false;
48
49 memset(imms, 0, sizeof(imms));
50 immCount = 0;
51 }
52
53 void
addImmediate(ImmediateValue * imm)54 BuildUtil::addImmediate(ImmediateValue *imm)
55 {
56 if (immCount > (NV50_IR_BUILD_IMM_HT_SIZE * 3) / 4)
57 return;
58
59 unsigned int pos = u32Hash(imm->reg.data.u32);
60
61 while (imms[pos])
62 pos = (pos + 1) % NV50_IR_BUILD_IMM_HT_SIZE;
63 imms[pos] = imm;
64 immCount++;
65 }
66
67 Instruction *
mkOp1(operation op,DataType ty,Value * dst,Value * src)68 BuildUtil::mkOp1(operation op, DataType ty, Value *dst, Value *src)
69 {
70 Instruction *insn = new_Instruction(func, op, ty);
71
72 insn->setDef(0, dst);
73 insn->setSrc(0, src);
74
75 insert(insn);
76 return insn;
77 }
78
79 Instruction *
mkOp2(operation op,DataType ty,Value * dst,Value * src0,Value * src1)80 BuildUtil::mkOp2(operation op, DataType ty, Value *dst,
81 Value *src0, Value *src1)
82 {
83 Instruction *insn = new_Instruction(func, op, ty);
84
85 insn->setDef(0, dst);
86 insn->setSrc(0, src0);
87 insn->setSrc(1, src1);
88
89 insert(insn);
90 return insn;
91 }
92
93 Instruction *
mkOp3(operation op,DataType ty,Value * dst,Value * src0,Value * src1,Value * src2)94 BuildUtil::mkOp3(operation op, DataType ty, Value *dst,
95 Value *src0, Value *src1, Value *src2)
96 {
97 Instruction *insn = new_Instruction(func, op, ty);
98
99 insn->setDef(0, dst);
100 insn->setSrc(0, src0);
101 insn->setSrc(1, src1);
102 insn->setSrc(2, src2);
103
104 insert(insn);
105 return insn;
106 }
107
108 Instruction *
mkLoad(DataType ty,Value * dst,Symbol * mem,Value * ptr)109 BuildUtil::mkLoad(DataType ty, Value *dst, Symbol *mem, Value *ptr)
110 {
111 Instruction *insn = new_Instruction(func, OP_LOAD, ty);
112
113 insn->setDef(0, dst);
114 insn->setSrc(0, mem);
115 if (ptr)
116 insn->setIndirect(0, 0, ptr);
117
118 insert(insn);
119 return insn;
120 }
121
122 Instruction *
mkStore(operation op,DataType ty,Symbol * mem,Value * ptr,Value * stVal)123 BuildUtil::mkStore(operation op, DataType ty, Symbol *mem, Value *ptr,
124 Value *stVal)
125 {
126 Instruction *insn = new_Instruction(func, op, ty);
127
128 insn->setSrc(0, mem);
129 insn->setSrc(1, stVal);
130 if (ptr)
131 insn->setIndirect(0, 0, ptr);
132
133 insert(insn);
134 return insn;
135 }
136
137 Instruction *
mkFetch(Value * dst,DataType ty,DataFile file,int32_t offset,Value * attrRel,Value * primRel)138 BuildUtil::mkFetch(Value *dst, DataType ty, DataFile file, int32_t offset,
139 Value *attrRel, Value *primRel)
140 {
141 Symbol *sym = mkSymbol(file, 0, ty, offset);
142
143 Instruction *insn = mkOp1(OP_VFETCH, ty, dst, sym);
144
145 insn->setIndirect(0, 0, attrRel);
146 insn->setIndirect(0, 1, primRel);
147
148 // already inserted
149 return insn;
150 }
151
152 Instruction *
mkInterp(unsigned mode,Value * dst,int32_t offset,Value * rel)153 BuildUtil::mkInterp(unsigned mode, Value *dst, int32_t offset, Value *rel)
154 {
155 operation op = OP_LINTERP;
156 DataType ty = TYPE_F32;
157
158 if ((mode & NV50_IR_INTERP_MODE_MASK) == NV50_IR_INTERP_FLAT)
159 ty = TYPE_U32;
160 else
161 if ((mode & NV50_IR_INTERP_MODE_MASK) == NV50_IR_INTERP_PERSPECTIVE)
162 op = OP_PINTERP;
163
164 Symbol *sym = mkSymbol(FILE_SHADER_INPUT, 0, ty, offset);
165
166 Instruction *insn = mkOp1(op, ty, dst, sym);
167 insn->setIndirect(0, 0, rel);
168 insn->setInterpolate(mode);
169 return insn;
170 }
171
172 Instruction *
mkMov(Value * dst,Value * src,DataType ty)173 BuildUtil::mkMov(Value *dst, Value *src, DataType ty)
174 {
175 Instruction *insn = new_Instruction(func, OP_MOV, ty);
176
177 insn->setDef(0, dst);
178 insn->setSrc(0, src);
179
180 insert(insn);
181 return insn;
182 }
183
184 Instruction *
mkMovToReg(int id,Value * src)185 BuildUtil::mkMovToReg(int id, Value *src)
186 {
187 Instruction *insn = new_Instruction(func, OP_MOV, typeOfSize(src->reg.size));
188
189 insn->setDef(0, new_LValue(func, FILE_GPR));
190 insn->getDef(0)->reg.data.id = id;
191 insn->setSrc(0, src);
192
193 insert(insn);
194 return insn;
195 }
196
197 Instruction *
mkMovFromReg(Value * dst,int id)198 BuildUtil::mkMovFromReg(Value *dst, int id)
199 {
200 Instruction *insn = new_Instruction(func, OP_MOV, typeOfSize(dst->reg.size));
201
202 insn->setDef(0, dst);
203 insn->setSrc(0, new_LValue(func, FILE_GPR));
204 insn->getSrc(0)->reg.data.id = id;
205
206 insert(insn);
207 return insn;
208 }
209
210 Instruction *
mkCvt(operation op,DataType dstTy,Value * dst,DataType srcTy,Value * src)211 BuildUtil::mkCvt(operation op,
212 DataType dstTy, Value *dst, DataType srcTy, Value *src)
213 {
214 Instruction *insn = new_Instruction(func, op, dstTy);
215
216 insn->setType(dstTy, srcTy);
217 insn->setDef(0, dst);
218 insn->setSrc(0, src);
219
220 insert(insn);
221 return insn;
222 }
223
224 CmpInstruction *
mkCmp(operation op,CondCode cc,DataType dstTy,Value * dst,DataType srcTy,Value * src0,Value * src1,Value * src2)225 BuildUtil::mkCmp(operation op, CondCode cc, DataType dstTy, Value *dst,
226 DataType srcTy, Value *src0, Value *src1, Value *src2)
227 {
228 CmpInstruction *insn = new_CmpInstruction(func, op);
229
230 insn->setType((dst->reg.file == FILE_PREDICATE ||
231 dst->reg.file == FILE_FLAGS) ? TYPE_U8 : dstTy, srcTy);
232 insn->setCondition(cc);
233 insn->setDef(0, dst);
234 insn->setSrc(0, src0);
235 insn->setSrc(1, src1);
236 if (src2)
237 insn->setSrc(2, src2);
238
239 if (dst->reg.file == FILE_FLAGS)
240 insn->flagsDef = 0;
241
242 insert(insn);
243 return insn;
244 }
245
246 TexInstruction *
mkTex(operation op,TexTarget targ,uint16_t tic,uint16_t tsc,const std::vector<Value * > & def,const std::vector<Value * > & src)247 BuildUtil::mkTex(operation op, TexTarget targ,
248 uint16_t tic, uint16_t tsc,
249 const std::vector<Value *> &def,
250 const std::vector<Value *> &src)
251 {
252 TexInstruction *tex = new_TexInstruction(func, op);
253
254 for (size_t d = 0; d < def.size() && def[d]; ++d)
255 tex->setDef(d, def[d]);
256 for (size_t s = 0; s < src.size() && src[s]; ++s)
257 tex->setSrc(s, src[s]);
258
259 tex->setTexture(targ, tic, tsc);
260
261 insert(tex);
262 return tex;
263 }
264
265 Instruction *
mkQuadop(uint8_t q,Value * def,uint8_t l,Value * src0,Value * src1)266 BuildUtil::mkQuadop(uint8_t q, Value *def, uint8_t l, Value *src0, Value *src1)
267 {
268 Instruction *quadop = mkOp2(OP_QUADOP, TYPE_F32, def, src0, src1);
269 quadop->subOp = q;
270 quadop->lanes = l;
271 return quadop;
272 }
273
274 Instruction *
mkSelect(Value * pred,Value * dst,Value * trSrc,Value * flSrc)275 BuildUtil::mkSelect(Value *pred, Value *dst, Value *trSrc, Value *flSrc)
276 {
277 LValue *def0 = getSSA();
278 LValue *def1 = getSSA();
279
280 mkMov(def0, trSrc)->setPredicate(CC_P, pred);
281 mkMov(def1, flSrc)->setPredicate(CC_NOT_P, pred);
282
283 return mkOp2(OP_UNION, typeOfSize(dst->reg.size), dst, def0, def1);
284 }
285
286 Instruction *
mkSplit(Value * h[2],uint8_t halfSize,Value * val)287 BuildUtil::mkSplit(Value *h[2], uint8_t halfSize, Value *val)
288 {
289 Instruction *insn = NULL;
290
291 const DataType fTy = typeOfSize(halfSize * 2);
292
293 if (val->reg.file == FILE_IMMEDIATE)
294 val = mkMov(getSSA(halfSize * 2), val, fTy)->getDef(0);
295
296 if (isMemoryFile(val->reg.file)) {
297 h[0] = cloneShallow(getFunction(), val);
298 h[1] = cloneShallow(getFunction(), val);
299 h[0]->reg.size = halfSize;
300 h[1]->reg.size = halfSize;
301 h[1]->reg.data.offset += halfSize;
302 } else {
303 // The value might already be the result of a split, and further
304 // splitting it can lead to issues regarding spill-offsets computations
305 // among others. By forcing a move between the two splits, this can be
306 // avoided.
307 Instruction* valInsn = val->getInsn();
308 if (valInsn && valInsn->op == OP_SPLIT)
309 val = mkMov(getSSA(halfSize * 2), val, fTy)->getDef(0);
310
311 h[0] = getSSA(halfSize, val->reg.file);
312 h[1] = getSSA(halfSize, val->reg.file);
313 insn = mkOp1(OP_SPLIT, fTy, h[0], val);
314 insn->setDef(1, h[1]);
315 }
316 return insn;
317 }
318
319 FlowInstruction *
mkFlow(operation op,void * targ,CondCode cc,Value * pred)320 BuildUtil::mkFlow(operation op, void *targ, CondCode cc, Value *pred)
321 {
322 FlowInstruction *insn = new_FlowInstruction(func, op, targ);
323
324 if (pred)
325 insn->setPredicate(cc, pred);
326
327 insert(insn);
328 return insn;
329 }
330
331 void
mkClobber(DataFile f,uint32_t rMask,int unit)332 BuildUtil::mkClobber(DataFile f, uint32_t rMask, int unit)
333 {
334 static const uint16_t baseSize2[16] =
335 {
336 0x0000, 0x0010, 0x0011, 0x0020, 0x0012, 0x1210, 0x1211, 0x1220,
337 0x0013, 0x1310, 0x1311, 0x1320, 0x0022, 0x2210, 0x2211, 0x0040,
338 };
339
340 int base = 0;
341
342 for (; rMask; rMask >>= 4, base += 4) {
343 const uint32_t mask = rMask & 0xf;
344 if (!mask)
345 continue;
346 int base1 = (baseSize2[mask] >> 0) & 0xf;
347 int size1 = (baseSize2[mask] >> 4) & 0xf;
348 int base2 = (baseSize2[mask] >> 8) & 0xf;
349 int size2 = (baseSize2[mask] >> 12) & 0xf;
350 Instruction *insn = mkOp(OP_NOP, TYPE_NONE, NULL);
351 if (true) { // size1 can't be 0
352 LValue *reg = new_LValue(func, f);
353 reg->reg.size = size1 << unit;
354 reg->reg.data.id = base + base1;
355 insn->setDef(0, reg);
356 }
357 if (size2) {
358 LValue *reg = new_LValue(func, f);
359 reg->reg.size = size2 << unit;
360 reg->reg.data.id = base + base2;
361 insn->setDef(1, reg);
362 }
363 }
364 }
365
366 ImmediateValue *
mkImm(uint16_t u)367 BuildUtil::mkImm(uint16_t u)
368 {
369 ImmediateValue *imm = new_ImmediateValue(prog, (uint32_t)0);
370
371 imm->reg.size = 2;
372 imm->reg.type = TYPE_U16;
373 imm->reg.data.u32 = u;
374
375 return imm;
376 }
377
378 ImmediateValue *
mkImm(uint32_t u)379 BuildUtil::mkImm(uint32_t u)
380 {
381 unsigned int pos = u32Hash(u);
382
383 while (imms[pos] && imms[pos]->reg.data.u32 != u)
384 pos = (pos + 1) % NV50_IR_BUILD_IMM_HT_SIZE;
385
386 ImmediateValue *imm = imms[pos];
387 if (!imm) {
388 imm = new_ImmediateValue(prog, u);
389 addImmediate(imm);
390 }
391 return imm;
392 }
393
394 ImmediateValue *
mkImm(uint64_t u)395 BuildUtil::mkImm(uint64_t u)
396 {
397 ImmediateValue *imm = new_ImmediateValue(prog, (uint32_t)0);
398
399 imm->reg.size = 8;
400 imm->reg.type = TYPE_U64;
401 imm->reg.data.u64 = u;
402
403 return imm;
404 }
405
406 ImmediateValue *
mkImm(float f)407 BuildUtil::mkImm(float f)
408 {
409 union {
410 float f32;
411 uint32_t u32;
412 } u;
413 u.f32 = f;
414 return mkImm(u.u32);
415 }
416
417 ImmediateValue *
mkImm(double d)418 BuildUtil::mkImm(double d)
419 {
420 return new_ImmediateValue(prog, d);
421 }
422
423 Value *
loadImm(Value * dst,float f)424 BuildUtil::loadImm(Value *dst, float f)
425 {
426 return mkOp1v(OP_MOV, TYPE_F32, dst ? dst : getScratch(), mkImm(f));
427 }
428
429 Value *
loadImm(Value * dst,double d)430 BuildUtil::loadImm(Value *dst, double d)
431 {
432 return mkOp1v(OP_MOV, TYPE_F64, dst ? dst : getScratch(8), mkImm(d));
433 }
434
435 Value *
loadImm(Value * dst,uint16_t u)436 BuildUtil::loadImm(Value *dst, uint16_t u)
437 {
438 return mkOp1v(OP_MOV, TYPE_U16, dst ? dst : getScratch(2), mkImm(u));
439 }
440
441 Value *
loadImm(Value * dst,uint32_t u)442 BuildUtil::loadImm(Value *dst, uint32_t u)
443 {
444 return mkOp1v(OP_MOV, TYPE_U32, dst ? dst : getScratch(), mkImm(u));
445 }
446
447 Value *
loadImm(Value * dst,uint64_t u)448 BuildUtil::loadImm(Value *dst, uint64_t u)
449 {
450 return mkOp1v(OP_MOV, TYPE_U64, dst ? dst : getScratch(8), mkImm(u));
451 }
452
453 Symbol *
mkSymbol(DataFile file,int8_t fileIndex,DataType ty,uint32_t baseAddr)454 BuildUtil::mkSymbol(DataFile file, int8_t fileIndex, DataType ty,
455 uint32_t baseAddr)
456 {
457 Symbol *sym = new_Symbol(prog, file, fileIndex);
458
459 sym->setOffset(baseAddr);
460 sym->reg.type = ty;
461 sym->reg.size = typeSizeof(ty);
462
463 return sym;
464 }
465
466 Symbol *
mkSysVal(SVSemantic svName,uint32_t svIndex)467 BuildUtil::mkSysVal(SVSemantic svName, uint32_t svIndex)
468 {
469 Symbol *sym = new_Symbol(prog, FILE_SYSTEM_VALUE, 0);
470
471 assert(svIndex < 4 || svName == SV_CLIP_DISTANCE);
472
473 switch (svName) {
474 case SV_POSITION:
475 case SV_FACE:
476 case SV_YDIR:
477 case SV_POINT_SIZE:
478 case SV_POINT_COORD:
479 case SV_CLIP_DISTANCE:
480 case SV_TESS_OUTER:
481 case SV_TESS_INNER:
482 case SV_TESS_COORD:
483 sym->reg.type = TYPE_F32;
484 break;
485 default:
486 sym->reg.type = TYPE_U32;
487 break;
488 }
489 sym->reg.size = typeSizeof(sym->reg.type);
490
491 sym->reg.data.sv.sv = svName;
492 sym->reg.data.sv.index = svIndex;
493
494 return sym;
495 }
496
497 Symbol *
mkTSVal(TSSemantic tsName)498 BuildUtil::mkTSVal(TSSemantic tsName)
499 {
500 Symbol *sym = new_Symbol(prog, FILE_THREAD_STATE, 0);
501 sym->reg.type = TYPE_U32;
502 sym->reg.size = typeSizeof(sym->reg.type);
503 sym->reg.data.ts = tsName;
504 return sym;
505 }
506
507
508 Instruction *
split64BitOpPostRA(Function * fn,Instruction * i,Value * zero,Value * carry)509 BuildUtil::split64BitOpPostRA(Function *fn, Instruction *i,
510 Value *zero,
511 Value *carry)
512 {
513 DataType hTy;
514 int srcNr;
515
516 switch (i->dType) {
517 case TYPE_U64: hTy = TYPE_U32; break;
518 case TYPE_S64: hTy = TYPE_S32; break;
519 case TYPE_F64:
520 if (i->op == OP_MOV) {
521 hTy = TYPE_U32;
522 break;
523 }
524 FALLTHROUGH;
525 default:
526 return NULL;
527 }
528
529 switch (i->op) {
530 case OP_MOV: srcNr = 1; break;
531 case OP_ADD:
532 case OP_SUB:
533 if (!carry)
534 return NULL;
535 srcNr = 2;
536 break;
537 case OP_SELP: srcNr = 3; break;
538 default:
539 // TODO when needed
540 return NULL;
541 }
542
543 i->setType(hTy);
544 i->setDef(0, cloneShallow(fn, i->getDef(0)));
545 i->getDef(0)->reg.size = 4;
546 Instruction *lo = i;
547 Instruction *hi = cloneForward(fn, i);
548 lo->bb->insertAfter(lo, hi);
549
550 hi->getDef(0)->reg.data.id++;
551
552 for (int s = 0; s < srcNr; ++s) {
553 if (lo->getSrc(s)->reg.size < 8) {
554 if (s == 2)
555 hi->setSrc(s, lo->getSrc(s));
556 else
557 hi->setSrc(s, zero);
558 } else {
559 if (lo->getSrc(s)->refCount() > 1)
560 lo->setSrc(s, cloneShallow(fn, lo->getSrc(s)));
561 lo->getSrc(s)->reg.size /= 2;
562 hi->setSrc(s, cloneShallow(fn, lo->getSrc(s)));
563
564 switch (hi->src(s).getFile()) {
565 case FILE_IMMEDIATE:
566 hi->getSrc(s)->reg.data.u64 >>= 32;
567 break;
568 case FILE_MEMORY_CONST:
569 case FILE_MEMORY_SHARED:
570 case FILE_SHADER_INPUT:
571 case FILE_SHADER_OUTPUT:
572 hi->getSrc(s)->reg.data.offset += 4;
573 break;
574 default:
575 assert(hi->src(s).getFile() == FILE_GPR);
576 hi->getSrc(s)->reg.data.id++;
577 break;
578 }
579 }
580 }
581 if (srcNr == 2) {
582 lo->setFlagsDef(1, carry);
583 hi->setFlagsSrc(hi->srcCount(), carry);
584 }
585 return hi;
586 }
587
588 } // namespace nv50_ir
589