1 /* -*- mesa-c++ -*- 2 * Copyright 2021 Collabora LTD 3 * Author: Gert Wollny <[email protected]> 4 * SPDX-License-Identifier: MIT 5 */ 6 7 #pragma once 8 9 #include "sfn_alu_defines.h" 10 #include "sfn_defines.h" 11 #include "sfn_virtualvalues.h" 12 13 #include <iostream> 14 #include <list> 15 #include <set> 16 17 namespace r600 { 18 19 class ConstInstrVisitor; 20 21 class InstrVisitor; 22 class AluInstr; 23 class AluGroup; 24 class TexInstr; 25 class ExportInstr; 26 class FetchInstr; 27 class ControlFlowInstr; 28 class IfInstr; 29 class ScratchIOInstr; 30 class StreamOutInstr; 31 class MemRingOutInstr; 32 class EmitVertexInstr; 33 class GDSInstr; 34 class WriteTFInstr; 35 class LDSAtomicInstr; 36 class LDSReadInstr; 37 class RatInstr; 38 39 bool 40 int_from_string_with_prefix_optional(const std::string& str, 41 const std::string& prefix, 42 int& value); 43 int 44 int_from_string_with_prefix(const std::string& str, const std::string& prefix); 45 int 46 sel_and_szw_from_string(const std::string& str, RegisterVec4::Swizzle& swz, bool& is_ssa); 47 48 class Instr : public Allocate { 49 public: 50 enum Flags { 51 always_keep, 52 dead, 53 scheduled, 54 vpm, 55 force_cf, 56 ack_rat_return_write, 57 helper, 58 no_lds_or_addr_group, 59 nflags 60 }; 61 62 Instr(); 63 64 Instr(const Instr& orig) = default; 65 66 virtual ~Instr(); 67 68 using Pointer = R600_POINTER_TYPE(Instr); 69 70 void print(std::ostream& os) const; 71 bool equal_to(const Instr& lhs) const; 72 73 virtual void accept(ConstInstrVisitor& visitor) const = 0; 74 virtual void accept(InstrVisitor& visitor) = 0; end_group()75 virtual bool end_group() const { return true; } 76 77 virtual bool is_last() const; 78 set_always_keep()79 void set_always_keep() { m_instr_flags.set(always_keep); } 80 bool set_dead(); set_scheduled()81 virtual void set_scheduled() 82 { 83 m_instr_flags.set(scheduled); 84 forward_set_scheduled(); 85 } is_dead()86 bool is_dead() const { return m_instr_flags.test(dead); } is_scheduled()87 bool is_scheduled() const { return m_instr_flags.test(scheduled); } keep()88 bool keep() const { return m_instr_flags.test(always_keep); } can_start_alu_block()89 bool can_start_alu_block() { return m_instr_flags.test(no_lds_or_addr_group);} group_force_alu_cf()90 bool group_force_alu_cf() { return m_instr_flags.test(force_cf);} 91 has_instr_flag(Flags f)92 bool has_instr_flag(Flags f) const { return m_instr_flags.test(f); } set_instr_flag(Flags f)93 void set_instr_flag(Flags f) { m_instr_flags.set(f); } 94 95 virtual bool replace_source(PRegister old_src, PVirtualValue new_src); 96 virtual bool replace_dest(PRegister new_dest, AluInstr *move_instr); 97 nesting_corr()98 virtual int nesting_corr() const { return 0; } 99 end_block()100 virtual bool end_block() const { return false; } nesting_offset()101 virtual int nesting_offset() const { return 0; } 102 103 void set_blockid(int id, int index); block_id()104 int block_id() const { return m_block_id; } index()105 int index() const { return m_index; } 106 107 void add_required_instr(Instr *instr); 108 void replace_required_instr(Instr *old_instr, Instr *new_instr); 109 110 bool ready() const; 111 slots()112 virtual uint32_t slots() const { return 0; }; 113 114 using InstrList = std::list<Instr *, Allocator<Instr *>>; 115 dependend_instr()116 const InstrList& dependend_instr() { return m_dependend_instr; } 117 as_alu()118 virtual AluInstr *as_alu() { return nullptr; } allowed_src_chan_mask()119 virtual uint8_t allowed_src_chan_mask() const { return 0; } 120 update_indirect_addr(PRegister old_reg,PRegister addr)121 virtual void update_indirect_addr(PRegister old_reg, PRegister addr) { 122 (void)old_reg; 123 (void)addr; 124 unreachable("Instruction type has no indirect address"); 125 }; required_instr()126 const InstrList& required_instr() const { return m_required_instr; } 127 as_alu_group()128 virtual AluGroup *as_alu_group() { return nullptr;} 129 130 protected: 131 132 133 private: 134 virtual void forward_set_blockid(int id, int index); 135 136 virtual bool do_ready() const = 0; 137 138 virtual void do_print(std::ostream& os) const = 0; 139 virtual bool propagate_death(); forward_set_scheduled()140 virtual void forward_set_scheduled() {} 141 142 InstrList m_required_instr; 143 InstrList m_dependend_instr; 144 145 int m_use_count; 146 int m_block_id; 147 int m_index; 148 std::bitset<nflags> m_instr_flags{0}; 149 }; 150 using PInst = Instr::Pointer; 151 152 class Block : public Instr { 153 public: 154 enum Type { 155 cf, 156 alu, 157 tex, 158 vtx, 159 gds, 160 unknown 161 }; 162 163 using Instructions = std::list<Instr *, Allocator<Instr *>>; 164 using Pointer = R600_POINTER_TYPE(Block); 165 using iterator = Instructions::iterator; 166 using reverse_iterator = Instructions::reverse_iterator; 167 using const_iterator = Instructions::const_iterator; 168 169 Block(int nesting_depth, int id); 170 Block(const Block& orig) = delete; 171 172 void push_back(PInst instr); begin()173 iterator begin() { return m_instructions.begin(); } end()174 iterator end() { return m_instructions.end(); } rbegin()175 reverse_iterator rbegin() { return m_instructions.rbegin(); } rend()176 reverse_iterator rend() { return m_instructions.rend(); } 177 begin()178 const_iterator begin() const { return m_instructions.begin(); } end()179 const_iterator end() const { return m_instructions.end(); } 180 empty()181 bool empty() const { return m_instructions.empty(); } 182 183 void erase(iterator node); 184 185 iterator insert(const iterator pos, Instr *instr); 186 187 bool is_equal_to(const Block& lhs) const; 188 189 void accept(ConstInstrVisitor& visitor) const override; 190 void accept(InstrVisitor& visitor) override; 191 nesting_depth()192 int nesting_depth() const { return m_nesting_depth; } 193 id()194 int id() const { return m_id; } 195 type()196 auto type() const {return m_block_type; } 197 void set_type(Type t, r600_chip_class chip_class); remaining_slots()198 int32_t remaining_slots() const { return m_remaining_slots;} 199 200 bool try_reserve_kcache(const AluGroup& instr); 201 bool try_reserve_kcache(const AluInstr& group); 202 last_lds_instr()203 auto last_lds_instr() { return m_last_lds_instr; } set_last_lds_instr(Instr * instr)204 void set_last_lds_instr(Instr *instr) { m_last_lds_instr = instr; } 205 206 void lds_group_start(AluInstr *alu); 207 void lds_group_end(); lds_group_active()208 bool lds_group_active() { return m_lds_group_start != nullptr; } 209 size()210 size_t size() const { return m_instructions.size(); } 211 kcache_reservation_failed()212 bool kcache_reservation_failed() const { return m_kcache_alloc_failed; } 213 inc_rat_emitted()214 int inc_rat_emitted() { return ++m_emitted_rat_instr; } 215 set_expected_ar_uses(uint32_t n)216 void set_expected_ar_uses(uint32_t n) {m_expected_ar_uses = n;} expected_ar_uses()217 auto expected_ar_uses() const {return m_expected_ar_uses;} dec_expected_ar_uses()218 void dec_expected_ar_uses() { 219 assert(m_expected_ar_uses > 0); 220 --m_expected_ar_uses; 221 } 222 223 static void set_chipclass(r600_chip_class chip_class); 224 225 private: 226 bool try_reserve_kcache(const UniformValue& u, 227 std::array<KCacheLine, 4>& kcache) const; 228 do_ready()229 bool do_ready() const override { return true; }; 230 void do_print(std::ostream& os) const override; 231 Instructions m_instructions; 232 int m_nesting_depth; 233 int m_id; 234 int m_next_index; 235 236 Type m_block_type{unknown}; 237 uint32_t m_remaining_slots{0xffff}; 238 239 std::array<KCacheLine, 4> m_kcache; 240 bool m_kcache_alloc_failed{false}; 241 242 Instr *m_last_lds_instr{nullptr}; 243 244 int m_lds_group_requirement{0}; 245 AluInstr *m_lds_group_start{nullptr}; 246 static unsigned s_max_kcache_banks; 247 int m_emitted_rat_instr{0}; 248 uint32_t m_expected_ar_uses{0}; 249 }; 250 251 class Resource { 252 public: Resource(Instr * user,int base,PRegister offset)253 Resource(Instr *user, int base, PRegister offset): 254 m_base(base), 255 m_offset(offset), 256 m_user(user) 257 { 258 if (m_offset) { 259 m_offset->add_use(m_user); 260 } 261 } replace_resource_offset(PRegister old_offset,PRegister new_offset)262 bool replace_resource_offset(PRegister old_offset, PRegister new_offset) 263 { 264 if (m_offset && old_offset->equal_to(*m_offset)) { 265 m_offset->del_use(m_user); 266 m_offset = new_offset; 267 m_offset->add_use(m_user); 268 return true; 269 } 270 return false; 271 } set_resource_offset(PRegister offset)272 void set_resource_offset(PRegister offset) 273 { 274 if (m_offset) 275 m_offset->del_use(m_user); 276 m_offset = offset; 277 if (m_offset) { 278 m_offset->add_use(m_user); 279 } 280 } 281 resource_is_equal(const Resource & other)282 bool resource_is_equal(const Resource& other) const 283 { 284 if (m_base != other.m_base) 285 return false; 286 if (m_offset && other.m_offset) 287 return m_offset->equal_to(*other.m_offset); 288 return !m_offset && !other.m_offset; 289 } 290 resource_id()291 auto resource_id() const { return m_base; } 292 resource_offset()293 auto resource_offset() const { return m_offset; } 294 295 auto resource_index_mode() const -> EBufferIndexMode 296 { 297 if (!m_offset || !m_offset->has_flag(Register::addr_or_idx)) 298 return bim_none; 299 300 switch (m_offset->sel()) { 301 case 1: 302 return bim_zero; 303 case 2: 304 return bim_one; 305 default: 306 unreachable("Invalid resource offset, scheduler must substitute registers"); 307 } 308 } 309 resource_ready(int block_id,int index)310 bool resource_ready(int block_id, int index) const 311 { 312 return !m_offset || m_offset->ready(block_id, index); 313 } 314 315 protected: print_resource_offset(std::ostream & os)316 void print_resource_offset(std::ostream& os) const 317 { 318 if (m_offset) 319 os << " + " << *m_offset; 320 } 321 322 private: 323 int m_base{0}; 324 PRegister m_offset{nullptr}; 325 Instr *m_user; 326 }; 327 328 class InstrWithVectorResult : public Instr, public Resource { 329 public: 330 InstrWithVectorResult(const RegisterVec4& dest, 331 const RegisterVec4::Swizzle& dest_swizzle, 332 int resource_base, 333 PRegister resource_offset); 334 set_dest_swizzle(const RegisterVec4::Swizzle & swz)335 void set_dest_swizzle(const RegisterVec4::Swizzle& swz) { m_dest_swizzle = swz; } dest_swizzle(int i)336 int dest_swizzle(int i) const { return m_dest_swizzle[i]; } all_dest_swizzle()337 const RegisterVec4::Swizzle& all_dest_swizzle() const { return m_dest_swizzle; } dst()338 const RegisterVec4& dst() const { return m_dest; } 339 340 void update_indirect_addr(PRegister old_reg, PRegister addr) override; 341 342 protected: 343 InstrWithVectorResult(const InstrWithVectorResult& orig); 344 345 void print_dest(std::ostream& os) const; 346 bool comp_dest(const RegisterVec4& dest, 347 const RegisterVec4::Swizzle& dest_swizzle) const; 348 349 private: 350 RegisterVec4 m_dest; 351 RegisterVec4::Swizzle m_dest_swizzle; 352 }; 353 354 inline bool 355 operator==(const Instr& lhs, const Instr& rhs) 356 { 357 return lhs.equal_to(rhs); 358 } 359 360 inline bool 361 operator!=(const Instr& lhs, const Instr& rhs) 362 { 363 return !(lhs == rhs); 364 } 365 366 inline std::ostream& 367 operator<<(std::ostream& os, const Instr& instr) 368 { 369 instr.print(os); 370 return os; 371 } 372 373 template <typename T, typename = std::enable_if_t<std::is_base_of_v<Instr, T>>> 374 std::ostream& 375 operator<<(std::ostream& os, const T& instr) 376 { 377 instr.print(os); 378 return os; 379 } 380 381 class ConstInstrVisitor { 382 public: 383 virtual void visit(const AluInstr& instr) = 0; 384 virtual void visit(const AluGroup& instr) = 0; 385 virtual void visit(const TexInstr& instr) = 0; 386 virtual void visit(const ExportInstr& instr) = 0; 387 virtual void visit(const FetchInstr& instr) = 0; 388 virtual void visit(const Block& instr) = 0; 389 virtual void visit(const ControlFlowInstr& instr) = 0; 390 virtual void visit(const IfInstr& instr) = 0; 391 virtual void visit(const ScratchIOInstr& instr) = 0; 392 virtual void visit(const StreamOutInstr& instr) = 0; 393 virtual void visit(const MemRingOutInstr& instr) = 0; 394 virtual void visit(const EmitVertexInstr& instr) = 0; 395 virtual void visit(const GDSInstr& instr) = 0; 396 virtual void visit(const WriteTFInstr& instr) = 0; 397 virtual void visit(const LDSAtomicInstr& instr) = 0; 398 virtual void visit(const LDSReadInstr& instr) = 0; 399 virtual void visit(const RatInstr& instr) = 0; 400 }; 401 402 class InstrVisitor { 403 public: 404 virtual void visit(AluInstr *instr) = 0; 405 virtual void visit(AluGroup *instr) = 0; 406 virtual void visit(TexInstr *instr) = 0; 407 virtual void visit(ExportInstr *instr) = 0; 408 virtual void visit(FetchInstr *instr) = 0; 409 virtual void visit(Block *instr) = 0; 410 virtual void visit(ControlFlowInstr *instr) = 0; 411 virtual void visit(IfInstr *instr) = 0; 412 virtual void visit(ScratchIOInstr *instr) = 0; 413 virtual void visit(StreamOutInstr *instr) = 0; 414 virtual void visit(MemRingOutInstr *instr) = 0; 415 virtual void visit(EmitVertexInstr *instr) = 0; 416 virtual void visit(GDSInstr *instr) = 0; 417 virtual void visit(WriteTFInstr *instr) = 0; 418 virtual void visit(LDSAtomicInstr *instr) = 0; 419 virtual void visit(LDSReadInstr *instr) = 0; 420 virtual void visit(RatInstr *instr) = 0; 421 }; 422 423 } // namespace r600 424