1 //
2 // Copyright 2012-2016 Francisco Jerez
3 // Copyright 2012-2016 Advanced Micro Devices, Inc.
4 // Copyright 2015 Zoltan Gilian
5 //
6 // Permission is hereby granted, free of charge, to any person obtaining a
7 // copy of this software and associated documentation files (the "Software"),
8 // to deal in the Software without restriction, including without limitation
9 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 // and/or sell copies of the Software, and to permit persons to whom the
11 // Software is furnished to do so, subject to the following conditions:
12 //
13 // The above copyright notice and this permission notice shall be included in
14 // all copies or substantial portions of the Software.
15 //
16 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 // OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 // ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 // OTHER DEALINGS IN THE SOFTWARE.
23 //
24
25 ///
26 /// \file
27 /// Codegen back-end-independent part of the construction of an executable
28 /// clover::binary, including kernel argument metadata extraction and
29 /// formatting of the pre-generated binary code in a form that can be
30 /// understood by pipe drivers.
31 ///
32
33 #include <llvm/IR/Type.h>
34 #include <llvm/Support/Allocator.h>
35
36 #include "llvm/codegen.hpp"
37 #include "llvm/compat.hpp"
38 #include "llvm/metadata.hpp"
39
40 #include "CL/cl.h"
41
42 #include "pipe/p_state.h"
43 #include "util/u_math.h"
44
45 #include <clang/Basic/TargetInfo.h>
46
47 using clover::binary;
48 using clover::detokenize;
49 using namespace clover::llvm;
50
51 using ::llvm::Module;
52 using ::llvm::Function;
53 using ::llvm::Type;
54 using ::llvm::isa;
55 using ::llvm::cast;
56 using ::llvm::dyn_cast;
57
58 namespace {
59 enum binary::argument::type
get_image_type(const std::string & type,const std::string & qual)60 get_image_type(const std::string &type,
61 const std::string &qual) {
62 if (type == "image1d_t" || type == "image2d_t" || type == "image3d_t") {
63 if (qual == "read_only")
64 return binary::argument::image_rd;
65 else if (qual == "write_only")
66 return binary::argument::image_wr;
67 }
68
69 unreachable("Unsupported image type");
70 }
71
create_arg_info(const std::string & arg_name,const std::string & type_name,const std::string & type_qualifier,const uint64_t address_qualifier,const std::string & access_qualifier)72 binary::arg_info create_arg_info(const std::string &arg_name,
73 const std::string &type_name,
74 const std::string &type_qualifier,
75 const uint64_t address_qualifier,
76 const std::string &access_qualifier) {
77
78 cl_kernel_arg_type_qualifier cl_type_qualifier =
79 CL_KERNEL_ARG_TYPE_NONE;
80 if (type_qualifier.find("const") != std::string::npos)
81 cl_type_qualifier |= CL_KERNEL_ARG_TYPE_CONST;
82 if (type_qualifier.find("restrict") != std::string::npos)
83 cl_type_qualifier |= CL_KERNEL_ARG_TYPE_RESTRICT;
84 if (type_qualifier.find("volatile") != std::string::npos)
85 cl_type_qualifier |= CL_KERNEL_ARG_TYPE_VOLATILE;
86
87 cl_kernel_arg_address_qualifier cl_address_qualifier =
88 CL_KERNEL_ARG_ADDRESS_PRIVATE;
89 if (address_qualifier == 1)
90 cl_address_qualifier = CL_KERNEL_ARG_ADDRESS_GLOBAL;
91 else if (address_qualifier == 2)
92 cl_address_qualifier = CL_KERNEL_ARG_ADDRESS_CONSTANT;
93 else if (address_qualifier == 3)
94 cl_address_qualifier = CL_KERNEL_ARG_ADDRESS_LOCAL;
95
96 cl_kernel_arg_access_qualifier cl_access_qualifier =
97 CL_KERNEL_ARG_ACCESS_NONE;
98 if (access_qualifier == "read_only")
99 cl_access_qualifier = CL_KERNEL_ARG_ACCESS_READ_ONLY;
100 else if (access_qualifier == "write_only")
101 cl_access_qualifier = CL_KERNEL_ARG_ACCESS_WRITE_ONLY;
102 else if (access_qualifier == "read_write")
103 cl_access_qualifier = CL_KERNEL_ARG_ACCESS_READ_WRITE;
104
105 return binary::arg_info(arg_name, type_name, cl_type_qualifier,
106 cl_address_qualifier, cl_access_qualifier);
107 }
108
109 std::vector<size_t>
get_reqd_work_group_size(const Module & mod,const std::string & kernel_name)110 get_reqd_work_group_size(const Module &mod,
111 const std::string &kernel_name) {
112 const Function &f = *mod.getFunction(kernel_name);
113 auto vector_metadata = get_uint_vector_kernel_metadata(f, "reqd_work_group_size");
114
115 return vector_metadata.empty() ? std::vector<size_t>({0, 0, 0}) : vector_metadata;
116 }
117
118
119 std::string
kernel_attributes(const Module & mod,const std::string & kernel_name)120 kernel_attributes(const Module &mod, const std::string &kernel_name) {
121 std::vector<std::string> attributes;
122
123 const Function &f = *mod.getFunction(kernel_name);
124
125 auto vec_type_hint = get_type_kernel_metadata(f, "vec_type_hint");
126 if (!vec_type_hint.empty())
127 attributes.emplace_back("vec_type_hint(" + vec_type_hint + ")");
128
129 auto work_group_size_hint = get_uint_vector_kernel_metadata(f, "work_group_size_hint");
130 if (!work_group_size_hint.empty()) {
131 std::string s = "work_group_size_hint(";
132 s += detokenize(work_group_size_hint, ",");
133 s += ")";
134 attributes.emplace_back(s);
135 }
136
137 auto reqd_work_group_size = get_uint_vector_kernel_metadata(f, "reqd_work_group_size");
138 if (!reqd_work_group_size.empty()) {
139 std::string s = "reqd_work_group_size(";
140 s += detokenize(reqd_work_group_size, ",");
141 s += ")";
142 attributes.emplace_back(s);
143 }
144
145 auto nosvm = get_str_kernel_metadata(f, "nosvm");
146 if (!nosvm.empty())
147 attributes.emplace_back("nosvm");
148
149 return detokenize(attributes, " ");
150 }
151
152 // Parse the type which are pointers to CL vector types with no prefix.
153 // so e.g. char/uchar, short/ushort, int/uint, long/ulong
154 // half/float/double, followed by the vector length, followed by *.
155 // uint8 is 8x32-bit integer, short4 is 4x16-bit integer etc.
156 // Since this is a pointer only path, assert the * is on the end.
157 ::llvm::Type *
ptr_arg_to_llvm_type(const Module & mod,std::string type_name)158 ptr_arg_to_llvm_type(const Module &mod, std::string type_name) {
159 int len = type_name.length();
160 assert (type_name[len-1] == '*');
161 ::llvm::Type *base_type = NULL;
162 if (type_name.find("void") != std::string::npos)
163 base_type = ::llvm::Type::getVoidTy(mod.getContext());
164 else if (type_name.find("char") != std::string::npos)
165 base_type = ::llvm::Type::getInt8Ty(mod.getContext());
166 else if (type_name.find("short") != std::string::npos)
167 base_type = ::llvm::Type::getInt16Ty(mod.getContext());
168 else if (type_name.find("int") != std::string::npos)
169 base_type = ::llvm::Type::getInt32Ty(mod.getContext());
170 else if (type_name.find("long") != std::string::npos)
171 base_type = ::llvm::Type::getInt64Ty(mod.getContext());
172 else if (type_name.find("half") != std::string::npos)
173 base_type = ::llvm::Type::getHalfTy(mod.getContext());
174 else if (type_name.find("float") != std::string::npos)
175 base_type = ::llvm::Type::getFloatTy(mod.getContext());
176 else if (type_name.find("double") != std::string::npos)
177 base_type = ::llvm::Type::getDoubleTy(mod.getContext());
178
179 assert(base_type);
180 if (type_name.find("2") != std::string::npos)
181 base_type = ::llvm::FixedVectorType::get(base_type, 2);
182 else if (type_name.find("3") != std::string::npos)
183 base_type = ::llvm::FixedVectorType::get(base_type, 3);
184 else if (type_name.find("4") != std::string::npos)
185 base_type = ::llvm::FixedVectorType::get(base_type, 4);
186 else if (type_name.find("8") != std::string::npos)
187 base_type = ::llvm::FixedVectorType::get(base_type, 8);
188 else if (type_name.find("16") != std::string::npos)
189 base_type = ::llvm::FixedVectorType::get(base_type, 16);
190 return base_type;
191 }
192
193 std::vector<binary::argument>
make_kernel_args(const Module & mod,const std::string & kernel_name,const clang::CompilerInstance & c)194 make_kernel_args(const Module &mod, const std::string &kernel_name,
195 const clang::CompilerInstance &c) {
196 std::vector<binary::argument> args;
197 const Function &f = *mod.getFunction(kernel_name);
198 ::llvm::DataLayout dl(&mod);
199 const auto size_type =
200 dl.getSmallestLegalIntType(mod.getContext(), sizeof(cl_uint) * 8);
201 const unsigned size_align = compat::get_abi_type_alignment(dl, size_type);
202
203 for (const auto &arg : f.args()) {
204 const auto arg_type = arg.getType();
205
206 // OpenCL 1.2 specification, Ch. 6.1.5: "A built-in data
207 // type that is not a power of two bytes in size must be
208 // aligned to the next larger power of two.
209 // This rule applies to built-in types only, not structs or unions."
210 const unsigned arg_api_size = dl.getTypeAllocSize(arg_type);
211
212 const unsigned target_size = dl.getTypeStoreSize(arg_type);
213 const unsigned target_align = compat::get_abi_type_alignment(dl, arg_type);
214
215 const auto type_name = get_str_argument_metadata(f, arg,
216 "kernel_arg_type");
217 if (type_name == "image2d_t" || type_name == "image3d_t") {
218 // Image.
219 const auto access_qual = get_str_argument_metadata(
220 f, arg, "kernel_arg_access_qual");
221 args.emplace_back(get_image_type(type_name, access_qual),
222 target_size, target_size,
223 target_align, binary::argument::zero_ext);
224
225 } else if (type_name == "sampler_t") {
226 args.emplace_back(binary::argument::sampler, arg_api_size,
227 target_size, target_align,
228 binary::argument::zero_ext);
229
230 } else if (type_name == "__llvm_image_size") {
231 // Image size implicit argument.
232 args.emplace_back(binary::argument::scalar, sizeof(cl_uint),
233 dl.getTypeStoreSize(size_type),
234 size_align,
235 binary::argument::zero_ext,
236 binary::argument::image_size);
237
238 } else if (type_name == "__llvm_image_format") {
239 // Image format implicit argument.
240 args.emplace_back(binary::argument::scalar, sizeof(cl_uint),
241 dl.getTypeStoreSize(size_type),
242 size_align,
243 binary::argument::zero_ext,
244 binary::argument::image_format);
245
246 } else {
247 // Other types.
248 const auto actual_type =
249 isa< ::llvm::PointerType>(arg_type) && arg.hasByValAttr() ?
250 ptr_arg_to_llvm_type(mod, type_name) : arg_type;
251
252 if (actual_type->isPointerTy()) {
253 const unsigned address_space =
254 cast< ::llvm::PointerType>(actual_type)->getAddressSpace();
255
256 const auto &map = c.getTarget().getAddressSpaceMap();
257 const auto offset =
258 static_cast<unsigned>(clang::LangAS::opencl_local);
259 if (address_space == map[offset]) {
260 const auto pointee_type = ptr_arg_to_llvm_type(mod, type_name);
261
262 args.emplace_back(binary::argument::local, arg_api_size,
263 target_size,
264 (pointee_type->isVoidTy()) ? 8 :
265 compat::get_abi_type_alignment(dl, pointee_type),
266 binary::argument::zero_ext);
267 } else {
268 // XXX: Correctly handle constant address space. There is no
269 // way for r600g to pass a handle for constant buffers back
270 // to clover like it can for global buffers, so
271 // creating constant arguments will break r600g. For now,
272 // continue treating constant buffers as global buffers
273 // until we can come up with a way to create handles for
274 // constant buffers.
275 args.emplace_back(binary::argument::global, arg_api_size,
276 target_size, target_align,
277 binary::argument::zero_ext);
278 }
279
280 } else {
281 const bool needs_sign_ext = f.getAttributes().hasParamAttr(
282 arg.getArgNo(), ::llvm::Attribute::SExt);
283
284 args.emplace_back(binary::argument::scalar, arg_api_size,
285 target_size, target_align,
286 (needs_sign_ext ? binary::argument::sign_ext :
287 binary::argument::zero_ext));
288 }
289
290 // Add kernel argument infos if built with -cl-kernel-arg-info.
291 if (c.getCodeGenOpts().EmitOpenCLArgMetadata) {
292 args.back().info = create_arg_info(
293 get_str_argument_metadata(f, arg, "kernel_arg_name"),
294 type_name,
295 get_str_argument_metadata(f, arg, "kernel_arg_type_qual"),
296 get_uint_argument_metadata(f, arg, "kernel_arg_addr_space"),
297 get_str_argument_metadata(f, arg, "kernel_arg_access_qual"));
298 }
299 }
300 }
301
302 // Append implicit arguments. XXX - The types, ordering and
303 // vector size of the implicit arguments should depend on the
304 // target according to the selected calling convention.
305 args.emplace_back(binary::argument::scalar, sizeof(cl_uint),
306 dl.getTypeStoreSize(size_type),
307 size_align,
308 binary::argument::zero_ext,
309 binary::argument::grid_dimension);
310
311 args.emplace_back(binary::argument::scalar, sizeof(cl_uint),
312 dl.getTypeStoreSize(size_type),
313 size_align,
314 binary::argument::zero_ext,
315 binary::argument::grid_offset);
316
317 return args;
318 }
319
320 binary::section
make_text_section(const std::vector<char> & code)321 make_text_section(const std::vector<char> &code) {
322 const pipe_binary_program_header header { uint32_t(code.size()) };
323 binary::section text { 0, binary::section::text_executable,
324 header.num_bytes, {} };
325
326 text.data.insert(text.data.end(), reinterpret_cast<const char *>(&header),
327 reinterpret_cast<const char *>(&header) + sizeof(header));
328 text.data.insert(text.data.end(), code.begin(), code.end());
329
330 return text;
331 }
332 }
333
334 binary
build_module_common(const Module & mod,const std::vector<char> & code,const std::map<std::string,unsigned> & offsets,const clang::CompilerInstance & c)335 clover::llvm::build_module_common(const Module &mod,
336 const std::vector<char> &code,
337 const std::map<std::string,
338 unsigned> &offsets,
339 const clang::CompilerInstance &c) {
340 binary b;
341
342 for (const auto &llvm_name : map(std::mem_fn(&Function::getName),
343 get_kernels(mod))) {
344 const ::std::string name(llvm_name);
345 if (offsets.count(name))
346 b.syms.emplace_back(name, kernel_attributes(mod, name),
347 get_reqd_work_group_size(mod, name),
348 0, offsets.at(name),
349 make_kernel_args(mod, name, c));
350 }
351
352 b.secs.push_back(make_text_section(code));
353 return b;
354 }
355