1 /* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15
16 #include "tensorflow/lite/delegates/gpu/gl/compiler/object_accessor.h"
17
18 #include <string>
19 #include <utility>
20 #include <variant>
21
22 #include "absl/strings/ascii.h"
23 #include "absl/strings/str_cat.h"
24 #include "absl/strings/str_format.h"
25 #include "absl/strings/str_join.h"
26 #include "absl/strings/str_split.h"
27 #include "absl/types/variant.h"
28 #include "tensorflow/lite/delegates/gpu/common/data_type.h"
29 #include "tensorflow/lite/delegates/gpu/common/types.h"
30
31 namespace tflite {
32 namespace gpu {
33 namespace gl {
34 namespace object_accessor_internal {
35
36 // Splits name[index1, index2...] into 'name' and {'index1', 'index2'...}.
ParseElement(absl::string_view input)37 IndexedElement ParseElement(absl::string_view input) {
38 auto i = input.find('[');
39 if (i == std::string::npos || input.back() != ']') {
40 return {};
41 }
42 return {input.substr(0, i),
43 absl::StrSplit(input.substr(i + 1, input.size() - i - 2), ',',
44 absl::SkipWhitespace())};
45 }
46
47 } // namespace object_accessor_internal
48
49 namespace {
50
MaybeConvertToHalf(DataType data_type,absl::string_view value,std::string * output)51 void MaybeConvertToHalf(DataType data_type, absl::string_view value,
52 std::string* output) {
53 if (data_type == DataType::FLOAT16) {
54 absl::StrAppend(output, "Vec4ToHalf(", value, ")");
55 } else {
56 absl::StrAppend(output, value);
57 }
58 }
59
MaybeConvertFromHalf(DataType data_type,absl::string_view value,std::string * output)60 void MaybeConvertFromHalf(DataType data_type, absl::string_view value,
61 std::string* output) {
62 if (data_type == DataType::FLOAT16) {
63 absl::StrAppend(output, "Vec4FromHalf(", value, ")");
64 } else {
65 absl::StrAppend(output, value);
66 }
67 }
68
69 struct ReadFromTextureGenerator {
operator ()tflite::gpu::gl::__anonf2a2b3c90111::ReadFromTextureGenerator70 RewriteStatus operator()(size_t) const {
71 if (element.indices.size() != 1) {
72 result->append("WRONG_NUMBER_OF_INDICES");
73 return RewriteStatus::ERROR;
74 }
75 // 1D textures are emulated as 2D textures
76 if (sampler_textures) {
77 absl::StrAppend(result, "texelFetch(", element.object_name, ", ivec2(",
78 element.indices[0], ", 0), 0)");
79 } else {
80 absl::StrAppend(result, "imageLoad(", element.object_name, ", ivec2(",
81 element.indices[0], ", 0))");
82 }
83 return RewriteStatus::SUCCESS;
84 }
85
86 template <typename Shape>
operator ()tflite::gpu::gl::__anonf2a2b3c90111::ReadFromTextureGenerator87 RewriteStatus operator()(const Shape&) const {
88 if (element.indices.size() != Shape::size()) {
89 result->append("WRONG_NUMBER_OF_INDICES");
90 return RewriteStatus::ERROR;
91 }
92 if (sampler_textures) {
93 absl::StrAppend(result, "texelFetch(", element.object_name, ", ivec",
94 Shape::size(), "(", absl::StrJoin(element.indices, ", "),
95 "), 0)");
96 } else {
97 absl::StrAppend(result, "imageLoad(", element.object_name, ", ivec",
98 Shape::size(), "(", absl::StrJoin(element.indices, ", "),
99 "))");
100 }
101 return RewriteStatus::SUCCESS;
102 }
103
104 const object_accessor_internal::IndexedElement& element;
105 const bool sampler_textures;
106 std::string* result;
107 };
108
109 struct ReadFromBufferGenerator {
operator ()tflite::gpu::gl::__anonf2a2b3c90111::ReadFromBufferGenerator110 RewriteStatus operator()(size_t) const {
111 if (element.indices.size() != 1) {
112 result->append("WRONG_NUMBER_OF_INDICES");
113 return RewriteStatus::ERROR;
114 }
115 MaybeConvertFromHalf(
116 data_type,
117 absl::StrCat(element.object_name, ".data[", element.indices[0], "]"),
118 result);
119 return RewriteStatus::SUCCESS;
120 }
121
operator ()tflite::gpu::gl::__anonf2a2b3c90111::ReadFromBufferGenerator122 RewriteStatus operator()(const uint2& size) const {
123 if (element.indices.size() == 1) {
124 // access by linear index. Use method above to generate accessor.
125 return (*this)(1U);
126 }
127 if (element.indices.size() != 2) {
128 result->append("WRONG_NUMBER_OF_INDICES");
129 return RewriteStatus::ERROR;
130 }
131 MaybeConvertFromHalf(
132 data_type,
133 absl::StrCat(element.object_name, ".data[", element.indices[0], " + $",
134 element.object_name, "_w$ * (", element.indices[1], ")]"),
135 result);
136 *requires_sizes = true;
137 return RewriteStatus::SUCCESS;
138 }
139
operator ()tflite::gpu::gl::__anonf2a2b3c90111::ReadFromBufferGenerator140 RewriteStatus operator()(const uint3& size) const {
141 if (element.indices.size() == 1) {
142 // access by linear index. Use method above to generate accessor.
143 return (*this)(1U);
144 }
145 if (element.indices.size() != 3) {
146 result->append("WRONG_NUMBER_OF_INDICES");
147 return RewriteStatus::ERROR;
148 }
149 MaybeConvertFromHalf(
150 data_type,
151 absl::StrCat(element.object_name, ".data[", element.indices[0], " + $",
152 element.object_name, "_w$ * (", element.indices[1], " + $",
153 element.object_name, "_h$ * (", element.indices[2], "))]"),
154 result);
155 *requires_sizes = true;
156 return RewriteStatus::SUCCESS;
157 }
158
159 DataType data_type;
160 const object_accessor_internal::IndexedElement& element;
161 std::string* result;
162
163 // indicates that generated code accessed _w and/or _h index variables.
164 bool* requires_sizes;
165 };
166
167 // Generates code for reading an element from an object.
GenerateReadAccessor(const Object & object,const object_accessor_internal::IndexedElement & element,bool sampler_textures,std::string * result,bool * requires_sizes)168 RewriteStatus GenerateReadAccessor(
169 const Object& object,
170 const object_accessor_internal::IndexedElement& element,
171 bool sampler_textures, std::string* result, bool* requires_sizes) {
172 switch (object.object_type) {
173 case ObjectType::BUFFER:
174 return std::visit(ReadFromBufferGenerator{object.data_type, element,
175 result, requires_sizes},
176 object.size);
177 case ObjectType::TEXTURE:
178 return std::visit(
179 ReadFromTextureGenerator{element, sampler_textures, result},
180 object.size);
181 case ObjectType::UNKNOWN:
182 return RewriteStatus::ERROR;
183 }
184 }
185
186 struct WriteToBufferGenerator {
operator ()tflite::gpu::gl::__anonf2a2b3c90111::WriteToBufferGenerator187 RewriteStatus operator()(size_t) const {
188 if (element.indices.size() != 1) {
189 result->append("WRONG_NUMBER_OF_INDICES");
190 return RewriteStatus::ERROR;
191 }
192 absl::StrAppend(result, element.object_name, ".data[", element.indices[0],
193 "] = ");
194 MaybeConvertToHalf(data_type, value, result);
195 return RewriteStatus::SUCCESS;
196 }
197
operator ()tflite::gpu::gl::__anonf2a2b3c90111::WriteToBufferGenerator198 RewriteStatus operator()(const uint2& size) const {
199 if (element.indices.size() == 1) {
200 // access by linear index. Use method above to generate accessor.
201 return (*this)(1U);
202 }
203 if (element.indices.size() != 2) {
204 result->append("WRONG_NUMBER_OF_INDICES");
205 return RewriteStatus::ERROR;
206 }
207 absl::StrAppend(result, element.object_name, ".data[", element.indices[0],
208 " + $", element.object_name, "_w$ * (", element.indices[1],
209 ")] = ");
210 MaybeConvertToHalf(data_type, value, result);
211 *requires_sizes = true;
212 return RewriteStatus::SUCCESS;
213 }
214
operator ()tflite::gpu::gl::__anonf2a2b3c90111::WriteToBufferGenerator215 RewriteStatus operator()(const uint3& size) const {
216 if (element.indices.size() == 1) {
217 // access by linear index. Use method above to generate accessor.
218 return (*this)(1U);
219 }
220 if (element.indices.size() != 3) {
221 result->append("WRONG_NUMBER_OF_INDICES");
222 return RewriteStatus::ERROR;
223 }
224 absl::StrAppend(result, element.object_name, ".data[", element.indices[0],
225 " + $", element.object_name, "_w$ * (", element.indices[1],
226 " + $", element.object_name, "_h$ * (", element.indices[2],
227 "))] = ");
228 MaybeConvertToHalf(data_type, value, result);
229 *requires_sizes = true;
230 return RewriteStatus::SUCCESS;
231 }
232
233 DataType data_type;
234 const object_accessor_internal::IndexedElement& element;
235 absl::string_view value;
236 std::string* result;
237
238 // indicates that generated code accessed _w and/or _h index variables.
239 bool* requires_sizes;
240 };
241
242 struct WriteToTextureGenerator {
operator ()tflite::gpu::gl::__anonf2a2b3c90111::WriteToTextureGenerator243 RewriteStatus operator()(size_t) const {
244 if (element.indices.size() != 1) {
245 result->append("WRONG_NUMBER_OF_INDICES");
246 return RewriteStatus::ERROR;
247 }
248 // 1D textures are emulated as 2D textures
249 absl::StrAppend(result, "imageStore(", element.object_name, ", ivec2(",
250 element.indices[0], ", 0), ", value, ")");
251 return RewriteStatus::SUCCESS;
252 }
253
254 template <typename Shape>
operator ()tflite::gpu::gl::__anonf2a2b3c90111::WriteToTextureGenerator255 RewriteStatus operator()(const Shape&) const {
256 if (element.indices.size() != Shape::size()) {
257 result->append("WRONG_NUMBER_OF_INDICES");
258 return RewriteStatus::ERROR;
259 }
260 absl::StrAppend(result, "imageStore(", element.object_name, ", ivec",
261 Shape::size(), "(", absl::StrJoin(element.indices, ", "),
262 "), ", value, ")");
263 return RewriteStatus::SUCCESS;
264 }
265
266 const object_accessor_internal::IndexedElement& element;
267 absl::string_view value;
268 std::string* result;
269 };
270
271 // Generates code for writing value an element in an object.
GenerateWriteAccessor(const Object & object,const object_accessor_internal::IndexedElement & element,absl::string_view value,std::string * result,bool * requires_sizes)272 RewriteStatus GenerateWriteAccessor(
273 const Object& object,
274 const object_accessor_internal::IndexedElement& element,
275 absl::string_view value, std::string* result, bool* requires_sizes) {
276 switch (object.object_type) {
277 case ObjectType::BUFFER:
278 return std::visit(WriteToBufferGenerator{object.data_type, element, value,
279 result, requires_sizes},
280 object.size);
281 case ObjectType::TEXTURE:
282 return std::visit(WriteToTextureGenerator{element, value, result},
283 object.size);
284 case ObjectType::UNKNOWN:
285 return RewriteStatus::ERROR;
286 }
287 }
288
ToAccessModifier(AccessType access,bool use_readonly_modifier)289 std::string ToAccessModifier(AccessType access, bool use_readonly_modifier) {
290 switch (access) {
291 case AccessType::READ:
292 return use_readonly_modifier ? " readonly" : "";
293 case AccessType::WRITE:
294 return " writeonly";
295 case AccessType::READ_WRITE:
296 return " restrict";
297 }
298 return " unknown_access";
299 }
300
ToBufferType(DataType data_type)301 std::string ToBufferType(DataType data_type) {
302 switch (data_type) {
303 case DataType::UINT8:
304 case DataType::UINT16:
305 case DataType::UINT32:
306 return "uvec4";
307 case DataType::UINT64:
308 return "u64vec4_not_available_in_glsl";
309 case DataType::INT8:
310 case DataType::INT16:
311 case DataType::INT32:
312 return "ivec4";
313 case DataType::INT64:
314 return "i64vec4_not_available_in_glsl";
315 case DataType::FLOAT16:
316 return "uvec2";
317 case DataType::BOOL:
318 case DataType::FLOAT32:
319 return "vec4";
320 case DataType::FLOAT64:
321 return "dvec4";
322 case DataType::UNKNOWN:
323 return "unknown_buffer_type";
324 // Do NOT add `default:'; we want build failure for new enum values.
325 }
326 }
327
328 struct TextureImageTypeGetter {
operator ()tflite::gpu::gl::__anonf2a2b3c90111::TextureImageTypeGetter329 std::string operator()(size_t) const {
330 // 1D textures are emulated as 2D textures
331 return (*this)(uint2());
332 }
333
operator ()tflite::gpu::gl::__anonf2a2b3c90111::TextureImageTypeGetter334 std::string operator()(const uint2&) const {
335 switch (type) {
336 case DataType::UINT16:
337 case DataType::UINT32:
338 return "uimage2D";
339 case DataType::INT16:
340 case DataType::INT32:
341 return "iimage2D";
342 case DataType::FLOAT16:
343 case DataType::FLOAT32:
344 return "image2D";
345 default:
346 return "unknown_image_2d";
347 }
348 }
349
operator ()tflite::gpu::gl::__anonf2a2b3c90111::TextureImageTypeGetter350 std::string operator()(const uint3&) const {
351 switch (type) {
352 case DataType::UINT16:
353 case DataType::UINT32:
354 return "uimage2DArray";
355 case DataType::INT16:
356 case DataType::INT32:
357 return "iimage2DArray";
358 case DataType::FLOAT16:
359 case DataType::FLOAT32:
360 return "image2DArray";
361 default:
362 return "unknown_image_2d_array";
363 }
364 }
365
366 DataType type;
367 };
368
369 struct TextureSamplerTypeGetter {
operator ()tflite::gpu::gl::__anonf2a2b3c90111::TextureSamplerTypeGetter370 std::string operator()(size_t) const {
371 // 1D textures are emulated as 2D textures
372 return (*this)(uint2());
373 }
374
operator ()tflite::gpu::gl::__anonf2a2b3c90111::TextureSamplerTypeGetter375 std::string operator()(const uint2&) const {
376 switch (type) {
377 case DataType::FLOAT16:
378 case DataType::FLOAT32:
379 return "sampler2D";
380 case DataType::INT32:
381 case DataType::INT16:
382 return "isampler2D";
383 case DataType::UINT32:
384 case DataType::UINT16:
385 return "usampler2D";
386 default:
387 return "unknown_sampler2D";
388 }
389 }
390
operator ()tflite::gpu::gl::__anonf2a2b3c90111::TextureSamplerTypeGetter391 std::string operator()(const uint3&) const {
392 switch (type) {
393 case DataType::FLOAT16:
394 case DataType::FLOAT32:
395 return "sampler2DArray";
396 case DataType::INT32:
397 case DataType::INT16:
398 return "isampler2DArray";
399 case DataType::UINT32:
400 case DataType::UINT16:
401 return "usampler2DArray";
402 default:
403 return "unknown_sampler2DArray";
404 }
405 }
406
407 DataType type;
408 };
409
ToImageType(const Object & object,bool sampler_textures)410 std::string ToImageType(const Object& object, bool sampler_textures) {
411 if (sampler_textures && (object.access == AccessType::READ)) {
412 return std::visit(TextureSamplerTypeGetter{object.data_type}, object.size);
413 } else {
414 return std::visit(TextureImageTypeGetter{object.data_type}, object.size);
415 }
416 }
417
ToImageLayoutQualifier(DataType type)418 std::string ToImageLayoutQualifier(DataType type) {
419 switch (type) {
420 case DataType::UINT16:
421 return "rgba16ui";
422 case DataType::UINT32:
423 return "rgba32ui";
424 case DataType::INT16:
425 return "rgba16i";
426 case DataType::INT32:
427 return "rgba32i";
428 case DataType::FLOAT16:
429 return "rgba16f";
430 case DataType::FLOAT32:
431 return "rgba32f";
432 default:
433 return "unknown_image_layout";
434 }
435 }
436
ToImagePrecision(DataType type)437 std::string ToImagePrecision(DataType type) {
438 switch (type) {
439 case DataType::UINT16:
440 case DataType::INT16:
441 case DataType::FLOAT16:
442 return "mediump";
443 case DataType::UINT32:
444 case DataType::INT32:
445 case DataType::FLOAT32:
446 return "highp";
447 default:
448 return "unknown_image_precision";
449 }
450 }
451
452 struct SizeParametersAdder {
operator ()tflite::gpu::gl::__anonf2a2b3c90111::SizeParametersAdder453 void operator()(size_t) const {}
454
operator ()tflite::gpu::gl::__anonf2a2b3c90111::SizeParametersAdder455 void operator()(const uint2& size) const {
456 variable_accessor->AddUniformParameter(
457 {absl::StrCat(object_name, "_w"), static_cast<int32_t>(size.x)});
458 }
459
460 // p1 and p2 are padding. For some reason buffer does not map correctly
461 // without it.
operator ()tflite::gpu::gl::__anonf2a2b3c90111::SizeParametersAdder462 void operator()(const uint3& size) const {
463 variable_accessor->AddUniformParameter(
464 {absl::StrCat(object_name, "_w"), static_cast<int32_t>(size.x)});
465 variable_accessor->AddUniformParameter(
466 {absl::StrCat(object_name, "_h"), static_cast<int32_t>(size.y)});
467 }
468
469 absl::string_view object_name;
470 VariableAccessor* variable_accessor;
471 };
472
473 // Adds necessary parameters to parameter accessor that represent object size
474 // needed for indexed access.
475 // - 1D : empty
476 // - 2D : 'int object_name_w'
477 // - 3D : 'int object_name_w' + 'int object_name_h'
AddSizeParameters(absl::string_view object_name,const Object & object,VariableAccessor * parameters)478 void AddSizeParameters(absl::string_view object_name, const Object& object,
479 VariableAccessor* parameters) {
480 std::visit(SizeParametersAdder{object_name, parameters}, object.size);
481 }
482
GenerateObjectDeclaration(absl::string_view name,const Object & object,std::string * declaration,bool is_mali,bool sampler_textures)483 void GenerateObjectDeclaration(absl::string_view name, const Object& object,
484 std::string* declaration, bool is_mali,
485 bool sampler_textures) {
486 switch (object.object_type) {
487 case ObjectType::BUFFER:
488 // readonly modifier used to fix shader compilation for Mali on Android 8,
489 // see b/111601761
490 absl::StrAppend(declaration, "layout(binding = ", object.binding, ")",
491 ToAccessModifier(object.access, !is_mali), " buffer B",
492 object.binding, " { ", ToBufferType(object.data_type),
493 " data[]; } ", name, ";\n");
494 break;
495 case ObjectType::TEXTURE:
496 if (sampler_textures && (object.access == AccessType::READ)) {
497 absl::StrAppend(declaration, "layout(binding = ", object.binding,
498 ") uniform ", ToImagePrecision(object.data_type), " ",
499 ToImageType(object, sampler_textures), " ", name,
500 ";\n");
501 } else {
502 absl::StrAppend(
503 declaration, "layout(", ToImageLayoutQualifier(object.data_type),
504 ", binding = ", object.binding, ")",
505 ToAccessModifier(object.access, true), " uniform ",
506 ToImagePrecision(object.data_type), " ",
507 ToImageType(object, sampler_textures), " ", name, ";\n");
508 }
509 break;
510 case ObjectType::UNKNOWN:
511 // do nothing.
512 break;
513 }
514 }
515
516 } // namespace
517
Rewrite(absl::string_view input,std::string * output)518 RewriteStatus ObjectAccessor::Rewrite(absl::string_view input,
519 std::string* output) {
520 // Splits 'a =b' into {'a','b'}.
521 std::pair<absl::string_view, absl::string_view> n =
522 absl::StrSplit(input, absl::MaxSplits('=', 1), absl::SkipWhitespace());
523 if (n.first.empty()) {
524 return RewriteStatus::NOT_RECOGNIZED;
525 }
526 if (n.second.empty()) {
527 return RewriteRead(absl::StripAsciiWhitespace(n.first), output);
528 }
529 return RewriteWrite(absl::StripAsciiWhitespace(n.first),
530 absl::StripAsciiWhitespace(n.second), output);
531 }
532
RewriteRead(absl::string_view location,std::string * output)533 RewriteStatus ObjectAccessor::RewriteRead(absl::string_view location,
534 std::string* output) {
535 auto element = object_accessor_internal::ParseElement(location);
536 if (element.object_name.empty()) {
537 return RewriteStatus::NOT_RECOGNIZED;
538 }
539 auto it = name_to_object_.find(
540 std::string(element.object_name.data(), element.object_name.size()));
541 if (it == name_to_object_.end()) {
542 return RewriteStatus::NOT_RECOGNIZED;
543 }
544 bool requires_sizes = false;
545 auto status = GenerateReadAccessor(it->second, element, sampler_textures_,
546 output, &requires_sizes);
547 if (requires_sizes) {
548 AddSizeParameters(it->first, it->second, variable_accessor_);
549 }
550 return status;
551 }
552
RewriteWrite(absl::string_view location,absl::string_view value,std::string * output)553 RewriteStatus ObjectAccessor::RewriteWrite(absl::string_view location,
554 absl::string_view value,
555 std::string* output) {
556 // name[index1, index2...] = value
557 auto element = object_accessor_internal::ParseElement(location);
558 if (element.object_name.empty()) {
559 return RewriteStatus::NOT_RECOGNIZED;
560 }
561 auto it = name_to_object_.find(
562 std::string(element.object_name.data(), element.object_name.size()));
563 if (it == name_to_object_.end()) {
564 return RewriteStatus::NOT_RECOGNIZED;
565 }
566 bool requires_sizes = false;
567 auto status = GenerateWriteAccessor(it->second, element, value, output,
568 &requires_sizes);
569 if (requires_sizes) {
570 AddSizeParameters(it->first, it->second, variable_accessor_);
571 }
572 return status;
573 }
574
AddObject(const std::string & name,Object object)575 bool ObjectAccessor::AddObject(const std::string& name, Object object) {
576 if (object.object_type == ObjectType::UNKNOWN) {
577 return false;
578 }
579 return name_to_object_.insert({name, std::move(object)}).second;
580 }
581
GetObjectDeclarations() const582 std::string ObjectAccessor::GetObjectDeclarations() const {
583 std::string declarations;
584 for (auto& o : name_to_object_) {
585 GenerateObjectDeclaration(o.first, o.second, &declarations, is_mali_,
586 sampler_textures_);
587 }
588 return declarations;
589 }
590
GetFunctionsDeclarations() const591 std::string ObjectAccessor::GetFunctionsDeclarations() const {
592 // If there is a single object SSBO with F16, then we need to output macros
593 // as well.
594 for (const auto& o : name_to_object_) {
595 if (o.second.data_type == DataType::FLOAT16 &&
596 o.second.object_type == ObjectType::BUFFER) {
597 return absl::StrCat(
598 "#define Vec4FromHalf(v) vec4(unpackHalf2x16(v.x), "
599 "unpackHalf2x16(v.y))\n",
600 "#define Vec4ToHalf(v) uvec2(packHalf2x16(v.xy), "
601 "packHalf2x16(v.zw))");
602 }
603 }
604 return "";
605 }
606
GetObjects() const607 std::vector<Object> ObjectAccessor::GetObjects() const {
608 std::vector<Object> objects;
609 objects.reserve(name_to_object_.size());
610 for (auto& o : name_to_object_) {
611 objects.push_back(o.second);
612 }
613 return objects;
614 }
615
616 } // namespace gl
617 } // namespace gpu
618 } // namespace tflite
619