1 /* Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 #ifndef TENSORFLOW_LITE_TOOLS_OPTIMIZE_OPERATOR_PROPERTY_H_ 16 #define TENSORFLOW_LITE_TOOLS_OPTIMIZE_OPERATOR_PROPERTY_H_ 17 18 #include <functional> 19 #include <initializer_list> 20 21 #include "tensorflow/lite/model.h" 22 #include "tensorflow/lite/schema/schema_generated.h" 23 24 namespace tflite { 25 namespace optimize { 26 namespace operator_property { 27 28 // The scales of a certain tensor can be derived from the multiplications of all 29 // the scales. For example, for bias in conv, derived_scale = {{0, 1}, {}, {}} 30 // and for lstm gate bias, the derived scale is {{}, {0}, {2^-10}} 31 struct DerivedScale { 32 // MSVC2015 version 14.0 and below doesn't support struct initialization with 33 // initializer lists so emulate the behavior using a float initializer list. 34 #if _MSC_VER <= 1900 DerivedScaleDerivedScale35 DerivedScale() {} 36 // Construct this object with a list of initializer lists. All list elements 37 // are cast to float values to avoid ambiguous construction of a union-style 38 // object that could take either std::initializer_list<float> or 39 // std::initializer_list<int>. DerivedScaleDerivedScale40 DerivedScale(std::initializer_list<std::initializer_list<float>> values) { 41 assert(values.size() == 3); 42 std::vector<std::initializer_list<float>> items(values); 43 for (auto& it : items[0]) { 44 input_tensors.push_back(static_cast<int>(it)); 45 } 46 for (auto& it : items[1]) { 47 intermediate_tensors.push_back(static_cast<int>(it)); 48 } 49 factors.assign(items[2]); 50 } 51 #endif // _MSC_VER <= 1900 52 53 std::vector<int> input_tensors = {}; 54 std::vector<int> intermediate_tensors = {}; 55 // This is a list of extra factors that are not associated with any other 56 // tensor. 57 std::vector<float> factors = {}; 58 }; 59 60 struct TensorProperty { 61 // per_axis also implies symmetric currently. 62 bool per_axis = false; 63 // TODO(jianlijianli): remove dimension index and read it from tensor instead. 64 int per_axis_index = 0; 65 bool symmetric = false; 66 67 // Constraints. 68 bool restriction = false; 69 // scale/zero_point hardcoded. 70 std::pair<float, int> restricted_value_int8 = {0.0f, 0}; 71 std::pair<float, int> restricted_value_int16 = {0.0f, 0}; 72 73 // Use derived scale. 74 bool use_derived_scale = false; 75 // The derived scale. 76 DerivedScale derived_scale; 77 78 // The number of bits for this tensor. It could be 8, 16, 32 or even not power 79 // of two. 80 int number_of_bits = 8; 81 82 // Extend the range to power of two. 83 bool extend_to_power_of_two = false; 84 85 // State tensor. 86 bool state_tensor = false; 87 }; 88 89 struct OperatorProperty { 90 // Is a quantized operations currently supported. 91 bool quantizable = true; 92 // Is a quantized operations currently supported for 16x8 93 bool quantizable_int16 = true; 94 // Op has arbitrary number of inputs, such as concat. 95 bool arbitrary_inputs = false; 96 // Op has arbitrary number of outputs, such as slice. 97 bool arbitrary_outputs = false; 98 // Input indexes -> input tensor property. 99 // Must be topologically sorted since there are derived scales. 100 std::vector<std::pair<int, TensorProperty>> inputs = {}; 101 // Output indexes -> output tensor property. 102 std::vector<std::pair<int, TensorProperty>> outputs = {}; 103 // Bias indexes. 104 // TODO(jianlijianli): remove this by putting biases into inputs as well since 105 // we now can model "derived scale". 106 std::vector<int> biases = {}; 107 108 // Intermediate indexes -> intermediate tensor property. 109 std::vector<std::pair<int, TensorProperty>> intermediates = {}; 110 111 // Force output to reuse the same scale and zero point of input when the 112 // certain type support must require the same scale and zero point 113 // requirement. 114 std::function<bool(TensorType)> restrict_same_input_output_scale = 115 [](TensorType) { return false; }; 116 117 // Use same min of min and max of max for each group. 118 // Incompatible with restrict_same_input_output_scale and restricted_value. 119 // Currently it only supports scale pair of {input_index, output_index}. 120 std::vector<std::vector<int>> restrict_scale = {}; 121 122 // Op version. 123 int version = 1; 124 125 // When we quantize activations into 16 bit and weights into 8 bit, 126 // we want to quantize all inputs, including constant tensors, 127 // for the operators like Add, Mul into 16-bit as well. The constant 128 // inputs are quantized as weights and this variable indicates 129 // that we want to do quantizations of these tensors as activations. 130 bool quantize_input_as_activations = false; 131 }; 132 133 // The op as well as it variants. 134 struct OpVariant { 135 BuiltinOperator op_code; 136 bool use_layer_norm = false; 137 bool use_projection = false; 138 bool use_peephole = false; 139 // An attribute to indicate if quantization is supported for this Op. 140 // This attribute is equivalent to the "quantizable" attribute in 141 // "OperatorProperty". It added here since OpVariants peeks inside the Op and 142 // determines its quantization related properties. 143 bool is_quantizable = true; 144 }; 145 146 OperatorProperty GetOperatorProperty(const ModelT* model, int subgraph_index, 147 int op_index); 148 OperatorProperty GetOperatorProperty(OpVariant op_variant); 149 150 } // namespace operator_property 151 } // namespace optimize 152 } // namespace tflite 153 154 #endif // TENSORFLOW_LITE_TOOLS_OPTIMIZE_OPERATOR_PROPERTY_H_ 155