1 /* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
2 
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6 
7     http://www.apache.org/licenses/LICENSE-2.0
8 
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15 
16 package org.tensorflow.lite.task.core.vision;
17 
18 import android.graphics.Rect;
19 import com.google.auto.value.AutoValue;
20 
21 /**
22  * Options to configure the image processing pipeline, which operates before inference.
23  *
24  * <p>The Task Library Vision API performs image preprocessing on the input image over the region of
25  * interest, so that it fits model requirements (e.g. upright 224x224 RGB) and populate the
26  * corresponding input tensor. This is performed by (in this order):
27  *
28  * <ul>
29  *   <li>cropping the frame buffer to the region of interest (which, in most cases, just covers the
30  *       entire input image),
31  *   <li>resizing it (with bilinear interpolation, aspect-ratio *not* preserved) to the dimensions
32  *       of the model input tensor,
33  *   <li>converting it to the colorspace of the input tensor (i.e. RGB, which is the only supported
34  *       colorspace for now),
35  *   <li>rotating it according to its {@link Orientation} so that inference is performed on an
36  *       "upright" image.
37  * </ul>
38  *
39  * <p>IMPORTANT: as a consequence of cropping occurring first, the provided region of interest is
40  * expressed in the unrotated frame of reference coordinates system, i.e. in {@code [0,
41  * TensorImage.getWidth()) x [0, TensorImage.getHeight())}, which are the dimensions of the
42  * underlying image data before any orientation gets applied. If the region is out of these bounds,
43  * the inference method, such as {@link ImageClassifier#classify}, will return error.
44  */
45 @AutoValue
46 public abstract class ImageProcessingOptions {
47 
48   /**
49    * Orientation type that follows EXIF specification.
50    *
51    * <p>The name of each enum value defines the position of the 0th row and the 0th column of the
52    * image content. See the <a href="http://jpegclub.org/exif_orientation.html">EXIF orientation
53    * documentation</a> for details.
54    */
55   public enum Orientation {
56     TOP_LEFT(0),
57     TOP_RIGHT(1),
58     BOTTOM_RIGHT(2),
59     BOTTOM_LEFT(3),
60     LEFT_TOP(4),
61     RIGHT_TOP(5),
62     RIGHT_BOTTOM(6),
63     LEFT_BOTTOM(7);
64 
65     private final int value;
66 
Orientation(int value)67     Orientation(int value) {
68       this.value = value;
69     }
70 
getValue()71     public int getValue() {
72       return value;
73     }
74   };
75 
76   private static final Rect defaultRoi = new Rect();
77   private static final Orientation DEFAULT_ORIENTATION = Orientation.TOP_LEFT;
78 
getRoi()79   public abstract Rect getRoi();
80 
getOrientation()81   public abstract Orientation getOrientation();
82 
builder()83   public static Builder builder() {
84     return new AutoValue_ImageProcessingOptions.Builder()
85         .setRoi(defaultRoi)
86         .setOrientation(DEFAULT_ORIENTATION);
87   }
88 
89   /** Builder for {@link ImageProcessingOptions}. */
90   @AutoValue.Builder
91   public abstract static class Builder {
92 
93     /**
94      * Sets the region of interest (ROI) of the image. Defaults to the entire image.
95      *
96      * <p>Cropping according to this region of interest is prepended to the pre-processing
97      * operations.
98      */
setRoi(Rect roi)99     public abstract Builder setRoi(Rect roi);
100 
101     /**
102      * Sets the orientation of the image. Defaults to {@link Orientation#TOP_LEFT}.
103      *
104      * <p>Rotation will be applied accordingly so that inference is performed on an "upright" image.
105      */
setOrientation(Orientation orientation)106     public abstract Builder setOrientation(Orientation orientation);
107 
getRoi()108     abstract Rect getRoi();
109 
autoBuild()110     abstract ImageProcessingOptions autoBuild();
111 
build()112     public ImageProcessingOptions build() {
113       setRoi(new Rect(getRoi())); // Make a defensive copy, since Rect is mutable.
114       return autoBuild();
115     }
116   }
117 }
118