1 /*
2 * Copyright (C) 2020 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #define LOG_TAG "Camera3-RotCropMapper"
18 #define ATRACE_TAG ATRACE_TAG_CAMERA
19 //#define LOG_NDEBUG 0
20
21 #include <algorithm>
22 #include <cmath>
23
24 #include "device3/RotateAndCropMapper.h"
25
26 namespace android {
27
28 namespace camera3 {
29
initRemappedKeys()30 void RotateAndCropMapper::initRemappedKeys() {
31 mRemappedKeys.insert(
32 kMeteringRegionsToCorrect.begin(),
33 kMeteringRegionsToCorrect.end());
34 mRemappedKeys.insert(
35 kResultPointsToCorrectNoClamp.begin(),
36 kResultPointsToCorrectNoClamp.end());
37
38 mRemappedKeys.insert(ANDROID_SCALER_ROTATE_AND_CROP);
39 mRemappedKeys.insert(ANDROID_SCALER_CROP_REGION);
40 mRemappedKeys.insert(ANDROID_LOGICAL_MULTI_CAMERA_ACTIVE_PHYSICAL_SENSOR_CROP_REGION);
41 }
42
isNeeded(const CameraMetadata * deviceInfo)43 bool RotateAndCropMapper::isNeeded(const CameraMetadata* deviceInfo) {
44 auto entry = deviceInfo->find(ANDROID_SCALER_AVAILABLE_ROTATE_AND_CROP_MODES);
45 for (size_t i = 0; i < entry.count; i++) {
46 if (entry.data.u8[i] == ANDROID_SCALER_ROTATE_AND_CROP_AUTO) return true;
47 }
48 return false;
49 }
50
RotateAndCropMapper(const CameraMetadata * deviceInfo)51 RotateAndCropMapper::RotateAndCropMapper(const CameraMetadata* deviceInfo) {
52 initRemappedKeys();
53
54 auto entry = deviceInfo->find(ANDROID_SENSOR_INFO_ACTIVE_ARRAY_SIZE);
55 if (entry.count != 4) return;
56
57 mArrayWidth = entry.data.i32[2];
58 mArrayHeight = entry.data.i32[3];
59 mArrayAspect = static_cast<float>(mArrayWidth) / mArrayHeight;
60 mRotateAspect = 1.f/mArrayAspect;
61 }
62
63 /**
64 * Adjust capture request when rotate and crop AUTO is enabled
65 */
updateCaptureRequest(CameraMetadata * request)66 status_t RotateAndCropMapper::updateCaptureRequest(CameraMetadata *request) {
67 auto entry = request->find(ANDROID_SCALER_ROTATE_AND_CROP);
68 if (entry.count == 0) return OK;
69 uint8_t rotateMode = entry.data.u8[0];
70 if (rotateMode == ANDROID_SCALER_ROTATE_AND_CROP_NONE) return OK;
71
72 int32_t cx = 0;
73 int32_t cy = 0;
74 int32_t cw = mArrayWidth;
75 int32_t ch = mArrayHeight;
76 entry = request->find(ANDROID_SCALER_CROP_REGION);
77 if (entry.count == 4) {
78 cx = entry.data.i32[0];
79 cy = entry.data.i32[1];
80 cw = entry.data.i32[2];
81 ch = entry.data.i32[3];
82 }
83
84 // User inputs are relative to the rotated-and-cropped view, so convert back
85 // to active array coordinates. To be more specific, the application is
86 // calculating coordinates based on the crop rectangle and the active array,
87 // even though the view the user sees is the cropped-and-rotated one. So we
88 // need to adjust the coordinates so that a point that would be on the
89 // top-left corner of the crop region is mapped to the top-left corner of
90 // the rotated-and-cropped fov within the crop region, and the same for the
91 // bottom-right corner.
92 //
93 // Since the zoom ratio control scales everything uniformly (so an app does
94 // not need to adjust anything if it wants to put a metering region on the
95 // top-left quadrant of the preview FOV, when changing zoomRatio), it does
96 // not need to be factored into this calculation at all.
97 //
98 // ->+x active array aw
99 // |+--------------------------------------------------------------------+
100 // v| |
101 // +y| a 1 cw 2 b |
102 // | +=========*HHHHHHHHHHHHHHH*===========+ |
103 // | I H rw H I |
104 // | I H H I |
105 // | I H H I |
106 //ah | ch I H rh H I crop region |
107 // | I H H I |
108 // | I H H I |
109 // | I H rotate region H I |
110 // | +=========*HHHHHHHHHHHHHHH*===========+ |
111 // | d 4 3 c |
112 // | |
113 // +--------------------------------------------------------------------+
114 //
115 // aw , ah = active array width,height
116 // cw , ch = crop region width,height
117 // rw , rh = rotated-and-cropped region width,height
118 // aw / ah = array aspect = rh / rw = 1 / rotated aspect
119 // Coordinate mappings:
120 // ROTATE_AND_CROP_90: point a -> point 2
121 // point c -> point 4 = +x -> +y, +y -> -x
122 // ROTATE_AND_CROP_180: point a -> point c
123 // point c -> point a = +x -> -x, +y -> -y
124 // ROTATE_AND_CROP_270: point a -> point 4
125 // point c -> point 2 = +x -> -y, +y -> +x
126
127 float cropAspect = static_cast<float>(cw) / ch;
128 float transformMat[4] = {0, 0,
129 0, 0};
130 float xShift = 0;
131 float yShift = 0;
132
133 if (rotateMode == ANDROID_SCALER_ROTATE_AND_CROP_180) {
134 transformMat[0] = -1;
135 transformMat[3] = -1;
136 xShift = cw;
137 yShift = ch;
138 } else {
139 float rw = cropAspect > mRotateAspect ?
140 ch * mRotateAspect : // pillarbox, not full width
141 cw; // letterbox or 1:1, full width
142 float rh = cropAspect >= mRotateAspect ?
143 ch : // pillarbox or 1:1, full height
144 cw / mRotateAspect; // letterbox, not full height
145 switch (rotateMode) {
146 case ANDROID_SCALER_ROTATE_AND_CROP_270:
147 transformMat[1] = -rw / ch; // +y -> -x
148 transformMat[2] = rh / cw; // +x -> +y
149 xShift = (cw + rw) / 2; // left edge of crop to right edge of rotated
150 yShift = (ch - rh) / 2; // top edge of crop to top edge of rotated
151 break;
152 case ANDROID_SCALER_ROTATE_AND_CROP_90:
153 transformMat[1] = rw / ch; // +y -> +x
154 transformMat[2] = -rh / cw; // +x -> -y
155 xShift = (cw - rw) / 2; // left edge of crop to left edge of rotated
156 yShift = (ch + rh) / 2; // top edge of crop to bottom edge of rotated
157 break;
158 default:
159 ALOGE("%s: Unexpected rotate mode: %d", __FUNCTION__, rotateMode);
160 return BAD_VALUE;
161 }
162 }
163
164 for (auto regionTag : kMeteringRegionsToCorrect) {
165 entry = request->find(regionTag);
166 for (size_t i = 0; i < entry.count; i += 5) {
167 int32_t weight = entry.data.i32[i + 4];
168 if (weight == 0) {
169 continue;
170 }
171 transformPoints(entry.data.i32 + i, 2, transformMat, xShift, yShift, cx, cy);
172 swapRectToMinFirst(entry.data.i32 + i);
173 }
174 }
175
176 return OK;
177 }
178
179 /**
180 * Adjust capture result when rotate and crop AUTO is enabled
181 */
updateCaptureResult(CameraMetadata * result)182 status_t RotateAndCropMapper::updateCaptureResult(CameraMetadata *result) {
183 auto entry = result->find(ANDROID_SCALER_ROTATE_AND_CROP);
184 if (entry.count == 0) return OK;
185 uint8_t rotateMode = entry.data.u8[0];
186 if (rotateMode == ANDROID_SCALER_ROTATE_AND_CROP_NONE) return OK;
187
188 int32_t cx = 0;
189 int32_t cy = 0;
190 int32_t cw = mArrayWidth;
191 int32_t ch = mArrayHeight;
192 entry = result->find(ANDROID_SCALER_CROP_REGION);
193 if (entry.count == 4) {
194 cx = entry.data.i32[0];
195 cy = entry.data.i32[1];
196 cw = entry.data.i32[2];
197 ch = entry.data.i32[3];
198 }
199
200 // HAL inputs are relative to the full active array, so convert back to
201 // rotated-and-cropped coordinates for apps. To be more specific, the
202 // application is calculating coordinates based on the crop rectangle and
203 // the active array, even though the view the user sees is the
204 // cropped-and-rotated one. So we need to adjust the coordinates so that a
205 // point that would be on the top-left corner of the rotate-and-cropped
206 // region is mapped to the top-left corner of the crop region, and the same
207 // for the bottom-right corner.
208 //
209 // Since the zoom ratio control scales everything uniformly (so an app does
210 // not need to adjust anything if it wants to put a metering region on the
211 // top-left quadrant of the preview FOV, when changing zoomRatio), it does
212 // not need to be factored into this calculation at all.
213 //
214 // Also note that round-tripping between original request and final result
215 // fields can't be perfect, since the intermediate values have to be
216 // integers on a smaller range than the original crop region range. That
217 // means that multiple input values map to a single output value in
218 // adjusting a request, so when adjusting a result, the original answer may
219 // not be obtainable. Given that aspect ratios are rarely > 16/9, the
220 // round-trip values should generally only be off by 1 at most.
221 //
222 // ->+x active array aw
223 // |+--------------------------------------------------------------------+
224 // v| |
225 // +y| a 1 cw 2 b |
226 // | +=========*HHHHHHHHHHHHHHH*===========+ |
227 // | I H rw H I |
228 // | I H H I |
229 // | I H H I |
230 //ah | ch I H rh H I crop region |
231 // | I H H I |
232 // | I H H I |
233 // | I H rotate region H I |
234 // | +=========*HHHHHHHHHHHHHHH*===========+ |
235 // | d 4 3 c |
236 // | |
237 // +--------------------------------------------------------------------+
238 //
239 // aw , ah = active array width,height
240 // cw , ch = crop region width,height
241 // rw , rh = rotated-and-cropped region width,height
242 // aw / ah = array aspect = rh / rw = 1 / rotated aspect
243 // Coordinate mappings:
244 // ROTATE_AND_CROP_90: point 2 -> point a
245 // point 4 -> point c = +x -> -y, +y -> +x
246 // ROTATE_AND_CROP_180: point c -> point a
247 // point a -> point c = +x -> -x, +y -> -y
248 // ROTATE_AND_CROP_270: point 4 -> point a
249 // point 2 -> point c = +x -> +y, +y -> -x
250
251 float cropAspect = static_cast<float>(cw) / ch;
252 float transformMat[4] = {0, 0,
253 0, 0};
254 float xShift = 0;
255 float yShift = 0;
256 float rx = 0; // top-left corner of rotated region
257 float ry = 0;
258 if (rotateMode == ANDROID_SCALER_ROTATE_AND_CROP_180) {
259 transformMat[0] = -1;
260 transformMat[3] = -1;
261 xShift = cw;
262 yShift = ch;
263 rx = cx;
264 ry = cy;
265 } else {
266 float rw = cropAspect > mRotateAspect ?
267 ch * mRotateAspect : // pillarbox, not full width
268 cw; // letterbox or 1:1, full width
269 float rh = cropAspect >= mRotateAspect ?
270 ch : // pillarbox or 1:1, full height
271 cw / mRotateAspect; // letterbox, not full height
272 rx = cx + (cw - rw) / 2;
273 ry = cy + (ch - rh) / 2;
274 switch (rotateMode) {
275 case ANDROID_SCALER_ROTATE_AND_CROP_270:
276 transformMat[1] = ch / rw; // +y -> +x
277 transformMat[2] = -cw / rh; // +x -> -y
278 xShift = -(cw - rw) / 2; // left edge of rotated to left edge of cropped
279 yShift = ry - cy + ch; // top edge of rotated to bottom edge of cropped
280 break;
281 case ANDROID_SCALER_ROTATE_AND_CROP_90:
282 transformMat[1] = -ch / rw; // +y -> -x
283 transformMat[2] = cw / rh; // +x -> +y
284 xShift = (cw + rw) / 2; // left edge of rotated to left edge of cropped
285 yShift = (ch - rh) / 2; // top edge of rotated to bottom edge of cropped
286 break;
287 default:
288 ALOGE("%s: Unexpected rotate mode: %d", __FUNCTION__, rotateMode);
289 return BAD_VALUE;
290 }
291 }
292
293 for (auto regionTag : kMeteringRegionsToCorrect) {
294 entry = result->find(regionTag);
295 for (size_t i = 0; i < entry.count; i += 5) {
296 int32_t weight = entry.data.i32[i + 4];
297 if (weight == 0) {
298 continue;
299 }
300 transformPoints(entry.data.i32 + i, 2, transformMat, xShift, yShift, rx, ry);
301 swapRectToMinFirst(entry.data.i32 + i);
302 }
303 }
304
305 for (auto pointsTag: kResultPointsToCorrectNoClamp) {
306 entry = result->find(pointsTag);
307 transformPoints(entry.data.i32, entry.count / 2, transformMat, xShift, yShift, rx, ry);
308 if (pointsTag == ANDROID_STATISTICS_FACE_RECTANGLES) {
309 for (size_t i = 0; i < entry.count; i += 4) {
310 swapRectToMinFirst(entry.data.i32 + i);
311 }
312 }
313 }
314
315 return OK;
316 }
317
transformPoints(int32_t * pts,size_t count,float transformMat[4],float xShift,float yShift,float ox,float oy)318 void RotateAndCropMapper::transformPoints(int32_t* pts, size_t count, float transformMat[4],
319 float xShift, float yShift, float ox, float oy) {
320 for (size_t i = 0; i < count * 2; i += 2) {
321 float x0 = pts[i] - ox;
322 float y0 = pts[i + 1] - oy;
323 int32_t nx = std::round(transformMat[0] * x0 + transformMat[1] * y0 + xShift + ox);
324 int32_t ny = std::round(transformMat[2] * x0 + transformMat[3] * y0 + yShift + oy);
325
326 pts[i] = std::min(std::max(nx, 0), mArrayWidth);
327 pts[i + 1] = std::min(std::max(ny, 0), mArrayHeight);
328 }
329 }
330
swapRectToMinFirst(int32_t * rect)331 void RotateAndCropMapper::swapRectToMinFirst(int32_t* rect) {
332 if (rect[0] > rect[2]) {
333 auto tmp = rect[0];
334 rect[0] = rect[2];
335 rect[2] = tmp;
336 }
337 if (rect[1] > rect[3]) {
338 auto tmp = rect[1];
339 rect[1] = rect[3];
340 rect[3] = tmp;
341 }
342 }
343
344 } // namespace camera3
345
346 } // namespace android
347