1 /*
2 * Copyright (C) 2012-2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #define LOG_TAG "Camera2-CallbackProcessor"
18 #define ATRACE_TAG ATRACE_TAG_CAMERA
19 //#define LOG_NDEBUG 0
20
21 #include <com_android_graphics_libgui_flags.h>
22 #include <gui/Surface.h>
23 #include <utils/Log.h>
24 #include <utils/Trace.h>
25
26 #include "common/CameraDeviceBase.h"
27 #include "api1/Camera2Client.h"
28 #include "api1/client2/CallbackProcessor.h"
29
30 #define ALIGN(x, mask) ( ((x) + (mask) - 1) & ~((mask) - 1) )
31
32 namespace android {
33 namespace camera2 {
34
35 using android::camera3::CAMERA_STREAM_ROTATION_0;
36
CallbackProcessor(sp<Camera2Client> client)37 CallbackProcessor::CallbackProcessor(sp<Camera2Client> client):
38 Thread(false),
39 mClient(client),
40 mDevice(client->getCameraDevice()),
41 mId(client->getCameraId()),
42 mCallbackAvailable(false),
43 mCallbackPaused(true),
44 mCallbackToApp(false),
45 mCallbackStreamId(NO_STREAM) {
46 }
47
~CallbackProcessor()48 CallbackProcessor::~CallbackProcessor() {
49 ALOGV("%s: Exit", __FUNCTION__);
50 deleteStream();
51 }
52
onFrameAvailable(const BufferItem &)53 void CallbackProcessor::onFrameAvailable(const BufferItem& /*item*/) {
54 Mutex::Autolock l(mInputMutex);
55 if (!mCallbackAvailable) {
56 mCallbackAvailable = true;
57 mCallbackAvailableSignal.signal();
58 }
59 }
60
setCallbackWindow(const sp<Surface> & callbackWindow)61 status_t CallbackProcessor::setCallbackWindow(
62 const sp<Surface>& callbackWindow) {
63 ATRACE_CALL();
64 status_t res;
65
66 Mutex::Autolock l(mInputMutex);
67
68 sp<Camera2Client> client = mClient.promote();
69 if (client == 0) return OK;
70 sp<CameraDeviceBase> device = client->getCameraDevice();
71
72 // If the window is changing, clear out stream if it already exists
73 if (mCallbackWindow != callbackWindow && mCallbackStreamId != NO_STREAM) {
74 res = device->deleteStream(mCallbackStreamId);
75 if (res != OK) {
76 ALOGE("%s: Camera %d: Unable to delete old stream "
77 "for callbacks: %s (%d)", __FUNCTION__,
78 client->getCameraId(), strerror(-res), res);
79 return res;
80 }
81 mCallbackStreamId = NO_STREAM;
82 mCallbackConsumer.clear();
83 }
84 mCallbackWindow = callbackWindow;
85 mCallbackToApp = (mCallbackWindow != NULL);
86
87 return OK;
88 }
89
updateStream(const Parameters & params)90 status_t CallbackProcessor::updateStream(const Parameters ¶ms) {
91 ATRACE_CALL();
92 status_t res;
93
94 Mutex::Autolock l(mInputMutex);
95
96 sp<CameraDeviceBase> device = mDevice.promote();
97 if (device == 0) {
98 ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId);
99 return INVALID_OPERATION;
100 }
101
102 // If possible, use the flexible YUV format
103 int32_t callbackFormat = params.previewFormat;
104 if (mCallbackToApp) {
105 // TODO: etalvala: This should use the flexible YUV format as well, but
106 // need to reconcile HAL2/HAL3 requirements.
107 callbackFormat = HAL_PIXEL_FORMAT_YV12;
108 } else if(params.fastInfo.useFlexibleYuv &&
109 (params.previewFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP ||
110 params.previewFormat == HAL_PIXEL_FORMAT_YV12) ) {
111 callbackFormat = HAL_PIXEL_FORMAT_YCbCr_420_888;
112 }
113
114 if (!mCallbackToApp && mCallbackConsumer == 0) {
115 // Create CPU buffer queue endpoint, since app hasn't given us one
116 // Make it async to avoid disconnect deadlocks
117 #if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ)
118 mCallbackConsumer = new CpuConsumer(kCallbackHeapCount);
119 mCallbackConsumer->setFrameAvailableListener(this);
120 mCallbackConsumer->setName(String8("Camera2-CallbackConsumer"));
121 mCallbackWindow = mCallbackConsumer->getSurface();
122 #else
123 sp<IGraphicBufferProducer> producer;
124 sp<IGraphicBufferConsumer> consumer;
125 BufferQueue::createBufferQueue(&producer, &consumer);
126 mCallbackConsumer = new CpuConsumer(consumer, kCallbackHeapCount);
127 mCallbackConsumer->setFrameAvailableListener(this);
128 mCallbackConsumer->setName(String8("Camera2-CallbackConsumer"));
129 mCallbackWindow = new Surface(producer);
130 #endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ)
131 }
132
133 if (mCallbackStreamId != NO_STREAM) {
134 // Check if stream parameters have to change
135 CameraDeviceBase::StreamInfo streamInfo;
136 res = device->getStreamInfo(mCallbackStreamId, &streamInfo);
137 if (res != OK) {
138 ALOGE("%s: Camera %d: Error querying callback output stream info: "
139 "%s (%d)", __FUNCTION__, mId,
140 strerror(-res), res);
141 return res;
142 }
143 if (streamInfo.width != (uint32_t)params.previewWidth ||
144 streamInfo.height != (uint32_t)params.previewHeight ||
145 !streamInfo.matchFormat((uint32_t)callbackFormat)) {
146 // Since size should only change while preview is not running,
147 // assuming that all existing use of old callback stream is
148 // completed.
149 ALOGV("%s: Camera %d: Deleting stream %d since the buffer "
150 "parameters changed", __FUNCTION__, mId, mCallbackStreamId);
151 res = device->deleteStream(mCallbackStreamId);
152 if (res != OK) {
153 ALOGE("%s: Camera %d: Unable to delete old output stream "
154 "for callbacks: %s (%d)", __FUNCTION__,
155 mId, strerror(-res), res);
156 return res;
157 }
158 mCallbackStreamId = NO_STREAM;
159 }
160 }
161
162 if (mCallbackStreamId == NO_STREAM) {
163 ALOGV("Creating callback stream: %d x %d, format 0x%x, API format 0x%x",
164 params.previewWidth, params.previewHeight,
165 callbackFormat, params.previewFormat);
166 res = device->createStream(mCallbackWindow,
167 params.previewWidth, params.previewHeight, callbackFormat,
168 HAL_DATASPACE_V0_JFIF, CAMERA_STREAM_ROTATION_0, &mCallbackStreamId,
169 std::string(), std::unordered_set<int32_t>{ANDROID_SENSOR_PIXEL_MODE_DEFAULT});
170 if (res != OK) {
171 ALOGE("%s: Camera %d: Can't create output stream for callbacks: "
172 "%s (%d)", __FUNCTION__, mId,
173 strerror(-res), res);
174 return res;
175 }
176 }
177
178 return OK;
179 }
180
deleteStream()181 status_t CallbackProcessor::deleteStream() {
182 ATRACE_CALL();
183 sp<CameraDeviceBase> device;
184 status_t res;
185 {
186 Mutex::Autolock l(mInputMutex);
187
188 if (mCallbackStreamId == NO_STREAM) {
189 return OK;
190 }
191 device = mDevice.promote();
192 if (device == 0) {
193 ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId);
194 return INVALID_OPERATION;
195 }
196 }
197 res = device->waitUntilDrained();
198 if (res != OK) {
199 ALOGE("%s: Error waiting for HAL to drain: %s (%d)",
200 __FUNCTION__, strerror(-res), res);
201 return res;
202 }
203
204 res = device->deleteStream(mCallbackStreamId);
205 if (res != OK) {
206 ALOGE("%s: Unable to delete callback stream: %s (%d)",
207 __FUNCTION__, strerror(-res), res);
208 return res;
209 }
210
211 {
212 Mutex::Autolock l(mInputMutex);
213
214 mCallbackHeap.clear();
215 mCallbackWindow.clear();
216 mCallbackConsumer.clear();
217
218 mCallbackStreamId = NO_STREAM;
219 }
220 return OK;
221 }
222
getStreamId() const223 int CallbackProcessor::getStreamId() const {
224 Mutex::Autolock l(mInputMutex);
225 return mCallbackStreamId;
226 }
227
unpauseCallback()228 void CallbackProcessor::unpauseCallback() {
229 mCallbackPaused = false;
230 }
231
pauseCallback()232 void CallbackProcessor::pauseCallback() {
233 mCallbackPaused = true;
234 }
235
dump(int,const Vector<String16> &) const236 void CallbackProcessor::dump(int /*fd*/, const Vector<String16>& /*args*/) const {
237 }
238
threadLoop()239 bool CallbackProcessor::threadLoop() {
240 status_t res;
241
242 {
243 Mutex::Autolock l(mInputMutex);
244 while (!mCallbackAvailable) {
245 res = mCallbackAvailableSignal.waitRelative(mInputMutex,
246 kWaitDuration);
247 if (res == TIMED_OUT) return true;
248 }
249 mCallbackAvailable = false;
250 }
251
252 do {
253 sp<Camera2Client> client = mClient.promote();
254 if (client == 0 || mCallbackPaused) {
255 res = discardNewCallback();
256 } else {
257 res = processNewCallback(client);
258 }
259 } while (res == OK);
260
261 return true;
262 }
263
discardNewCallback()264 status_t CallbackProcessor::discardNewCallback() {
265 ATRACE_CALL();
266 status_t res;
267 CpuConsumer::LockedBuffer imgBuffer;
268 res = mCallbackConsumer->lockNextBuffer(&imgBuffer);
269 if (res != OK) {
270 if (res != BAD_VALUE) {
271 ALOGE("%s: Camera %d: Error receiving next callback buffer: "
272 "%s (%d)", __FUNCTION__, mId, strerror(-res), res);
273 }
274 return res;
275 }
276 mCallbackConsumer->unlockBuffer(imgBuffer);
277 return OK;
278 }
279
processNewCallback(sp<Camera2Client> & client)280 status_t CallbackProcessor::processNewCallback(sp<Camera2Client> &client) {
281 ATRACE_CALL();
282 status_t res;
283
284 sp<Camera2Heap> callbackHeap;
285 bool useFlexibleYuv = false;
286 int32_t previewFormat = 0;
287 size_t heapIdx;
288
289 {
290 /* acquire SharedParameters before mMutex so we don't dead lock
291 with Camera2Client code calling into StreamingProcessor */
292 SharedParameters::Lock l(client->getParameters());
293 Mutex::Autolock m(mInputMutex);
294 CpuConsumer::LockedBuffer imgBuffer;
295 if (mCallbackStreamId == NO_STREAM) {
296 ALOGV("%s: Camera %d:No stream is available"
297 , __FUNCTION__, mId);
298 return INVALID_OPERATION;
299 }
300
301 ALOGV("%s: Getting buffer", __FUNCTION__);
302 res = mCallbackConsumer->lockNextBuffer(&imgBuffer);
303 if (res != OK) {
304 if (res != BAD_VALUE) {
305 ALOGE("%s: Camera %d: Error receiving next callback buffer: "
306 "%s (%d)", __FUNCTION__, mId, strerror(-res), res);
307 }
308 return res;
309 }
310 ALOGV("%s: Camera %d: Preview callback available", __FUNCTION__,
311 mId);
312
313 if ( l.mParameters.state != Parameters::PREVIEW
314 && l.mParameters.state != Parameters::RECORD
315 && l.mParameters.state != Parameters::VIDEO_SNAPSHOT) {
316 ALOGV("%s: Camera %d: No longer streaming",
317 __FUNCTION__, mId);
318 mCallbackConsumer->unlockBuffer(imgBuffer);
319 return OK;
320 }
321
322 if (! (l.mParameters.previewCallbackFlags &
323 CAMERA_FRAME_CALLBACK_FLAG_ENABLE_MASK) ) {
324 ALOGV("%s: No longer enabled, dropping", __FUNCTION__);
325 mCallbackConsumer->unlockBuffer(imgBuffer);
326 return OK;
327 }
328 if ((l.mParameters.previewCallbackFlags &
329 CAMERA_FRAME_CALLBACK_FLAG_ONE_SHOT_MASK) &&
330 !l.mParameters.previewCallbackOneShot) {
331 ALOGV("%s: One shot mode, already sent, dropping", __FUNCTION__);
332 mCallbackConsumer->unlockBuffer(imgBuffer);
333 return OK;
334 }
335
336 if (imgBuffer.width != static_cast<uint32_t>(l.mParameters.previewWidth) ||
337 imgBuffer.height != static_cast<uint32_t>(l.mParameters.previewHeight)) {
338 ALOGW("%s: The preview size has changed to %d x %d from %d x %d, this buffer is"
339 " no longer valid, dropping",__FUNCTION__,
340 l.mParameters.previewWidth, l.mParameters.previewHeight,
341 imgBuffer.width, imgBuffer.height);
342 mCallbackConsumer->unlockBuffer(imgBuffer);
343 return OK;
344 }
345
346 previewFormat = l.mParameters.previewFormat;
347 useFlexibleYuv = l.mParameters.fastInfo.useFlexibleYuv &&
348 (previewFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP ||
349 previewFormat == HAL_PIXEL_FORMAT_YV12);
350
351 int32_t expectedFormat = useFlexibleYuv ?
352 HAL_PIXEL_FORMAT_YCbCr_420_888 : previewFormat;
353
354 if (imgBuffer.format != expectedFormat) {
355 ALOGE("%s: Camera %d: Unexpected format for callback: "
356 "0x%x, expected 0x%x", __FUNCTION__, mId,
357 imgBuffer.format, expectedFormat);
358 mCallbackConsumer->unlockBuffer(imgBuffer);
359 return INVALID_OPERATION;
360 }
361
362 // In one-shot mode, stop sending callbacks after the first one
363 if (l.mParameters.previewCallbackFlags &
364 CAMERA_FRAME_CALLBACK_FLAG_ONE_SHOT_MASK) {
365 ALOGV("%s: clearing oneshot", __FUNCTION__);
366 l.mParameters.previewCallbackOneShot = false;
367 }
368
369 uint32_t destYStride = 0;
370 uint32_t destCStride = 0;
371 if (useFlexibleYuv) {
372 if (previewFormat == HAL_PIXEL_FORMAT_YV12) {
373 // Strides must align to 16 for YV12
374 destYStride = ALIGN(imgBuffer.width, 16);
375 destCStride = ALIGN(destYStride / 2, 16);
376 } else {
377 // No padding for NV21
378 ALOG_ASSERT(previewFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP,
379 "Unexpected preview format 0x%x", previewFormat);
380 destYStride = imgBuffer.width;
381 destCStride = destYStride / 2;
382 }
383 } else {
384 destYStride = imgBuffer.stride;
385 // don't care about cStride
386 }
387
388 size_t bufferSize = Camera2Client::calculateBufferSize(
389 imgBuffer.width, imgBuffer.height,
390 previewFormat, destYStride);
391 size_t currentBufferSize = (mCallbackHeap == 0) ?
392 0 : (mCallbackHeap->mHeap->getSize() / kCallbackHeapCount);
393 if (bufferSize != currentBufferSize) {
394 mCallbackHeap.clear();
395 mCallbackHeap = new Camera2Heap(bufferSize, kCallbackHeapCount,
396 "Camera2Client::CallbackHeap");
397 if (mCallbackHeap->mHeap->getSize() == 0) {
398 ALOGE("%s: Camera %d: Unable to allocate memory for callbacks",
399 __FUNCTION__, mId);
400 mCallbackConsumer->unlockBuffer(imgBuffer);
401 return INVALID_OPERATION;
402 }
403
404 mCallbackHeapHead = 0;
405 mCallbackHeapFree = kCallbackHeapCount;
406 }
407
408 if (mCallbackHeapFree == 0) {
409 ALOGE("%s: Camera %d: No free callback buffers, dropping frame",
410 __FUNCTION__, mId);
411 mCallbackConsumer->unlockBuffer(imgBuffer);
412 return OK;
413 }
414
415 heapIdx = mCallbackHeapHead;
416
417 mCallbackHeapHead = (mCallbackHeapHead + 1) % kCallbackHeapCount;
418 mCallbackHeapFree--;
419
420 // TODO: Get rid of this copy by passing the gralloc queue all the way
421 // to app
422
423 ssize_t offset;
424 size_t size;
425 sp<IMemoryHeap> heap =
426 mCallbackHeap->mBuffers[heapIdx]->getMemory(&offset,
427 &size);
428 uint8_t *data = (uint8_t*)heap->getBase() + offset;
429
430 if (!useFlexibleYuv) {
431 // Can just memcpy when HAL format matches API format
432 memcpy(data, imgBuffer.data, bufferSize);
433 } else {
434 res = convertFromFlexibleYuv(previewFormat, data, imgBuffer,
435 destYStride, destCStride);
436 if (res != OK) {
437 ALOGE("%s: Camera %d: Can't convert between 0x%x and 0x%x formats!",
438 __FUNCTION__, mId, imgBuffer.format, previewFormat);
439 mCallbackConsumer->unlockBuffer(imgBuffer);
440 return BAD_VALUE;
441 }
442 }
443
444 ALOGV("%s: Freeing buffer", __FUNCTION__);
445 mCallbackConsumer->unlockBuffer(imgBuffer);
446
447 // mCallbackHeap may get freed up once input mutex is released
448 callbackHeap = mCallbackHeap;
449 }
450
451 // Call outside parameter lock to allow re-entrancy from notification
452 {
453 Camera2Client::SharedCameraCallbacks::Lock
454 l(client->mSharedCameraCallbacks);
455 if (l.mRemoteCallback != 0) {
456 ALOGV("%s: Camera %d: Invoking client data callback",
457 __FUNCTION__, mId);
458 l.mRemoteCallback->dataCallback(CAMERA_MSG_PREVIEW_FRAME,
459 callbackHeap->mBuffers[heapIdx], NULL);
460 }
461 }
462
463 // Only increment free if we're still using the same heap
464 mCallbackHeapFree++;
465
466 ALOGV("%s: exit", __FUNCTION__);
467
468 return OK;
469 }
470
convertFromFlexibleYuv(int32_t previewFormat,uint8_t * dst,const CpuConsumer::LockedBuffer & src,uint32_t dstYStride,uint32_t dstCStride) const471 status_t CallbackProcessor::convertFromFlexibleYuv(int32_t previewFormat,
472 uint8_t *dst,
473 const CpuConsumer::LockedBuffer &src,
474 uint32_t dstYStride,
475 uint32_t dstCStride) const {
476
477 if (previewFormat != HAL_PIXEL_FORMAT_YCrCb_420_SP &&
478 previewFormat != HAL_PIXEL_FORMAT_YV12) {
479 ALOGE("%s: Camera %d: Unexpected preview format when using "
480 "flexible YUV: 0x%x", __FUNCTION__, mId, previewFormat);
481 return INVALID_OPERATION;
482 }
483
484 // Copy Y plane, adjusting for stride
485 const uint8_t *ySrc = src.data;
486 uint8_t *yDst = dst;
487 for (size_t row = 0; row < src.height; row++) {
488 memcpy(yDst, ySrc, src.width);
489 ySrc += src.stride;
490 yDst += dstYStride;
491 }
492
493 // Copy/swizzle chroma planes, 4:2:0 subsampling
494 const uint8_t *cbSrc = src.dataCb;
495 const uint8_t *crSrc = src.dataCr;
496 size_t chromaHeight = src.height / 2;
497 size_t chromaWidth = src.width / 2;
498 ssize_t chromaGap = src.chromaStride -
499 (chromaWidth * src.chromaStep);
500 size_t dstChromaGap = dstCStride - chromaWidth;
501
502 if (previewFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP) {
503 // Flexible YUV chroma to NV21 chroma
504 uint8_t *crcbDst = yDst;
505 // Check for shortcuts
506 if (cbSrc == crSrc + 1 && src.chromaStep == 2) {
507 ALOGV("%s: Fast NV21->NV21", __FUNCTION__);
508 // Source has semiplanar CrCb chroma layout, can copy by rows
509 for (size_t row = 0; row < chromaHeight; row++) {
510 memcpy(crcbDst, crSrc, src.width);
511 crcbDst += src.width;
512 crSrc += src.chromaStride;
513 }
514 } else {
515 ALOGV("%s: Generic->NV21", __FUNCTION__);
516 // Generic copy, always works but not very efficient
517 for (size_t row = 0; row < chromaHeight; row++) {
518 for (size_t col = 0; col < chromaWidth; col++) {
519 *(crcbDst++) = *crSrc;
520 *(crcbDst++) = *cbSrc;
521 crSrc += src.chromaStep;
522 cbSrc += src.chromaStep;
523 }
524 crSrc += chromaGap;
525 cbSrc += chromaGap;
526 }
527 }
528 } else {
529 // flexible YUV chroma to YV12 chroma
530 ALOG_ASSERT(previewFormat == HAL_PIXEL_FORMAT_YV12,
531 "Unexpected preview format 0x%x", previewFormat);
532 uint8_t *crDst = yDst;
533 uint8_t *cbDst = yDst + chromaHeight * dstCStride;
534 if (src.chromaStep == 1) {
535 ALOGV("%s: Fast YV12->YV12", __FUNCTION__);
536 // Source has planar chroma layout, can copy by row
537 for (size_t row = 0; row < chromaHeight; row++) {
538 memcpy(crDst, crSrc, chromaWidth);
539 crDst += dstCStride;
540 crSrc += src.chromaStride;
541 }
542 for (size_t row = 0; row < chromaHeight; row++) {
543 memcpy(cbDst, cbSrc, chromaWidth);
544 cbDst += dstCStride;
545 cbSrc += src.chromaStride;
546 }
547 } else {
548 ALOGV("%s: Generic->YV12", __FUNCTION__);
549 // Generic copy, always works but not very efficient
550 for (size_t row = 0; row < chromaHeight; row++) {
551 for (size_t col = 0; col < chromaWidth; col++) {
552 *(crDst++) = *crSrc;
553 *(cbDst++) = *cbSrc;
554 crSrc += src.chromaStep;
555 cbSrc += src.chromaStep;
556 }
557 crSrc += chromaGap;
558 cbSrc += chromaGap;
559 crDst += dstChromaGap;
560 cbDst += dstChromaGap;
561 }
562 }
563 }
564
565 return OK;
566 }
567
568 }; // namespace camera2
569 }; // namespace android
570