1 /* 2 * Copyright 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package androidx.camera.extensions.impl.advanced; 18 19 import androidx.camera.extensions.impl.advanced.JpegEncoder; 20 21 import static androidx.camera.extensions.impl.advanced.JpegEncoder.JPEG_DEFAULT_QUALITY; 22 import static androidx.camera.extensions.impl.advanced.JpegEncoder.JPEG_DEFAULT_ROTATION; 23 24 import android.annotation.SuppressLint; 25 import android.content.Context; 26 import android.graphics.ColorSpace; 27 import android.graphics.ImageFormat; 28 import android.hardware.camera2.CameraCharacteristics; 29 import android.hardware.camera2.CameraCharacteristics.Key; 30 import android.hardware.camera2.CameraDevice; 31 import android.hardware.camera2.CameraMetadata; 32 import android.hardware.camera2.CaptureFailure; 33 import android.hardware.camera2.CaptureRequest; 34 import android.hardware.camera2.CaptureResult; 35 import android.hardware.camera2.TotalCaptureResult; 36 import android.hardware.camera2.params.ColorSpaceProfiles; 37 import android.hardware.camera2.params.DynamicRangeProfiles; 38 import android.hardware.camera2.params.StreamConfigurationMap; 39 import android.hardware.DataSpace; 40 import android.media.Image; 41 import android.media.Image.Plane; 42 import android.media.ImageWriter; 43 import android.os.Build; 44 import android.util.Log; 45 import android.util.Pair; 46 import android.util.Range; 47 import android.util.Size; 48 import android.view.Surface; 49 50 import androidx.annotation.GuardedBy; 51 import androidx.annotation.NonNull; 52 import androidx.annotation.Nullable; 53 54 import java.nio.ByteBuffer; 55 import java.util.ArrayList; 56 import java.util.Arrays; 57 import java.util.concurrent.atomic.AtomicInteger; 58 import java.util.concurrent.Executor; 59 import java.util.HashMap; 60 import java.util.HashSet; 61 import java.util.LinkedHashMap; 62 import java.util.List; 63 import java.util.Map; 64 import java.util.Set; 65 66 @SuppressLint("UnknownNullness") 67 public abstract class BaseAdvancedExtenderImpl implements AdvancedExtenderImpl { 68 69 static { 70 try { 71 System.loadLibrary("encoderjpeg_jni"); 72 } catch (UnsatisfiedLinkError e) { 73 Log.e("BaseAdvancedExtenderImpl", "libencoderjpeg_jni not loaded"); 74 } 75 } 76 77 protected CameraCharacteristics mCameraCharacteristics; 78 protected String mCameraId; 79 80 protected static final Key REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES_MAP = 81 new Key<long[]>("android.request.availableDynamicRangeProfilesMap", long[].class); 82 protected static final Key REQUEST_AVAILABLE_COLOR_SPACE_PROFILES_MAP = 83 new Key<long[]>("android.request.availableColorSpaceProfilesMap", long[].class); 84 BaseAdvancedExtenderImpl()85 public BaseAdvancedExtenderImpl() { 86 } 87 88 @Override isExtensionAvailable(String cameraId, Map<String, CameraCharacteristics> characteristicsMap)89 public abstract boolean isExtensionAvailable(String cameraId, 90 Map<String, CameraCharacteristics> characteristicsMap); 91 92 @Override init(String cameraId, Map<String, CameraCharacteristics> characteristicsMap)93 public void init(String cameraId, 94 Map<String, CameraCharacteristics> characteristicsMap) { 95 mCameraCharacteristics = characteristicsMap.get(cameraId); 96 mCameraId = cameraId; 97 } 98 99 @Override getEstimatedCaptureLatencyRange( String cameraId, Size size, int imageFormat)100 public Range<Long> getEstimatedCaptureLatencyRange( 101 String cameraId, Size size, int imageFormat) { 102 return null; 103 } 104 filterOutputResolutions(List<Integer> formats)105 protected Map<Integer, List<Size>> filterOutputResolutions(List<Integer> formats) { 106 Map<Integer, List<Size>> formatResolutions = new HashMap<>(); 107 108 StreamConfigurationMap map = 109 mCameraCharacteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); 110 111 if (map != null) { 112 for (Integer format : formats) { 113 if (map.getOutputSizes(format) != null) { 114 formatResolutions.put(format, Arrays.asList(map.getOutputSizes(format))); 115 } 116 } 117 } 118 119 return formatResolutions; 120 } 121 122 @Override getSupportedPreviewOutputResolutions(String cameraId)123 public Map<Integer, List<Size>> getSupportedPreviewOutputResolutions(String cameraId) { 124 return filterOutputResolutions(Arrays.asList(ImageFormat.PRIVATE, ImageFormat.YUV_420_888)); 125 } 126 127 @Override getSupportedCaptureOutputResolutions(String cameraId)128 public Map<Integer, List<Size>> getSupportedCaptureOutputResolutions(String cameraId) { 129 return filterOutputResolutions(Arrays.asList(ImageFormat.JPEG, ImageFormat.YUV_420_888, 130 ImageFormat.DEPTH_JPEG, ImageFormat.JPEG_R, ImageFormat.YCBCR_P010)); 131 } 132 133 @Override getSupportedPostviewResolutions(Size captureSize)134 public Map<Integer, List<Size>> getSupportedPostviewResolutions(Size captureSize) { 135 return new HashMap<Integer, List<Size>>(); 136 } 137 138 @Override getSupportedYuvAnalysisResolutions( String cameraId)139 public List<Size> getSupportedYuvAnalysisResolutions( 140 String cameraId) { 141 return null; 142 } 143 144 public class BaseAdvancedSessionProcessor implements SessionProcessorImpl { 145 protected String TAG = "BaseAdvancedSessionProcessor"; 146 147 protected static final int DEFAULT_CAPTURE_ID = 0; 148 protected static final int BASIC_CAPTURE_PROCESS_MAX_IMAGES = 3; 149 protected static final int MAX_NUM_IMAGES = 1; 150 151 protected Camera2OutputConfigImpl mPreviewOutputConfig; 152 protected Camera2OutputConfigImpl mCaptureOutputConfig; 153 protected Camera2OutputConfigImpl mPostviewOutputConfig; 154 155 protected OutputSurfaceImpl mPreviewOutputSurfaceConfig; 156 protected OutputSurfaceImpl mCaptureOutputSurfaceConfig; 157 protected OutputSurfaceImpl mPostviewOutputSurfaceConfig; 158 159 protected final Object mLock = new Object(); 160 @GuardedBy("mLock") 161 protected Map<CaptureRequest.Key<?>, Object> mParameters = new LinkedHashMap<>(); 162 163 protected final Object mLockImageWriter = new Object(); 164 @GuardedBy("mLockImageWriter") 165 protected ImageWriter mCaptureSurfaceImageWriter; 166 167 @GuardedBy("mLockImageWriter") 168 protected ImageWriter mPostviewSurfaceImageWriter; 169 170 protected CaptureResultImageMatcher mImageCaptureCaptureResultImageMatcher = 171 new CaptureResultImageMatcher(); 172 protected CaptureResultImageMatcher mPostviewCaptureCaptureResultImageMatcher = 173 new CaptureResultImageMatcher(); 174 protected HashMap<Integer, Pair<ImageReferenceImpl, TotalCaptureResult>> mCaptureResults = 175 new HashMap<>(); 176 protected HashMap<Integer, Pair<ImageReferenceImpl, TotalCaptureResult>> mPostviewResults = 177 new HashMap<>(); 178 protected RequestProcessorImpl mRequestProcessor; 179 180 protected List<Integer> mCaptureIdList = List.of(DEFAULT_CAPTURE_ID); 181 182 protected AtomicInteger mNextCaptureSequenceId = new AtomicInteger(1); 183 184 protected boolean mProcessCapture = true; 185 appendTag(String tag)186 protected void appendTag(String tag) { 187 TAG += tag; 188 } 189 190 @Override 191 @NonNull initSession(@onNull String cameraId, @NonNull Map<String, CameraCharacteristics> cameraCharacteristicsMap, @NonNull Context context, @NonNull OutputSurfaceConfigurationImpl surfaceConfigs)192 public Camera2SessionConfigImpl initSession(@NonNull String cameraId, 193 @NonNull Map<String, CameraCharacteristics> cameraCharacteristicsMap, 194 @NonNull Context context, 195 @NonNull OutputSurfaceConfigurationImpl surfaceConfigs) { 196 197 Log.d(TAG, "initSession cameraId=" + cameraId); 198 199 mPreviewOutputSurfaceConfig = surfaceConfigs.getPreviewOutputSurface(); 200 mCaptureOutputSurfaceConfig = surfaceConfigs.getImageCaptureOutputSurface(); 201 mPostviewOutputSurfaceConfig = surfaceConfigs.getPostviewOutputSurface(); 202 203 Camera2SessionConfigImplBuilder builder = 204 new Camera2SessionConfigImplBuilder() 205 .setSessionTemplateId(CameraDevice.TEMPLATE_PREVIEW); 206 207 // Preview 208 if (mPreviewOutputSurfaceConfig.getSurface() != null) { 209 Camera2OutputConfigImplBuilder previewOutputConfigBuilder; 210 211 previewOutputConfigBuilder = 212 Camera2OutputConfigImplBuilder.newSurfaceConfig( 213 mPreviewOutputSurfaceConfig.getSurface()); 214 215 previewOutputConfigBuilder.setDynamicRangeProfile( 216 mPreviewOutputSurfaceConfig.getDynamicRangeProfile()); 217 218 mPreviewOutputConfig = previewOutputConfigBuilder.build(); 219 220 builder.addOutputConfig(mPreviewOutputConfig); 221 } 222 223 // Image Capture 224 if (mCaptureOutputSurfaceConfig.getSurface() != null) { 225 226 // For this sample, DEPTH_JPEG, JPEG_R or YCBCR_P010 will not be processed 227 if (isJpegR(mCaptureOutputSurfaceConfig) || 228 isDepthJpeg(mCaptureOutputSurfaceConfig)) { 229 Camera2OutputConfigImplBuilder captureOutputConfigBuilder; 230 231 captureOutputConfigBuilder = 232 Camera2OutputConfigImplBuilder.newSurfaceConfig( 233 mCaptureOutputSurfaceConfig.getSurface()); 234 235 mCaptureOutputConfig = captureOutputConfigBuilder.build(); 236 237 builder.addOutputConfig(mCaptureOutputConfig); 238 mProcessCapture = false; 239 } else if (mCaptureOutputSurfaceConfig.getImageFormat() == ImageFormat.YCBCR_P010) { 240 Camera2OutputConfigImplBuilder captureOutputConfigBuilder; 241 242 captureOutputConfigBuilder = 243 Camera2OutputConfigImplBuilder.newSurfaceConfig( 244 mCaptureOutputSurfaceConfig.getSurface()); 245 246 captureOutputConfigBuilder.setDynamicRangeProfile( 247 mCaptureOutputSurfaceConfig.getDynamicRangeProfile()); 248 249 mCaptureOutputConfig = captureOutputConfigBuilder.build(); 250 251 builder.addOutputConfig(mCaptureOutputConfig); 252 mProcessCapture = false; 253 } else { 254 Camera2OutputConfigImplBuilder captureOutputConfigBuilder; 255 256 captureOutputConfigBuilder = 257 Camera2OutputConfigImplBuilder.newImageReaderConfig( 258 mCaptureOutputSurfaceConfig.getSize(), 259 ImageFormat.YUV_420_888, 260 BASIC_CAPTURE_PROCESS_MAX_IMAGES, 261 mCaptureOutputSurfaceConfig.getUsage()); 262 263 mCaptureOutputConfig = captureOutputConfigBuilder.build(); 264 265 builder.addOutputConfig(mCaptureOutputConfig); 266 } 267 } 268 269 // Postview 270 if (mPostviewOutputSurfaceConfig.getSurface() != null) { 271 Camera2OutputConfigImplBuilder postviewOutputConfigBuilder; 272 273 postviewOutputConfigBuilder = 274 Camera2OutputConfigImplBuilder.newImageReaderConfig( 275 mPostviewOutputSurfaceConfig.getSize(), 276 ImageFormat.YUV_420_888, 277 BASIC_CAPTURE_PROCESS_MAX_IMAGES, 278 mPostviewOutputSurfaceConfig.getUsage()); 279 280 mPostviewOutputConfig = postviewOutputConfigBuilder.build(); 281 282 builder.addOutputConfig(mPostviewOutputConfig); 283 } 284 285 builder.setColorSpace(surfaceConfigs.getColorSpace()); 286 addSessionParameter(builder); 287 288 return builder.build(); 289 } 290 291 @Override initSession(@onNull String cameraId, @NonNull Map<String, CameraCharacteristics> cameraCharacteristicsMap, @NonNull Context context, @NonNull OutputSurfaceImpl previewSurfaceConfig, @NonNull OutputSurfaceImpl imageCaptureSurfaceConfig, @Nullable OutputSurfaceImpl imageAnalysisSurfaceConfig)292 public Camera2SessionConfigImpl initSession(@NonNull String cameraId, 293 @NonNull Map<String, CameraCharacteristics> cameraCharacteristicsMap, 294 @NonNull Context context, 295 @NonNull OutputSurfaceImpl previewSurfaceConfig, 296 @NonNull OutputSurfaceImpl imageCaptureSurfaceConfig, 297 @Nullable OutputSurfaceImpl imageAnalysisSurfaceConfig) { 298 299 // Since this sample impl uses version 1.4, the other initSession method will be 300 // called. This is just a sample for earlier versions if wanting to redirect this call. 301 OutputSurfaceConfigurationImplImpl surfaceConfigs = 302 new OutputSurfaceConfigurationImplImpl(previewSurfaceConfig, 303 imageCaptureSurfaceConfig, imageAnalysisSurfaceConfig, 304 null /*postviewSurfaceConfig*/); 305 306 return initSession(cameraId, cameraCharacteristicsMap, context, surfaceConfigs); 307 } 308 addSessionParameter(Camera2SessionConfigImplBuilder builder)309 protected void addSessionParameter(Camera2SessionConfigImplBuilder builder) { 310 // default empty implementation 311 } 312 313 @Override deInitSession()314 public void deInitSession() { 315 synchronized (mLockImageWriter) { 316 if (mCaptureSurfaceImageWriter != null) { 317 mCaptureSurfaceImageWriter.close(); 318 mCaptureSurfaceImageWriter = null; 319 } 320 321 if (mPostviewSurfaceImageWriter != null) { 322 mPostviewSurfaceImageWriter.close(); 323 mPostviewSurfaceImageWriter = null; 324 } 325 } 326 } 327 328 @Override setParameters(@onNull Map<CaptureRequest.Key<?>, Object> parameters)329 public void setParameters(@NonNull Map<CaptureRequest.Key<?>, Object> parameters) { 330 synchronized (mLock) { 331 for (CaptureRequest.Key<?> key : parameters.keySet()) { 332 Object value = parameters.get(key); 333 if (value != null) { 334 mParameters.put(key, value); 335 } 336 } 337 } 338 } 339 applyParameters(RequestBuilder builder)340 protected void applyParameters(RequestBuilder builder) { 341 synchronized (mLock) { 342 for (CaptureRequest.Key<?> key : mParameters.keySet()) { 343 Object value = mParameters.get(key); 344 builder.setParameters(key, value); 345 } 346 } 347 } 348 isJpegR(OutputSurfaceImpl surfaceConfig)349 protected boolean isJpegR(OutputSurfaceImpl surfaceConfig) { 350 // The surface configuration format for JPEG_R can be specified in either format. 351 // Camera2 uses HAL_PIXEL_FORMAT_BLOB and CameraX uses ImageFormat.JPEG_R. 352 return ((surfaceConfig.getImageFormat() == JpegEncoder.HAL_PIXEL_FORMAT_BLOB) && 353 (surfaceConfig.getDataspace() == DataSpace.DATASPACE_JPEG_R)) || 354 (surfaceConfig.getImageFormat() == ImageFormat.JPEG_R); 355 } 356 isDepthJpeg(OutputSurfaceImpl surfaceConfig)357 protected boolean isDepthJpeg(OutputSurfaceImpl surfaceConfig) { 358 // The surface configuration format for DepthJpeg can be specified in either format. 359 // Camera2 uses HAL_PIXEL_FORMAT_BLOB and CameraX uses ImageFormat.DEPTH_JPEG. 360 return ((surfaceConfig.getImageFormat() == JpegEncoder.HAL_PIXEL_FORMAT_BLOB) && 361 (surfaceConfig.getDataspace() == DataSpace.DATASPACE_DYNAMIC_DEPTH)) || 362 (surfaceConfig.getImageFormat() == ImageFormat.DEPTH_JPEG); 363 } 364 isJpeg(OutputSurfaceImpl surfaceConfig)365 protected boolean isJpeg(OutputSurfaceImpl surfaceConfig) { 366 return (JpegEncoder.imageFormatToPublic(surfaceConfig.getImageFormat()) == 367 ImageFormat.JPEG) || (surfaceConfig.getImageFormat() == ImageFormat.JPEG); 368 } 369 addTriggerRequestKeys(RequestBuilder builder, Map<CaptureRequest.Key<?>, Object> triggers)370 protected void addTriggerRequestKeys(RequestBuilder builder, 371 Map<CaptureRequest.Key<?>, Object> triggers) { 372 HashSet<CaptureRequest.Key> supportedCaptureRequestKeys = 373 new HashSet<>(getAvailableCaptureRequestKeys()); 374 375 for (CaptureRequest.Key<?> key : triggers.keySet()) { 376 if (supportedCaptureRequestKeys.contains(key)) { 377 Object value = triggers.get(key); 378 builder.setParameters(key, value); 379 } 380 } 381 } 382 383 @Override startTrigger(Map<CaptureRequest.Key<?>, Object> triggers, CaptureCallback captureCallback)384 public int startTrigger(Map<CaptureRequest.Key<?>, Object> triggers, 385 CaptureCallback captureCallback) { 386 RequestBuilder builder = new RequestBuilder(mPreviewOutputConfig.getId(), 387 CameraDevice.TEMPLATE_PREVIEW, 0); 388 addTriggerRequestKeys(builder, triggers); 389 390 final int seqId = mNextCaptureSequenceId.getAndIncrement(); 391 392 RequestProcessorImpl.Callback callback = new RequestProcessorImpl.Callback() { 393 @Override 394 public void onCaptureStarted(RequestProcessorImpl.Request request, long frameNumber, 395 long timestamp) { 396 captureCallback.onCaptureStarted(seqId, timestamp); 397 } 398 399 @Override 400 public void onCaptureProgressed(RequestProcessorImpl.Request request, 401 CaptureResult partialResult) { 402 403 } 404 405 @Override 406 public void onCaptureCompleted(RequestProcessorImpl.Request request, 407 TotalCaptureResult totalCaptureResult) { 408 addCaptureResultKeys(seqId, totalCaptureResult, captureCallback); 409 410 captureCallback.onCaptureProcessStarted(seqId); 411 } 412 413 @Override 414 public void onCaptureFailed(RequestProcessorImpl.Request request, 415 CaptureFailure captureFailure) { 416 captureCallback.onCaptureFailed(seqId, captureFailure.getReason()); 417 } 418 419 @Override 420 public void onCaptureBufferLost(RequestProcessorImpl.Request request, 421 long frameNumber, int outputStreamId) { 422 captureCallback.onCaptureFailed(seqId, CaptureFailure.REASON_ERROR); 423 } 424 425 @Override 426 public void onCaptureSequenceCompleted(int sequenceId, long frameNumber) { 427 captureCallback.onCaptureSequenceCompleted(seqId); 428 } 429 430 @Override 431 public void onCaptureSequenceAborted(int sequenceId) { 432 captureCallback.onCaptureSequenceAborted(seqId); 433 } 434 }; 435 436 mRequestProcessor.submit(builder.build(), callback); 437 438 return seqId; 439 } 440 441 @Override onCaptureSessionStart(@onNull RequestProcessorImpl requestProcessor)442 public void onCaptureSessionStart(@NonNull RequestProcessorImpl requestProcessor) { 443 mRequestProcessor = requestProcessor; 444 445 if (mCaptureOutputSurfaceConfig.getSurface() != null) { 446 synchronized (mLockImageWriter) { 447 if (mProcessCapture) { 448 if (isJpeg(mCaptureOutputSurfaceConfig)) { 449 mCaptureSurfaceImageWriter = new ImageWriter 450 .Builder(mCaptureOutputSurfaceConfig.getSurface()) 451 .setImageFormat(ImageFormat.JPEG) 452 .setMaxImages(MAX_NUM_IMAGES) 453 // For JPEG format, width x height should be set to (w*h) x 1 454 // since the JPEG image is returned as a 1D byte array 455 .setWidthAndHeight( 456 mCaptureOutputSurfaceConfig.getSize().getWidth() 457 * mCaptureOutputSurfaceConfig.getSize().getHeight(), 458 1) 459 .build(); 460 } else { 461 mCaptureSurfaceImageWriter = new ImageWriter 462 .Builder(mCaptureOutputSurfaceConfig.getSurface()) 463 .setImageFormat(mCaptureOutputSurfaceConfig.getImageFormat()) 464 .setMaxImages(MAX_NUM_IMAGES) 465 .build(); 466 } 467 } 468 } 469 } 470 471 if (mPostviewOutputSurfaceConfig != null 472 && mPostviewOutputSurfaceConfig.getSurface() != null) { 473 synchronized (mLockImageWriter) { 474 mPostviewSurfaceImageWriter = new ImageWriter 475 .Builder(mPostviewOutputSurfaceConfig.getSurface()) 476 .setImageFormat(mPostviewOutputSurfaceConfig.getImageFormat()) 477 .setMaxImages(MAX_NUM_IMAGES) 478 .build(); 479 } 480 } 481 } 482 483 @Override onCaptureSessionEnd()484 public void onCaptureSessionEnd() { 485 synchronized (this) { 486 mImageCaptureCaptureResultImageMatcher.clear(); 487 mPostviewCaptureCaptureResultImageMatcher.clear(); 488 } 489 490 mRequestProcessor = null; 491 } 492 493 @Override startRepeating(@onNull CaptureCallback captureCallback)494 public int startRepeating(@NonNull CaptureCallback captureCallback) { 495 RequestBuilder builder = new RequestBuilder(mPreviewOutputConfig.getId(), 496 CameraDevice.TEMPLATE_PREVIEW, 0); 497 applyParameters(builder); 498 final int seqId = mNextCaptureSequenceId.getAndIncrement(); 499 500 RequestProcessorImpl.Callback callback = new RequestProcessorImpl.Callback() { 501 @Override 502 public void onCaptureStarted(RequestProcessorImpl.Request request, long frameNumber, 503 long timestamp) { 504 captureCallback.onCaptureStarted(seqId, timestamp); 505 } 506 507 @Override 508 public void onCaptureProgressed(RequestProcessorImpl.Request request, 509 CaptureResult partialResult) { 510 511 } 512 513 @Override 514 public void onCaptureCompleted(RequestProcessorImpl.Request request, 515 TotalCaptureResult totalCaptureResult) { 516 addCaptureResultKeys(seqId, totalCaptureResult, captureCallback); 517 518 captureCallback.onCaptureProcessStarted(seqId); 519 } 520 521 @Override 522 public void onCaptureFailed(RequestProcessorImpl.Request request, 523 CaptureFailure captureFailure) { 524 captureCallback.onCaptureFailed(seqId, captureFailure.getReason()); 525 } 526 527 @Override 528 public void onCaptureBufferLost(RequestProcessorImpl.Request request, 529 long frameNumber, int outputStreamId) { 530 captureCallback.onCaptureFailed(seqId, CaptureFailure.REASON_ERROR); 531 } 532 533 @Override 534 public void onCaptureSequenceCompleted(int sequenceId, long frameNumber) { 535 captureCallback.onCaptureSequenceCompleted(seqId); 536 } 537 538 @Override 539 public void onCaptureSequenceAborted(int sequenceId) { 540 captureCallback.onCaptureSequenceAborted(seqId); 541 } 542 }; 543 544 mRequestProcessor.setRepeating(builder.build(), callback); 545 546 return seqId; 547 } 548 addCaptureResultKeys( @onNull int seqId, @NonNull TotalCaptureResult result, @NonNull CaptureCallback captureCallback)549 protected void addCaptureResultKeys( 550 @NonNull int seqId, 551 @NonNull TotalCaptureResult result, 552 @NonNull CaptureCallback captureCallback) { 553 HashMap<CaptureResult.Key, Object> captureResults = new HashMap<>(); 554 555 Long shutterTimestamp = result.get(CaptureResult.SENSOR_TIMESTAMP); 556 557 if (shutterTimestamp != null) { 558 559 List<CaptureResult.Key> captureResultKeys = getAvailableCaptureResultKeys(); 560 for (CaptureResult.Key key : captureResultKeys) { 561 if (result.get(key) != null) { 562 captureResults.put(key, result.get(key)); 563 } 564 } 565 566 captureCallback.onCaptureCompleted(shutterTimestamp, seqId, 567 captureResults); 568 } 569 } 570 addCaptureRequestParameters(List<RequestProcessorImpl.Request> requestList)571 protected void addCaptureRequestParameters(List<RequestProcessorImpl.Request> requestList) { 572 RequestBuilder build = new RequestBuilder(mCaptureOutputConfig.getId(), 573 CameraDevice.TEMPLATE_STILL_CAPTURE, DEFAULT_CAPTURE_ID); 574 applyParameters(build); 575 576 requestList.add(build.build()); 577 } 578 579 @Override startCaptureWithPostview(@onNull CaptureCallback captureCallback)580 public int startCaptureWithPostview(@NonNull CaptureCallback captureCallback) { 581 Log.d(TAG, "startCaptureWithPostview"); 582 return startCapture(captureCallback); 583 } 584 585 @Override startCapture(@onNull CaptureCallback captureCallback)586 public int startCapture(@NonNull CaptureCallback captureCallback) { 587 List<RequestProcessorImpl.Request> requestList = new ArrayList<>(); 588 addCaptureRequestParameters(requestList); 589 final int seqId = mNextCaptureSequenceId.getAndIncrement(); 590 591 RequestProcessorImpl.Callback callback = new RequestProcessorImpl.Callback() { 592 593 @Override 594 public void onCaptureStarted(RequestProcessorImpl.Request request, 595 long frameNumber, long timestamp) { 596 captureCallback.onCaptureStarted(seqId, timestamp); 597 } 598 599 @Override 600 public void onCaptureProgressed(RequestProcessorImpl.Request request, 601 CaptureResult partialResult) { 602 603 } 604 605 @Override 606 public void onCaptureCompleted(RequestProcessorImpl.Request request, 607 TotalCaptureResult totalCaptureResult) { 608 RequestBuilder.RequestProcessorRequest requestProcessorRequest = 609 (RequestBuilder.RequestProcessorRequest) request; 610 611 addCaptureResultKeys(seqId, totalCaptureResult, captureCallback); 612 613 if (mPostviewOutputSurfaceConfig != null) { 614 mPostviewCaptureCaptureResultImageMatcher.setCameraCaptureCallback( 615 totalCaptureResult, 616 requestProcessorRequest.getCaptureStageId()); 617 } 618 619 if (!mProcessCapture) { 620 captureCallback.onCaptureProcessStarted(seqId); 621 } else { 622 mImageCaptureCaptureResultImageMatcher.setCameraCaptureCallback( 623 totalCaptureResult, 624 requestProcessorRequest.getCaptureStageId()); 625 } 626 } 627 628 @Override 629 public void onCaptureFailed(RequestProcessorImpl.Request request, 630 CaptureFailure captureFailure) { 631 captureCallback.onCaptureFailed(seqId, captureFailure.getReason()); 632 } 633 634 @Override 635 public void onCaptureBufferLost(RequestProcessorImpl.Request request, 636 long frameNumber, int outputStreamId) { 637 captureCallback.onCaptureFailed(seqId, CaptureFailure.REASON_ERROR); 638 } 639 640 @Override 641 public void onCaptureSequenceCompleted(int sequenceId, long frameNumber) { 642 captureCallback.onCaptureSequenceCompleted(seqId); 643 captureCallback.onCaptureProcessProgressed(100); 644 } 645 646 @Override 647 public void onCaptureSequenceAborted(int sequenceId) { 648 captureCallback.onCaptureSequenceAborted(seqId); 649 } 650 }; 651 652 Log.d(TAG, "startCapture"); 653 654 mRequestProcessor.submit(requestList, callback); 655 656 if (mPostviewOutputSurfaceConfig != null && 657 mPostviewOutputSurfaceConfig.getSurface() != null) { 658 mRequestProcessor.setImageProcessor(mPostviewOutputConfig.getId(), 659 new ImageProcessorImpl() { 660 @Override 661 public void onNextImageAvailable(int outputStreamId, 662 long timestampNs, 663 @NonNull ImageReferenceImpl imgReferenceImpl, 664 @Nullable String physicalCameraId) { 665 mPostviewCaptureCaptureResultImageMatcher 666 .setInputImage(imgReferenceImpl); 667 } 668 }); 669 670 mPostviewCaptureCaptureResultImageMatcher.setImageReferenceListener( 671 new CaptureResultImageMatcher.ImageReferenceListener() { 672 @Override 673 public void onImageReferenceIncoming( 674 @NonNull ImageReferenceImpl imageReferenceImpl, 675 @NonNull TotalCaptureResult totalCaptureResult, 676 int captureId) { 677 processImageCapture(imageReferenceImpl, totalCaptureResult, 678 captureId, true /*isPostview*/); 679 } 680 }); 681 } 682 683 if (mCaptureOutputSurfaceConfig.getSurface() != null && mProcessCapture) { 684 mRequestProcessor.setImageProcessor(mCaptureOutputConfig.getId(), 685 new ImageProcessorImpl() { 686 @Override 687 public void onNextImageAvailable(int outputStreamId, 688 long timestampNs, 689 @NonNull ImageReferenceImpl imgReferenceImpl, 690 @Nullable String physicalCameraId) { 691 mImageCaptureCaptureResultImageMatcher 692 .setInputImage(imgReferenceImpl); 693 } 694 }); 695 696 mImageCaptureCaptureResultImageMatcher.setImageReferenceListener( 697 new CaptureResultImageMatcher.ImageReferenceListener() { 698 @Override 699 public void onImageReferenceIncoming( 700 @NonNull ImageReferenceImpl imageReferenceImpl, 701 @NonNull TotalCaptureResult totalCaptureResult, 702 int captureId) { 703 captureCallback.onCaptureProcessStarted(seqId); 704 processImageCapture(imageReferenceImpl, totalCaptureResult, 705 captureId, false /*isPostview*/); 706 } 707 }); 708 } 709 710 return seqId; 711 } 712 processImageCapture(@onNull ImageReferenceImpl imageReferenceImpl, @NonNull TotalCaptureResult totalCaptureResult, int captureId, boolean isPostview)713 protected void processImageCapture(@NonNull ImageReferenceImpl imageReferenceImpl, 714 @NonNull TotalCaptureResult totalCaptureResult, int captureId, 715 boolean isPostview) { 716 717 HashMap<Integer, Pair<ImageReferenceImpl, TotalCaptureResult>> captureResults = 718 isPostview ? mPostviewResults : mCaptureResults; 719 ImageWriter imageWriter = isPostview ? mPostviewSurfaceImageWriter : 720 mCaptureSurfaceImageWriter; 721 722 captureResults.put(captureId, new Pair<>(imageReferenceImpl, totalCaptureResult)); 723 724 if (captureResults.keySet().containsAll(mCaptureIdList)) { 725 List<Pair<ImageReferenceImpl, TotalCaptureResult>> imageDataPairs = 726 new ArrayList<>(captureResults.values()); 727 728 Image resultImage = null; 729 int captureSurfaceWriterImageFormat = ImageFormat.UNKNOWN; 730 synchronized (mLockImageWriter) { 731 resultImage = imageWriter.dequeueInputImage(); 732 captureSurfaceWriterImageFormat = imageWriter.getFormat(); 733 } 734 735 if (captureSurfaceWriterImageFormat == ImageFormat.JPEG) { 736 // Simple processing sample that encodes image from YUV to JPEG 737 Image yuvImage = imageDataPairs.get(DEFAULT_CAPTURE_ID).first.get(); 738 739 Integer jpegOrientation = JPEG_DEFAULT_ROTATION; 740 741 synchronized (mLock) { 742 if (mParameters.get(CaptureRequest.JPEG_ORIENTATION) != null) { 743 jpegOrientation = 744 (Integer) mParameters.get(CaptureRequest.JPEG_ORIENTATION); 745 } 746 } 747 748 JpegEncoder.encodeToJpeg(yuvImage, resultImage, jpegOrientation, 749 JPEG_DEFAULT_QUALITY); 750 751 resultImage.setTimestamp(yuvImage.getTimestamp()); 752 753 } else { 754 // Simple processing sample that transfers bytes and returns image as is 755 ByteBuffer yByteBuffer = resultImage.getPlanes()[0].getBuffer(); 756 ByteBuffer uByteBuffer = resultImage.getPlanes()[2].getBuffer(); 757 ByteBuffer vByteBuffer = resultImage.getPlanes()[1].getBuffer(); 758 759 yByteBuffer.put(imageDataPairs.get( 760 DEFAULT_CAPTURE_ID).first.get().getPlanes()[0].getBuffer()); 761 uByteBuffer.put(imageDataPairs.get( 762 DEFAULT_CAPTURE_ID).first.get().getPlanes()[2].getBuffer()); 763 vByteBuffer.put(imageDataPairs.get( 764 DEFAULT_CAPTURE_ID).first.get().getPlanes()[1].getBuffer()); 765 766 resultImage.setTimestamp(imageDataPairs.get( 767 DEFAULT_CAPTURE_ID).first.get().getTimestamp()); 768 } 769 770 synchronized (mLockImageWriter) { 771 imageWriter.queueInputImage(resultImage); 772 } 773 774 for (Pair<ImageReferenceImpl, TotalCaptureResult> val : captureResults.values()) { 775 val.first.decrement(); 776 } 777 } else { 778 Log.w(TAG, "Unable to process, waiting for all images"); 779 } 780 } 781 782 @Override stopRepeating()783 public void stopRepeating() { 784 mRequestProcessor.stopRepeating(); 785 } 786 787 @Override abortCapture(int captureSequenceId)788 public void abortCapture(int captureSequenceId) { 789 790 } 791 792 @Override getRealtimeCaptureLatency()793 public Pair<Long, Long> getRealtimeCaptureLatency() { 794 return null; 795 } 796 } 797 798 public static class OutputSurfaceConfigurationImplImpl implements OutputSurfaceConfigurationImpl { 799 private OutputSurfaceImpl mOutputPreviewSurfaceImpl; 800 private OutputSurfaceImpl mOutputImageCaptureSurfaceImpl; 801 private OutputSurfaceImpl mOutputImageAnalysisSurfaceImpl; 802 private OutputSurfaceImpl mOutputPostviewSurfaceImpl; 803 OutputSurfaceConfigurationImplImpl(OutputSurfaceImpl previewSurfaceConfig, OutputSurfaceImpl imageCaptureSurfaceConfig, OutputSurfaceImpl imageAnalysisSurfaceConfig, OutputSurfaceImpl postviewSurfaceConfig)804 public OutputSurfaceConfigurationImplImpl(OutputSurfaceImpl previewSurfaceConfig, 805 OutputSurfaceImpl imageCaptureSurfaceConfig, 806 OutputSurfaceImpl imageAnalysisSurfaceConfig, 807 OutputSurfaceImpl postviewSurfaceConfig) { 808 mOutputPreviewSurfaceImpl = previewSurfaceConfig; 809 mOutputImageCaptureSurfaceImpl = imageCaptureSurfaceConfig; 810 mOutputImageAnalysisSurfaceImpl = imageAnalysisSurfaceConfig; 811 mOutputPostviewSurfaceImpl = postviewSurfaceConfig; 812 } 813 814 @Override getPreviewOutputSurface()815 public OutputSurfaceImpl getPreviewOutputSurface() { 816 return mOutputPreviewSurfaceImpl; 817 } 818 819 @Override getImageCaptureOutputSurface()820 public OutputSurfaceImpl getImageCaptureOutputSurface() { 821 return mOutputImageCaptureSurfaceImpl; 822 } 823 824 @Override getImageAnalysisOutputSurface()825 public OutputSurfaceImpl getImageAnalysisOutputSurface() { 826 return mOutputImageAnalysisSurfaceImpl; 827 } 828 829 @Override getPostviewOutputSurface()830 public OutputSurfaceImpl getPostviewOutputSurface() { 831 return mOutputPostviewSurfaceImpl; 832 } 833 } 834 835 @Override createSessionProcessor()836 public abstract SessionProcessorImpl createSessionProcessor(); 837 838 @Override getAvailableCaptureRequestKeys()839 public List<CaptureRequest.Key> getAvailableCaptureRequestKeys() { 840 final CaptureRequest.Key [] CAPTURE_REQUEST_SET = {CaptureRequest.JPEG_QUALITY, 841 CaptureRequest.JPEG_ORIENTATION}; 842 return Arrays.asList(CAPTURE_REQUEST_SET); 843 } 844 845 @Override getAvailableCaptureResultKeys()846 public List<CaptureResult.Key> getAvailableCaptureResultKeys() { 847 final CaptureResult.Key [] CAPTURE_RESULT_SET = {CaptureResult.JPEG_QUALITY, 848 CaptureResult.JPEG_ORIENTATION}; 849 return Arrays.asList(CAPTURE_RESULT_SET); 850 } 851 852 @Override isCaptureProcessProgressAvailable()853 public boolean isCaptureProcessProgressAvailable() { 854 return true; 855 } 856 857 @Override isPostviewAvailable()858 public boolean isPostviewAvailable() { 859 return false; 860 } 861 862 @Override getAvailableCharacteristicsKeyValues()863 public List<Pair<CameraCharacteristics.Key, Object>> getAvailableCharacteristicsKeyValues() { 864 int[] caps = mCameraCharacteristics 865 .get(CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES); 866 867 Set<Integer> unsupportedCapabilities = new HashSet<>(Arrays.asList( 868 CameraMetadata.REQUEST_AVAILABLE_CAPABILITIES_DEPTH_OUTPUT, 869 CameraMetadata.REQUEST_AVAILABLE_CAPABILITIES_RAW, 870 CameraMetadata.REQUEST_AVAILABLE_CAPABILITIES_PRIVATE_REPROCESSING, 871 CameraMetadata.REQUEST_AVAILABLE_CAPABILITIES_YUV_REPROCESSING, 872 CameraMetadata.REQUEST_AVAILABLE_CAPABILITIES_CONSTRAINED_HIGH_SPEED_VIDEO, 873 CameraMetadata.REQUEST_AVAILABLE_CAPABILITIES_OFFLINE_PROCESSING, 874 CameraMetadata.REQUEST_AVAILABLE_CAPABILITIES_REMOSAIC_REPROCESSING, 875 CameraMetadata.REQUEST_AVAILABLE_CAPABILITIES_MONOCHROME, 876 CameraMetadata.REQUEST_AVAILABLE_CAPABILITIES_SECURE_IMAGE_DATA, 877 CameraMetadata.REQUEST_AVAILABLE_CAPABILITIES_SYSTEM_CAMERA, 878 CameraMetadata.REQUEST_AVAILABLE_CAPABILITIES_ULTRA_HIGH_RESOLUTION_SENSOR 879 )); 880 881 List<Integer> filtered = new ArrayList<>(); 882 for (int c : caps) { 883 if (unsupportedCapabilities.contains(c)) { 884 continue; 885 } 886 filtered.add(c); 887 } 888 int[] extensionsCaps = new int[filtered.size()]; 889 for (int i = 0; i < filtered.size(); i++) { 890 extensionsCaps[i] = filtered.get(i); 891 } 892 893 long[] dynamicRangeProfileArray = new long[]{ 894 DynamicRangeProfiles.HLG10, 895 DynamicRangeProfiles.HLG10 | DynamicRangeProfiles.STANDARD, 896 0L}; 897 long[] colorSpacesProfileArray = new long[]{ 898 ColorSpace.Named.BT2020_HLG.ordinal(), 899 ImageFormat.YCBCR_P010, 900 DynamicRangeProfiles.HLG10}; 901 902 Range<Float> zoomRange = mCameraCharacteristics 903 .get(CameraCharacteristics.CONTROL_ZOOM_RATIO_RANGE); 904 float zoomRangeLower = Math.max(1f, zoomRange.getLower()); 905 float zoomRangeUpper = Math.min(10f, zoomRange.getUpper()); 906 return Arrays.asList( 907 Pair.create(CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES, 908 extensionsCaps), 909 Pair.create(CameraCharacteristics.CONTROL_ZOOM_RATIO_RANGE, 910 Range.create(zoomRangeLower, zoomRangeUpper)), 911 Pair.create(CameraCharacteristics.CONTROL_AF_AVAILABLE_MODES, 912 new int[]{ 913 CameraMetadata.CONTROL_AF_MODE_OFF, 914 CameraMetadata.CONTROL_AF_MODE_AUTO 915 }), 916 Pair.create(REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES_MAP, 917 dynamicRangeProfileArray), 918 Pair.create(REQUEST_AVAILABLE_COLOR_SPACE_PROFILES_MAP, 919 colorSpacesProfileArray) 920 ); 921 } 922 } 923