1 /*
2  * Copyright 2022 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package androidx.camera.extensions.impl.advanced;
18 
19 import androidx.camera.extensions.impl.advanced.JpegEncoder;
20 import androidx.camera.extensions.impl.advanced.BaseAdvancedExtenderImpl.BaseAdvancedSessionProcessor;
21 
22 import static  androidx.camera.extensions.impl.advanced.JpegEncoder.JPEG_DEFAULT_QUALITY;
23 import static androidx.camera.extensions.impl.advanced.JpegEncoder.JPEG_DEFAULT_ROTATION;
24 
25 import android.annotation.SuppressLint;
26 import android.content.Context;
27 import android.graphics.ImageFormat;
28 import android.hardware.camera2.CameraCharacteristics;
29 import android.hardware.camera2.CameraDevice;
30 import android.hardware.camera2.CaptureFailure;
31 import android.hardware.camera2.CaptureRequest;
32 import android.hardware.camera2.CaptureResult;
33 import android.hardware.camera2.TotalCaptureResult;
34 import android.hardware.camera2.params.MeteringRectangle;
35 import android.hardware.camera2.params.StreamConfigurationMap;
36 import android.media.Image;
37 import android.media.Image.Plane;
38 import android.media.ImageWriter;
39 import android.util.Log;
40 import android.util.Pair;
41 import android.util.Range;
42 import android.util.Size;
43 import android.view.Surface;
44 
45 import androidx.annotation.GuardedBy;
46 import androidx.annotation.NonNull;
47 import androidx.annotation.Nullable;
48 
49 import java.io.Closeable;
50 import java.nio.ByteBuffer;
51 import java.util.ArrayList;
52 import java.util.Arrays;
53 import java.util.concurrent.atomic.AtomicInteger;
54 import java.util.concurrent.Executor;
55 import java.util.concurrent.TimeUnit;
56 import java.util.HashMap;
57 import java.util.LinkedHashMap;
58 import java.util.List;
59 import java.util.Map;
60 
61 @SuppressLint("UnknownNullness")
62 public class HdrAdvancedExtenderImpl extends BaseAdvancedExtenderImpl {
63 
HdrAdvancedExtenderImpl()64     public HdrAdvancedExtenderImpl() {
65     }
66 
67     @Override
isExtensionAvailable(String cameraId, Map<String, CameraCharacteristics> characteristicsMap)68     public boolean isExtensionAvailable(String cameraId,
69             Map<String, CameraCharacteristics> characteristicsMap) {
70         CameraCharacteristics cameraCharacteristics = characteristicsMap.get(cameraId);
71 
72         if (cameraCharacteristics == null) {
73             return false;
74         }
75 
76         boolean zoomRatioSupported =
77             CameraCharacteristicAvailability.supportsZoomRatio(cameraCharacteristics);
78         boolean hasFocuser =
79             CameraCharacteristicAvailability.hasFocuser(cameraCharacteristics);
80 
81         // Requires API 23 for ImageWriter
82         return (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.M) &&
83                 zoomRatioSupported && hasFocuser;
84     }
85 
86     public class HDRAdvancedSessionProcessor extends BaseAdvancedSessionProcessor {
87         protected static final int UNDER_EXPOSED_CAPTURE_ID = 0;
88         protected static final int NORMAL_EXPOSED_CAPTURE_ID = 1;
89         protected static final int OVER_EXPOSED_CAPTURE_ID = 2;
90 
91         List<Integer> mCaptureIdsList = List.of(UNDER_EXPOSED_CAPTURE_ID,
92                 NORMAL_EXPOSED_CAPTURE_ID, OVER_EXPOSED_CAPTURE_ID);
93 
HDRAdvancedSessionProcessor()94         public HDRAdvancedSessionProcessor() {
95             appendTag("::HDR");
96         }
97 
98         @Override
addCaptureRequestParameters(List<RequestProcessorImpl.Request> requestList)99         protected void addCaptureRequestParameters(List<RequestProcessorImpl.Request> requestList) {
100             // Under exposed capture
101             RequestBuilder builderUnder = new RequestBuilder(mCaptureOutputConfig.getId(),
102                     CameraDevice.TEMPLATE_STILL_CAPTURE, UNDER_EXPOSED_CAPTURE_ID);
103             // Turn off AE so that ISO sensitivity can be controlled
104             builderUnder.setParameters(CaptureRequest.CONTROL_AE_MODE,
105                     CaptureRequest.CONTROL_AE_MODE_OFF);
106             builderUnder.setParameters(CaptureRequest.SENSOR_EXPOSURE_TIME,
107                     TimeUnit.MILLISECONDS.toNanos(8));
108             applyParameters(builderUnder);
109 
110             // Normal exposed capture
111             RequestBuilder builderNormal = new RequestBuilder(mCaptureOutputConfig.getId(),
112                     CameraDevice.TEMPLATE_STILL_CAPTURE, NORMAL_EXPOSED_CAPTURE_ID);
113             builderNormal.setParameters(CaptureRequest.SENSOR_EXPOSURE_TIME,
114                     TimeUnit.MILLISECONDS.toNanos(16));
115             applyParameters(builderNormal);
116 
117             // Over exposed capture
118             RequestBuilder builderOver = new RequestBuilder(mCaptureOutputConfig.getId(),
119                     CameraDevice.TEMPLATE_STILL_CAPTURE, OVER_EXPOSED_CAPTURE_ID);
120             builderOver.setParameters(CaptureRequest.SENSOR_EXPOSURE_TIME,
121                     TimeUnit.MILLISECONDS.toNanos(32));
122             applyParameters(builderOver);
123 
124             requestList.add(builderUnder.build());
125             requestList.add(builderNormal.build());
126             requestList.add(builderOver.build());
127         }
128 
129         @Override
startCapture(@onNull CaptureCallback captureCallback)130         public int startCapture(@NonNull CaptureCallback captureCallback) {
131             List<RequestProcessorImpl.Request> requestList = new ArrayList<>();
132             if (mProcessCapture) {
133                 addCaptureRequestParameters(requestList);
134             } else {
135                 super.addCaptureRequestParameters(requestList);
136             }
137             final int seqId = mNextCaptureSequenceId.getAndIncrement();
138 
139             RequestProcessorImpl.Callback callback = new RequestProcessorImpl.Callback() {
140                 boolean mCaptureStarted = false;
141 
142                 @Override
143                 public void onCaptureStarted(RequestProcessorImpl.Request request,
144                         long frameNumber, long timestamp) {
145                     if (!mCaptureStarted || !mProcessCapture) {
146                         mCaptureStarted = true;
147                         captureCallback.onCaptureStarted(seqId, timestamp);
148                     }
149                 }
150 
151                 @Override
152                 public void onCaptureProgressed(RequestProcessorImpl.Request request,
153                         CaptureResult partialResult) {
154 
155                 }
156 
157                 @Override
158                 public void onCaptureCompleted(RequestProcessorImpl.Request request,
159                         TotalCaptureResult totalCaptureResult) {
160                     RequestBuilder.RequestProcessorRequest requestProcessorRequest =
161                             (RequestBuilder.RequestProcessorRequest) request;
162 
163                     if (!mProcessCapture) {
164                         captureCallback.onCaptureProcessStarted(seqId);
165                         addCaptureResultKeys(seqId, totalCaptureResult, captureCallback);
166                     } else {
167                         mImageCaptureCaptureResultImageMatcher.setCameraCaptureCallback(
168                                 totalCaptureResult,
169                                 requestProcessorRequest.getCaptureStageId());
170                     }
171                 }
172 
173                 @Override
174                 public void onCaptureFailed(RequestProcessorImpl.Request request,
175                         CaptureFailure captureFailure) {
176                     captureCallback.onCaptureFailed(seqId);
177                 }
178 
179                 @Override
180                 public void onCaptureBufferLost(RequestProcessorImpl.Request request,
181                         long frameNumber, int outputStreamId) {
182                     captureCallback.onCaptureFailed(seqId);
183                 }
184 
185                 @Override
186                 public void onCaptureSequenceCompleted(int sequenceId, long frameNumber) {
187                     captureCallback.onCaptureSequenceCompleted(seqId);
188                     captureCallback.onCaptureProcessProgressed(100);
189                 }
190 
191                 @Override
192                 public void onCaptureSequenceAborted(int sequenceId) {
193                     captureCallback.onCaptureSequenceAborted(seqId);
194                 }
195             };
196 
197             Log.d(TAG, "startCapture");
198 
199             mRequestProcessor.submit(requestList, callback);
200 
201             if (mCaptureOutputSurfaceConfig.getSurface() != null && mProcessCapture) {
202                 mRequestProcessor.setImageProcessor(mCaptureOutputConfig.getId(),
203                         new ImageProcessorImpl() {
204                             boolean mCaptureStarted = false;
205                                 @Override
206                                 public void onNextImageAvailable(int outputStreamId,
207                                         long timestampNs,
208                                         @NonNull ImageReferenceImpl imgReferenceImpl,
209                                         @Nullable String physicalCameraId) {
210                                     mImageCaptureCaptureResultImageMatcher
211                                             .setInputImage(imgReferenceImpl);
212 
213                                     if (!mCaptureStarted) {
214                                         mCaptureStarted = true;
215                                         captureCallback.onCaptureProcessStarted(seqId);
216                                     }
217                                 }
218                 });
219 
220                 mImageCaptureCaptureResultImageMatcher.setImageReferenceListener(
221                         new CaptureResultImageMatcher.ImageReferenceListener() {
222                                     @Override
223                                     public void onImageReferenceIncoming(
224                                             @NonNull ImageReferenceImpl imageReferenceImpl,
225                                             @NonNull TotalCaptureResult totalCaptureResult,
226                                             int captureId) {
227                                         processImageCapture(imageReferenceImpl, totalCaptureResult,
228                                                 captureId, seqId, captureCallback);
229                                     }
230                 });
231             }
232 
233             return seqId;
234         }
235 
processImageCapture(@onNull ImageReferenceImpl imageReferenceImpl, @NonNull TotalCaptureResult totalCaptureResult, int captureId, int seqId, @NonNull CaptureCallback captureCallback)236         private void processImageCapture(@NonNull ImageReferenceImpl imageReferenceImpl,
237             @NonNull TotalCaptureResult totalCaptureResult,
238             int captureId,
239             int seqId,
240             @NonNull CaptureCallback captureCallback) {
241 
242             mCaptureResults.put(captureId, new Pair<>(imageReferenceImpl, totalCaptureResult));
243 
244             if (mCaptureResults.keySet().containsAll(mCaptureIdsList)) {
245                 List<Pair<ImageReferenceImpl, TotalCaptureResult>> imageDataPairs =
246                         new ArrayList<>(mCaptureResults.values());
247 
248                 Image resultImage = null;
249                 int captureSurfaceWriterImageFormat = ImageFormat.UNKNOWN;
250                 synchronized (mLockImageWriter) {
251                     resultImage = mCaptureSurfaceImageWriter.dequeueInputImage();
252                     captureSurfaceWriterImageFormat = mCaptureSurfaceImageWriter.getFormat();
253                 }
254 
255                 if (captureSurfaceWriterImageFormat == ImageFormat.JPEG) {
256                     Image yuvImage = imageDataPairs.get(NORMAL_EXPOSED_CAPTURE_ID).first.get();
257 
258                     Integer jpegOrientation = JPEG_DEFAULT_ROTATION;
259 
260                     synchronized (mLock) {
261                         if (mParameters.get(CaptureRequest.JPEG_ORIENTATION) != null) {
262                             jpegOrientation =
263                                     (Integer) mParameters.get(CaptureRequest.JPEG_ORIENTATION);
264                         }
265                     }
266 
267                     JpegEncoder.encodeToJpeg(yuvImage, resultImage, jpegOrientation,
268                             JPEG_DEFAULT_QUALITY);
269 
270                     addCaptureResultKeys(seqId, imageDataPairs.get(UNDER_EXPOSED_CAPTURE_ID)
271                             .second, captureCallback);
272                     resultImage.setTimestamp(imageDataPairs.get(UNDER_EXPOSED_CAPTURE_ID)
273                             .first.get().getTimestamp());
274 
275                 } else {
276                     ByteBuffer yByteBuffer = resultImage.getPlanes()[0].getBuffer();
277                     ByteBuffer uByteBuffer = resultImage.getPlanes()[2].getBuffer();
278                     ByteBuffer vByteBuffer = resultImage.getPlanes()[1].getBuffer();
279 
280                     yByteBuffer.put(imageDataPairs.get(
281                         NORMAL_EXPOSED_CAPTURE_ID).first.get().getPlanes()[0].getBuffer());
282                     uByteBuffer.put(imageDataPairs.get(
283                         NORMAL_EXPOSED_CAPTURE_ID).first.get().getPlanes()[2].getBuffer());
284                     vByteBuffer.put(imageDataPairs.get(
285                         NORMAL_EXPOSED_CAPTURE_ID).first.get().getPlanes()[1].getBuffer());
286 
287                     addCaptureResultKeys(seqId, imageDataPairs.get(UNDER_EXPOSED_CAPTURE_ID)
288                             .second, captureCallback);
289                     resultImage.setTimestamp(imageDataPairs.get(
290                         UNDER_EXPOSED_CAPTURE_ID).first.get().getTimestamp());
291                 }
292 
293                 synchronized (mLockImageWriter) {
294                     mCaptureSurfaceImageWriter.queueInputImage(resultImage);
295                 }
296 
297                 for (Pair<ImageReferenceImpl, TotalCaptureResult> val : mCaptureResults.values()) {
298                     val.first.decrement();
299                 }
300 
301                 mCaptureResults.clear();
302             } else {
303                 Log.w(TAG, "Unable to process, waiting for all images");
304             }
305         }
306     }
307 
308     @Override
createSessionProcessor()309     public SessionProcessorImpl createSessionProcessor() {
310         return new HDRAdvancedSessionProcessor();
311     }
312 
313     @Override
getAvailableCaptureRequestKeys()314     public List<CaptureRequest.Key> getAvailableCaptureRequestKeys() {
315         final CaptureRequest.Key [] CAPTURE_REQUEST_SET = {CaptureRequest.CONTROL_ZOOM_RATIO,
316             CaptureRequest.CONTROL_AF_MODE, CaptureRequest.CONTROL_AF_REGIONS,
317             CaptureRequest.CONTROL_AF_TRIGGER, CaptureRequest.JPEG_QUALITY,
318             CaptureRequest.JPEG_ORIENTATION};
319         return Arrays.asList(CAPTURE_REQUEST_SET);
320     }
321 
322     @Override
getAvailableCaptureResultKeys()323     public List<CaptureResult.Key> getAvailableCaptureResultKeys() {
324         final CaptureResult.Key [] CAPTURE_RESULT_SET = {CaptureResult.CONTROL_ZOOM_RATIO,
325             CaptureResult.CONTROL_AF_MODE, CaptureResult.CONTROL_AF_REGIONS,
326             CaptureResult.CONTROL_AF_TRIGGER, CaptureResult.CONTROL_AF_STATE,
327             CaptureResult.JPEG_QUALITY, CaptureResult.JPEG_ORIENTATION};
328         return Arrays.asList(CAPTURE_RESULT_SET);
329     }
330 }
331