1 /*
2  * Copyright 2022 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package androidx.camera.extensions.impl.advanced;
18 
19 import android.hardware.camera2.CaptureResult;
20 import android.hardware.camera2.TotalCaptureResult;
21 import android.media.Image;
22 import android.util.LongSparseArray;
23 
24 import androidx.annotation.GuardedBy;
25 import androidx.annotation.NonNull;
26 
27 import java.util.ArrayList;
28 import java.util.HashMap;
29 import java.util.List;
30 import java.util.Map;
31 
32 public class CaptureResultImageMatcher {
33     private static final String TAG = "CaptureResultImageReader";
34     private final Object mLock = new Object();
35 
36     @GuardedBy("mLock")
37     private boolean mClosed = false;
38 
39     /** ImageInfos haven't been matched with Image. */
40     @GuardedBy("mLock")
41     private final LongSparseArray<TotalCaptureResult> mPendingImageInfos = new LongSparseArray<>();
42 
43     Map<TotalCaptureResult, Integer> mCaptureStageIdMap = new HashMap<>();
44 
45 
46     /** Images haven't been matched with ImageInfo. */
47     @GuardedBy("mLock")
48     private final LongSparseArray<ImageReferenceImpl> mPendingImages = new LongSparseArray<>();
49 
50     ImageReferenceListener mImageReferenceListener;
51 
CaptureResultImageMatcher()52     public CaptureResultImageMatcher() {
53 
54     }
55 
clear()56     public void clear() {
57         synchronized (mLock) {
58             mPendingImageInfos.clear();
59             for (int i = 0; i < mPendingImages.size(); i++) {
60                 long key = mPendingImages.keyAt(i);
61                 mPendingImages.get(key).decrement();
62             }
63             mPendingImages.clear();
64             mCaptureStageIdMap.clear();
65             mClosed = false;
66         }
67     }
68 
setImageReferenceListener( @onNull ImageReferenceListener imageReferenceImplListener)69     public void setImageReferenceListener(
70             @NonNull ImageReferenceListener imageReferenceImplListener) {
71         synchronized (mLock) {
72             mImageReferenceListener = imageReferenceImplListener;
73         }
74     }
75 
setInputImage(@onNull ImageReferenceImpl imageReferenceImpl)76     public void setInputImage(@NonNull ImageReferenceImpl imageReferenceImpl) {
77         synchronized (mLock) {
78             if (mClosed) {
79                 return;
80             }
81 
82             Image image = imageReferenceImpl.get();
83             // Add the incoming Image to pending list and do the matching logic.
84             mPendingImages.put(image.getTimestamp(), imageReferenceImpl);
85             matchImages();
86         }
87     }
88 
setCameraCaptureCallback(@onNull TotalCaptureResult captureResult)89     public void setCameraCaptureCallback(@NonNull TotalCaptureResult captureResult) {
90         setCameraCaptureCallback(captureResult, 0);
91     }
92 
setCameraCaptureCallback(@onNull TotalCaptureResult captureResult, int captureStageId)93     public void setCameraCaptureCallback(@NonNull TotalCaptureResult captureResult,
94             int captureStageId) {
95         synchronized (mLock) {
96             if (mClosed) {
97                 return;
98             }
99 
100             long timestamp = getTimeStampFromCaptureResult(captureResult);
101 
102             // Add the incoming CameraCaptureResult to pending list and do the matching logic.
103             mPendingImageInfos.put(timestamp, captureResult);
104             mCaptureStageIdMap.put(captureResult, captureStageId);
105             matchImages();
106         }
107     }
108 
109 
getTimeStampFromCaptureResult(TotalCaptureResult captureResult)110     private long getTimeStampFromCaptureResult(TotalCaptureResult captureResult) {
111         Long timestamp = captureResult.get(CaptureResult.SENSOR_TIMESTAMP);
112         long timestampValue = -1;
113         if (timestamp != null) {
114             timestampValue = timestamp;
115         }
116 
117         return timestampValue;
118     }
119 
120 
notifyImage(ImageReferenceImpl imageReferenceImpl, TotalCaptureResult totalCaptureResult)121     private void notifyImage(ImageReferenceImpl imageReferenceImpl,
122             TotalCaptureResult totalCaptureResult) {
123         synchronized (mLock) {
124             if (mImageReferenceListener != null) {
125                 mImageReferenceListener.onImageReferenceIncoming(imageReferenceImpl,
126                         totalCaptureResult, mCaptureStageIdMap.get(totalCaptureResult));
127             } else {
128                 imageReferenceImpl.decrement();
129             }
130         }
131     }
132 
133     // Remove the stale {@link ImageProxy} and {@link ImageInfo} from the pending queue if there are
134     // any missing which can happen if the camera is momentarily shut off.
135     // The ImageProxy and ImageInfo timestamps are assumed to be monotonically increasing. This
136     // means any ImageProxy or ImageInfo which has a timestamp older (smaller in value) than the
137     // oldest timestamp in the other queue will never get matched, so they should be removed.
138     //
139     // This should only be called at the end of matchImages(). The assumption is that there are no
140     // matching timestamps.
removeStaleData()141     private void removeStaleData() {
142         synchronized (mLock) {
143             // No stale data to remove
144             if (mPendingImages.size() == 0 || mPendingImageInfos.size() == 0) {
145                 return;
146             }
147 
148             Long minImageProxyTimestamp = mPendingImages.keyAt(0);
149             Long minImageInfoTimestamp = mPendingImageInfos.keyAt(0);
150 
151             // If timestamps are equal then matchImages did not correctly match up the ImageInfo
152             // and ImageProxy
153             if (minImageInfoTimestamp.equals(minImageProxyTimestamp)) {
154                 throw new IllegalArgumentException();
155             }
156 
157             if (minImageInfoTimestamp > minImageProxyTimestamp) {
158                 for (int i = mPendingImages.size() - 1; i >= 0; i--) {
159                     if (mPendingImages.keyAt(i) < minImageInfoTimestamp) {
160                         ImageReferenceImpl imageReferenceImpl = mPendingImages.valueAt(i);
161                         imageReferenceImpl.decrement();
162                         mPendingImages.removeAt(i);
163                     }
164                 }
165             } else {
166                 for (int i = mPendingImageInfos.size() - 1; i >= 0; i--) {
167                     if (mPendingImageInfos.keyAt(i) < minImageProxyTimestamp) {
168                         mPendingImageInfos.removeAt(i);
169                     }
170                 }
171             }
172 
173         }
174     }
175 
176     // Match incoming Image from the ImageReader with the corresponding ImageInfo.
matchImages()177     private void matchImages() {
178         synchronized (mLock) {
179             // Iterate in reverse order so that ImageInfo can be removed in place
180             for (int i = mPendingImageInfos.size() - 1; i >= 0; i--) {
181                 TotalCaptureResult captureResult = mPendingImageInfos.valueAt(i);
182                 long timestamp = getTimeStampFromCaptureResult(captureResult);
183 
184                 ImageReferenceImpl imageReferenceImpl = mPendingImages.get(timestamp);
185 
186                 if (imageReferenceImpl != null) {
187                     mPendingImages.remove(timestamp);
188                     mPendingImageInfos.removeAt(i);
189                     // Got a match. Add the ImageProxy to matched list and invoke
190                     // onImageAvailableListener.
191                     notifyImage(imageReferenceImpl, captureResult);
192                 }
193             }
194 
195             removeStaleData();
196         }
197     }
198 
199     public interface ImageReferenceListener {
onImageReferenceIncoming(@onNull ImageReferenceImpl imageReferenceImpl, @NonNull TotalCaptureResult totalCaptureResult, int captureStageId)200         void onImageReferenceIncoming(@NonNull ImageReferenceImpl imageReferenceImpl,
201                 @NonNull TotalCaptureResult totalCaptureResult, int captureStageId);
202     }
203 
204 }