1 /* 2 * Copyright 2016 The TensorFlow Authors. All Rights Reserved. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package org.tensorflow.demo; 18 19 import android.Manifest; 20 import android.app.Activity; 21 import android.app.Fragment; 22 import android.content.Context; 23 import android.content.pm.PackageManager; 24 import android.hardware.Camera; 25 import android.hardware.camera2.CameraAccessException; 26 import android.hardware.camera2.CameraCharacteristics; 27 import android.hardware.camera2.CameraManager; 28 import android.hardware.camera2.params.StreamConfigurationMap; 29 import android.media.Image; 30 import android.media.Image.Plane; 31 import android.media.ImageReader; 32 import android.media.ImageReader.OnImageAvailableListener; 33 import android.os.Build; 34 import android.os.Bundle; 35 import android.os.Handler; 36 import android.os.HandlerThread; 37 import android.os.Trace; 38 import android.util.Size; 39 import android.view.KeyEvent; 40 import android.view.Surface; 41 import android.view.WindowManager; 42 import android.widget.Toast; 43 import java.nio.ByteBuffer; 44 import org.tensorflow.demo.env.ImageUtils; 45 import org.tensorflow.demo.env.Logger; 46 import org.tensorflow.demo.R; // Explicit import needed for internal Google builds. 47 48 public abstract class CameraActivity extends Activity 49 implements OnImageAvailableListener, Camera.PreviewCallback { 50 private static final Logger LOGGER = new Logger(); 51 52 private static final int PERMISSIONS_REQUEST = 1; 53 54 private static final String PERMISSION_CAMERA = Manifest.permission.CAMERA; 55 private static final String PERMISSION_STORAGE = Manifest.permission.WRITE_EXTERNAL_STORAGE; 56 57 private boolean debug = false; 58 59 private Handler handler; 60 private HandlerThread handlerThread; 61 private boolean useCamera2API; 62 private boolean isProcessingFrame = false; 63 private byte[][] yuvBytes = new byte[3][]; 64 private int[] rgbBytes = null; 65 private int yRowStride; 66 67 protected int previewWidth = 0; 68 protected int previewHeight = 0; 69 70 private Runnable postInferenceCallback; 71 private Runnable imageConverter; 72 73 @Override onCreate(final Bundle savedInstanceState)74 protected void onCreate(final Bundle savedInstanceState) { 75 LOGGER.d("onCreate " + this); 76 super.onCreate(null); 77 getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); 78 79 setContentView(R.layout.activity_camera); 80 81 if (hasPermission()) { 82 setFragment(); 83 } else { 84 requestPermission(); 85 } 86 } 87 88 private byte[] lastPreviewFrame; 89 getRgbBytes()90 protected int[] getRgbBytes() { 91 imageConverter.run(); 92 return rgbBytes; 93 } 94 getLuminanceStride()95 protected int getLuminanceStride() { 96 return yRowStride; 97 } 98 getLuminance()99 protected byte[] getLuminance() { 100 return yuvBytes[0]; 101 } 102 103 /** 104 * Callback for android.hardware.Camera API 105 */ 106 @Override onPreviewFrame(final byte[] bytes, final Camera camera)107 public void onPreviewFrame(final byte[] bytes, final Camera camera) { 108 if (isProcessingFrame) { 109 LOGGER.w("Dropping frame!"); 110 return; 111 } 112 113 try { 114 // Initialize the storage bitmaps once when the resolution is known. 115 if (rgbBytes == null) { 116 Camera.Size previewSize = camera.getParameters().getPreviewSize(); 117 previewHeight = previewSize.height; 118 previewWidth = previewSize.width; 119 rgbBytes = new int[previewWidth * previewHeight]; 120 onPreviewSizeChosen(new Size(previewSize.width, previewSize.height), 90); 121 } 122 } catch (final Exception e) { 123 LOGGER.e(e, "Exception!"); 124 return; 125 } 126 127 isProcessingFrame = true; 128 lastPreviewFrame = bytes; 129 yuvBytes[0] = bytes; 130 yRowStride = previewWidth; 131 132 imageConverter = 133 new Runnable() { 134 @Override 135 public void run() { 136 ImageUtils.convertYUV420SPToARGB8888(bytes, previewWidth, previewHeight, rgbBytes); 137 } 138 }; 139 140 postInferenceCallback = 141 new Runnable() { 142 @Override 143 public void run() { 144 camera.addCallbackBuffer(bytes); 145 isProcessingFrame = false; 146 } 147 }; 148 processImage(); 149 } 150 151 /** 152 * Callback for Camera2 API 153 */ 154 @Override onImageAvailable(final ImageReader reader)155 public void onImageAvailable(final ImageReader reader) { 156 //We need wait until we have some size from onPreviewSizeChosen 157 if (previewWidth == 0 || previewHeight == 0) { 158 return; 159 } 160 if (rgbBytes == null) { 161 rgbBytes = new int[previewWidth * previewHeight]; 162 } 163 try { 164 final Image image = reader.acquireLatestImage(); 165 166 if (image == null) { 167 return; 168 } 169 170 if (isProcessingFrame) { 171 image.close(); 172 return; 173 } 174 isProcessingFrame = true; 175 Trace.beginSection("imageAvailable"); 176 final Plane[] planes = image.getPlanes(); 177 fillBytes(planes, yuvBytes); 178 yRowStride = planes[0].getRowStride(); 179 final int uvRowStride = planes[1].getRowStride(); 180 final int uvPixelStride = planes[1].getPixelStride(); 181 182 imageConverter = 183 new Runnable() { 184 @Override 185 public void run() { 186 ImageUtils.convertYUV420ToARGB8888( 187 yuvBytes[0], 188 yuvBytes[1], 189 yuvBytes[2], 190 previewWidth, 191 previewHeight, 192 yRowStride, 193 uvRowStride, 194 uvPixelStride, 195 rgbBytes); 196 } 197 }; 198 199 postInferenceCallback = 200 new Runnable() { 201 @Override 202 public void run() { 203 image.close(); 204 isProcessingFrame = false; 205 } 206 }; 207 208 processImage(); 209 } catch (final Exception e) { 210 LOGGER.e(e, "Exception!"); 211 Trace.endSection(); 212 return; 213 } 214 Trace.endSection(); 215 } 216 217 @Override onStart()218 public synchronized void onStart() { 219 LOGGER.d("onStart " + this); 220 super.onStart(); 221 } 222 223 @Override onResume()224 public synchronized void onResume() { 225 LOGGER.d("onResume " + this); 226 super.onResume(); 227 228 handlerThread = new HandlerThread("inference"); 229 handlerThread.start(); 230 handler = new Handler(handlerThread.getLooper()); 231 } 232 233 @Override onPause()234 public synchronized void onPause() { 235 LOGGER.d("onPause " + this); 236 237 if (!isFinishing()) { 238 LOGGER.d("Requesting finish"); 239 finish(); 240 } 241 242 handlerThread.quitSafely(); 243 try { 244 handlerThread.join(); 245 handlerThread = null; 246 handler = null; 247 } catch (final InterruptedException e) { 248 LOGGER.e(e, "Exception!"); 249 } 250 251 super.onPause(); 252 } 253 254 @Override onStop()255 public synchronized void onStop() { 256 LOGGER.d("onStop " + this); 257 super.onStop(); 258 } 259 260 @Override onDestroy()261 public synchronized void onDestroy() { 262 LOGGER.d("onDestroy " + this); 263 super.onDestroy(); 264 } 265 runInBackground(final Runnable r)266 protected synchronized void runInBackground(final Runnable r) { 267 if (handler != null) { 268 handler.post(r); 269 } 270 } 271 272 @Override onRequestPermissionsResult( final int requestCode, final String[] permissions, final int[] grantResults)273 public void onRequestPermissionsResult( 274 final int requestCode, final String[] permissions, final int[] grantResults) { 275 if (requestCode == PERMISSIONS_REQUEST) { 276 if (grantResults.length > 0 277 && grantResults[0] == PackageManager.PERMISSION_GRANTED 278 && grantResults[1] == PackageManager.PERMISSION_GRANTED) { 279 setFragment(); 280 } else { 281 requestPermission(); 282 } 283 } 284 } 285 hasPermission()286 private boolean hasPermission() { 287 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { 288 return checkSelfPermission(PERMISSION_CAMERA) == PackageManager.PERMISSION_GRANTED && 289 checkSelfPermission(PERMISSION_STORAGE) == PackageManager.PERMISSION_GRANTED; 290 } else { 291 return true; 292 } 293 } 294 requestPermission()295 private void requestPermission() { 296 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { 297 if (shouldShowRequestPermissionRationale(PERMISSION_CAMERA) || 298 shouldShowRequestPermissionRationale(PERMISSION_STORAGE)) { 299 Toast.makeText(CameraActivity.this, 300 "Camera AND storage permission are required for this demo", Toast.LENGTH_LONG).show(); 301 } 302 requestPermissions(new String[] {PERMISSION_CAMERA, PERMISSION_STORAGE}, PERMISSIONS_REQUEST); 303 } 304 } 305 306 // Returns true if the device supports the required hardware level, or better. isHardwareLevelSupported( CameraCharacteristics characteristics, int requiredLevel)307 private boolean isHardwareLevelSupported( 308 CameraCharacteristics characteristics, int requiredLevel) { 309 int deviceLevel = characteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL); 310 if (deviceLevel == CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY) { 311 return requiredLevel == deviceLevel; 312 } 313 // deviceLevel is not LEGACY, can use numerical sort 314 return requiredLevel <= deviceLevel; 315 } 316 chooseCamera()317 private String chooseCamera() { 318 final CameraManager manager = (CameraManager) getSystemService(Context.CAMERA_SERVICE); 319 try { 320 for (final String cameraId : manager.getCameraIdList()) { 321 final CameraCharacteristics characteristics = manager.getCameraCharacteristics(cameraId); 322 323 // We don't use a front facing camera in this sample. 324 final Integer facing = characteristics.get(CameraCharacteristics.LENS_FACING); 325 if (facing != null && facing == CameraCharacteristics.LENS_FACING_FRONT) { 326 continue; 327 } 328 329 final StreamConfigurationMap map = 330 characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); 331 332 if (map == null) { 333 continue; 334 } 335 336 // Fallback to camera1 API for internal cameras that don't have full support. 337 // This should help with legacy situations where using the camera2 API causes 338 // distorted or otherwise broken previews. 339 useCamera2API = (facing == CameraCharacteristics.LENS_FACING_EXTERNAL) 340 || isHardwareLevelSupported(characteristics, 341 CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_FULL); 342 LOGGER.i("Camera API lv2?: %s", useCamera2API); 343 return cameraId; 344 } 345 } catch (CameraAccessException e) { 346 LOGGER.e(e, "Not allowed to access camera"); 347 } 348 349 return null; 350 } 351 setFragment()352 protected void setFragment() { 353 String cameraId = chooseCamera(); 354 if (cameraId == null) { 355 Toast.makeText(this, "No Camera Detected", Toast.LENGTH_SHORT).show(); 356 finish(); 357 } 358 359 Fragment fragment; 360 if (useCamera2API) { 361 CameraConnectionFragment camera2Fragment = 362 CameraConnectionFragment.newInstance( 363 new CameraConnectionFragment.ConnectionCallback() { 364 @Override 365 public void onPreviewSizeChosen(final Size size, final int rotation) { 366 previewHeight = size.getHeight(); 367 previewWidth = size.getWidth(); 368 CameraActivity.this.onPreviewSizeChosen(size, rotation); 369 } 370 }, 371 this, 372 getLayoutId(), 373 getDesiredPreviewFrameSize()); 374 375 camera2Fragment.setCamera(cameraId); 376 fragment = camera2Fragment; 377 } else { 378 fragment = 379 new LegacyCameraConnectionFragment(this, getLayoutId(), getDesiredPreviewFrameSize()); 380 } 381 382 getFragmentManager() 383 .beginTransaction() 384 .replace(R.id.container, fragment) 385 .commit(); 386 } 387 fillBytes(final Plane[] planes, final byte[][] yuvBytes)388 protected void fillBytes(final Plane[] planes, final byte[][] yuvBytes) { 389 // Because of the variable row stride it's not possible to know in 390 // advance the actual necessary dimensions of the yuv planes. 391 for (int i = 0; i < planes.length; ++i) { 392 final ByteBuffer buffer = planes[i].getBuffer(); 393 if (yuvBytes[i] == null) { 394 LOGGER.d("Initializing buffer %d at size %d", i, buffer.capacity()); 395 yuvBytes[i] = new byte[buffer.capacity()]; 396 } 397 buffer.get(yuvBytes[i]); 398 } 399 } 400 isDebug()401 public boolean isDebug() { 402 return debug; 403 } 404 requestRender()405 public void requestRender() { 406 final OverlayView overlay = (OverlayView) findViewById(R.id.debug_overlay); 407 if (overlay != null) { 408 overlay.postInvalidate(); 409 } 410 } 411 addCallback(final OverlayView.DrawCallback callback)412 public void addCallback(final OverlayView.DrawCallback callback) { 413 final OverlayView overlay = (OverlayView) findViewById(R.id.debug_overlay); 414 if (overlay != null) { 415 overlay.addCallback(callback); 416 } 417 } 418 onSetDebug(final boolean debug)419 public void onSetDebug(final boolean debug) {} 420 421 @Override onKeyDown(final int keyCode, final KeyEvent event)422 public boolean onKeyDown(final int keyCode, final KeyEvent event) { 423 if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN || keyCode == KeyEvent.KEYCODE_VOLUME_UP 424 || keyCode == KeyEvent.KEYCODE_BUTTON_L1 || keyCode == KeyEvent.KEYCODE_DPAD_CENTER) { 425 debug = !debug; 426 requestRender(); 427 onSetDebug(debug); 428 return true; 429 } 430 return super.onKeyDown(keyCode, event); 431 } 432 readyForNextImage()433 protected void readyForNextImage() { 434 if (postInferenceCallback != null) { 435 postInferenceCallback.run(); 436 } 437 } 438 getScreenOrientation()439 protected int getScreenOrientation() { 440 switch (getWindowManager().getDefaultDisplay().getRotation()) { 441 case Surface.ROTATION_270: 442 return 270; 443 case Surface.ROTATION_180: 444 return 180; 445 case Surface.ROTATION_90: 446 return 90; 447 default: 448 return 0; 449 } 450 } 451 processImage()452 protected abstract void processImage(); 453 onPreviewSizeChosen(final Size size, final int rotation)454 protected abstract void onPreviewSizeChosen(final Size size, final int rotation); getLayoutId()455 protected abstract int getLayoutId(); getDesiredPreviewFrameSize()456 protected abstract Size getDesiredPreviewFrameSize(); 457 } 458