1# Copyright 2022 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the 'License'); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an 'AS IS' BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14"""Utility functions to enable capture read noise analysis.""" 15 16import csv 17import logging 18import math 19import os 20import pickle 21import camera_properties_utils 22import capture_request_utils 23import error_util 24import image_processing_utils 25import its_session_utils 26from matplotlib import pyplot as plt 27from matplotlib.ticker import NullLocator 28from matplotlib.ticker import ScalarFormatter 29import noise_model_constants 30import noise_model_utils 31import numpy as np 32 33_LINEAR_FIT_NUM_SAMPLES = 100 # Number of samples to plot for the linear fit 34_PLOT_AXIS_TICKS = 5 # Number of ticks to display on the plot axis 35_FIG_DPI = 100 # Read noise plots dpi. 36# Valid raw format for capturing read noise images. 37_VALID_RAW_FORMATS = ('raw', 'raw10', 'rawQuadBayer', 'raw10QuadBayer') 38 39 40def save_read_noise_data_as_csv(read_noise_data, iso_low, iso_high, file, 41 color_channels_names): 42 """Creates and saves a CSV file containing read noise data. 43 44 Args: 45 read_noise_data: A list of lists of dictionaries, where each dictionary 46 contains read noise data for a single color channel. 47 iso_low: The minimum ISO sensitivity to include in the CSV file. 48 iso_high: The maximum ISO sensitivity to include in the CSV file. 49 file: The path to the CSV file to create. 50 color_channels_names: A list of color channels to include in the CSV file. 51 """ 52 with open(file, 'w+') as f: 53 writer = csv.writer(f) 54 55 results = list( 56 filter( 57 lambda x: x[0]['iso'] >= iso_low and x[0]['iso'] <= iso_high, 58 read_noise_data, 59 ) 60 ) 61 62 # Create headers for csv file 63 headers = ['iso', 'iso^2'] 64 headers.extend([f'mean_{color}' for color in color_channels_names]) 65 headers.extend([f'var_{color}' for color in color_channels_names]) 66 headers.extend([f'norm_var_{color}' for color in color_channels_names]) 67 68 writer.writerow(headers) 69 70 # Create data rows 71 for data_row in results: 72 row = [data_row[0]['iso']] 73 row.append(data_row[0]['iso']**2) 74 row.extend([stats['mean'] for stats in data_row]) 75 row.extend([stats['var'] for stats in data_row]) 76 row.extend([stats['norm_var'] for stats in data_row]) 77 78 writer.writerow(row) 79 80 writer.writerow([]) # divider line 81 82 # Create row containing the offset coefficients calculated by np.polyfit 83 coeff_headers = ['', 'offset_coefficient_a', 'offset_coefficient_b'] 84 writer.writerow(coeff_headers) 85 86 offset_a, offset_b = get_read_noise_coefficients(results, iso_low, iso_high) 87 for i in range(len(color_channels_names)): 88 writer.writerow([color_channels_names[i], offset_a[i], offset_b[i]]) 89 90 91def plot_read_noise_data(read_noise_data, iso_low, iso_high, file_path, 92 color_channel_names, plot_colors): 93 """Plots the read noise data for the given ISO range. 94 95 Args: 96 read_noise_data: Quad Bayer read noise data object. 97 iso_low: The minimum iso value to include. 98 iso_high: The maximum iso value to include. 99 file_path: File path for the plot image. 100 color_channel_names: The name list of each color channel. 101 plot_colors: The color list for plotting. 102 """ 103 num_channels = len(color_channel_names) 104 is_quad_bayer = num_channels == noise_model_constants.NUM_QUAD_BAYER_CHANNELS 105 # Create the figure for plotting the read noise to ISO^2 curve. 106 fig, ((red, green_r), (green_b, blue)) = plt.subplots(2, 2, figsize=(22, 22)) 107 subplots = [red, green_r, green_b, blue] 108 fig.gca() 109 fig.suptitle('Read Noise to ISO^2', x=0.54, y=0.99) 110 111 # Get the ISO values for the current range. 112 filtered_data = list( 113 filter( 114 lambda x: x[0]['iso'] >= iso_low and x[0]['iso'] <= iso_high, 115 read_noise_data, 116 ) 117 ) 118 119 # Get X-axis values (ISO^2) for current_range. 120 iso_sq = [data[0]['iso']**2 for data in filtered_data] 121 122 # Get X-axis values for the calculated linear fit for the read noise 123 iso_sq_values = np.linspace(iso_low**2, iso_high**2, _LINEAR_FIT_NUM_SAMPLES) 124 125 # Get the line fit coeff for plotting the linear fit of read noise to iso^2 126 coeff_a, coeff_b = get_read_noise_coefficients( 127 filtered_data, iso_low, iso_high 128 ) 129 130 # Plot the read noise to iso^2 data 131 for pidx, color_channel in enumerate(color_channel_names): 132 norm_vars = [data[pidx]['norm_var'] for data in filtered_data] 133 134 # Plot the measured read noise to ISO^2 values 135 if is_quad_bayer: 136 subplot = subplots[pidx // 4] 137 else: 138 subplot = subplots[pidx] 139 140 subplot.plot( 141 iso_sq, 142 norm_vars, 143 color=plot_colors[pidx], 144 marker='o', 145 markeredgecolor=plot_colors[pidx], 146 linestyle='None', 147 label=color_channel, 148 alpha=0.3, 149 ) 150 151 # Plot the line fit calculated from the read noise values 152 subplot.plot( 153 iso_sq_values, 154 coeff_a[pidx] * iso_sq_values + coeff_b[pidx], 155 color=plot_colors[pidx], 156 ) 157 158 # Create a numpy array containing all normalized variance values for the 159 # current range, this will be used for labelling the X-axis 160 y_values = np.array( 161 [[color['norm_var'] for color in x] for x in filtered_data] 162 ) 163 164 x_ticks = np.linspace(iso_low**2, iso_high**2, _PLOT_AXIS_TICKS) 165 y_ticks = np.linspace(np.min(y_values), np.max(y_values), _PLOT_AXIS_TICKS) 166 167 for i, subplot in enumerate(subplots): 168 subplot.set_title(noise_model_constants.BAYER_COLORS[i]) 169 subplot.set_xlabel('ISO^2') 170 subplot.set_ylabel('Read Noise') 171 172 subplot.set_xticks(x_ticks) 173 subplot.xaxis.set_minor_locator(NullLocator()) 174 subplot.xaxis.set_major_formatter(ScalarFormatter()) 175 176 subplot.set_yticks(y_ticks) 177 subplot.yaxis.set_minor_locator(NullLocator()) 178 subplot.yaxis.set_major_formatter(ScalarFormatter()) 179 180 subplot.legend() 181 plt.tight_layout() 182 183 fig.savefig(file_path, dpi=_FIG_DPI) 184 185 186def _generate_read_noise_stats(img, iso, white_level, cfa_order): 187 """Generates read noise data for a given image. 188 189 The read noise data of each channel is added in the order of cfa_order. 190 As a result, the read noise data channels are reordered as the following. 191 (1) For standard Bayer: R, Gr, Gb, B. 192 (2) For quad Bayer: R0, R1, R2, R3, 193 Gr0, Gr1, Gr2, Gr3, 194 Gb0, Gb1, Gb2, Gb3, 195 B0, B1, B2, B3. 196 197 Args: 198 img: The input image. 199 iso: The ISO sensitivity used to capture the image. 200 white_level: The white level of the image. 201 cfa_order: The color filter arrangement (CFA) order of the image. 202 203 Returns: 204 A list of dictionaries, where each dictionary contains information for a 205 single color channel in the image. 206 """ 207 result = [] 208 209 num_channels = len(cfa_order) 210 channel_img = image_processing_utils.subsample(img, num_channels) 211 212 # Create a list of dictionaries of read noise stats for each color channel 213 # in the image. 214 # The stats is reordered according to the color filter arrangement order. 215 for ch in cfa_order: 216 mean = np.mean(channel_img[:, :, ch]) 217 var = np.var(channel_img[:, :, ch]) 218 norm_var = var / ((white_level - mean)**2) 219 result.append({ 220 'iso': iso, 221 'mean': mean, 222 'var': var, 223 'norm_var': norm_var 224 }) 225 226 return result 227 228 229def get_read_noise_coefficients(read_noise_data, iso_low=0, iso_high=1000000): 230 """Calculates read noise coefficients that best fit the read noise data. 231 232 Args: 233 read_noise_data: Read noise data object. 234 iso_low: The lower bound of the ISO range to consider. 235 iso_high: The upper bound of the ISO range to consider. 236 237 Returns: 238 A tuple of two numpy arrays, where the first array contains read noise 239 coefficient a and the second array contains read noise coefficient b. 240 """ 241 # Filter the values by the given ISO range. 242 read_noise_data_filtered = list( 243 filter( 244 lambda x: x[0]['iso'] >= iso_low and x[0]['iso'] <= iso_high, 245 read_noise_data, 246 ) 247 ) 248 249 read_noise_coefficients_a = [] 250 read_noise_coefficients_b = [] 251 252 # Get ISO^2 values used for X-axis in polyfit 253 iso_sq = [data[0]['iso'] ** 2 for data in read_noise_data_filtered] 254 255 # Find the linear equation coefficients for each color channel 256 num_channels = len(read_noise_data_filtered[0]) 257 for i in range(num_channels): 258 norm_var = [data[i]['norm_var'] for data in read_noise_data_filtered] 259 coeffs = np.polyfit(iso_sq, norm_var, 1) 260 261 read_noise_coefficients_a.append(coeffs[0]) 262 read_noise_coefficients_b.append(coeffs[1]) 263 264 read_noise_coefficients_a = np.asarray(read_noise_coefficients_a) 265 read_noise_coefficients_b = np.asarray(read_noise_coefficients_b) 266 return read_noise_coefficients_a, read_noise_coefficients_b 267 268 269def _capture_read_noise_for_iso_range(cam, raw_format, low_iso, high_iso, 270 steps_per_stop, dest_file): 271 """Captures read noise data at the lowest advertised exposure value. 272 273 This function captures a series of images at different ISO sensitivities, 274 starting at `low_iso` and ending at `high_iso`. The number of steps between 275 each ISO sensitivity is equal to `steps`. Then read noise stats data is 276 computed. Finally, stats data of color channels are reordered into the 277 canonical order before saving it to `dest_file`. 278 279 Args: 280 cam: Camera for the current ItsSession. 281 raw_format: The format of read noise image. 282 low_iso: The lowest iso value in range. 283 high_iso: The highest iso value in range. 284 steps_per_stop: Steps to take per stop. 285 dest_file: The path where read noise stats should be saved. 286 287 Returns: 288 Read noise stats list for each sensitivity. 289 """ 290 if raw_format not in _VALID_RAW_FORMATS: 291 supported_formats_str = ', '.join(_VALID_RAW_FORMATS) 292 raise error_util.CameraItsError( 293 f'Invalid raw format {raw_format}. ' 294 f'Current supported raw formats: {supported_formats_str}.' 295 ) 296 297 props = cam.get_camera_properties() 298 props = cam.override_with_hidden_physical_camera_props(props) 299 300 format_check_result = False 301 if raw_format in ('raw', 'rawQuadBayer'): 302 format_check_result = camera_properties_utils.raw16(props) 303 elif raw_format in ('raw10', 'raw10QuadBayer'): 304 format_check_result = camera_properties_utils.raw10(props) 305 306 camera_properties_utils.skip_unless( 307 format_check_result and 308 camera_properties_utils.manual_sensor(props) and 309 camera_properties_utils.read_3a(props) and 310 camera_properties_utils.per_frame_control(props)) 311 min_exposure_ns, _ = props['android.sensor.info.exposureTimeRange'] 312 min_fd = props['android.lens.info.minimumFocusDistance'] 313 white_level = props['android.sensor.info.whiteLevel'] 314 is_quad_bayer = 'QuadBayer' in raw_format 315 cfa_order = image_processing_utils.get_canonical_cfa_order( 316 props, is_quad_bayer 317 ) 318 pre_iso_cap = None 319 iso = low_iso 320 iso_multiplier = math.pow(2, 1.0 / steps_per_stop) 321 stats_list = [] 322 # This operation can last a very long time, if it happens to fail halfway 323 # through, this section of code will allow us to pick up where we left off 324 if os.path.exists(dest_file): 325 # If there already exists a read noise stats file, retrieve them. 326 with open(dest_file, 'rb') as f: 327 stats_list = pickle.load(f) 328 # Set the starting iso to the last iso of read noise stats. 329 pre_iso_cap = stats_list[-1][0]['iso'] 330 iso = noise_model_utils.get_next_iso(pre_iso_cap, high_iso, iso_multiplier) 331 332 if round(iso) <= high_iso: 333 # Wait until camera is repositioned for read noise data collection. 334 input( 335 f'\nPress <ENTER> after concealing camera {cam.get_camera_name()} ' 336 'in complete darkness.\n' 337 ) 338 339 fmt = {'format': raw_format} 340 logging.info('Capturing read noise images with format %s.', raw_format) 341 while round(iso) <= high_iso: 342 req = capture_request_utils.manual_capture_request( 343 round(iso), min_exposure_ns 344 ) 345 req['android.lens.focusDistance'] = min_fd 346 cap = cam.do_capture(req, fmt) 347 iso_cap = cap['metadata']['android.sensor.sensitivity'] 348 349 # Different iso values may result in captures with the same iso_cap value, 350 # so skip this capture if it's redundant. 351 if iso_cap == pre_iso_cap: 352 logging.info( 353 'Skip current capture because of the same iso %d with the previous' 354 ' capture.', 355 iso_cap, 356 ) 357 iso = noise_model_utils.get_next_iso(iso, high_iso, iso_multiplier) 358 continue 359 360 pre_iso_cap = iso_cap 361 w = cap['width'] 362 h = cap['height'] 363 364 if raw_format in ('raw10', 'raw10QuadBayer'): 365 img = image_processing_utils.unpack_raw10_image( 366 cap['data'].reshape(h, w * 5 // 4) 367 ) 368 elif raw_format in ('raw', 'rawQuadBayer'): 369 img = np.ndarray( 370 shape=(h * w,), dtype='<u2', buffer=cap['data'][0: w * h * 2] 371 ) 372 img = img.astype(dtype=np.uint16).reshape(h, w) 373 374 # Add reordered read noise stats to read noise stats list. 375 stats = _generate_read_noise_stats(img, iso_cap, white_level, cfa_order) 376 stats_list.append(stats) 377 378 logging.info('iso: %.2f, mean: %.2f, var: %.2f, min: %d, max: %d', iso_cap, 379 np.mean(img), np.var(img), np.min(img), np.max(img)) 380 381 with open(dest_file, 'wb+') as f: 382 pickle.dump(stats_list, f) 383 384 iso = noise_model_utils.get_next_iso(iso, high_iso, iso_multiplier) 385 386 logging.info('Read noise stats pickled into file %s.', dest_file) 387 388 return stats_list 389 390 391def calibrate_read_noise( 392 device_id: str, 393 camera_id: str, 394 hidden_physical_id: str, 395 read_noise_folder_prefix: str, 396 read_noise_file_name: str, 397 steps_per_stop: int, 398 raw_format: str = 'raw', 399 is_two_stage_model: bool = False, 400) -> str: 401 """Calibrates the read noise of the camera. 402 403 Read noise is a type of noise that occurs in digital cameras when the image 404 sensor converts light to an electronic signal. Calibrating read noise is the 405 first step in the 2-stage noise model calibration. 406 407 Args: 408 device_id: The device ID of the camera. 409 camera_id: The camera ID of the camera. 410 hidden_physical_id: The hidden physical ID of the camera. 411 read_noise_folder_prefix: The prefix of the read noise folder. 412 read_noise_file_name: The name of the read noise file. 413 steps_per_stop: The number of steps per stop. 414 raw_format: The format of raw capture, which can be one of raw, raw10, 415 rawQuadBayer and raw10QuadBayer. 416 is_two_stage_model: A boolean flag indicating if the noise model is 417 calibrated in the two-stage mode. 418 419 Returns: 420 The path to the read noise file. 421 """ 422 if not is_two_stage_model: 423 return '' 424 # If two-stage model is enabled, check/collect read noise data. 425 with its_session_utils.ItsSession( 426 device_id=device_id, 427 camera_id=camera_id, 428 hidden_physical_id=hidden_physical_id, 429 ) as cam: 430 props = cam.get_camera_properties() 431 props = cam.override_with_hidden_physical_camera_props(props) 432 433 # Get sensor analog ISO range. 434 sens_min, _ = props['android.sensor.info.sensitivityRange'] 435 sens_max_analog = props['android.sensor.maxAnalogSensitivity'] 436 # Maximum sensitivity for measuring noise model. 437 sens_max_meas = sens_max_analog 438 439 # Prepare read noise folder. 440 camera_name = cam.get_camera_name() 441 read_noise_folder = os.path.join( 442 read_noise_folder_prefix, device_id.replace(':', '_'), camera_name 443 ) 444 read_noise_file_path = os.path.join(read_noise_folder, read_noise_file_name) 445 if not os.path.exists(read_noise_folder): 446 os.makedirs(read_noise_folder) 447 logging.info('Read noise data folder: %s', read_noise_folder) 448 449 # Collect or retrieve read noise data. 450 if not os.path.isfile(read_noise_file_path): 451 logging.info('Collecting read noise data for %s', camera_name) 452 # Read noise data file does not exist, collect read noise data. 453 _capture_read_noise_for_iso_range( 454 cam, 455 raw_format, 456 sens_min, 457 sens_max_meas, 458 steps_per_stop, 459 read_noise_file_path, 460 ) 461 else: 462 # If data exists, check if it covers the full range. 463 with open(read_noise_file_path, 'rb') as f: 464 read_noise_data = pickle.load(f) 465 # The +5 offset takes write to read error into account. 466 if read_noise_data[-1][0]['iso'] + 5 < sens_max_meas: 467 logging.error( 468 ( 469 '\nNot enough ISO data points exist. ' 470 '\nMax ISO measured: %.2f' 471 '\nMax ISO possible: %.2f' 472 ), 473 read_noise_data[-1][0]['iso'], 474 sens_max_meas, 475 ) 476 # Not all data points were captured, continue capture. 477 _capture_read_noise_for_iso_range( 478 cam, 479 raw_format, 480 sens_min, 481 sens_max_meas, 482 steps_per_stop, 483 read_noise_file_path, 484 ) 485 486 return read_noise_file_path 487