1# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. 2# 3# Use of this source code is governed by a BSD-style license 4# that can be found in the LICENSE file in the root of the source 5# tree. An additional intellectual property rights grant can be found 6# in the file PATENTS. All contributing project authors may 7# be found in the AUTHORS file in the root of the source tree. 8"""APM module simulator. 9""" 10 11import logging 12import os 13 14from . import annotations 15from . import data_access 16from . import echo_path_simulation 17from . import echo_path_simulation_factory 18from . import eval_scores 19from . import exceptions 20from . import input_mixer 21from . import input_signal_creator 22from . import signal_processing 23from . import test_data_generation 24 25 26class ApmModuleSimulator(object): 27 """Audio processing module (APM) simulator class. 28 """ 29 30 _TEST_DATA_GENERATOR_CLASSES = ( 31 test_data_generation.TestDataGenerator.REGISTERED_CLASSES) 32 _EVAL_SCORE_WORKER_CLASSES = eval_scores.EvaluationScore.REGISTERED_CLASSES 33 34 _PREFIX_APM_CONFIG = 'apmcfg-' 35 _PREFIX_CAPTURE = 'capture-' 36 _PREFIX_RENDER = 'render-' 37 _PREFIX_ECHO_SIMULATOR = 'echosim-' 38 _PREFIX_TEST_DATA_GEN = 'datagen-' 39 _PREFIX_TEST_DATA_GEN_PARAMS = 'datagen_params-' 40 _PREFIX_SCORE = 'score-' 41 42 def __init__(self, 43 test_data_generator_factory, 44 evaluation_score_factory, 45 ap_wrapper, 46 evaluator, 47 external_vads=None): 48 if external_vads is None: 49 external_vads = {} 50 self._test_data_generator_factory = test_data_generator_factory 51 self._evaluation_score_factory = evaluation_score_factory 52 self._audioproc_wrapper = ap_wrapper 53 self._evaluator = evaluator 54 self._annotator = annotations.AudioAnnotationsExtractor( 55 annotations.AudioAnnotationsExtractor.VadType.ENERGY_THRESHOLD 56 | annotations.AudioAnnotationsExtractor.VadType.WEBRTC_COMMON_AUDIO 57 | annotations.AudioAnnotationsExtractor.VadType.WEBRTC_APM, 58 external_vads) 59 60 # Init. 61 self._test_data_generator_factory.SetOutputDirectoryPrefix( 62 self._PREFIX_TEST_DATA_GEN_PARAMS) 63 self._evaluation_score_factory.SetScoreFilenamePrefix( 64 self._PREFIX_SCORE) 65 66 # Properties for each run. 67 self._base_output_path = None 68 self._output_cache_path = None 69 self._test_data_generators = None 70 self._evaluation_score_workers = None 71 self._config_filepaths = None 72 self._capture_input_filepaths = None 73 self._render_input_filepaths = None 74 self._echo_path_simulator_class = None 75 76 @classmethod 77 def GetPrefixApmConfig(cls): 78 return cls._PREFIX_APM_CONFIG 79 80 @classmethod 81 def GetPrefixCapture(cls): 82 return cls._PREFIX_CAPTURE 83 84 @classmethod 85 def GetPrefixRender(cls): 86 return cls._PREFIX_RENDER 87 88 @classmethod 89 def GetPrefixEchoSimulator(cls): 90 return cls._PREFIX_ECHO_SIMULATOR 91 92 @classmethod 93 def GetPrefixTestDataGenerator(cls): 94 return cls._PREFIX_TEST_DATA_GEN 95 96 @classmethod 97 def GetPrefixTestDataGeneratorParameters(cls): 98 return cls._PREFIX_TEST_DATA_GEN_PARAMS 99 100 @classmethod 101 def GetPrefixScore(cls): 102 return cls._PREFIX_SCORE 103 104 def Run(self, 105 config_filepaths, 106 capture_input_filepaths, 107 test_data_generator_names, 108 eval_score_names, 109 output_dir, 110 render_input_filepaths=None, 111 echo_path_simulator_name=( 112 echo_path_simulation.NoEchoPathSimulator.NAME)): 113 """Runs the APM simulation. 114 115 Initializes paths and required instances, then runs all the simulations. 116 The render input can be optionally added. If added, the number of capture 117 input audio tracks and the number of render input audio tracks have to be 118 equal. The two lists are used to form pairs of capture and render input. 119 120 Args: 121 config_filepaths: set of APM configuration files to test. 122 capture_input_filepaths: set of capture input audio track files to test. 123 test_data_generator_names: set of test data generator names to test. 124 eval_score_names: set of evaluation score names to test. 125 output_dir: base path to the output directory for wav files and outcomes. 126 render_input_filepaths: set of render input audio track files to test. 127 echo_path_simulator_name: name of the echo path simulator to use when 128 render input is provided. 129 """ 130 assert render_input_filepaths is None or ( 131 len(capture_input_filepaths) == len(render_input_filepaths)), ( 132 'render input set size not matching input set size') 133 assert render_input_filepaths is None or echo_path_simulator_name in ( 134 echo_path_simulation.EchoPathSimulator.REGISTERED_CLASSES), ( 135 'invalid echo path simulator') 136 self._base_output_path = os.path.abspath(output_dir) 137 138 # Output path used to cache the data shared across simulations. 139 self._output_cache_path = os.path.join(self._base_output_path, 140 '_cache') 141 142 # Instance test data generators. 143 self._test_data_generators = [ 144 self._test_data_generator_factory.GetInstance( 145 test_data_generators_class=( 146 self._TEST_DATA_GENERATOR_CLASSES[name])) 147 for name in (test_data_generator_names) 148 ] 149 150 # Instance evaluation score workers. 151 self._evaluation_score_workers = [ 152 self._evaluation_score_factory.GetInstance( 153 evaluation_score_class=self._EVAL_SCORE_WORKER_CLASSES[name]) 154 for (name) in eval_score_names 155 ] 156 157 # Set APM configuration file paths. 158 self._config_filepaths = self._CreatePathsCollection(config_filepaths) 159 160 # Set probing signal file paths. 161 if render_input_filepaths is None: 162 # Capture input only. 163 self._capture_input_filepaths = self._CreatePathsCollection( 164 capture_input_filepaths) 165 self._render_input_filepaths = None 166 else: 167 # Set both capture and render input signals. 168 self._SetTestInputSignalFilePaths(capture_input_filepaths, 169 render_input_filepaths) 170 171 # Set the echo path simulator class. 172 self._echo_path_simulator_class = ( 173 echo_path_simulation.EchoPathSimulator. 174 REGISTERED_CLASSES[echo_path_simulator_name]) 175 176 self._SimulateAll() 177 178 def _SimulateAll(self): 179 """Runs all the simulations. 180 181 Iterates over the combinations of APM configurations, probing signals, and 182 test data generators. This method is mainly responsible for the creation of 183 the cache and output directories required in order to call _Simulate(). 184 """ 185 without_render_input = self._render_input_filepaths is None 186 187 # Try different APM config files. 188 for config_name in self._config_filepaths: 189 config_filepath = self._config_filepaths[config_name] 190 191 # Try different capture-render pairs. 192 for capture_input_name in self._capture_input_filepaths: 193 # Output path for the capture signal annotations. 194 capture_annotations_cache_path = os.path.join( 195 self._output_cache_path, 196 self._PREFIX_CAPTURE + capture_input_name) 197 data_access.MakeDirectory(capture_annotations_cache_path) 198 199 # Capture. 200 capture_input_filepath = self._capture_input_filepaths[ 201 capture_input_name] 202 if not os.path.exists(capture_input_filepath): 203 # If the input signal file does not exist, try to create using the 204 # available input signal creators. 205 self._CreateInputSignal(capture_input_filepath) 206 assert os.path.exists(capture_input_filepath) 207 self._ExtractCaptureAnnotations( 208 capture_input_filepath, capture_annotations_cache_path) 209 210 # Render and simulated echo path (optional). 211 render_input_filepath = None if without_render_input else ( 212 self._render_input_filepaths[capture_input_name]) 213 render_input_name = '(none)' if without_render_input else ( 214 self._ExtractFileName(render_input_filepath)) 215 echo_path_simulator = (echo_path_simulation_factory. 216 EchoPathSimulatorFactory.GetInstance( 217 self._echo_path_simulator_class, 218 render_input_filepath)) 219 220 # Try different test data generators. 221 for test_data_generators in self._test_data_generators: 222 logging.info( 223 'APM config preset: <%s>, capture: <%s>, render: <%s>,' 224 'test data generator: <%s>, echo simulator: <%s>', 225 config_name, capture_input_name, render_input_name, 226 test_data_generators.NAME, echo_path_simulator.NAME) 227 228 # Output path for the generated test data. 229 test_data_cache_path = os.path.join( 230 capture_annotations_cache_path, 231 self._PREFIX_TEST_DATA_GEN + test_data_generators.NAME) 232 data_access.MakeDirectory(test_data_cache_path) 233 logging.debug('test data cache path: <%s>', 234 test_data_cache_path) 235 236 # Output path for the echo simulator and APM input mixer output. 237 echo_test_data_cache_path = os.path.join( 238 test_data_cache_path, 239 'echosim-{}'.format(echo_path_simulator.NAME)) 240 data_access.MakeDirectory(echo_test_data_cache_path) 241 logging.debug('echo test data cache path: <%s>', 242 echo_test_data_cache_path) 243 244 # Full output path. 245 output_path = os.path.join( 246 self._base_output_path, 247 self._PREFIX_APM_CONFIG + config_name, 248 self._PREFIX_CAPTURE + capture_input_name, 249 self._PREFIX_RENDER + render_input_name, 250 self._PREFIX_ECHO_SIMULATOR + echo_path_simulator.NAME, 251 self._PREFIX_TEST_DATA_GEN + test_data_generators.NAME) 252 data_access.MakeDirectory(output_path) 253 logging.debug('output path: <%s>', output_path) 254 255 self._Simulate(test_data_generators, 256 capture_input_filepath, 257 render_input_filepath, test_data_cache_path, 258 echo_test_data_cache_path, output_path, 259 config_filepath, echo_path_simulator) 260 261 @staticmethod 262 def _CreateInputSignal(input_signal_filepath): 263 """Creates a missing input signal file. 264 265 The file name is parsed to extract input signal creator and params. If a 266 creator is matched and the parameters are valid, a new signal is generated 267 and written in `input_signal_filepath`. 268 269 Args: 270 input_signal_filepath: Path to the input signal audio file to write. 271 272 Raises: 273 InputSignalCreatorException 274 """ 275 filename = os.path.splitext( 276 os.path.split(input_signal_filepath)[-1])[0] 277 filename_parts = filename.split('-') 278 279 if len(filename_parts) < 2: 280 raise exceptions.InputSignalCreatorException( 281 'Cannot parse input signal file name') 282 283 signal, metadata = input_signal_creator.InputSignalCreator.Create( 284 filename_parts[0], filename_parts[1].split('_')) 285 286 signal_processing.SignalProcessingUtils.SaveWav( 287 input_signal_filepath, signal) 288 data_access.Metadata.SaveFileMetadata(input_signal_filepath, metadata) 289 290 def _ExtractCaptureAnnotations(self, 291 input_filepath, 292 output_path, 293 annotation_name=""): 294 self._annotator.Extract(input_filepath) 295 self._annotator.Save(output_path, annotation_name) 296 297 def _Simulate(self, test_data_generators, clean_capture_input_filepath, 298 render_input_filepath, test_data_cache_path, 299 echo_test_data_cache_path, output_path, config_filepath, 300 echo_path_simulator): 301 """Runs a single set of simulation. 302 303 Simulates a given combination of APM configuration, probing signal, and 304 test data generator. It iterates over the test data generator 305 internal configurations. 306 307 Args: 308 test_data_generators: TestDataGenerator instance. 309 clean_capture_input_filepath: capture input audio track file to be 310 processed by a test data generator and 311 not affected by echo. 312 render_input_filepath: render input audio track file to test. 313 test_data_cache_path: path for the generated test audio track files. 314 echo_test_data_cache_path: path for the echo simulator. 315 output_path: base output path for the test data generator. 316 config_filepath: APM configuration file to test. 317 echo_path_simulator: EchoPathSimulator instance. 318 """ 319 # Generate pairs of noisy input and reference signal files. 320 test_data_generators.Generate( 321 input_signal_filepath=clean_capture_input_filepath, 322 test_data_cache_path=test_data_cache_path, 323 base_output_path=output_path) 324 325 # Extract metadata linked to the clean input file (if any). 326 apm_input_metadata = None 327 try: 328 apm_input_metadata = data_access.Metadata.LoadFileMetadata( 329 clean_capture_input_filepath) 330 except IOError as e: 331 apm_input_metadata = {} 332 apm_input_metadata['test_data_gen_name'] = test_data_generators.NAME 333 apm_input_metadata['test_data_gen_config'] = None 334 335 # For each test data pair, simulate a call and evaluate. 336 for config_name in test_data_generators.config_names: 337 logging.info(' - test data generator config: <%s>', config_name) 338 apm_input_metadata['test_data_gen_config'] = config_name 339 340 # Paths to the test data generator output. 341 # Note that the reference signal does not depend on the render input 342 # which is optional. 343 noisy_capture_input_filepath = ( 344 test_data_generators.noisy_signal_filepaths[config_name]) 345 reference_signal_filepath = ( 346 test_data_generators.reference_signal_filepaths[config_name]) 347 348 # Output path for the evaluation (e.g., APM output file). 349 evaluation_output_path = test_data_generators.apm_output_paths[ 350 config_name] 351 352 # Paths to the APM input signals. 353 echo_path_filepath = echo_path_simulator.Simulate( 354 echo_test_data_cache_path) 355 apm_input_filepath = input_mixer.ApmInputMixer.Mix( 356 echo_test_data_cache_path, noisy_capture_input_filepath, 357 echo_path_filepath) 358 359 # Extract annotations for the APM input mix. 360 apm_input_basepath, apm_input_filename = os.path.split( 361 apm_input_filepath) 362 self._ExtractCaptureAnnotations( 363 apm_input_filepath, apm_input_basepath, 364 os.path.splitext(apm_input_filename)[0] + '-') 365 366 # Simulate a call using APM. 367 self._audioproc_wrapper.Run( 368 config_filepath=config_filepath, 369 capture_input_filepath=apm_input_filepath, 370 render_input_filepath=render_input_filepath, 371 output_path=evaluation_output_path) 372 373 try: 374 # Evaluate. 375 self._evaluator.Run( 376 evaluation_score_workers=self._evaluation_score_workers, 377 apm_input_metadata=apm_input_metadata, 378 apm_output_filepath=self._audioproc_wrapper. 379 output_filepath, 380 reference_input_filepath=reference_signal_filepath, 381 render_input_filepath=render_input_filepath, 382 output_path=evaluation_output_path, 383 ) 384 385 # Save simulation metadata. 386 data_access.Metadata.SaveAudioTestDataPaths( 387 output_path=evaluation_output_path, 388 clean_capture_input_filepath=clean_capture_input_filepath, 389 echo_free_capture_filepath=noisy_capture_input_filepath, 390 echo_filepath=echo_path_filepath, 391 render_filepath=render_input_filepath, 392 capture_filepath=apm_input_filepath, 393 apm_output_filepath=self._audioproc_wrapper. 394 output_filepath, 395 apm_reference_filepath=reference_signal_filepath, 396 apm_config_filepath=config_filepath, 397 ) 398 except exceptions.EvaluationScoreException as e: 399 logging.warning('the evaluation failed: %s', e.message) 400 continue 401 402 def _SetTestInputSignalFilePaths(self, capture_input_filepaths, 403 render_input_filepaths): 404 """Sets input and render input file paths collections. 405 406 Pairs the input and render input files by storing the file paths into two 407 collections. The key is the file name of the input file. 408 409 Args: 410 capture_input_filepaths: list of file paths. 411 render_input_filepaths: list of file paths. 412 """ 413 self._capture_input_filepaths = {} 414 self._render_input_filepaths = {} 415 assert len(capture_input_filepaths) == len(render_input_filepaths) 416 for capture_input_filepath, render_input_filepath in zip( 417 capture_input_filepaths, render_input_filepaths): 418 name = self._ExtractFileName(capture_input_filepath) 419 self._capture_input_filepaths[name] = os.path.abspath( 420 capture_input_filepath) 421 self._render_input_filepaths[name] = os.path.abspath( 422 render_input_filepath) 423 424 @classmethod 425 def _CreatePathsCollection(cls, filepaths): 426 """Creates a collection of file paths. 427 428 Given a list of file paths, makes a collection with one item for each file 429 path. The value is absolute path, the key is the file name without 430 extenstion. 431 432 Args: 433 filepaths: list of file paths. 434 435 Returns: 436 A dict. 437 """ 438 filepaths_collection = {} 439 for filepath in filepaths: 440 name = cls._ExtractFileName(filepath) 441 filepaths_collection[name] = os.path.abspath(filepath) 442 return filepaths_collection 443 444 @classmethod 445 def _ExtractFileName(cls, filepath): 446 return os.path.splitext(os.path.split(filepath)[-1])[0] 447