1#!/usr/bin/env python 2# Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. 3# 4# Use of this source code is governed by a BSD-style license 5# that can be found in the LICENSE file in the root of the source 6# tree. An additional intellectual property rights grant can be found 7# in the file PATENTS. All contributing project authors may 8# be found in the AUTHORS file in the root of the source tree. 9 10from __future__ import absolute_import 11from __future__ import division 12from __future__ import print_function 13import json 14import optparse 15import os 16import shutil 17import subprocess 18import sys 19import tempfile 20 21SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 22 23# Chrome browsertests will throw away stderr; avoid that output gets lost. 24sys.stderr = sys.stdout 25 26 27def _ParseArgs(): 28 """Registers the command-line options.""" 29 usage = 'usage: %prog [options]' 30 parser = optparse.OptionParser(usage=usage) 31 32 parser.add_option('--label', 33 type='string', 34 default='MY_TEST', 35 help=('Label of the test, used to identify different ' 36 'tests. Default: %default')) 37 parser.add_option('--ref_video', 38 type='string', 39 help='Reference video to compare with (YUV).') 40 parser.add_option('--test_video', 41 type='string', 42 help=('Test video to be compared with the reference ' 43 'video (YUV).')) 44 parser.add_option('--frame_analyzer', 45 type='string', 46 help='Path to the frame analyzer executable.') 47 parser.add_option('--aligned_output_file', 48 type='string', 49 help='Path for output aligned YUV or Y4M file.') 50 parser.add_option('--vmaf', type='string', help='Path to VMAF executable.') 51 parser.add_option('--vmaf_model', 52 type='string', 53 help='Path to VMAF model.') 54 parser.add_option('--vmaf_phone_model', 55 action='store_true', 56 help='Whether to use phone model in VMAF.') 57 parser.add_option( 58 '--yuv_frame_width', 59 type='int', 60 default=640, 61 help='Width of the YUV file\'s frames. Default: %default') 62 parser.add_option( 63 '--yuv_frame_height', 64 type='int', 65 default=480, 66 help='Height of the YUV file\'s frames. Default: %default') 67 parser.add_option('--chartjson_result_file', 68 type='str', 69 default=None, 70 help='Where to store perf results in chartjson format.') 71 options, _ = parser.parse_args() 72 73 if not options.ref_video: 74 parser.error('You must provide a path to the reference video!') 75 if not os.path.exists(options.ref_video): 76 parser.error('Cannot find the reference video at %s' % 77 options.ref_video) 78 79 if not options.test_video: 80 parser.error('You must provide a path to the test video!') 81 if not os.path.exists(options.test_video): 82 parser.error('Cannot find the test video at %s' % options.test_video) 83 84 if not options.frame_analyzer: 85 parser.error( 86 'You must provide the path to the frame analyzer executable!') 87 if not os.path.exists(options.frame_analyzer): 88 parser.error('Cannot find frame analyzer executable at %s!' % 89 options.frame_analyzer) 90 91 if options.vmaf and not options.vmaf_model: 92 parser.error('You must provide a path to a VMAF model to use VMAF.') 93 94 return options 95 96 97def _DevNull(): 98 """On Windows, sometimes the inherited stdin handle from the parent process 99 fails. Workaround this by passing null to stdin to the subprocesses commands. 100 This function can be used to create the null file handler. 101 """ 102 return open(os.devnull, 'r') 103 104 105def _RunFrameAnalyzer(options, yuv_directory=None): 106 """Run frame analyzer to compare the videos and print output.""" 107 cmd = [ 108 options.frame_analyzer, 109 '--label=%s' % options.label, 110 '--reference_file=%s' % options.ref_video, 111 '--test_file=%s' % options.test_video, 112 '--width=%d' % options.yuv_frame_width, 113 '--height=%d' % options.yuv_frame_height, 114 ] 115 if options.chartjson_result_file: 116 cmd.append('--chartjson_result_file=%s' % 117 options.chartjson_result_file) 118 if options.aligned_output_file: 119 cmd.append('--aligned_output_file=%s' % options.aligned_output_file) 120 if yuv_directory: 121 cmd.append('--yuv_directory=%s' % yuv_directory) 122 frame_analyzer = subprocess.Popen(cmd, 123 stdin=_DevNull(), 124 stdout=sys.stdout, 125 stderr=sys.stderr) 126 frame_analyzer.wait() 127 if frame_analyzer.returncode != 0: 128 print('Failed to run frame analyzer.') 129 return frame_analyzer.returncode 130 131 132def _RunVmaf(options, yuv_directory, logfile): 133 """ Run VMAF to compare videos and print output. 134 135 The yuv_directory is assumed to have been populated with a reference and test 136 video in .yuv format, with names according to the label. 137 """ 138 cmd = [ 139 options.vmaf, 140 'yuv420p', 141 str(options.yuv_frame_width), 142 str(options.yuv_frame_height), 143 os.path.join(yuv_directory, "ref.yuv"), 144 os.path.join(yuv_directory, "test.yuv"), 145 options.vmaf_model, 146 '--log', 147 logfile, 148 '--log-fmt', 149 'json', 150 ] 151 if options.vmaf_phone_model: 152 cmd.append('--phone-model') 153 154 vmaf = subprocess.Popen(cmd, 155 stdin=_DevNull(), 156 stdout=sys.stdout, 157 stderr=sys.stderr) 158 vmaf.wait() 159 if vmaf.returncode != 0: 160 print('Failed to run VMAF.') 161 return 1 162 163 # Read per-frame scores from VMAF output and print. 164 with open(logfile) as f: 165 vmaf_data = json.load(f) 166 vmaf_scores = [] 167 for frame in vmaf_data['frames']: 168 vmaf_scores.append(frame['metrics']['vmaf']) 169 print('RESULT VMAF: %s=' % options.label, vmaf_scores) 170 171 return 0 172 173 174def main(): 175 """The main function. 176 177 A simple invocation is: 178 ./webrtc/rtc_tools/compare_videos.py 179 --ref_video=<path_and_name_of_reference_video> 180 --test_video=<path_and_name_of_test_video> 181 --frame_analyzer=<path_and_name_of_the_frame_analyzer_executable> 182 183 Running vmaf requires the following arguments: 184 --vmaf, --vmaf_model, --yuv_frame_width, --yuv_frame_height 185 """ 186 options = _ParseArgs() 187 188 if options.vmaf: 189 try: 190 # Directory to save temporary YUV files for VMAF in frame_analyzer. 191 yuv_directory = tempfile.mkdtemp() 192 _, vmaf_logfile = tempfile.mkstemp() 193 194 # Run frame analyzer to compare the videos and print output. 195 if _RunFrameAnalyzer(options, yuv_directory=yuv_directory) != 0: 196 return 1 197 198 # Run VMAF for further video comparison and print output. 199 return _RunVmaf(options, yuv_directory, vmaf_logfile) 200 finally: 201 shutil.rmtree(yuv_directory) 202 os.remove(vmaf_logfile) 203 else: 204 return _RunFrameAnalyzer(options) 205 206 return 0 207 208 209if __name__ == '__main__': 210 sys.exit(main()) 211