1 /*
2  * Copyright (C) 2023 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.media.videoquality.bdrate;
18 
19 import com.google.common.annotations.VisibleForTesting;
20 import com.google.common.base.Preconditions;
21 import com.google.gson.Gson;
22 import com.google.gson.GsonBuilder;
23 import com.google.gson.JsonParseException;
24 import com.google.gson.reflect.TypeToken;
25 
26 import org.kohsuke.args4j.CmdLineException;
27 import org.kohsuke.args4j.CmdLineParser;
28 import org.kohsuke.args4j.Option;
29 
30 import java.io.BufferedReader;
31 import java.io.IOException;
32 import java.nio.charset.StandardCharsets;
33 import java.nio.file.Files;
34 import java.nio.file.Path;
35 import java.nio.file.Paths;
36 import java.text.DecimalFormat;
37 import java.text.NumberFormat;
38 import java.util.ArrayList;
39 import java.util.Arrays;
40 import java.util.Iterator;
41 import java.util.List;
42 import java.util.logging.ConsoleHandler;
43 import java.util.logging.FileHandler;
44 import java.util.logging.Formatter;
45 import java.util.logging.Level;
46 import java.util.logging.LogRecord;
47 import java.util.logging.Logger;
48 import java.util.logging.SimpleFormatter;
49 
50 /**
51  * Class for calculating BD-RATE as part of the Performance Class - Video Encoding Quality CTS
52  * test.
53  *
54  * verifyBdRate() method returns one of the following exit-codes:
55  *
56  * <ul>
57  *   <li>0 - The VEQ test has passed and the BD-RATE was within the threshold defined by the
58  *       reference configuration.
59  *   <li>1 - The configuration files could not be loaded and thus, BD-RATE could not be calculated.
60  *   <li>2 - BD-RATE could not be calculated because one of the required conditions for calculation
61  *       was not met.
62  *   <li>3 - The VEQ test has failed due to the calculated BD-RATE being greater than the allowed
63  *       threshold defined by the reference configuration.
64  * </ul>
65  */
66 public class BdRateMain {
67     private static final Logger LOGGER = Logger.getLogger(BdRateMain.class.getName());
68 
69     private static final NumberFormat NUMBER_FORMAT = new DecimalFormat("0.00");
70     private final Gson mGson;
71     private final BdRateCalculator mBdRateCalculator;
72     private final BdQualityCalculator mBdQualityCalculator;
73 
74     private static Path mRefJsonFile;
75     private static Path mTestVmafFile;
76 
77     private static Formatter mFormatter = new SimpleFormatter() {
78         @Override
79         public String format(LogRecord record) {
80             return record.getMessage() + "\n";
81         }
82     };
83 
84     private enum Result {
85         SUCCESS, INVALID_ARGS, INVALID_DATA, FAILED
86     }
87 
BdRateMain( Gson gson, BdRateCalculator bdRateCalculator, BdQualityCalculator bdQualityCalculator)88     public BdRateMain(
89             Gson gson, BdRateCalculator bdRateCalculator, BdQualityCalculator bdQualityCalculator) {
90         mGson = gson;
91         mBdRateCalculator = bdRateCalculator;
92         mBdQualityCalculator = bdQualityCalculator;
93     }
94 
run()95     public Result run() {
96         LOGGER.info(String.format("Running cts-media-videoquality-bdrate library"));
97         LOGGER.info(String.format("Reading reference configuration JSON file: %s", mRefJsonFile));
98         ReferenceConfig refConfig = null;
99         try {
100             refConfig = loadReferenceConfig(mRefJsonFile, mGson);
101         } catch (IOException | JsonParseException e) {
102             LOGGER.log(Level.SEVERE,
103                     "Invalid Arguments: Failed to load reference configuration file!", e);
104             return Result.INVALID_ARGS;
105         }
106 
107         LOGGER.info(String.format("Reading test result text file: %s", mTestVmafFile));
108         VeqTestResult veqTestResult = null;
109         try {
110             veqTestResult = loadTestResult(mTestVmafFile);
111         } catch (IOException | IllegalArgumentException e) {
112             LOGGER.log(Level.SEVERE, "Invalid Arguments: Failed to load VEQ Test Result file!", e);
113             return Result.INVALID_ARGS;
114         }
115 
116         if (!veqTestResult.referenceFile().equals(refConfig.referenceFile())) {
117             LOGGER.log(Level.SEVERE,
118                     "Test Result file and Reference JSON file are not for the same reference file"
119                             + ".");
120             return Result.INVALID_ARGS;
121         }
122 
123         logCurves(
124                 "Successfully loaded rate-distortion data: ",
125                 refConfig.referenceCurve(),
126                 veqTestResult.curve());
127         LOGGER.info(
128                 String.format(
129                         "Checking Video Encoding Quality (VEQ) for %s", refConfig.referenceFile()));
130 
131         return checkVeq(
132                 mBdRateCalculator,
133                 mBdQualityCalculator,
134                 refConfig.referenceCurve(),
135                 veqTestResult.curve(),
136                 refConfig.referenceThreshold());
137     }
138 
139     /**
140      * Checks the video encoding quality of the target curve against the reference curve using
141      * Bjontegaard-Delta (BD) values.
142      */
143     @VisibleForTesting
checkVeq( BdRateCalculator bdRateCalculator, BdQualityCalculator bdQualityCalculator, RateDistortionCurve baseline, RateDistortionCurve target, double threshold)144     static Result checkVeq(
145             BdRateCalculator bdRateCalculator,
146             BdQualityCalculator bdQualityCalculator,
147             RateDistortionCurve baseline,
148             RateDistortionCurve target,
149             double threshold) {
150         RateDistortionCurvePair curvePair =
151                 RateDistortionCurvePair.createClusteredPair(baseline, target);
152 
153         if (curvePair.canCalculateBdRate()) {
154             LOGGER.info("Calculating BD-RATE...");
155 
156             double bdRateResult = bdRateCalculator.calculate(curvePair);
157             LOGGER.info(
158                     String.format("BD-RATE: %.04f (%.02f%%)", bdRateResult, bdRateResult * 100));
159 
160             if (bdRateResult > threshold) {
161                 LOGGER.log(Level.SEVERE, String.format(
162                         "Failed Video Encoding Quality (VEQ) test, calculated BD-RATE was (%.04f)"
163                                 + " which was greater than the test-defined threshold of (%.04f)",
164                         bdRateResult, threshold));
165                 return Result.FAILED;
166             }
167         } else if (curvePair.canCalculateBdQuality()) {
168             LOGGER.warning("Unable to calculate BD-RATE, falling back to checking BD-QUALITY...");
169 
170             double bdQualityResult = bdQualityCalculator.calculate(curvePair);
171             LOGGER.info(String.format("BD-QUALITY: %.02f", bdQualityResult));
172 
173             double percentageQualityChange =
174                     bdQualityResult
175                             / Arrays.stream(curvePair.baseline().getDistortionsArray())
176                             .average()
177                             .getAsDouble();
178 
179             // Since distortion is measured as a higher == better value, invert
180             // the percentage so that it can be compared equivalently with the threshold.
181             if (percentageQualityChange * -1 > threshold) {
182                 LOGGER.log(Level.SEVERE, String.format(
183                         "Failed Video Encoding Quality (VEQ) test, calculated BD-Quality was (%"
184                                 + ".04f) which was greater than the test-defined threshold of (%"
185                                 + ".04f)",
186                         bdQualityResult, threshold));
187                 return Result.FAILED;
188             }
189         } else {
190             LOGGER.log(Level.SEVERE,
191                     "Cannot calculate BD-RATE or BD-QUALITY. Reference configuration likely does "
192                             + "not match the test result data.");
193             return Result.INVALID_DATA;
194         }
195         return Result.SUCCESS;
196     }
197 
logCurves( String message, RateDistortionCurve referenceCurve, RateDistortionCurve targetCurve)198     private static void logCurves(
199             String message, RateDistortionCurve referenceCurve, RateDistortionCurve targetCurve) {
200         ArrayList<String> rows = new ArrayList<>();
201         rows.add(message);
202         rows.add(
203                 String.format(
204                         "|%15s|%15s|%15s|%15s|",
205                         "Reference Rate", "Reference Dist", "Target Rate", "Target Dist"));
206         rows.add("=".repeat(rows.get(1).length()));
207 
208         Iterator<RateDistortionPoint> referencePoints = referenceCurve.points().iterator();
209         Iterator<RateDistortionPoint> targetPoints = targetCurve.points().iterator();
210 
211         while (referencePoints.hasNext() || targetPoints.hasNext()) {
212             String refRate = "";
213             String refDist = "";
214             if (referencePoints.hasNext()) {
215                 RateDistortionPoint refPoint = referencePoints.next();
216                 refRate = NUMBER_FORMAT.format(refPoint.rate());
217                 refDist = NUMBER_FORMAT.format(refPoint.distortion());
218             }
219 
220             String targetRate = "";
221             String targetDist = "";
222             if (targetPoints.hasNext()) {
223                 RateDistortionPoint targetPoint = targetPoints.next();
224                 targetRate = NUMBER_FORMAT.format(targetPoint.rate());
225                 targetDist = NUMBER_FORMAT.format(targetPoint.distortion());
226             }
227 
228             rows.add(
229                     String.format(
230                             "|%15s|%15s|%15s|%15s|", refRate, refDist, targetRate, targetDist));
231         }
232 
233         LOGGER.info(String.join("\n", rows));
234     }
235 
loadReferenceConfig(Path path, Gson gson)236     private static ReferenceConfig loadReferenceConfig(Path path, Gson gson) throws IOException {
237         Preconditions.checkArgument(Files.exists(path));
238 
239         // Each config file contains a single ReferenceConfig in a list,
240         // the first one is returned here.
241         try (BufferedReader reader = Files.newBufferedReader(path)) {
242             TypeToken<List<ReferenceConfig>> configsType = new TypeToken<>() {};
243             ArrayList<ReferenceConfig> configs = gson.fromJson(reader, configsType.getType());
244             return configs.get(0);
245         }
246     }
247 
loadTestResult(Path path)248     private static VeqTestResult loadTestResult(Path path) throws IOException {
249         Preconditions.checkState(Files.exists(path));
250 
251         String testResult = new String(Files.readAllBytes(path), StandardCharsets.UTF_8);
252         return VeqTestResult.parseFromTestResult(testResult);
253     }
254 
verifyBdRate(String refJsonFilePath, String testVmafFilePath, String resultFilePath)255     public static int verifyBdRate(String refJsonFilePath, String testVmafFilePath,
256             String resultFilePath) throws IOException {
257         mRefJsonFile = Paths.get(refJsonFilePath);
258         mTestVmafFile = Paths.get(testVmafFilePath);
259 
260         // Setup the logger.
261         FileHandler fileHandler = new FileHandler(resultFilePath);
262         fileHandler.setFormatter(mFormatter);
263         Logger rootLogger = Logger.getLogger("");
264         rootLogger.addHandler(fileHandler);
265         rootLogger.setLevel(Level.FINEST);
266 
267         Result res = new BdRateMain(
268                 new GsonBuilder()
269                         .registerTypeAdapter(
270                                 ReferenceConfig.class,
271                                 new ReferenceConfig.Deserializer())
272                         .create(),
273                 BdRateCalculator.create(),
274                 BdQualityCalculator.create())
275                 .run();
276 
277         LOGGER.info("Passed Video Encoding Quality (VEQ) test.");
278         fileHandler.close();
279         return res.ordinal();
280     }
281 }
282