1 package org.unicode.cldr.test; 2 3 import com.ibm.icu.dev.tool.shared.UOption; 4 import com.ibm.icu.dev.util.ElapsedTimer; 5 import com.ibm.icu.impl.Relation; 6 import com.ibm.icu.impl.Row; 7 import com.ibm.icu.lang.UCharacter; 8 import com.ibm.icu.text.Collator; 9 import com.ibm.icu.text.UnicodeSet; 10 import com.ibm.icu.util.ULocale; 11 import java.io.File; 12 import java.io.IOException; 13 import java.io.PrintWriter; 14 import java.util.ArrayList; 15 import java.util.Arrays; 16 import java.util.Collection; 17 import java.util.Comparator; 18 import java.util.EnumSet; 19 import java.util.HashMap; 20 import java.util.Iterator; 21 import java.util.List; 22 import java.util.Map; 23 import java.util.Set; 24 import java.util.TreeMap; 25 import java.util.TreeSet; 26 import java.util.concurrent.ConcurrentHashMap; 27 import java.util.regex.Matcher; 28 import java.util.stream.Stream; 29 import org.unicode.cldr.draft.FileUtilities; 30 import org.unicode.cldr.test.CheckCLDR.CheckStatus; 31 import org.unicode.cldr.test.CheckCLDR.CheckStatus.Subtype; 32 import org.unicode.cldr.test.CheckCLDR.CompoundCheckCLDR; 33 import org.unicode.cldr.test.CheckCLDR.FormatDemo; 34 import org.unicode.cldr.test.CheckCLDR.Options; 35 import org.unicode.cldr.test.CheckCLDR.Phase; 36 import org.unicode.cldr.test.CheckCLDR.SimpleDemo; 37 import org.unicode.cldr.test.TestCache.TestResultBundle; 38 import org.unicode.cldr.tool.Option; 39 import org.unicode.cldr.tool.Option.Params; 40 import org.unicode.cldr.tool.ShowData; 41 import org.unicode.cldr.tool.TablePrinter; 42 import org.unicode.cldr.util.CLDRConfig; 43 import org.unicode.cldr.util.CLDRConfig.Environment; 44 import org.unicode.cldr.util.CLDRFile; 45 import org.unicode.cldr.util.CLDRFile.Status; 46 import org.unicode.cldr.util.CLDRPaths; 47 import org.unicode.cldr.util.CLDRTool; 48 import org.unicode.cldr.util.CldrUtility; 49 import org.unicode.cldr.util.Counter; 50 import org.unicode.cldr.util.CoverageInfo; 51 import org.unicode.cldr.util.Factory; 52 import org.unicode.cldr.util.LanguageTagParser; 53 import org.unicode.cldr.util.Level; 54 import org.unicode.cldr.util.LocaleIDParser; 55 import org.unicode.cldr.util.LogicalGrouping; 56 import org.unicode.cldr.util.Organization; 57 import org.unicode.cldr.util.Pair; 58 import org.unicode.cldr.util.PathDescription; 59 import org.unicode.cldr.util.PathHeader; 60 import org.unicode.cldr.util.PathHeader.SectionId; 61 import org.unicode.cldr.util.PathUtilities; 62 import org.unicode.cldr.util.PatternCache; 63 import org.unicode.cldr.util.SimpleFactory; 64 import org.unicode.cldr.util.StandardCodes; 65 import org.unicode.cldr.util.StringId; 66 import org.unicode.cldr.util.SupplementalDataInfo; 67 import org.unicode.cldr.util.UnicodeSetPrettyPrinter; 68 import org.unicode.cldr.util.VoteResolver; 69 import org.unicode.cldr.util.VoteResolver.CandidateInfo; 70 import org.unicode.cldr.util.VoteResolver.UnknownVoterException; 71 import org.unicode.cldr.util.VoterInfoList; 72 import org.unicode.cldr.util.XMLSource; 73 import org.unicode.cldr.util.XMLSource.SourceLocation; 74 import org.unicode.cldr.util.XPathParts; 75 76 /** 77 * Console test for CheckCLDR. <br> 78 * Some common source directories: 79 * 80 * <pre> 81 * -s C:/cvsdata/unicode/cldr/incoming/vetted/main 82 * -s C:/cvsdata/unicode/cldr/incoming/proposed/main 83 * -s C:/cvsdata/unicode/cldr/incoming/proposed/main 84 * -s C:/cvsdata/unicode/cldr/testdata/main 85 * </pre> 86 * 87 * @author markdavis 88 */ 89 @CLDRTool(alias = "check", description = "Run CheckCLDR against CLDR data") 90 public class ConsoleCheckCLDR { 91 private static final CLDRConfig CLDR_CONFIG = CLDRConfig.getInstance(); 92 public static boolean showStackTrace = false; 93 public static boolean errorsOnly = false; 94 static boolean SHOW_LOCALE = true; 95 static boolean SHOW_EXAMPLES = false; 96 private static boolean CLDR_GITHUB_ANNOTATIONS = 97 (Boolean.parseBoolean(System.getProperty("CLDR_GITHUB_ANNOTATIONS", "false"))); 98 99 // TODO get ride of these in favor of MyOptions 100 101 private static final int COVERAGE = 2, 102 EXAMPLES = 3, 103 FILE_FILTER = 4, 104 TEST_FILTER = 5, 105 DATE_FORMATS = 6, 106 ORGANIZATION = 7, 107 SHOWALL = 8, 108 PATH_FILTER = 9, 109 ERRORS_ONLY = 10, 110 CHECK_ON_SUBMIT = 11, 111 NO_ALIASES = 12, 112 SOURCE_DIRECTORY = 13, 113 USER = 14, 114 PHASE = 15, 115 GENERATE_HTML = 16, 116 VOTE_RESOLVE = 17, 117 ID_VIEW = 18, 118 SUBTYPE_FILTER = 19, 119 BAILEY = 21, 120 SINGLE_THREAD = 24; 121 122 static final String SOURCE_DIRS = 123 CLDRPaths.MAIN_DIRECTORY 124 + "," 125 + CLDRPaths.ANNOTATIONS_DIRECTORY 126 + "," 127 + CLDRPaths.SEED_DIRECTORY; 128 129 enum MyOptions { 130 coverage( 131 new Params() 132 .setHelp("Set the coverage: eg -c comprehensive") 133 .setMatch("comprehensive|modern|moderate|basic")), // UOption.REQUIRES_ARG 134 examples( 135 new Params() 136 .setHelp("Turn on examples (actually a summary of the demo)") 137 .setFlag('x')), // , 'x', UOption.NO_ARG), 138 file_filter( 139 new Params() 140 .setHelp( 141 "Pick the locales (files) to check: arg is a regular expression, eg -f fr, or -f fr.*, or -f (fr|en-.*)") 142 .setDefault(".*") 143 .setMatch(".*")), // , 'f', UOption.REQUIRES_ARG).setDefault(".*"), 144 test_filter( 145 new Params() 146 .setHelp( 147 "Filter the Checks: arg is a regular expression, eg -t.*number.*. To check all BUT a given test, use the style -t ((?!.*CheckZones).*)") 148 .setDefault(".*") 149 .setMatch(".*")), // , 't', UOption.REQUIRES_ARG).setDefault(".*"), 150 date_formats( 151 new Params() 152 .setHelp("Turn on special date format checks")), // , 'd', UOption.NO_ARG), 153 organization( 154 new Params() 155 .setHelp( 156 "Organization: ibm, google, ....; Uses Locales.txt for to filter locales and set coverage levels") 157 .setDefault(".*") 158 .setMatch(".*")), // , 'o', UOption.REQUIRES_ARG), 159 showall( 160 new Params() 161 .setHelp("Show all paths, including aliased") 162 .setFlag('a')), // , 'a', UOption.NO_ARG), 163 path_filter( 164 new Params() 165 .setHelp("Pick the paths to check, eg -p.*languages.*") 166 .setDefault(".*") 167 .setMatch(".*")), // , 'p', UOption.REQUIRES_ARG).setDefault(".*"), 168 errors_only( 169 new Params() 170 .setHelp("Show errors only (with -ef, only final processing errors)")), // , 171 // 'e', UOption.NO_ARG), 172 check_on_submit(new Params().setHelp("").setFlag('k')), // , 'k', UOption.NO_ARG), 173 noaliases(new Params().setHelp("No aliases")), // , 'n', UOption.NO_ARG), 174 source_directory( 175 new Params() 176 .setHelp("Fully qualified source directories. (Conflicts with -S.)") 177 .setDefault(SOURCE_DIRS) 178 .setMatch(".*")), // , 's', UOption.REQUIRES_ARG).setDefault(SOURCE_DIRS), 179 user( 180 new Params() 181 .setHelp("User, eg -uu148") 182 .setMatch(".*")), // , 'u', UOption.REQUIRES_ARG), 183 phase( 184 new Params() 185 .setHelp("?") 186 .setMatch(Phase.class) 187 .setFlag('z')), // , 'z', UOption.REQUIRES_ARG), 188 generate_html( 189 new Params() 190 .setHelp("Generate HTML-style chart in directory.") 191 .setDefault(CLDRPaths.CHART_DIRECTORY + "/errors/") 192 .setMatch(".*")), // , 'g', 193 // UOption.OPTIONAL_ARG).setDefault(CLDRPaths.CHART_DIRECTORY + "/errors/"), 194 vote_resolution(new Params().setHelp("")), // , 'v', UOption.NO_ARG), 195 id_view(new Params().setHelp("")), // , 'i', UOption.NO_ARG), 196 subtype_filter( 197 new Params() 198 .setHelp("error/warning subtype filter, eg unexpectedOrderOfEraYear") 199 .setDefault(".*") 200 .setMatch(".*") 201 .setFlag('y')), // , 'y', UOption.REQUIRES_ARG), 202 source_all( 203 new Params() 204 .setHelp( 205 "Partially qualified directories. Standard subdirectories added if not specified (/main, /annotations, /subdivisions). (Conflicts with -s.)") 206 .setMatch(".*") 207 .setFlag('S') 208 .setDefault("common,seed,exemplars")), // , 'S', <changed>), 209 bailey( 210 new Params() 211 .setHelp( 212 "check bailey values (" 213 + CldrUtility.INHERITANCE_MARKER 214 + ")")), // , 'b', UOption.NO_ARG) 215 exemplarError(new Params().setFlag('E').setHelp("include to force strict Exemplar check")), 216 missingPaths( 217 new Params() 218 .setHelp( 219 "include to show missing and provisional paths, at the specified level")), 220 singleThread(new Params().setHelp("Run in single-thread mode.").setFlag('1')); 221 222 // BOILERPLATE TO COPY 223 final Option option; 224 MyOptions(Params params)225 private MyOptions(Params params) { 226 option = new Option(this, params); 227 } 228 229 private static Option.Options myOptions = new Option.Options(); 230 231 static { 232 for (MyOptions option : MyOptions.values()) { myOptions.add(option, option.option)233 myOptions.add(option, option.option); 234 } 235 } 236 parse(String[] args, boolean showArguments)237 private static Set<String> parse(String[] args, boolean showArguments) { 238 return myOptions.parse(MyOptions.values()[0], args, true); 239 } 240 } 241 242 // TODO get rid of these in favor of MyOptions 243 244 private static final UOption[] options = { 245 UOption.HELP_H(), 246 UOption.HELP_QUESTION_MARK(), 247 UOption.create("coverage", 'c', UOption.REQUIRES_ARG), 248 UOption.create("examples", 'x', UOption.NO_ARG), 249 UOption.create("file_filter", 'f', UOption.REQUIRES_ARG).setDefault(".*"), 250 UOption.create("test_filter", 't', UOption.REQUIRES_ARG).setDefault(".*"), 251 UOption.create("date_formats", 'd', UOption.NO_ARG), 252 UOption.create("organization", 'o', UOption.REQUIRES_ARG), 253 UOption.create("showall", 'a', UOption.NO_ARG), 254 UOption.create("path_filter", 'p', UOption.REQUIRES_ARG).setDefault(".*"), 255 UOption.create("errors_only", 'e', UOption.NO_ARG), 256 UOption.create("check-on-submit", 'k', UOption.NO_ARG), 257 UOption.create("noaliases", 'n', UOption.NO_ARG), 258 UOption.create("source_directory", 's', UOption.REQUIRES_ARG).setDefault(SOURCE_DIRS), 259 UOption.create("user", 'u', UOption.REQUIRES_ARG), 260 UOption.create("phase", 'z', UOption.REQUIRES_ARG), 261 UOption.create("generate_html", 'g', UOption.OPTIONAL_ARG) 262 .setDefault(CLDRPaths.CHART_DIRECTORY + "/errors/"), 263 UOption.create("vote resolution", 'v', UOption.NO_ARG), 264 UOption.create("id view", 'i', UOption.NO_ARG), 265 UOption.create("subtype_filter", 'y', UOption.REQUIRES_ARG), 266 UOption.create("source_all", 'S', UOption.OPTIONAL_ARG).setDefault("common,seed,exemplars"), 267 UOption.create("bailey", 'b', UOption.NO_ARG), 268 UOption.create("exemplarError", 'E', UOption.NO_ARG), 269 UOption.create("missingPaths", 'm', UOption.NO_ARG), 270 UOption.create("singleThread", '1', UOption.NO_ARG) 271 }; 272 273 private static final Comparator<String> baseFirstCollator = 274 new Comparator<>() { 275 LanguageTagParser languageTagParser1 = new LanguageTagParser(); 276 LanguageTagParser languageTagParser2 = new LanguageTagParser(); 277 278 @Override 279 public int compare(String o1, String o2) { 280 String ls1 = languageTagParser1.set(o1).getLanguageScript(); 281 String ls2 = languageTagParser2.set(o2).getLanguageScript(); 282 int result = ls1.compareTo(ls2); 283 if (result != 0) return result; 284 return o1.compareTo(o2); 285 } 286 }; 287 private static final boolean PATH_IN_COUNT = false; 288 289 static Counter<ErrorType> subtotalCount = new Counter<>(true); // new ErrorCount(); 290 static Counter<ErrorType> totalCount = new Counter<>(true); 291 292 private enum RawStatus { 293 missing, 294 provisional, 295 present 296 } 297 calculateSubtypeFilter(final String subtypeFilterString)298 private static EnumSet<Subtype> calculateSubtypeFilter(final String subtypeFilterString) { 299 EnumSet<Subtype> subtypeFilter = null; 300 if (subtypeFilterString != null) { 301 subtypeFilter = EnumSet.noneOf(Subtype.class); 302 Matcher m = PatternCache.get(subtypeFilterString).matcher(""); 303 for (Subtype value : Subtype.values()) { 304 if (m.reset(value.toString()).find() || m.reset(value.name()).find()) { 305 subtypeFilter.add(value); 306 } 307 } 308 if (subtypeFilter.size() == 0) { 309 throw new IllegalArgumentException("No subtype match for " + subtypeFilterString); 310 } 311 } 312 return subtypeFilter; 313 } 314 calculatePathFilter(final String pathFilterString)315 static Matcher calculatePathFilter(final String pathFilterString) { 316 if (!pathFilterString.equals(".*")) { 317 return PatternCache.get(pathFilterString).matcher(""); 318 } else { 319 return null; 320 } 321 } 322 calculateCoverageLevel(final String coverageLevelInput, boolean forHtml)323 static Level calculateCoverageLevel(final String coverageLevelInput, boolean forHtml) { 324 Level coverageLevel = null; 325 if (forHtml) { 326 coverageLevel = Level.MODERN; // reset 327 } else if (coverageLevelInput != null) { 328 coverageLevel = Level.get(coverageLevelInput); 329 if (coverageLevel == Level.UNDETERMINED) { 330 throw new IllegalArgumentException( 331 "-c" 332 + coverageLevelInput 333 + "\t is invalid: must be one of: " 334 + "basic,moderate,..."); 335 } 336 } 337 return coverageLevel; 338 } 339 340 /** 341 * This will be the test framework way of using these tests. 342 * 343 * @param args 344 * @throws Throwable 345 */ main(String[] args)346 public static void main(String[] args) throws Throwable { 347 // turn off logging to not mess up html and other output. 348 CheckCLDR.setLoggerLevel(java.util.logging.Level.OFF); 349 MyOptions.parse(args, true); 350 ElapsedTimer totalTimer = new ElapsedTimer(); 351 UOption.parseArgs(args, options); 352 String factoryFilter = options[FILE_FILTER].value; 353 if (factoryFilter.equals("key")) { 354 factoryFilter = 355 "(en|ru|nl|fr|de|it|pl|es|tr|th|ja|zh|ko|ar|bg|sr|uk|ca|hr|cs|da|fil|fi|hu|id|lv|lt|no|pt|ro|sk|sl|sv|vi|el|he|fa|hi|am|af|et|is|ms|sw|zu|bn|mr|ta|eu|gl|ur|gu|kn|ml|te|zh_Hant|pt_PT|en_GB)"; 356 } 357 String checkFilter = options[TEST_FILTER].value; 358 final String subtypeFilterString = options[SUBTYPE_FILTER].value; 359 final EnumSet<Subtype> subtypeFilter = calculateSubtypeFilter(subtypeFilterString); 360 361 errorsOnly = options[ERRORS_ONLY].doesOccur; 362 boolean showMissing = MyOptions.missingPaths.option.doesOccur(); 363 364 SHOW_EXAMPLES = options[EXAMPLES].doesOccur; 365 boolean showAll = options[SHOWALL].doesOccur; 366 boolean checkFlexibleDates = options[DATE_FORMATS].doesOccur; 367 final String pathFilterString = options[PATH_FILTER].value; 368 final Matcher pathFilter = calculatePathFilter(pathFilterString); 369 boolean checkOnSubmit = options[CHECK_ON_SUBMIT].doesOccur; 370 boolean noaliases = options[NO_ALIASES].doesOccur; 371 372 final Level coverageLevel = 373 calculateCoverageLevel(options[COVERAGE].value, options[GENERATE_HTML].doesOccur); 374 375 Organization organization = 376 options[ORGANIZATION].value == null 377 ? null 378 : Organization.fromString(options[ORGANIZATION].value); 379 if (organization != null) { 380 Set<Organization> organizations = StandardCodes.make().getLocaleCoverageOrganizations(); 381 if (!organizations.contains(organization)) { 382 throw new IllegalArgumentException( 383 "-o" + organization + "\t is invalid: must be one of: " + organizations); 384 } 385 } 386 final CLDRConfig cldrConf = CLDR_CONFIG; 387 cldrConf.setEnvironment(Environment.UNITTEST); 388 final Phase phase; 389 if (options[PHASE].doesOccur) { 390 String phaseVal = options[PHASE].value; 391 try { 392 // no null check for argument; if it is is null, Phase.forString would return the 393 // one from CLDRConfig 394 phase = Phase.forString(phaseVal); 395 } catch (IllegalArgumentException e) { 396 StringBuilder sb = new StringBuilder("Incorrect Phase value"); 397 if (phaseVal != null && !phaseVal.isEmpty()) { 398 sb.append(" '"); 399 sb.append(phaseVal); 400 sb.append("'"); 401 } 402 sb.append(": should be one of "); 403 for (Phase curPhase : Phase.values()) { 404 // implicitly does a toString; 405 sb.append(curPhase); 406 sb.append(", "); 407 } 408 int lastIdx = sb.lastIndexOf(","); 409 // remove the last comma, if it occurs 410 if (lastIdx > -1) { 411 String tmpBuf = sb.substring(0, lastIdx); 412 sb.setLength(0); 413 sb.append(tmpBuf); 414 } 415 sb.append("."); 416 // TODO: Reporting should be similar to an error (wrong parameter...), and not 417 // actually an Exception 418 throw new IllegalArgumentException(sb.toString(), e); 419 } 420 } else { 421 phase = cldrConf.getPhase(); 422 } 423 424 boolean baileyTest = options[BAILEY].doesOccur; 425 426 File sourceDirectories[] = null; 427 428 if (MyOptions.source_all.option.doesOccur()) { 429 if (MyOptions.source_directory.option.doesOccur()) { 430 throw new IllegalArgumentException("Don't use -s and -S together."); 431 } 432 sourceDirectories = 433 cldrConf.addStandardSubdirectories( 434 cldrConf.getCLDRDataDirectories( 435 MyOptions.source_all.option.getValue())); 436 } else { 437 String[] sdirs = options[SOURCE_DIRECTORY].value.split(",\\s*"); 438 sourceDirectories = new File[sdirs.length]; 439 for (int i = 0; i < sdirs.length; ++i) { 440 sourceDirectories[i] = 441 new File( 442 CldrUtility.checkValidDirectory( 443 sdirs[i], "Fix with -s. Use -h for help.")); 444 } 445 } 446 447 if (options[GENERATE_HTML].doesOccur) { 448 ErrorFile.generated_html_directory = options[GENERATE_HTML].value; 449 ErrorFile.generated_html_count = 450 FileUtilities.openUTF8Writer(ErrorFile.generated_html_directory, "count.txt"); 451 } 452 453 final boolean sequential = 454 SHOW_EXAMPLES 455 || options[GENERATE_HTML].doesOccur 456 || options[SINGLE_THREAD].doesOccur; 457 458 idView = options[ID_VIEW].doesOccur; 459 460 if (options[VOTE_RESOLVE].doesOccur) { 461 resolveVotesDirectory = 462 CldrUtility.checkValidFile( 463 CLDRPaths.BASE_DIRECTORY + "incoming/vetted/votes/", true, null); 464 voterInfoList.setVoterToInfo( 465 CldrUtility.checkValidFile( 466 CLDRPaths.BASE_DIRECTORY + "incoming/vetted/usersa/usersa.xml", 467 false, 468 null)); 469 voteResolver = new VoteResolver<>(voterInfoList); 470 } 471 472 String user = options[USER].value; 473 474 System.out.println("Source directories:\n"); 475 for (File f : sourceDirectories) { 476 System.out.println( 477 " " + f.getPath() + "\t(" + PathUtilities.getNormalizedPathString(f) + ")"); 478 } 479 480 // set up the test 481 Factory cldrFactory = 482 SimpleFactory.make(sourceDirectories, factoryFilter) 483 .setSupplementalDirectory(new File(CLDRPaths.SUPPLEMENTAL_DIRECTORY)); 484 final TestCache testCache = cldrFactory.getTestCache(); 485 testCache.setNameMatcher(checkFilter); 486 487 { 488 // we create an extraneous CompoundCheckCLDR here just to check the filters 489 CompoundCheckCLDR checkCldr = CheckCLDR.getCheckAll(cldrFactory, checkFilter); 490 if (checkCldr.getFilteredTestList().size() == 0) { 491 throw new IllegalArgumentException("The filter doesn't match any tests."); 492 } 493 System.out.println("filtered tests: " + checkCldr.getFilteredTests()); 494 } 495 496 Factory backCldrFactory = CLDRConfig.getInstance().getMainAndAnnotationsFactory(); 497 english = backCldrFactory.make("en", true); 498 499 CheckCLDR.setDisplayInformation(english); 500 setExampleGenerator(new ExampleGenerator(english, english)); 501 PathShower pathShower = new PathShower(); 502 503 // call on the files 504 Set<String> locales = new TreeSet<>(baseFirstCollator); 505 locales.addAll(cldrFactory.getAvailable()); 506 507 Set<String> fatalErrors = new TreeSet<>(); 508 509 showHeaderLine(); 510 511 supplementalDataInfo = SupplementalDataInfo.getInstance(CLDRPaths.SUPPLEMENTAL_DIRECTORY); 512 513 LocaleIDParser localeIDParser = new LocaleIDParser(); 514 PathHeader.Factory pathHeaderFactory = PathHeader.getFactory(english); 515 516 final Map<String, Level> locale_status = 517 StandardCodes.make().getLocaleToLevel(organization); 518 519 final List<String> specialPurposeLocales = new ArrayList<>(Arrays.asList("en_US_POSIX")); 520 521 // TODO: englishPaths doesn't seem to be used. 522 // final HashSet<String> ep = new HashSet<>(); 523 // final CLDRFile displayFile = CheckCLDR.getDisplayInformation(); 524 // addPrettyPaths( 525 // displayFile, pathFilter, pathHeaderFactory, noaliases, true, ep); 526 // addPrettyPaths( 527 // displayFile, 528 // displayFile.getExtraPaths(), 529 // pathFilter, 530 // pathHeaderFactory, 531 // noaliases, 532 // true, 533 // ep); 534 // final Set<String> englishPaths = Collections.unmodifiableSet(ep); // for robustness 535 536 // Set up our stream to use. It will be parallel usually, or sequential for HTML. 537 Stream<String> stream; 538 539 if (sequential) { 540 System.err.println("# Note: running in sequential mode."); 541 stream = locales.stream(); 542 } else { 543 stream = locales.parallelStream(); 544 } 545 546 // now, run it 547 stream.forEach( 548 localeID -> { 549 if (ErrorFile.writeError != null) { 550 return; // get out, it's an error. 551 } 552 553 List<CheckStatus> result = new ArrayList<>(); 554 Set<PathHeader> paths = new TreeSet<>(); // CLDRFile.ldmlComparator); 555 Map<String, String> m = new TreeMap<>(); 556 Map<String, String> options = new HashMap<>(); 557 FlexibleDateFromCLDR fset = new FlexibleDateFromCLDR(); 558 559 if (CLDRFile.isSupplementalName(localeID)) return; 560 if (supplementalDataInfo.getDefaultContentLocales().contains(localeID)) { 561 System.out.println("# Skipping default content locale: " + localeID); 562 return; 563 } 564 565 // We don't really need to check the POSIX locale, as it is a special purpose 566 // locale 567 if (specialPurposeLocales.contains(localeID)) { 568 System.out.println("# Skipping special purpose locale: " + localeID); 569 return; 570 } 571 572 boolean isLanguageLocale = 573 localeID.equals(localeIDParser.set(localeID).getLanguageScript()); 574 options.clear(); 575 576 if (MyOptions.exemplarError.option.doesOccur()) { 577 options.put(Options.Option.exemplarErrors.toString(), "true"); 578 } 579 580 // if the organization is set, skip any locale that doesn't have a value in 581 // Locales.txt 582 Level level = coverageLevel; 583 if (level == null) { 584 level = Level.MODERN; 585 } 586 if (organization != null) { 587 if (locale_status == null) return; 588 level = locale_status.get(localeID); 589 if (level == null) return; 590 if (level.compareTo(Level.BASIC) < 0) return; 591 } else if (!isLanguageLocale) { 592 // otherwise, skip all language locales 593 options.put(Options.Option.CheckCoverage_skip.getKey(), "true"); 594 } 595 596 // if (organization != null) 597 // options.put(Options.Option.CoverageLevel_localeType.getKey(), 598 // organization.toString()); 599 options.put(Options.Option.phase.getKey(), phase.toString()); 600 601 // also need the locale in the options 602 options.put(Options.Option.locale.getKey(), localeID); 603 604 if (SHOW_LOCALE && sequential) System.out.println(); 605 606 CLDRFile file; 607 CLDRFile englishFile = english; 608 CLDRFile parent = null; 609 610 ElapsedTimer timer = new ElapsedTimer(); 611 try { 612 file = cldrFactory.make(localeID, true); 613 if (ErrorFile.voteFactory != null) { 614 ErrorFile.voteFile = ErrorFile.voteFactory.make(localeID, true); 615 } 616 final String parentID = LocaleIDParser.getParent(localeID); 617 if (parentID != null) { 618 parent = cldrFactory.make(parentID, true); 619 } 620 } catch (RuntimeException e) { 621 fatalErrors.add(localeID); 622 System.out.println("FATAL ERROR: " + localeID); 623 e.printStackTrace(System.out); 624 return; 625 } 626 627 TestResultBundle bundle = testCache.getBundle(new CheckCLDR.Options(options)); 628 629 // generate HTML if asked for 630 if (ErrorFile.generated_html_directory != null) { 631 String baseLanguage = localeIDParser.set(localeID).getLanguageScript(); 632 633 if (!baseLanguage.equals(ErrorFile.lastBaseLanguage)) { 634 ErrorFile.lastBaseLanguage = baseLanguage; 635 try { 636 ErrorFile.openErrorFile(localeID, baseLanguage); 637 } catch (IOException ioe) { 638 ErrorFile.writeError = ioe; 639 System.err.println( 640 "Exception " 641 + ioe 642 + " while trying to open file " 643 + localeID); 644 ioe.printStackTrace(); 645 return; 646 } 647 } 648 } 649 650 if (user != null) { 651 file = new CLDRFile.TestUser(file, user, isLanguageLocale); 652 if (parent != null) { 653 parent = new CLDRFile.TestUser(parent, user, isLanguageLocale); 654 } 655 } 656 // checkCldr.setCldrFileToCheck(file, new Options(options), result); 657 658 subtotalCount.clear(); 659 660 for (Iterator<CheckStatus> it3 = result.iterator(); it3.hasNext(); ) { 661 CheckStatus status = it3.next(); 662 String statusString = status.toString(); // com.ibm.icu.impl.Utility.escape( 663 CheckStatus.Type statusType = status.getType(); 664 665 if (errorsOnly) { 666 if (!statusType.equals(CheckStatus.errorType)) continue; 667 } 668 669 if (subtypeFilter != null) { 670 if (!subtypeFilter.contains(status.getSubtype())) { 671 continue; 672 } 673 } 674 675 if (checkOnSubmit) { 676 if (!status.isCheckOnSubmit() 677 || !statusType.equals(CheckStatus.errorType)) continue; 678 } 679 showValue( 680 file, 681 null, 682 localeID, 683 null, 684 null, 685 null, 686 null, 687 statusString, 688 status.getSubtype()); 689 } 690 paths.clear(); 691 692 CoverageInfo covInfo = cldrConf.getCoverageInfo(); 693 for (String path : file.fullIterable()) { 694 if (pathFilter != null && !pathFilter.reset(path).find()) { 695 continue; 696 } 697 if (level != null) { 698 Level currentLevel = covInfo.getCoverageLevel(path, localeID); 699 if (currentLevel.compareTo(level) > 0) { 700 continue; 701 } 702 } 703 final PathHeader pathHeader = pathHeaderFactory.fromPath(path); 704 if (pathHeader.getSectionId() != SectionId.Special) { 705 paths.add(pathHeader); 706 } 707 } 708 709 UnicodeSet missingExemplars = new UnicodeSet(); 710 UnicodeSet missingCurrencyExemplars = new UnicodeSet(); 711 if (checkFlexibleDates) { 712 fset.set(file); 713 } 714 pathShower.set(localeID); 715 716 // only create if we are going to use it 717 final ExampleGenerator exampleGenerator = 718 SHOW_EXAMPLES ? new ExampleGenerator(file, englishFile) : null; 719 720 int pathCount = 0; 721 Status otherPath = new Status(); 722 int rawMissingCount = 0; 723 int rawProvisionalCount = 0; 724 CLDRFile unresolved = file.getUnresolved(); 725 726 for (PathHeader pathHeader : paths) { 727 pathCount++; 728 String path = pathHeader.getOriginalPath(); 729 String prettyPath = 730 pathHeader.toString().replace('\t', '|').replace(' ', '_'); 731 if (!showAll && !file.isWinningPath(path)) { 732 continue; 733 } 734 final String topValue = unresolved.getStringValue(path); 735 RawStatus rawStatus = RawStatus.present; 736 737 if (topValue == null) { 738 rawStatus = RawStatus.missing; 739 rawMissingCount++; 740 } 741 742 if (!isLanguageLocale && !baileyTest) { 743 final String sourceLocaleID = file.getSourceLocaleID(path, otherPath); 744 if (!localeID.equals(sourceLocaleID)) { 745 continue; 746 } 747 // also skip aliases 748 if (!path.equals(otherPath.pathWhereFound)) { 749 continue; 750 } 751 } 752 if (path.contains("@alt") && path.contains("proposed")) { 753 continue; 754 } 755 String value = file.getStringValue(path); 756 if (baileyTest) { 757 value = CldrUtility.INHERITANCE_MARKER; 758 } 759 760 String fullPath = file.getFullXPath(path); 761 if (topValue != null) { 762 XPathParts fullParts = XPathParts.getFrozenInstance(fullPath); 763 String draftStatus = fullParts.getAttributeValue(-1, "draft"); 764 if (draftStatus != null && !draftStatus.equals("contributed")) { 765 rawProvisionalCount++; 766 rawStatus = RawStatus.provisional; 767 } 768 } 769 if (showMissing && rawStatus != RawStatus.present) { 770 String englishValue = englishFile.getStringValue(path); 771 if (englishValue == null) { 772 englishValue = "n/a"; 773 } 774 System.out.println( 775 getLocaleAndName(localeID) 776 + "\tRaw " 777 + rawStatus 778 + "\t" 779 + pathHeader 780 + "\t" 781 + englishValue 782 + "\t" 783 + path); 784 } 785 786 String example = ""; 787 if (SHOW_EXAMPLES && exampleGenerator != null) { 788 example = 789 ExampleGenerator.simplify( 790 exampleGenerator.getExampleHtml(path, value)); 791 showExamples( 792 file, prettyPath, localeID, path, value, fullPath, example); 793 } 794 if (checkFlexibleDates) { 795 fset.checkFlexibles(path, value, fullPath); 796 } 797 int limit = 1; 798 for (int jj = 0; jj < limit; ++jj) { 799 if (jj == 0) { 800 bundle.check(path, result, value); 801 } else { 802 bundle.getExamples(path, value, result); 803 } 804 805 boolean showedOne = false; 806 for (Iterator<CheckStatus> it3 = result.iterator(); it3.hasNext(); ) { 807 CheckStatus status = it3.next(); 808 String statusString = 809 status.toString(); // com.ibm.icu.impl.Utility.escape( 810 CheckStatus.Type statusType = status.getType(); 811 Object[] parameters = status.getParameters(); 812 813 if (parameters != null) { 814 if (parameters.length >= 1 815 && status.getCause().getClass() 816 == CheckForExemplars.class) { 817 try { 818 UnicodeSet set = 819 new UnicodeSet(parameters[0].toString()); 820 if (status.getMessage().contains("currency")) { 821 missingCurrencyExemplars.addAll(set); 822 } else { 823 missingExemplars.addAll(set); 824 } 825 } catch (RuntimeException e) { 826 } // skip if not parseable as set 827 } 828 } 829 830 if (errorsOnly && !statusType.equals(CheckStatus.errorType)) { 831 continue; 832 } 833 834 if (subtypeFilter != null) { 835 if (!subtypeFilter.contains(status.getSubtype())) { 836 continue; 837 } 838 } 839 if (checkOnSubmit) { 840 if (!status.isCheckOnSubmit() 841 || !statusType.equals(CheckStatus.errorType)) continue; 842 } 843 844 if (statusType.equals(CheckStatus.demoType)) { 845 SimpleDemo d = status.getDemo(); 846 if (d != null && d instanceof FormatDemo) { 847 FormatDemo fd = (FormatDemo) d; 848 m.clear(); 849 if (d.processPost(m)) 850 System.out.println("\tDemo:\t" + fd.getPlainText(m)); 851 } 852 continue; 853 } 854 855 if (parameters != null) { 856 for (int i = 0; i < parameters.length; ++i) { 857 if (showStackTrace && parameters[i] instanceof Throwable) { 858 ((Throwable) parameters[i]).printStackTrace(); 859 } 860 } 861 } 862 863 showValue( 864 file, 865 prettyPath, 866 localeID, 867 example, 868 path, 869 value, 870 fullPath, 871 statusString, 872 status.getSubtype()); 873 showedOne = true; 874 } 875 if (!showedOne && phase != Phase.FINAL_TESTING) { 876 if (!showedOne && showAll) { 877 showValue( 878 file, 879 prettyPath, 880 localeID, 881 example, 882 path, 883 value, 884 fullPath, 885 "ok", 886 Subtype.none); 887 showedOne = true; 888 } 889 } 890 } 891 } 892 893 if (resolveVotesDirectory != null) { 894 LocaleVotingData.resolveErrors(localeID); 895 } 896 897 showSummary( 898 localeID, 899 level, 900 "Items:\t" 901 + pathCount 902 + "\tRaw Missing:\t" 903 + rawMissingCount 904 + "\tRaw Provisional:\t" 905 + rawProvisionalCount); 906 907 if (missingExemplars.size() != 0) { 908 missingExemplars.removeAll( 909 new UnicodeSet("[[:Uppercase:]-[İ]]")); // remove uppercase #4670 910 if (missingExemplars.size() != 0) { 911 showSummary( 912 localeID, 913 level, 914 "Total missing from general exemplars:\t" 915 + missingExemplars.size() 916 + "\t" 917 + UnicodeSetPrettyPrinter.fromIcuLocale(localeID) 918 .format(missingExemplars)); 919 } 920 } 921 if (missingCurrencyExemplars.size() != 0) { 922 Collator col = Collator.getInstance(new ULocale(localeID)); 923 showSummary( 924 localeID, 925 level, 926 "Total missing from currency exemplars:\t" 927 + UnicodeSetPrettyPrinter.fromIcuLocale(localeID) 928 .format(missingCurrencyExemplars)); 929 } 930 for (ErrorType type : subtotalCount.keySet()) { 931 showSummary( 932 localeID, 933 level, 934 "Subtotal " + type + ":\t" + subtotalCount.getCount(type)); 935 } 936 937 if (checkFlexibleDates) { 938 fset.showFlexibles(); 939 } 940 if (SHOW_EXAMPLES && exampleGenerator != null) { 941 // ldml/dates/timeZoneNames/zone[@type="America/Argentina/San_Juan"]/exemplarCity 942 for (String zone : StandardCodes.make().getGoodAvailableCodes("tzid")) { 943 String path = 944 "//ldml/dates/timeZoneNames/zone[@type=\"" 945 + zone 946 + "\"]/exemplarCity"; 947 PathHeader pathHeader = pathHeaderFactory.fromPath(path); 948 String prettyPath = 949 pathHeader.toString().replace('\t', '|').replace(' ', '_'); 950 if (pathFilter != null && !pathFilter.reset(path).matches()) { 951 continue; 952 } 953 String fullPath = file.getStringValue(path); 954 if (fullPath != null) { 955 continue; 956 } 957 /* 958 * TODO: fix this code. Calling getExampleHtml with value = null will always return null, 959 * so what's this supposed to accomplish? 960 */ 961 String example = 962 ExampleGenerator.simplify( 963 exampleGenerator.getExampleHtml( 964 path, null /* value */)); 965 showExamples(file, prettyPath, localeID, path, null, fullPath, example); 966 } 967 } 968 System.out.println("# " + localeID + " Elapsed time: " + timer); 969 System.out.flush(); 970 }); 971 972 if (ErrorFile.errorFileWriter != null) { 973 ErrorFile.closeErrorFile(); 974 } 975 976 // an error occurred opening HTML, rethrow it. 977 if (ErrorFile.writeError != null) { 978 throw ErrorFile.writeError; 979 } 980 981 if (ErrorFile.generated_html_directory != null) { 982 ErrorFile.writeErrorCountsText(); 983 ErrorFile.writeErrorFileIndex(); 984 } 985 System.out.println(); 986 for (ErrorType type : totalCount.keySet()) { 987 System.out.println("# Total " + type + ":\t" + totalCount.getCount(type)); 988 } 989 990 System.out.println(); 991 System.out.println("# Total elapsed time: " + totalTimer); 992 if (fatalErrors.size() != 0) { 993 System.out.println("# FATAL ERRORS:"); 994 } 995 long errorCount = totalCount.getCount(ErrorType.error) + fatalErrors.size(); 996 if (errorCount != 0) { 997 System.out.println(); 998 System.out.println("<< FAILURE - Error count is " + errorCount + " . >>"); 999 System.exit(-1); 1000 } else { 1001 System.out.println(); 1002 System.out.println("<< SUCCESS - No errors found. >>"); 1003 } 1004 if (LogicalGrouping.GET_TYPE_COUNTS) { 1005 for (String s : LogicalGrouping.typeCount.keySet()) { 1006 System.out.println(s + "=" + LogicalGrouping.typeCount.get(s)); 1007 } 1008 } 1009 } // end of main() 1010 1011 static class LocaleVotingData { 1012 private int disputedCount = 0; 1013 Counter<Organization> missingOrganizationCounter = new Counter<>(true); 1014 Counter<Organization> goodOrganizationCounter = new Counter<>(true); 1015 Counter<Organization> conflictedOrganizations = new Counter<>(true); 1016 Counter<VoteResolver.Status> winningStatusCounter = new Counter<>(true); 1017 1018 static Map<String, LocaleVotingData> localeToErrors = new HashMap<>(); 1019 private static Map<Integer, String> idToPath; 1020 resolveErrors(String locale)1021 public static void resolveErrors(String locale) { 1022 localeToErrors.put(locale, new LocaleVotingData(locale)); 1023 } 1024 LocaleVotingData(String locale)1025 public LocaleVotingData(String locale) { 1026 1027 Map<Organization, VoteResolver.Level> orgToMaxVote = 1028 voterInfoList.getOrganizationToMaxVote(locale); 1029 1030 Map<Integer, Map<Integer, CandidateInfo>> info = 1031 VoteResolver.getBaseToAlternateToInfo( 1032 resolveVotesDirectory + locale + ".xml", voterInfoList); 1033 1034 Map<String, Integer> valueToItem = new HashMap<>(); 1035 1036 for (int basePath : info.keySet()) { 1037 final Map<Integer, CandidateInfo> itemInfo = info.get(basePath); 1038 1039 // find the last release status and value 1040 voteResolver.clear(); 1041 valueToItem.clear(); 1042 1043 for (int item : itemInfo.keySet()) { 1044 String itemValue = getValue(item); 1045 valueToItem.put(itemValue, item); 1046 1047 CandidateInfo candidateInfo = itemInfo.get(item); 1048 if (candidateInfo.oldStatus != null) { 1049 voteResolver.setBaseline(itemValue, candidateInfo.oldStatus); 1050 } 1051 voteResolver.add(itemValue); 1052 for (int voter : candidateInfo.voters) { 1053 try { 1054 voteResolver.add(itemValue, voter); 1055 } catch (UnknownVoterException e) { 1056 // skip 1057 } 1058 } 1059 } 1060 1061 EnumSet<Organization> basePathConflictedOrganizations = 1062 voteResolver.getConflictedOrganizations(); 1063 conflictedOrganizations.addAll(basePathConflictedOrganizations, 1); 1064 1065 VoteResolver.Status winningStatus = voteResolver.getWinningStatus(); 1066 String winningValue = voteResolver.getWinningValue(); 1067 1068 winningStatusCounter.add(winningStatus, 1); 1069 1070 if (winningStatus == VoteResolver.Status.approved) { 1071 continue; 1072 } 1073 1074 CandidateInfo candidateInfo = itemInfo.get(valueToItem.get(winningValue)); 1075 Map<Organization, VoteResolver.Level> orgToMaxVoteHere = 1076 voterInfoList.getOrganizationToMaxVote(candidateInfo.voters); 1077 1078 // if the winning item is less than contributed, record the organizations that 1079 // haven't given their 1080 // maximum vote to the winning item. 1081 if (winningStatus.compareTo(VoteResolver.Status.contributed) < 0) { 1082 // showPaths(basePath, itemInfo); 1083 for (Organization org : orgToMaxVote.keySet()) { 1084 VoteResolver.Level maxVote = orgToMaxVote.get(org); 1085 VoteResolver.Level maxVoteHere = orgToMaxVoteHere.get(org); 1086 if (maxVoteHere == null || maxVoteHere.compareTo(maxVote) < 0) { 1087 missingOrganizationCounter.add(org, 1); 1088 } 1089 } 1090 if (voteResolver.isDisputed()) { 1091 disputedCount++; 1092 String path = getIdToPath(basePath); 1093 ErrorFile.addDataToErrorFile( 1094 locale, path, ErrorType.disputed, Subtype.none); 1095 } 1096 } else { 1097 for (Organization org : orgToMaxVote.keySet()) { 1098 VoteResolver.Level maxVote = orgToMaxVote.get(org); 1099 VoteResolver.Level maxVoteHere = orgToMaxVoteHere.get(org); 1100 if (maxVoteHere == null || maxVoteHere.compareTo(maxVote) < 0) { 1101 } else { 1102 goodOrganizationCounter.add(org, 1); 1103 } 1104 } 1105 } 1106 } 1107 System.out.println( 1108 getLocaleAndName(locale) + "\tEnabled Organizations:\t" + orgToMaxVote); 1109 if (disputedCount != 0) { 1110 System.out.println( 1111 getLocaleAndName(locale) + "\tDisputed Items:\t" + disputedCount); 1112 } 1113 1114 if (missingOrganizationCounter.size() > 0) { 1115 System.out.println( 1116 getLocaleAndName(locale) 1117 + "\tMIA organizations:\t" 1118 + missingOrganizationCounter); 1119 System.out.println( 1120 getLocaleAndName(locale) 1121 + "\tConflicted organizations:\t" 1122 + conflictedOrganizations); 1123 System.out.println( 1124 getLocaleAndName(locale) 1125 + "\tCool organizations!:\t" 1126 + goodOrganizationCounter); 1127 } 1128 System.out.println( 1129 getLocaleAndName(locale) + "\tOptimal Status:\t" + winningStatusCounter); 1130 } 1131 getIdToPath(int basePath)1132 private static String getIdToPath(int basePath) { 1133 if (idToPath == null) { 1134 idToPath = VoteResolver.getIdToPath(resolveVotesDirectory + "xpathTable.xml"); 1135 } 1136 return idToPath.get(basePath); 1137 } 1138 get(String locale)1139 public static LocaleVotingData get(String locale) { 1140 return localeToErrors.get(locale); 1141 } 1142 getDisputedCount()1143 int getDisputedCount() { 1144 return disputedCount; 1145 } 1146 getConflictedHTML()1147 String getConflictedHTML() { 1148 String result = conflictedOrganizations.toString(); 1149 if (result.length() == 0) { 1150 return ""; 1151 } 1152 result = result.substring(1, result.length() - 1); 1153 result = result.replace(", ", "<br>"); 1154 return result; 1155 } 1156 } 1157 getValue(int item)1158 private static String getValue(int item) { 1159 return String.valueOf(item); 1160 } 1161 1162 static Matcher draftStatusMatcher = 1163 PatternCache.get("\\[@draft=\"(provisional|unconfirmed)\"]").matcher(""); 1164 1165 enum ErrorType { 1166 ok, 1167 error, 1168 disputed, 1169 warning, 1170 core, 1171 posix, 1172 minimal, 1173 basic, 1174 moderate, 1175 modern, 1176 comprehensive, 1177 optional, 1178 contributed, 1179 provisional, 1180 unconfirmed, 1181 unknown; 1182 static EnumSet<ErrorType> unapproved = 1183 EnumSet.range(ErrorType.contributed, ErrorType.unconfirmed); 1184 static EnumSet<ErrorType> coverage = EnumSet.range(ErrorType.posix, ErrorType.optional); 1185 static EnumSet<ErrorType> showInSummary = 1186 EnumSet.of( 1187 ErrorType.error, 1188 ErrorType.warning, 1189 ErrorType.posix, 1190 ErrorType.minimal, 1191 ErrorType.basic); 1192 fromStatusString(String statusString)1193 static ErrorType fromStatusString(String statusString) { 1194 ErrorType shortStatus = 1195 statusString.equals("ok") 1196 ? ErrorType.ok 1197 : statusString.startsWith("Error") 1198 ? ErrorType.error 1199 : statusString.equals("disputed") 1200 ? ErrorType.disputed 1201 : statusString.startsWith("Warning") 1202 ? ErrorType.warning 1203 : statusString.equals("contributed") 1204 ? ErrorType.contributed 1205 : statusString.equals("provisional") 1206 ? ErrorType.provisional 1207 : statusString.equals( 1208 "unconfirmed") 1209 ? ErrorType.unconfirmed 1210 : ErrorType.unknown; 1211 if (shortStatus == ErrorType.unknown) { 1212 throw new IllegalArgumentException("Unknown error type: " + statusString); 1213 } else if (shortStatus == ErrorType.warning) { 1214 if (coverageMatcher.reset(statusString).find()) { 1215 shortStatus = ErrorType.valueOf(coverageMatcher.group(1)); 1216 } 1217 } 1218 return shortStatus; 1219 } 1220 } 1221 1222 static class ErrorFile { 1223 /** cached error for later rethrow */ 1224 public static Throwable writeError = null; 1225 1226 public static String lastBaseLanguage = ""; 1227 private static final boolean SHOW_VOTING_INFO = false; 1228 public static CLDRFile voteFile; 1229 public static Factory voteFactory; 1230 openErrorFile(String localeID, String baseLanguage)1231 private static void openErrorFile(String localeID, String baseLanguage) throws IOException { 1232 if (ErrorFile.errorFileWriter != null) { 1233 ErrorFile.closeErrorFile(); 1234 } 1235 ErrorFile.errorFileWriter = 1236 FileUtilities.openUTF8Writer( 1237 ErrorFile.generated_html_directory, baseLanguage + ".html"); 1238 ErrorFile.errorFileTable = new TablePrinter(); 1239 errorFileCounter.clear(); 1240 ErrorFile.errorFileTable 1241 .setCaption("Problem Details") 1242 .addColumn("Problem") 1243 .setCellAttributes("align=\"left\" class=\"{0}\"") 1244 .setSortPriority(0) 1245 .setSpanRows(true) 1246 .setBreakSpans(true) 1247 .setRepeatHeader(true) 1248 .setHeaderCell(true) 1249 .addColumn("Subtype") 1250 .setCellAttributes("align=\"left\" class=\"{1}\"") 1251 .setSortPriority(1) 1252 .setSpanRows(true) 1253 .setBreakSpans(true) 1254 .setRepeatHeader(true) 1255 .setHeaderCell(true) 1256 .addColumn("Locale") 1257 .setCellAttributes("class=\"{1}\"") 1258 .setCellPattern("<a href=\"http://unicode.org/cldr/apps/survey?_={0}\">{0}</a>") 1259 .setSortPriority(2) 1260 .setSpanRows(true) 1261 .setBreakSpans(true) // .setRepeatDivider(true) 1262 .addColumn("Name") 1263 .setCellAttributes("class=\"{1}\"") 1264 .setSpanRows(true) 1265 .setBreakSpans(true) 1266 .addColumn("Section") 1267 .setCellAttributes("class=\"{1}\"") 1268 .setSortPriority(3) 1269 .setCellPattern( 1270 "<a href=\"http://unicode.org/cldr/apps/survey?_={3}&x={0}\">{0}</a>") 1271 .setSpanRows(true) 1272 .addColumn("Count") 1273 .setCellAttributes("class=\"{1}\" align=\"right\""); 1274 1275 showIndexHead("", localeID, ErrorFile.errorFileWriter); 1276 } 1277 1278 static TablePrinter errorFileTable = new TablePrinter(); 1279 static Counter<Row.R4<String, String, ErrorType, Subtype>> errorFileCounter = 1280 new Counter<>(true); 1281 addDataToErrorFile( String localeID, String path, ErrorType shortStatus, Subtype subtype)1282 private static void addDataToErrorFile( 1283 String localeID, String path, ErrorType shortStatus, Subtype subtype) { 1284 String section = path == null ? null : XPathToMenu.xpathToMenu(path); 1285 if (section == null) { 1286 section = "general"; 1287 } 1288 errorFileCounter.add(new Row.R4<>(localeID, section, shortStatus, subtype), 1); 1289 ErrorFile.sectionToProblemsToLocaleToCount.add( 1290 new Row.R4<>(section, shortStatus, subtype, localeID), 1); 1291 } 1292 closeErrorFile()1293 private static void closeErrorFile() { 1294 Set<String> locales = new TreeSet<>(); 1295 for (Row.R4<String, String, ErrorType, Subtype> item : errorFileCounter.keySet()) { 1296 String localeID = item.get0(); 1297 locales.add(localeID); 1298 String section = item.get1(); 1299 ErrorType shortStatus = item.get2(); 1300 Subtype subtype = item.get3(); 1301 errorFileTable 1302 .addRow() 1303 .addCell(shortStatus) 1304 .addCell(subtype) 1305 .addCell(localeID) 1306 .addCell(ConsoleCheckCLDR.getLocaleName(localeID)) 1307 .addCell(section) // menuPath == null ? "" : "<a href='" + link + "'>" + 1308 // menuPath + "</a>" 1309 .addCell(errorFileCounter.getCount(item)) 1310 .finishRow(); 1311 } 1312 1313 if (SHOW_VOTING_INFO) { 1314 TablePrinter data = 1315 new TablePrinter() 1316 .setCaption("Voting Information") 1317 .addColumn("Locale") 1318 .setHeaderCell(true) 1319 .addColumn("Name") 1320 .setHeaderCell(true) 1321 .addColumn("Organization") 1322 .addColumn("Missing") 1323 .addColumn("Conflicted"); 1324 for (String localeID : locales) { 1325 // now the voting info 1326 LocaleVotingData localeVotingData = 1327 LocaleVotingData.localeToErrors.get(localeID); 1328 if (localeVotingData != null) { 1329 // find all the orgs with data 1330 EnumSet<Organization> orgs = EnumSet.noneOf(Organization.class); 1331 orgs.addAll(localeVotingData.missingOrganizationCounter.keySet()); 1332 orgs.addAll(localeVotingData.conflictedOrganizations.keySet()); 1333 orgs.addAll(localeVotingData.goodOrganizationCounter.keySet()); 1334 for (Organization org : orgs) { 1335 data.addRow() 1336 .addCell(ConsoleCheckCLDR.getLinkedLocale(localeID)) 1337 .addCell(ConsoleCheckCLDR.getLocaleName(localeID)) 1338 .addCell(org) 1339 .addCell( 1340 localeVotingData.missingOrganizationCounter.getCount( 1341 org)) 1342 .addCell(localeVotingData.conflictedOrganizations.getCount(org)) 1343 .finishRow(); 1344 } 1345 } 1346 } 1347 ErrorFile.errorFileWriter.println(data.toTable()); 1348 ErrorFile.errorFileWriter.println("<p></p>"); 1349 } 1350 1351 // generated_html.println("<table border='1' style='border-collapse: collapse' 1352 // bordercolor='#CCCCFF'>"); 1353 // Locale Group Error Warning Missing Votes: Contributed Missing Votes: Provisional 1354 // Missing Votes: 1355 // Unconfirmed Missing Coverage: Posix Missing Coverage: Minimal Missing Coverage: Basic 1356 // Missing Coverage: 1357 // Moderate Missing Coverage: Modern 1358 ErrorFile.errorFileWriter.println(ErrorFile.errorFileTable.toTable()); 1359 ErrorFile.errorFileWriter.println(ShowData.dateFooter()); 1360 ErrorFile.errorFileWriter.println(CldrUtility.ANALYTICS); 1361 ErrorFile.errorFileWriter.println("</body></html>"); 1362 ErrorFile.errorFileWriter.close(); 1363 ErrorFile.errorFileTable = null; 1364 } 1365 1366 // ================ Index File =================== 1367 showErrorFileIndex(PrintWriter generated_html_index)1368 static void showErrorFileIndex(PrintWriter generated_html_index) { 1369 1370 // get organizations 1371 Relation<Organization, String> orgToLocales = getOrgToLocales(); 1372 1373 TablePrinter indexTablePrinter = 1374 new TablePrinter() 1375 .setCaption("Problem Summary") 1376 .setTableAttributes( 1377 "border='1' style='border-collapse: collapse' bordercolor='blue'") 1378 .addColumn("BASE") 1379 .setHidden(true) // .setRepeatDivider(true) 1380 .addColumn("Locale") 1381 .setCellPattern( 1382 "<a name=\"{0}\" href=\"{1}.html\">{0}</a>") // link to base, 1383 // anchor 1384 // with full 1385 .addColumn("Name"); 1386 if (SHOW_VOTING_INFO) { 1387 indexTablePrinter.addColumn("Summary").addColumn("Missing"); 1388 } 1389 for (Organization org : orgToLocales.keySet()) { 1390 indexTablePrinter.addColumn(org.toString().substring(0, 2)); 1391 } 1392 indexTablePrinter 1393 .addColumn("Disputed") 1394 .setHeaderAttributes("class='disputed'") 1395 .setCellAttributes("class='disputed'") 1396 .addColumn("Conflicted") 1397 .setHeaderAttributes("class='conflicted'") 1398 .setCellAttributes("class='conflicted'"); 1399 1400 for (ConsoleCheckCLDR.ErrorType type : ConsoleCheckCLDR.ErrorType.showInSummary) { 1401 String columnTitle = UCharacter.toTitleCase(type.toString(), null); 1402 final boolean coverage = ConsoleCheckCLDR.ErrorType.coverage.contains(type); 1403 if (coverage) { 1404 columnTitle = "MC: " + columnTitle; 1405 } else if (ConsoleCheckCLDR.ErrorType.unapproved.contains(type)) { 1406 columnTitle = "MV: " + columnTitle; 1407 } 1408 indexTablePrinter 1409 .addColumn(columnTitle) 1410 .setHeaderAttributes("class='" + type + "'") 1411 .setCellAttributes("class='" + type + "'"); 1412 } 1413 1414 // now fill in the data 1415 LanguageTagParser ltp = new LanguageTagParser(); 1416 for (String key : ErrorFile.errorFileIndexData.keySet()) { 1417 Pair<String, Counter<ErrorType>> pair = ErrorFile.errorFileIndexData.get(key); 1418 String htmlOpenedFileLanguage = pair.getFirst(); 1419 Counter<ErrorType> counts = pair.getSecond(); 1420 LocaleVotingData votingData = LocaleVotingData.get(htmlOpenedFileLanguage); 1421 if (counts.getTotal() == 0) { 1422 continue; 1423 } 1424 final String baseLanguage = ltp.set(htmlOpenedFileLanguage).getLanguage(); 1425 indexTablePrinter 1426 .addRow() 1427 .addCell(baseLanguage) 1428 .addCell(htmlOpenedFileLanguage) 1429 .addCell(ConsoleCheckCLDR.getLocaleName(htmlOpenedFileLanguage)); 1430 if (SHOW_VOTING_INFO) { 1431 indexTablePrinter 1432 .addCell( 1433 votingData == null 1434 ? "" 1435 : votingData.winningStatusCounter.toString()) 1436 .addCell( 1437 votingData == null 1438 ? "" 1439 : votingData.missingOrganizationCounter.toString()); 1440 } 1441 for (Organization org : orgToLocales.keySet()) { 1442 indexTablePrinter.addCell( 1443 orgToLocales.getAll(org).contains(htmlOpenedFileLanguage) 1444 ? org.toString().substring(0, 2) 1445 : ""); 1446 } 1447 indexTablePrinter 1448 .addCell( 1449 votingData == null 1450 ? "" 1451 : formatSkippingZero(votingData.getDisputedCount())) 1452 .addCell(votingData == null ? "" : votingData.getConflictedHTML()); 1453 for (ConsoleCheckCLDR.ErrorType type : ConsoleCheckCLDR.ErrorType.showInSummary) { 1454 indexTablePrinter.addCell(formatSkippingZero(counts.getCount(type))); 1455 } 1456 indexTablePrinter.finishRow(); 1457 } 1458 generated_html_index.println(indexTablePrinter.toTable()); 1459 generated_html_index.println(ShowData.dateFooter()); 1460 generated_html_index.println(CldrUtility.ANALYTICS); 1461 generated_html_index.println("</body></html>"); 1462 } 1463 1464 static Relation<Organization, String> orgToLocales; 1465 getOrgToLocales()1466 private static Relation<Organization, String> getOrgToLocales() { 1467 if (orgToLocales == null) { 1468 orgToLocales = Relation.of(new TreeMap<Organization, Set<String>>(), TreeSet.class); 1469 StandardCodes sc = StandardCodes.make(); 1470 for (Organization org : sc.getLocaleCoverageOrganizations()) { 1471 for (String locale : sc.getLocaleCoverageLocales(org)) { 1472 Level x = sc.getLocaleCoverageLevel(org, locale); 1473 if (x.compareTo(Level.BASIC) > 0) { 1474 orgToLocales.put(org, locale); 1475 } 1476 } 1477 } 1478 } 1479 return orgToLocales; 1480 } 1481 showSections()1482 static void showSections() throws IOException { 1483 Relation<Organization, String> orgToLocales = getOrgToLocales(); 1484 TablePrinter indexTablePrinter = 1485 new TablePrinter() 1486 .setCaption("Problem Summary") 1487 .setTableAttributes( 1488 "border='1' style='border-collapse: collapse' bordercolor='blue'") 1489 .addColumn("Section") 1490 .setSpanRows(true) 1491 .setBreakSpans(true) // .setRepeatDivider(true) 1492 .addColumn("Problems") 1493 .setCellAttributes("style=\"text-align:left\" class=\"{2}\"") 1494 .setSpanRows(true) 1495 .addColumn("Subtype") 1496 .setCellAttributes("style=\"text-align:left\" class=\"{2}\"") 1497 .setSpanRows(true) 1498 .addColumn("Locale") 1499 .setCellAttributes("class=\"{2}\"") 1500 .addColumn("Code") 1501 .setCellAttributes("class=\"{2}\"") 1502 .setCellPattern( 1503 "<a href=\"http://unicode.org/cldr/apps/survey?_={0}&x={1}\">{0}</a>") // TODO: use CLDRConfig.urls() 1504 .addColumn("Count") 1505 .setCellAttributes("class=\"{2}\""); 1506 for (Organization org : orgToLocales.keySet()) { 1507 indexTablePrinter.addColumn(org.toString().substring(0, 2)); 1508 } 1509 1510 for (Row.R4<String, ErrorType, Subtype, String> sectionAndProblemsAndLocale : 1511 ErrorFile.sectionToProblemsToLocaleToCount.getKeysetSortedByKey()) { 1512 final ErrorType problem = sectionAndProblemsAndLocale.get1(); 1513 final Subtype subtype = sectionAndProblemsAndLocale.get2(); 1514 if (!ConsoleCheckCLDR.ErrorType.showInSummary.contains(problem)) { 1515 continue; 1516 } 1517 final String locale = sectionAndProblemsAndLocale.get3(); 1518 if (problem != ErrorType.error 1519 && problem != ErrorType.disputed 1520 && !orgToLocales.containsValue(locale)) { 1521 continue; 1522 } 1523 long count = 1524 ErrorFile.sectionToProblemsToLocaleToCount.getCount( 1525 sectionAndProblemsAndLocale); 1526 final String section = sectionAndProblemsAndLocale.get0(); 1527 indexTablePrinter 1528 .addRow() 1529 .addCell(section) 1530 .addCell(problem) 1531 .addCell(subtype) 1532 .addCell(ConsoleCheckCLDR.getLocaleName(locale)) 1533 .addCell(locale) 1534 .addCell(count); 1535 for (Organization org : orgToLocales.keySet()) { 1536 indexTablePrinter.addCell( 1537 orgToLocales.getAll(org).contains(locale) 1538 ? org.toString().substring(0, 2) 1539 : ""); 1540 } 1541 indexTablePrinter.finishRow(); 1542 } 1543 PrintWriter generated_html_index = 1544 FileUtilities.openUTF8Writer( 1545 ErrorFile.generated_html_directory, "sections.html"); 1546 ConsoleCheckCLDR.ErrorFile.showIndexHead( 1547 "Error Report Index by Section", "", generated_html_index); 1548 generated_html_index.println(indexTablePrinter.toTable()); 1549 generated_html_index.println(ShowData.dateFooter()); 1550 generated_html_index.println(CldrUtility.ANALYTICS); 1551 generated_html_index.println("</body></html>"); 1552 generated_html_index.close(); 1553 } 1554 formatSkippingZero(long count)1555 static String formatSkippingZero(long count) { 1556 if (count == 0) { 1557 return ""; 1558 } 1559 return String.valueOf(count); 1560 } 1561 showIndexHead(String title, String localeID, PrintWriter generated_html_index)1562 static void showIndexHead(String title, String localeID, PrintWriter generated_html_index) { 1563 final boolean notLocaleSpecific = localeID.length() == 0; 1564 if ((!notLocaleSpecific)) { 1565 title = "Errors in " + ConsoleCheckCLDR.getNameAndLocale(localeID, false); 1566 } 1567 generated_html_index.println( 1568 "<html>" 1569 + "<head><meta http-equiv='Content-Type' content='text/html; charset=utf-8'>" 1570 + CldrUtility.LINE_SEPARATOR 1571 + "<title>" 1572 + title 1573 + "</title>" 1574 + CldrUtility.LINE_SEPARATOR 1575 + "<link rel='stylesheet' href='errors.css' type='text/css'>" 1576 + CldrUtility.LINE_SEPARATOR 1577 + "<base target='_blank'>" 1578 + CldrUtility.LINE_SEPARATOR 1579 + "</head><body>" 1580 + CldrUtility.LINE_SEPARATOR 1581 + "<h1>" 1582 + title 1583 + "</h1>" 1584 + CldrUtility.LINE_SEPARATOR 1585 + "<p>" 1586 + "<a href='index.html" 1587 + (notLocaleSpecific ? "" : "#" + localeID) 1588 + "'>Index</a>" 1589 + " | " 1590 + "<a href='sections.html" 1591 + (notLocaleSpecific ? "" : "#" + localeID) 1592 + "'>Index by Section</a>" 1593 + " | " 1594 + "<a href='http://unicode.org/cldr/data/docs/survey/vetting.html'><b style='background-color: yellow;'><i>Help: How to Vet</i></b></a>" 1595 + "</p>" 1596 + "<p>The following errors have been detected in the locale" 1597 + (notLocaleSpecific 1598 ? "s. " 1599 + org.unicode.cldr.test.HelpMessages.getChartMessages( 1600 "error_index_header") 1601 : " " 1602 + ConsoleCheckCLDR.getNameAndLocale(localeID, false) 1603 + ". " 1604 + ErrorFile.ERROR_CHART_HEADER)); 1605 } 1606 writeErrorFileIndex()1607 private static void writeErrorFileIndex() throws IOException { 1608 PrintWriter generated_html_index = 1609 FileUtilities.openUTF8Writer(ErrorFile.generated_html_directory, "index.html"); 1610 ConsoleCheckCLDR.ErrorFile.showIndexHead( 1611 "Error Report Index", "", generated_html_index); 1612 ConsoleCheckCLDR.ErrorFile.showErrorFileIndex(generated_html_index); 1613 generated_html_index.close(); 1614 showSections(); 1615 } 1616 writeErrorCountsText()1617 private static void writeErrorCountsText() { 1618 // do the plain text file 1619 ErrorFile.generated_html_count.print(ConsoleCheckCLDR.lastHtmlLocaleID + ";\tcounts"); 1620 for (ConsoleCheckCLDR.ErrorType type : ConsoleCheckCLDR.ErrorType.showInSummary) { 1621 ErrorFile.generated_html_count.print( 1622 ";\t" + type + "=" + ErrorFile.htmlErrorsPerLocale.getCount(type)); 1623 } 1624 ErrorFile.generated_html_count.println(); 1625 ErrorFile.generated_html_count.flush(); 1626 1627 // now store the data for the index 1628 ErrorFile.errorFileIndexData.put( 1629 ConsoleCheckCLDR.lastHtmlLocaleID, 1630 new Pair<>(ConsoleCheckCLDR.lastHtmlLocaleID, ErrorFile.htmlErrorsPerLocale)); 1631 ErrorFile.htmlErrorsPerLocale = new Counter<>(); 1632 } 1633 1634 static Counter<ErrorType> htmlErrorsPerLocale = 1635 new Counter<>(); // ConsoleCheckCLDR.ErrorCount(); 1636 static PrintWriter generated_html_count = null; 1637 private static TreeMap<String, Pair<String, Counter<ErrorType>>> errorFileIndexData = 1638 new TreeMap<>(); 1639 1640 static PrintWriter errorFileWriter = null; 1641 private static final String ERROR_CHART_HEADER = 1642 org.unicode.cldr.test.HelpMessages.getChartMessages("error_locale_header"); 1643 static String generated_html_directory = null; 1644 public static Counter<Row.R4<String, ErrorType, Subtype, String>> 1645 sectionToProblemsToLocaleToCount = new Counter<>(); 1646 } 1647 showSummary(String localeID, Level level, String value)1648 private static void showSummary(String localeID, Level level, String value) { 1649 String line = "# " + getLocaleAndName(localeID) + "\tSummary\t" + level + "\t" + value; 1650 System.out.println(line); 1651 } 1652 showExamples( CLDRFile cldrFile, String prettyPath, String localeID, String path, String value, String fullPath, String example)1653 private static void showExamples( 1654 CLDRFile cldrFile, 1655 String prettyPath, 1656 String localeID, 1657 String path, 1658 String value, 1659 String fullPath, 1660 String example) { 1661 if (example != null) { 1662 showValue( 1663 cldrFile, 1664 prettyPath, 1665 localeID, 1666 example, 1667 path, 1668 value, 1669 fullPath, 1670 "ok", 1671 Subtype.none); 1672 } 1673 } 1674 addPrettyPaths( CLDRFile file, Matcher pathFilter, PathHeader.Factory pathHeaderFactory, boolean noaliases, boolean filterDraft, Collection<String> target)1675 private static void addPrettyPaths( 1676 CLDRFile file, 1677 Matcher pathFilter, 1678 PathHeader.Factory pathHeaderFactory, 1679 boolean noaliases, 1680 boolean filterDraft, 1681 Collection<String> target) { 1682 for (Iterator<String> pit = file.iterator(pathFilter); pit.hasNext(); ) { 1683 String path = pit.next(); 1684 addPrettyPath(file, pathHeaderFactory, noaliases, filterDraft, target, path); 1685 } 1686 } 1687 addPrettyPaths( CLDRFile file, Collection<String> paths, Matcher pathFilter, PathHeader.Factory pathHeaderFactory, boolean noaliases, boolean filterDraft, Collection<String> target)1688 private static void addPrettyPaths( 1689 CLDRFile file, 1690 Collection<String> paths, 1691 Matcher pathFilter, 1692 PathHeader.Factory pathHeaderFactory, 1693 boolean noaliases, 1694 boolean filterDraft, 1695 Collection<String> target) { 1696 for (String path : paths) { 1697 if (pathFilter != null && !pathFilter.reset(path).matches()) continue; 1698 addPrettyPath(file, pathHeaderFactory, noaliases, filterDraft, target, path); 1699 } 1700 } 1701 addPrettyPath( CLDRFile file, PathHeader.Factory pathHeaderFactory, boolean noaliases, boolean filterDraft, Collection<String> target, String path)1702 private static void addPrettyPath( 1703 CLDRFile file, 1704 PathHeader.Factory pathHeaderFactory, 1705 boolean noaliases, 1706 boolean filterDraft, 1707 Collection<String> target, 1708 String path) { 1709 if (noaliases 1710 && XMLSource.Alias.isAliasPath( 1711 path)) { // this is just for console testing, the survey tool 1712 // shouldn't do it. 1713 return; 1714 } 1715 if (filterDraft) { 1716 String newPath = CLDRFile.getNondraftNonaltXPath(path); 1717 if (!newPath.equals(path)) { 1718 String value = file.getStringValue(newPath); 1719 if (value != null) { 1720 return; 1721 } 1722 } 1723 } 1724 String prettyPath = pathHeaderFactory.fromPath(path).toString(); 1725 target.add(prettyPath); 1726 } 1727 setExampleGenerator(ExampleGenerator inputExampleGenerator)1728 private static synchronized void setExampleGenerator(ExampleGenerator inputExampleGenerator) { 1729 englishExampleGenerator = inputExampleGenerator; 1730 } 1731 getExampleGenerator()1732 private static synchronized ExampleGenerator getExampleGenerator() { 1733 return englishExampleGenerator; 1734 } 1735 1736 private static ExampleGenerator englishExampleGenerator; 1737 1738 static Matcher coverageMatcher = 1739 PatternCache.get("meet ([a-z]*) coverage").matcher(""); // HACK TODO fix 1740 showHeaderLine()1741 private static void showHeaderLine() { 1742 if (SHOW_LOCALE) { 1743 if (idView) { 1744 System.out.println( 1745 "Locale\tID\tDesc.\t〈Eng.Value〉\t【Eng.Ex.】\t〈Loc.Value〉\t【Loc.Ex】\t⁅error/warning type⁆\t❮Error/Warning Msg❯"); 1746 } else { 1747 System.out.println( 1748 "Locale\tStatus\t▸PPath◂\t〈Eng.Value〉\t【Eng.Ex.】\t〈Loc.Value〉\t«fill-in»\t【Loc.Ex】\t⁅error/warning type⁆\t❮Error/Warning Msg❯\tFull Path\tAliasedSource/Path?"); 1749 } 1750 } 1751 } 1752 1753 private static PathDescription pathDescription = null; 1754 getIdString(String path, String value)1755 private static String getIdString(String path, String value) { 1756 if (pathDescription == null) { 1757 pathDescription = 1758 new PathDescription( 1759 supplementalDataInfo, 1760 english, 1761 null, 1762 null, 1763 PathDescription.ErrorHandling.CONTINUE); 1764 } 1765 final String description = pathDescription.getDescription(path, value, null); 1766 return "\t" + StringId.getId(path) + "" + "\t" + description + ""; 1767 } 1768 showValue( CLDRFile cldrFile, String prettyPath, String localeID, String example, String path, String value, String fullPath, String statusString, Subtype subtype)1769 private static void showValue( 1770 CLDRFile cldrFile, 1771 String prettyPath, 1772 String localeID, 1773 String example, 1774 String path, 1775 String value, 1776 String fullPath, 1777 String statusString, 1778 Subtype subtype) { 1779 ErrorType shortStatus = ErrorType.fromStatusString(statusString); 1780 subtotalCount.add(shortStatus, 1); 1781 totalCount.add(shortStatus, 1); 1782 if (subtype == null) { 1783 subtype = Subtype.none; 1784 } 1785 final SourceLocation location = 1786 fullPath == null ? null : cldrFile.getSourceLocation(fullPath); 1787 1788 if (ErrorFile.errorFileWriter == null) { 1789 example = example == null ? "" : example; 1790 String englishExample = null; 1791 final String englishPathValue = path == null ? null : getEnglishPathValue(path); 1792 if (SHOW_EXAMPLES && path != null) { 1793 englishExample = 1794 ExampleGenerator.simplify( 1795 getExampleGenerator().getExampleHtml(path, englishPathValue)); 1796 } 1797 englishExample = englishExample == null ? "" : englishExample; 1798 String cleanPrettyPath = path == null ? null : prettyPath; 1799 Status status = new Status(); 1800 String sourceLocaleID = path == null ? null : cldrFile.getSourceLocaleID(path, status); 1801 String fillinValue = path == null ? null : cldrFile.getFillInValue(path); 1802 fillinValue = fillinValue == null ? "" : fillinValue.equals(value) ? "=" : fillinValue; 1803 1804 String pathLink = CLDR_CONFIG.urls().forXpath(localeID, path); 1805 1806 final String otherSource = 1807 path == null 1808 ? null 1809 : (sourceLocaleID.equals(localeID) ? "" : "\t" + sourceLocaleID); 1810 final String otherPath = 1811 path == null 1812 ? null 1813 : (status.pathWhereFound.equals(path) 1814 ? "" 1815 : "\t" + status.pathWhereFound); 1816 if (location != null) { 1817 System.err.println(location.toString() + shortStatus); // print full path here 1818 } 1819 String idViewString = 1820 idView ? (path == null ? "\tNO_ID" : getIdString(path, value)) : ""; 1821 System.out.println( 1822 getLocaleAndName(localeID) 1823 + (idViewString.isEmpty() 1824 ? 1825 // + "\t" + subtotalCount.getCount(shortStatus) 1826 "\t" 1827 + shortStatus 1828 + "\t▸" 1829 + cleanPrettyPath 1830 + "◂" 1831 + "\t〈" 1832 + englishPathValue 1833 + "〉" 1834 + "\t【" 1835 + englishExample 1836 + "】" 1837 + "\t〈" 1838 + value 1839 + "〉" 1840 + "\t«" 1841 + fillinValue 1842 + "»" 1843 + "\t【" 1844 + example 1845 + "】" 1846 + "\t⁅" 1847 + subtype 1848 + "⁆" 1849 + "\t❮" 1850 + statusString 1851 + "❯" 1852 + "\t" 1853 + pathLink 1854 + otherSource 1855 + otherPath 1856 : idViewString 1857 + "\t〈" 1858 + englishPathValue 1859 + "〉" 1860 + "\t【" 1861 + englishExample 1862 + "】" 1863 + "\t" 1864 + value 1865 + "〉" 1866 + "\t【" 1867 + example 1868 + "】" 1869 + "\t⁅" 1870 + subtype 1871 + "⁆" 1872 + "\t❮" 1873 + statusString 1874 + "❯")); 1875 } else if (ErrorFile.errorFileWriter != null) { 1876 if (shortStatus == ErrorType.contributed) { 1877 return; 1878 } 1879 if (shortStatus == ErrorType.posix) { 1880 shortStatus = ErrorType.minimal; 1881 } 1882 if (!localeID.equals(lastHtmlLocaleID)) { 1883 ErrorFile.writeErrorCountsText(); 1884 lastHtmlLocaleID = localeID; 1885 } 1886 addError(shortStatus); 1887 ErrorFile.addDataToErrorFile(localeID, path, shortStatus, subtype); 1888 } 1889 if (CLDR_GITHUB_ANNOTATIONS) { 1890 // Annotate anything that needs annotation 1891 if (shortStatus == ErrorType.error || shortStatus == ErrorType.warning) { 1892 String filePath = null; 1893 if (location != null) { 1894 // Use accurate location 1895 filePath = 1896 location.forGitHub( 1897 CLDRPaths.BASE_DIRECTORY); // Trim to CLDR_DIR for GitHub 1898 } else { 1899 // Fallback if SourceLocation fails 1900 filePath = 1901 "file=" 1902 + localeXpathToFilePath.computeIfAbsent( 1903 Pair.of(localeID, path), 1904 locPath -> guessFilePath(locPath)); 1905 } 1906 System.out.println( 1907 "::" 1908 + shortStatus 1909 + " " 1910 + filePath.trim() 1911 + ",title=" 1912 + subtype 1913 + ":: " 1914 + statusString); 1915 } 1916 } 1917 if (PATH_IN_COUNT && ErrorFile.generated_html_count != null) { 1918 ErrorFile.generated_html_count.println(lastHtmlLocaleID + ";\tpath:\t" + path); 1919 } 1920 } 1921 addError(ErrorType shortStatus)1922 private static void addError(ErrorType shortStatus) { 1923 if (ErrorType.showInSummary.contains(shortStatus)) { 1924 ErrorFile.htmlErrorsPerLocale.increment(shortStatus); 1925 } 1926 } 1927 1928 static String lastHtmlLocaleID = ""; 1929 private static VoterInfoList voterInfoList; 1930 private static VoteResolver<String> voteResolver; 1931 private static String resolveVotesDirectory; 1932 private static boolean idView; 1933 private static SupplementalDataInfo supplementalDataInfo; 1934 private static CLDRFile english; 1935 1936 public static class PathShower { 1937 String localeID; 1938 boolean newLocale = true; 1939 String lastPath; 1940 String[] lastSplitPath; 1941 boolean showEnglish; 1942 String splitChar = "/"; 1943 1944 static final String lead = "****************************************"; 1945 set(String localeID)1946 public void set(String localeID) { 1947 this.localeID = localeID; 1948 newLocale = true; 1949 LocaleIDParser localeIDParser = new LocaleIDParser(); 1950 showEnglish = !localeIDParser.set(localeID).getLanguageScript().equals("en"); 1951 lastPath = null; 1952 lastSplitPath = null; 1953 } 1954 getSplitChar()1955 public String getSplitChar() { 1956 return splitChar; 1957 } 1958 setSplitChar(String splitChar)1959 public PathShower setSplitChar(String splitChar) { 1960 this.splitChar = splitChar; 1961 return this; 1962 } 1963 } 1964 getEnglishPathValue(String path)1965 private static String getEnglishPathValue(String path) { 1966 String englishValue = CheckCLDR.getDisplayInformation().getWinningValue(path); 1967 if (englishValue == null) { 1968 String path2 = CLDRFile.getNondraftNonaltXPath(path); 1969 englishValue = CheckCLDR.getDisplayInformation().getWinningValue(path2); 1970 } 1971 return englishValue; 1972 } 1973 1974 /** 1975 * Utility for getting information. 1976 * 1977 * @param locale 1978 * @return 1979 */ getLocaleAndName(String locale)1980 private static String getLocaleAndName(String locale) { 1981 String localizedName = CheckCLDR.getDisplayInformation().getName(locale); 1982 if (localizedName == null || localizedName.equals(locale)) return locale; 1983 return locale + " [" + localizedName + "]"; 1984 } 1985 1986 /** 1987 * Utility for getting information. 1988 * 1989 * @param locale 1990 * @param linkToXml TODO 1991 * @return 1992 */ getNameAndLocale(String locale, boolean linkToXml)1993 private static String getNameAndLocale(String locale, boolean linkToXml) { 1994 String localizedName = CheckCLDR.getDisplayInformation().getName(locale); 1995 if (localizedName == null || localizedName.equals(locale)) return locale; 1996 if (linkToXml) { 1997 locale = 1998 "<a href='https://github.com/unicode-org/cldr/tree/main/common/main/" 1999 + locale 2000 + ".xml'>" 2001 + locale 2002 + "</a>"; 2003 } 2004 return localizedName + " [" + locale + "]"; 2005 } 2006 getLocaleName(String locale)2007 private static String getLocaleName(String locale) { 2008 String localizedName = CheckCLDR.getDisplayInformation().getName(locale); 2009 if (localizedName == null || localizedName.equals(locale)) return locale; 2010 return localizedName; 2011 } 2012 getLinkedLocale(String locale)2013 private static String getLinkedLocale(String locale) { 2014 return "<a href='http://unicode.org/cldr/apps/survey?_=" + locale + "'>" + locale + "</a>"; 2015 } 2016 2017 /** Approximate xml path */ guessFilePath(Pair<String, String> locPath)2018 private static String guessFilePath(Pair<String, String> locPath) { 2019 final File base = new File(CLDRPaths.BASE_DIRECTORY); 2020 final String loc = locPath.getFirst(); 2021 final String path = locPath.getSecond(); 2022 String subdir = "main"; 2023 if (path.startsWith("//ldml/annotations")) { 2024 subdir = "annotations"; 2025 } else if (path.startsWith("//ldml/subdivisions")) { 2026 subdir = "subdivisions"; 2027 } 2028 File inCommon = new File(base, "common"); 2029 File subsub = new File(inCommon, subdir); 2030 if (subsub.isDirectory()) { 2031 File subFile = new File(subsub, loc + ".xml"); 2032 if (subFile.canRead()) 2033 return subFile.getAbsolutePath().substring(base.getAbsolutePath().length() + 1); 2034 } 2035 2036 File inSeed = new File(base, "seed"); 2037 subsub = new File(inSeed, subdir); 2038 if (subsub.isDirectory()) { 2039 File subFile = new File(subsub, loc + ".xml"); 2040 if (subFile.canRead()) 2041 return subFile.getAbsolutePath().substring(base.getAbsolutePath().length() + 1); 2042 } 2043 return loc + ".xml"; 2044 } 2045 2046 static final ConcurrentHashMap<Pair<String, String>, String> localeXpathToFilePath = 2047 new ConcurrentHashMap<>(); 2048 } 2049