1 /* 2 ****************************************************************************** 3 * Copyright (C) 2005-2014, International Business Machines Corporation and * 4 * others. All Rights Reserved. * 5 ****************************************************************************** 6 */ 7 8 package org.unicode.cldr.test; 9 10 import com.google.common.collect.ImmutableSet; 11 import com.ibm.icu.dev.util.ElapsedTimer; 12 import com.ibm.icu.impl.Row.R3; 13 import com.ibm.icu.text.ListFormatter; 14 import com.ibm.icu.text.MessageFormat; 15 import java.text.ParsePosition; 16 import java.util.ArrayList; 17 import java.util.Arrays; 18 import java.util.HashMap; 19 import java.util.Iterator; 20 import java.util.List; 21 import java.util.Locale; 22 import java.util.Map; 23 import java.util.Set; 24 import java.util.TreeSet; 25 import java.util.logging.Logger; 26 import java.util.regex.Matcher; 27 import java.util.regex.Pattern; 28 import org.unicode.cldr.test.CheckCLDR.CheckStatus.Subtype; 29 import org.unicode.cldr.util.CLDRFile; 30 import org.unicode.cldr.util.CLDRInfo.CandidateInfo; 31 import org.unicode.cldr.util.CLDRInfo.PathValueInfo; 32 import org.unicode.cldr.util.CLDRInfo.UserInfo; 33 import org.unicode.cldr.util.CLDRLocale; 34 import org.unicode.cldr.util.CldrUtility; 35 import org.unicode.cldr.util.Factory; 36 import org.unicode.cldr.util.InternalCldrException; 37 import org.unicode.cldr.util.Level; 38 import org.unicode.cldr.util.PathHeader; 39 import org.unicode.cldr.util.PathHeader.SurveyToolStatus; 40 import org.unicode.cldr.util.PatternCache; 41 import org.unicode.cldr.util.RegexFileParser; 42 import org.unicode.cldr.util.RegexFileParser.RegexLineParser; 43 import org.unicode.cldr.util.StandardCodes; 44 import org.unicode.cldr.util.TransliteratorUtilities; 45 import org.unicode.cldr.util.VoteResolver; 46 import org.unicode.cldr.util.VoteResolver.Status; 47 48 /** 49 * This class provides a foundation for both console-driven CLDR tests, and Survey Tool Tests. 50 * 51 * <p>To add a test, subclass CLDRFile and override handleCheck and possibly setCldrFileToCheck. 52 * Then put the test into getCheckAll. 53 * 54 * <p>To use the test, take a look at the main in ConsoleCheckCLDR. Note that you need to call 55 * setDisplayInformation with the CLDRFile for the locale that you want the display information (eg 56 * names for codes) to be in.<br> 57 * Some options are passed in the Map options. Examples: boolean SHOW_TIMES = 58 * options.containsKey("SHOW_TIMES"); // for printing times for doing setCldrFileToCheck. 59 * 60 * <p>Some errors/warnings will be explicitly filtered out when calling CheckCLDR's check() method. 61 * The full list of filters can be found in org/unicode/cldr/util/data/CheckCLDR-exceptions.txt. 62 * 63 * @author davis 64 */ 65 public abstract class CheckCLDR implements CheckAccessor { 66 67 /** protected so subclasses can use it */ 68 protected static Logger logger = Logger.getLogger(CheckCLDR.class.getSimpleName()); 69 70 /** 71 * set the internal logger level. For ConsoleCheck. 72 * 73 * @returns the previous level 74 */ setLoggerLevel(java.util.logging.Level newLevel)75 public static java.util.logging.Level setLoggerLevel(java.util.logging.Level newLevel) { 76 // NB: we use the full package name here, to avoid conflict with other CLDR classes named 77 // Level 78 java.util.logging.Level oldLevel = logger.getLevel(); 79 logger.setLevel(newLevel); 80 return oldLevel; 81 } 82 83 /** serialize CheckCLDR as just its class name */ toString()84 public String toString() { 85 return getClass().getSimpleName(); 86 } 87 88 public static final boolean LIMITED_SUBMISSION = 89 false; // TODO: CLDR-13337: represent differently 90 91 private static CLDRFile displayInformation; 92 93 private CLDRFile cldrFileToCheck; 94 private CLDRFile englishFile = null; 95 96 private boolean skipTest = false; 97 private Phase phase; 98 private Map<Subtype, List<Pattern>> filtersForLocale = new HashMap<>(); 99 100 @Override getStringValue(String path)101 public String getStringValue(String path) { 102 return getCldrFileToCheck().getStringValue(path); 103 } 104 105 @Override getUnresolvedStringValue(String path)106 public String getUnresolvedStringValue(String path) { 107 return getCldrFileToCheck().getUnresolved().getStringValue(path); 108 } 109 110 @Override getLocaleID()111 public String getLocaleID() { 112 return getCldrFileToCheck().getLocaleID(); 113 } 114 115 @Override getCause()116 public CheckCLDR getCause() { 117 return this; 118 } 119 120 public enum InputMethod { 121 DIRECT, 122 BULK 123 } 124 125 public enum StatusAction { 126 /** Allow voting and add new values (in Change column). */ 127 ALLOW, 128 /** Allow voting and ticket (in Change column). */ 129 ALLOW_VOTING_AND_TICKET, 130 /** Allow voting but no add new values (in Change column). */ 131 ALLOW_VOTING_BUT_NO_ADD, 132 /** Only allow filing a ticket. */ 133 ALLOW_TICKET_ONLY, 134 /** Disallow (for various reasons) */ 135 FORBID_ERRORS(true), 136 FORBID_READONLY(true), 137 FORBID_UNLESS_DATA_SUBMISSION(true), 138 FORBID_NULL(true), 139 FORBID_ROOT(true), 140 FORBID_CODE(true), 141 FORBID_PERMANENT_WITHOUT_FORUM(true); 142 143 private final boolean isForbidden; 144 StatusAction()145 private StatusAction() { 146 isForbidden = false; 147 } 148 StatusAction(boolean isForbidden)149 private StatusAction(boolean isForbidden) { 150 this.isForbidden = isForbidden; 151 } 152 isForbidden()153 public boolean isForbidden() { 154 return isForbidden; 155 } 156 canShow()157 public boolean canShow() { 158 return !isForbidden; 159 } 160 } 161 162 private static final HashMap<String, Phase> PHASE_NAMES = new HashMap<>(); 163 164 public enum Phase { 165 BUILD, 166 SUBMISSION, 167 VETTING, 168 FINAL_TESTING("RESOLUTION"); 169 Phase(String... alternateName)170 Phase(String... alternateName) { 171 for (String name : alternateName) { 172 PHASE_NAMES.put(name.toUpperCase(Locale.ENGLISH), this); 173 } 174 } 175 forString(String value)176 public static Phase forString(String value) { 177 if (value == null) { 178 return org.unicode.cldr.util.CLDRConfig.getInstance().getPhase(); 179 } 180 value = value.toUpperCase(Locale.ENGLISH); 181 Phase result = PHASE_NAMES.get(value); 182 return result != null ? result : Phase.valueOf(value); 183 } 184 185 /** true if it's a 'unit test' phase. */ isUnitTest()186 public boolean isUnitTest() { 187 return this == BUILD || this == FINAL_TESTING; 188 } 189 190 /** 191 * Return whether or not to show a row, and if so, how. 192 * 193 * @param pathValueInfo 194 * @param inputMethod 195 * @param ph the path header 196 * @param userInfo null if there is no userInfo (nobody logged in). 197 * @return 198 */ getShowRowAction( PathValueInfo pathValueInfo, InputMethod inputMethod, PathHeader ph, UserInfo userInfo )199 public StatusAction getShowRowAction( 200 PathValueInfo pathValueInfo, 201 InputMethod inputMethod, 202 PathHeader ph, 203 UserInfo userInfo // can get voterInfo from this. 204 ) { 205 206 PathHeader.SurveyToolStatus status = ph.getSurveyToolStatus(); 207 /* 208 * Always forbid DEPRECATED items - don't show. 209 * 210 * Currently, bulk submission and TC voting are allowed even for SurveyToolStatus.HIDE, 211 * but not for SurveyToolStatus.DEPRECATED. If we ever want to treat HIDE and DEPRECATED 212 * the same here, then it would be simpler to call ph.shouldHide which is true for both. 213 */ 214 if (status == SurveyToolStatus.DEPRECATED) { 215 return StatusAction.FORBID_READONLY; 216 } 217 218 if (status == SurveyToolStatus.READ_ONLY) { 219 return StatusAction.ALLOW_TICKET_ONLY; 220 } 221 222 // if TC+, allow anything else, even suppressed items and errors 223 if (userInfo != null 224 && userInfo.getVoterInfo().getLevel().compareTo(VoteResolver.Level.tc) >= 0) { 225 return StatusAction.ALLOW; 226 } 227 228 // always forbid bulk import except in data submission. 229 if (inputMethod == InputMethod.BULK && (this != Phase.SUBMISSION && isUnitTest())) { 230 return StatusAction.FORBID_UNLESS_DATA_SUBMISSION; 231 } 232 233 if (status == SurveyToolStatus.HIDE) { 234 return StatusAction.FORBID_READONLY; 235 } 236 237 CandidateInfo winner = pathValueInfo.getCurrentItem(); 238 ValueStatus valueStatus = getValueStatus(winner, ValueStatus.NONE, null); 239 240 // if limited submission, and winner doesn't have an error, limit the values 241 242 if (LIMITED_SUBMISSION) { 243 if (!SubmissionLocales.allowEvenIfLimited( 244 pathValueInfo.getLocale().toString(), 245 pathValueInfo.getXpath(), 246 valueStatus == ValueStatus.ERROR, 247 pathValueInfo.getBaselineStatus() == Status.missing)) { 248 return StatusAction.FORBID_READONLY; 249 } 250 } 251 252 if (this == Phase.SUBMISSION || isUnitTest()) { 253 return (ph.canReadAndWrite()) 254 ? StatusAction.ALLOW 255 : StatusAction.ALLOW_VOTING_AND_TICKET; 256 } 257 258 // We are in vetting, not in submission 259 260 // Only allow ADD if we have an error or warning 261 // Only check winning value for errors/warnings per ticket #8677 262 if (valueStatus != ValueStatus.NONE) { 263 return (ph.canReadAndWrite()) 264 ? StatusAction.ALLOW 265 : StatusAction.ALLOW_VOTING_AND_TICKET; 266 } 267 268 // No warnings, so allow just voting. 269 return StatusAction.ALLOW_VOTING_BUT_NO_ADD; 270 } 271 272 /** 273 * getAcceptNewItemAction. MUST only be called if getShowRowAction(...).canShow() TODO 274 * Consider moving Phase, StatusAction, etc into CLDRInfo. 275 * 276 * @param enteredValue If null, means an abstention. If voting for an existing value, 277 * pathValueInfo.getValues().contains(enteredValue) MUST be true 278 * @param pathValueInfo 279 * @param inputMethod 280 * @param status 281 * @param userInfo 282 * @return 283 */ getAcceptNewItemAction( CandidateInfo enteredValue, PathValueInfo pathValueInfo, InputMethod inputMethod, PathHeader ph, UserInfo userInfo )284 public StatusAction getAcceptNewItemAction( 285 CandidateInfo enteredValue, 286 PathValueInfo pathValueInfo, 287 InputMethod inputMethod, 288 PathHeader ph, 289 UserInfo userInfo // can get voterInfo from this. 290 ) { 291 if (!ph.canReadAndWrite()) { 292 return StatusAction.FORBID_READONLY; 293 } 294 295 // only logged in users can add items. 296 if (userInfo == null) { 297 return StatusAction.FORBID_ERRORS; 298 } 299 300 // we can always abstain 301 if (enteredValue == null) { 302 return StatusAction.ALLOW; 303 } 304 305 // if TC+, allow anything else, even suppressed items and errors 306 if (userInfo.getVoterInfo().getLevel().compareTo(VoteResolver.Level.tc) >= 0) { 307 return StatusAction.ALLOW; 308 } 309 310 // Disallow errors. 311 ValueStatus valueStatus = 312 getValueStatus(enteredValue, ValueStatus.NONE, CheckStatus.crossCheckSubtypes); 313 if (valueStatus == ValueStatus.ERROR) { 314 return StatusAction.FORBID_ERRORS; 315 } 316 317 // Allow items if submission 318 if (this == Phase.SUBMISSION || isUnitTest()) { 319 return StatusAction.ALLOW; 320 } 321 322 // Voting for an existing value is ok 323 valueStatus = ValueStatus.NONE; 324 for (CandidateInfo value : pathValueInfo.getValues()) { 325 if (value == enteredValue) { 326 return StatusAction.ALLOW; 327 } 328 valueStatus = getValueStatus(value, valueStatus, CheckStatus.crossCheckSubtypes); 329 } 330 331 // If there were any errors/warnings on other values, allow 332 if (valueStatus != ValueStatus.NONE) { 333 return StatusAction.ALLOW; 334 } 335 336 // Otherwise (we are vetting, but with no errors or warnings) 337 // DISALLOW NEW STUFF 338 339 return StatusAction.FORBID_UNLESS_DATA_SUBMISSION; 340 } 341 342 public enum ValueStatus { 343 ERROR, 344 WARNING, 345 NONE 346 } 347 getValueStatus( CandidateInfo value, ValueStatus previous, Set<Subtype> changeErrorToWarning)348 public ValueStatus getValueStatus( 349 CandidateInfo value, ValueStatus previous, Set<Subtype> changeErrorToWarning) { 350 if (previous == ValueStatus.ERROR || value == null) { 351 return previous; 352 } 353 354 for (CheckStatus item : value.getCheckStatusList()) { 355 CheckStatus.Type type = item.getType(); 356 if (type.equals(CheckStatus.Type.Error)) { 357 if (changeErrorToWarning != null 358 && changeErrorToWarning.contains(item.getSubtype())) { 359 return ValueStatus.WARNING; 360 } else { 361 return ValueStatus.ERROR; 362 } 363 } else if (type.equals(CheckStatus.Type.Warning)) { 364 previous = ValueStatus.WARNING; 365 } 366 } 367 return previous; 368 } 369 } 370 371 public static final class Options implements Comparable<Options> { 372 373 public enum Option { 374 locale, 375 CoverageLevel_requiredLevel("CoverageLevel.requiredLevel"), 376 CoverageLevel_localeType("CoverageLevel.localeType"), 377 SHOW_TIMES, 378 phase, 379 lgWarningCheck, 380 CheckCoverage_skip("CheckCoverage.skip"), 381 exemplarErrors; 382 383 private String key; 384 getKey()385 public String getKey() { 386 return key; 387 } 388 Option(String key)389 Option(String key) { 390 this.key = key; 391 } 392 Option()393 Option() { 394 this.key = name(); 395 } 396 } 397 398 private static StandardCodes sc = StandardCodes.make(); 399 400 private final boolean DEBUG_OPTS = false; 401 402 String options[] = new String[Option.values().length]; 403 CLDRLocale locale = null; 404 405 private final String key; // for fast compare 406 407 /** 408 * Adopt some other map 409 * 410 * @param fromOptions 411 */ Options(Map<String, String> fromOptions)412 public Options(Map<String, String> fromOptions) { 413 clear(); 414 setAll(fromOptions); 415 key = null; // no key = slow compare 416 } 417 setAll(Map<String, String> fromOptions)418 private void setAll(Map<String, String> fromOptions) { 419 for (Map.Entry<String, String> e : fromOptions.entrySet()) { 420 set(e.getKey(), e.getValue()); 421 } 422 } 423 424 /** 425 * @param key 426 * @param value 427 */ set(String key, String value)428 public void set(String key, String value) { 429 // TODO- cache the map 430 for (Option o : Option.values()) { 431 if (o.getKey().equals(key)) { 432 set(o, value); 433 return; 434 } 435 } 436 throw new IllegalArgumentException( 437 "Unknown CLDR option: '" 438 + key 439 + "' - valid keys are: " 440 + Options.getValidKeys()); 441 } 442 getValidKeys()443 private static String getValidKeys() { 444 Set<String> allkeys = new TreeSet<>(); 445 for (Option o : Option.values()) { 446 allkeys.add(o.getKey()); 447 } 448 return ListFormatter.getInstance().format(allkeys); 449 } 450 Options()451 public Options() { 452 clear(); 453 key = "".intern(); // null Options. 454 } 455 456 /** 457 * Deep clone 458 * 459 * @param options2 460 */ Options(Options options2)461 public Options(Options options2) { 462 this.options = Arrays.copyOf(options2.options, options2.options.length); 463 this.key = options2.key; 464 this.locale = options2.locale; 465 } 466 Options(CLDRLocale locale)467 public Options(CLDRLocale locale) { 468 this.locale = locale; 469 options = new String[Option.values().length]; 470 set(Option.locale, locale.getBaseName()); 471 StringBuilder sb = new StringBuilder(); 472 sb.append(locale.getBaseName()).append('/'); 473 key = sb.toString().intern(); 474 } 475 Options( CLDRLocale locale, CheckCLDR.Phase testPhase, String requiredLevel, String localeType)476 public Options( 477 CLDRLocale locale, 478 CheckCLDR.Phase testPhase, 479 String requiredLevel, 480 String localeType) { 481 this.locale = locale; 482 options = new String[Option.values().length]; 483 StringBuilder sb = new StringBuilder(); 484 set(Option.locale, locale.getBaseName()); 485 sb.append(locale.getBaseName()).append('/'); 486 set(Option.CoverageLevel_requiredLevel, requiredLevel); 487 sb.append(requiredLevel).append('/'); 488 set(Option.CoverageLevel_localeType, localeType); 489 sb.append(localeType).append('/'); 490 set(Option.phase, testPhase.name().toLowerCase()); 491 sb.append(localeType).append('/'); 492 key = sb.toString().intern(); 493 } 494 495 @Override clone()496 public Options clone() { 497 return new Options(this); 498 } 499 500 @Override equals(Object other)501 public boolean equals(Object other) { 502 if (this == other) return true; 503 if (!(other instanceof Options)) return false; 504 if (this.key != null && ((Options) other).key != null) { 505 return (this.key == ((Options) other).key); 506 } else { 507 return this.compareTo((Options) other) == 0; 508 } 509 } 510 clear()511 private Options clear() { 512 for (int i = 0; i < options.length; i++) { 513 options[i] = null; 514 } 515 return this; 516 } 517 set(Option o, String v)518 private Options set(Option o, String v) { 519 options[o.ordinal()] = v; 520 if (DEBUG_OPTS) System.err.println("Setting " + o + " = " + v); 521 return this; 522 } 523 get(Option o)524 public String get(Option o) { 525 final String v = options[o.ordinal()]; 526 if (DEBUG_OPTS) System.err.println("Getting " + o + " = " + v); 527 return v; 528 } 529 getLocale()530 public CLDRLocale getLocale() { 531 if (locale != null) return locale; 532 return CLDRLocale.getInstance(get(Option.locale)); 533 } 534 535 /** 536 * Get the required coverage level for the specified locale, for this CheckCLDR object. 537 * 538 * @param localeID 539 * @return the Level 540 * <p>Called by CheckCoverage.setCldrFileToCheck and CheckDates.setCldrFileToCheck 541 */ getRequiredLevel(String localeID)542 public Level getRequiredLevel(String localeID) { 543 Level result; 544 // see if there is an explicit level 545 String localeType = get(Option.CoverageLevel_requiredLevel); 546 if (localeType != null) { 547 result = Level.get(localeType); 548 if (result != Level.UNDETERMINED) { 549 return result; 550 } 551 } 552 // otherwise, see if there is an organization level for the "Cldr" organization. 553 // This is not user-specific. 554 return sc.getLocaleCoverageLevel("Cldr", localeID); 555 } 556 contains(Option o)557 public boolean contains(Option o) { 558 String s = get(o); 559 return (s != null && !s.isEmpty()); 560 } 561 562 @Override compareTo(Options other)563 public int compareTo(Options other) { 564 if (other == this) return 0; 565 if (key != null && other.key != null) { 566 if (key == other.key) return 0; 567 return key.compareTo(other.key); 568 } 569 for (int i = 0; i < options.length; i++) { 570 final String s1 = options[i]; 571 final String s2 = other.options[i]; 572 if (s1 == null && s2 == null) { 573 // no difference 574 } else if (s1 == null) { 575 return -1; 576 } else if (s2 == null) { 577 return 1; 578 } else { 579 int rv = s1.compareTo(s2); 580 if (rv != 0) { 581 return rv; 582 } 583 } 584 } 585 return 0; 586 } 587 588 @Override hashCode()589 public int hashCode() { 590 if (key != null) return key.hashCode(); 591 592 int h = 1; 593 for (int i = 0; i < options.length; i++) { 594 if (options[i] == null) { 595 h *= 11; 596 } else { 597 h = (h * 11) + options[i].hashCode(); 598 } 599 } 600 return h; 601 } 602 603 @Override toString()604 public String toString() { 605 if (key != null) return "Options:" + key; 606 StringBuilder sb = new StringBuilder(); 607 for (Option o : Option.values()) { 608 if (options[o.ordinal()] != null) { 609 sb.append(o).append('=').append(options[o.ordinal()]).append(' '); 610 } 611 } 612 return sb.toString(); 613 } 614 } 615 isSkipTest()616 public boolean isSkipTest() { 617 return skipTest; 618 } 619 620 // this should only be set for the test in setCldrFileToCheck setSkipTest(boolean skipTest)621 public void setSkipTest(boolean skipTest) { 622 this.skipTest = skipTest; 623 } 624 625 /** 626 * Here is where the list of all checks is found. 627 * 628 * @param nameMatcher Regex pattern that determines which checks are run, based on their class 629 * name (such as .* for all checks, .*Collisions.* for CheckDisplayCollisions, etc.) 630 * @return 631 */ getCheckAll(Factory factory, String nameMatcher)632 public static CompoundCheckCLDR getCheckAll(Factory factory, String nameMatcher) { 633 return new CompoundCheckCLDR() 634 .setFilter(Pattern.compile(nameMatcher, Pattern.CASE_INSENSITIVE).matcher("")) 635 .add(new CheckAnnotations()) 636 // .add(new CheckAttributeValues(factory)) 637 .add(new CheckChildren(factory)) 638 .add(new CheckCoverage(factory)) 639 .add(new CheckDates(factory)) 640 .add(new CheckForCopy(factory)) 641 .add(new CheckDisplayCollisions(factory)) 642 .add(new CheckExemplars(factory)) 643 .add(new CheckForExemplars(factory)) 644 .add(new CheckForInheritanceMarkers()) 645 .add(new CheckNames()) 646 .add(new CheckNumbers(factory)) 647 // .add(new CheckZones()) // this doesn't work; many spurious errors that user can't 648 // correct 649 .add(new CheckMetazones()) 650 .add(new CheckLogicalGroupings(factory)) 651 .add(new CheckAlt()) 652 .add(new CheckAltOnly(factory)) 653 .add(new CheckCurrencies()) 654 .add(new CheckCasing()) 655 .add( 656 new CheckConsistentCasing( 657 factory)) // this doesn't work; many spurious errors that user can't 658 // correct 659 .add(new CheckQuotes()) 660 .add(new CheckUnits()) 661 .add(new CheckWidths()) 662 .add(new CheckPlaceHolders()) 663 .add(new CheckPersonNames()) 664 .add(new CheckNew(factory)) // this is at the end; it will check for other certain 665 // other errors and warnings and 666 // not add a message if there are any. 667 ; 668 } 669 670 /** These determine what language is used to display information. Must be set before use. */ getDisplayInformation()671 public static synchronized CLDRFile getDisplayInformation() { 672 return displayInformation; 673 } 674 setDisplayInformation(CLDRFile inputDisplayInformation)675 public static synchronized void setDisplayInformation(CLDRFile inputDisplayInformation) { 676 displayInformation = inputDisplayInformation; 677 } 678 679 /** Get the CLDRFile. */ getCldrFileToCheck()680 public final CLDRFile getCldrFileToCheck() { 681 return cldrFileToCheck; 682 } 683 684 /** 685 * Often subclassed for initializing. If so, make the first 2 lines: if (cldrFileToCheck == 686 * null) return this; super.handleSetCldrFileToCheck(cldrFileToCheck); do stuff 687 * 688 * <p>Called late via accept(). 689 * 690 * @param cldrFileToCheck 691 * @param options 692 * @param possibleErrors any deferred possibleErrors can be set here. They will be appended to 693 * every handleCheck() call. 694 * @return 695 */ handleSetCldrFileToCheck( CLDRFile cldrFileToCheck, Options options, List<CheckStatus> possibleErrors)696 public CheckCLDR handleSetCldrFileToCheck( 697 CLDRFile cldrFileToCheck, Options options, List<CheckStatus> possibleErrors) { 698 699 // nothing by default 700 return this; 701 } 702 703 /** 704 * Set the CLDRFile. Must be done before calling check. 705 * 706 * @param cldrFileToCheck 707 * @param options (not currently used) 708 * @param possibleErrors 709 */ setCldrFileToCheck( CLDRFile cldrFileToCheck, Options options, List<CheckStatus> possibleErrors)710 public CheckCLDR setCldrFileToCheck( 711 CLDRFile cldrFileToCheck, Options options, List<CheckStatus> possibleErrors) { 712 this.cldrFileToCheck = cldrFileToCheck; 713 reset(); 714 // clear the *cached* possible Errors. Not counting any set immediately by subclasses. 715 cachedPossibleErrors.clear(); 716 cachedOptions = new Options(options); 717 // we must load filters here, as they are used by check() 718 719 // Shortlist error filters for this locale. 720 loadFilters(); 721 String locale = cldrFileToCheck.getLocaleID(); 722 filtersForLocale.clear(); 723 for (R3<Pattern, Subtype, Pattern> filter : allFilters) { 724 if (filter.get0() == null || !filter.get0().matcher(locale).matches()) continue; 725 Subtype subtype = filter.get1(); 726 List<Pattern> xpaths = filtersForLocale.get(subtype); 727 if (xpaths == null) { 728 filtersForLocale.put(subtype, xpaths = new ArrayList<>()); 729 } 730 xpaths.add(filter.get2()); 731 } 732 733 // hook for checks that want to set possibleErrors early 734 handleCheckPossibleErrors(cldrFileToCheck, options, possibleErrors); 735 736 return this; 737 } 738 739 /** override this if you want to return errors immediately when setCldrFileToCheck is called */ handleCheckPossibleErrors( CLDRFile cldrFileToCheck, Options options, List<CheckStatus> possibleErrors)740 protected void handleCheckPossibleErrors( 741 CLDRFile cldrFileToCheck, Options options, List<CheckStatus> possibleErrors) { 742 // nothing by default. 743 } 744 745 /** override this if you want to reset state immediately when setCldrFileToCheck is called */ reset()746 protected void reset() { 747 initted = false; 748 } 749 750 /** 751 * Subclasses must call this, after any skip calculation to indicate that an xpath is relevant 752 * to them. 753 * 754 * @param result out-parameter to contain any deferred errors 755 * @return false if test is skipped and should exit 756 */ accept(List<CheckStatus> result)757 protected boolean accept(List<CheckStatus> result) { 758 if (!initted) { 759 if (this.cldrFileToCheck == null) { 760 throw new NullPointerException("accept() was called before setCldrFileToCheck()"); 761 } 762 // clear this again. 763 cachedPossibleErrors.clear(); 764 // call into the subclass 765 handleSetCldrFileToCheck(this.cldrFileToCheck, cachedOptions, cachedPossibleErrors); 766 initted = true; 767 } 768 // unconditionally append all cached possible errors 769 result.addAll(cachedPossibleErrors); 770 if (isSkipTest()) { 771 return false; 772 } 773 return true; 774 } 775 776 /** has accept() been called since setCldrFileToCheck() was called? */ 777 boolean initted = false; 778 779 /** cache of possible errors, for handleSetCldrFileToCheck */ 780 List<CheckStatus> cachedPossibleErrors = new ArrayList<>(); 781 782 Options cachedOptions = null; 783 784 /** Status value returned from check */ 785 public static class CheckStatus { 786 public static final Type alertType = Type.Comment, 787 warningType = Type.Warning, 788 errorType = Type.Error, 789 exampleType = Type.Example, 790 demoType = Type.Demo; 791 792 public enum Type { 793 Comment, 794 Warning, 795 Error, 796 Example, 797 Demo 798 } 799 800 public enum Subtype { 801 none, 802 noUnproposedVariant, 803 deprecatedAttribute, 804 illegalPlural, 805 invalidLocale, 806 incorrectCasing, 807 valueMustBeOverridden, 808 valueAlwaysOverridden, 809 nullChildFile, 810 internalError, 811 coverageLevel, 812 missingPluralInfo, 813 currencySymbolTooWide, 814 incorrectDatePattern, 815 abbreviatedDateFieldTooWide, 816 displayCollision, 817 illegalExemplarSet, 818 missingAuxiliaryExemplars, 819 extraPlaceholders, 820 missingPlaceholders, 821 shouldntHavePlaceholders, 822 couldNotAccessExemplars, 823 noExemplarCharacters, 824 modifiedEnglishValue, 825 invalidCurrencyMatchSet, 826 multipleMetazoneMappings, 827 noMetazoneMapping, 828 noMetazoneMappingAfter1970, 829 noMetazoneMappingBeforeNow, 830 cannotCreateZoneFormatter, 831 insufficientCoverage, 832 missingLanguageTerritoryInfo, 833 missingEuroCountryInfo, 834 deprecatedAttributeWithReplacement, 835 missingOrExtraDateField, 836 internalUnicodeSetFormattingError, 837 auxiliaryExemplarsOverlap, 838 missingPunctuationCharacters, 839 840 charactersNotInCurrencyExemplars, 841 asciiCharactersNotInCurrencyExemplars, 842 charactersNotInMainOrAuxiliaryExemplars, 843 asciiCharactersNotInMainOrAuxiliaryExemplars, 844 845 narrowDateFieldTooWide, 846 illegalCharactersInExemplars, 847 orientationDisagreesWithExemplars, 848 inconsistentDatePattern, 849 inconsistentTimePattern, 850 missingDatePattern, 851 illegalDatePattern, 852 missingMainExemplars, 853 mustNotStartOrEndWithSpace, 854 illegalCharactersInNumberPattern, 855 numberPatternNotCanonical, 856 currencyPatternMissingCurrencySymbol, 857 currencyPatternUnexpectedCurrencySymbol, 858 missingMinusSign, 859 badNumericType, 860 percentPatternMissingPercentSymbol, 861 illegalNumberFormat, 862 unexpectedAttributeValue, 863 metazoneContainsDigit, 864 tooManyGroupingSeparators, 865 inconsistentPluralFormat, 866 missingZeros, 867 sameAsEnglish, 868 sameAsCode, 869 dateSymbolCollision, 870 incompleteLogicalGroup, 871 extraMetazoneString, 872 inconsistentDraftStatus, 873 errorOrWarningInLogicalGroup, 874 valueTooWide, 875 valueTooNarrow, 876 nameContainsYear, 877 patternCannotContainDigits, 878 patternContainsInvalidCharacters, 879 parenthesesNotAllowed, 880 illegalNumberingSystem, 881 unexpectedOrderOfEraYear, 882 invalidPlaceHolder, 883 asciiQuotesNotAllowed, 884 badMinimumGroupingDigits, 885 inconsistentPeriods, 886 inheritanceMarkerNotAllowed, 887 invalidDurationUnitPattern, 888 invalidDelimiter, 889 illegalCharactersInPattern, 890 badParseLenient, 891 tooManyValues, 892 invalidSymbol, 893 invalidGenderCode, 894 mismatchedUnitComponent, 895 longPowerWithSubscripts, 896 gapsInPlaceholderNumbers, 897 duplicatePlaceholders, 898 largerDifferences, 899 missingNonAltPath, 900 badSamplePersonName, 901 missingLanguage, 902 namePlaceholderProblem, 903 missingSpaceBetweenNameFields, 904 illegalParameterValue, 905 illegalAnnotationCode, 906 illegalCharacter; 907 908 @Override toString()909 public String toString() { 910 // converts "thisThisThis" to "this this this" 911 return TO_STRING.matcher(name()).replaceAll(" $1").toLowerCase(); 912 } 913 914 static Pattern TO_STRING = PatternCache.get("([A-Z])"); 915 } 916 917 /** 918 * These error don't prevent entry during submission, since they become valid if a different 919 * row is changed. 920 */ 921 public static Set<Subtype> crossCheckSubtypes = 922 ImmutableSet.of( 923 Subtype.dateSymbolCollision, 924 Subtype.displayCollision, 925 Subtype.inconsistentDraftStatus, 926 Subtype.incompleteLogicalGroup, 927 Subtype.inconsistentPeriods, 928 Subtype.abbreviatedDateFieldTooWide, 929 Subtype.narrowDateFieldTooWide, 930 Subtype.coverageLevel); 931 932 public static Set<Subtype> errorCodesPath = 933 ImmutableSet.of( 934 Subtype.duplicatePlaceholders, 935 Subtype.extraPlaceholders, 936 Subtype.gapsInPlaceholderNumbers, 937 Subtype.invalidPlaceHolder, 938 Subtype.missingPlaceholders, 939 Subtype.shouldntHavePlaceholders); 940 941 private Type type; 942 private Subtype subtype = Subtype.none; 943 private String messageFormat; 944 private Object[] parameters; 945 private CheckAccessor cause; 946 private boolean checkOnSubmit = true; 947 CheckStatus()948 public CheckStatus() {} 949 isCheckOnSubmit()950 public boolean isCheckOnSubmit() { 951 return checkOnSubmit; 952 } 953 setCheckOnSubmit(boolean dependent)954 public CheckStatus setCheckOnSubmit(boolean dependent) { 955 this.checkOnSubmit = dependent; 956 return this; 957 } 958 getType()959 public Type getType() { 960 return type; 961 } 962 setMainType(CheckStatus.Type type)963 public CheckStatus setMainType(CheckStatus.Type type) { 964 this.type = type; 965 return this; 966 } 967 getMessage()968 public String getMessage() { 969 String message = messageFormat; 970 if (messageFormat != null && parameters != null) { 971 try { 972 String fixedApos = MessageFormat.autoQuoteApostrophe(messageFormat); 973 MessageFormat format = new MessageFormat(fixedApos); 974 message = format.format(parameters); 975 if (errorCodesPath.contains(subtype)) { 976 message += 977 "; see <a href='http://cldr.unicode.org/translation/error-codes#" 978 + subtype.name() 979 + "' target='cldr_error_codes'>" 980 + subtype 981 + "</a>."; 982 } 983 } catch (Exception e) { 984 message = messageFormat; 985 final String failMsg = 986 "MessageFormat Failure: " 987 + subtype 988 + "; " 989 + messageFormat 990 + "; " 991 + (parameters == null ? null : Arrays.asList(parameters)); 992 logger.log(java.util.logging.Level.SEVERE, e, () -> failMsg); 993 System.err.println(failMsg); 994 // throw new IllegalArgumentException(subtype + "; " + messageFormat + "; " 995 // + (parameters == null ? null : Arrays.asList(parameters)), e); 996 } 997 } 998 Exception[] exceptionParameters = getExceptionParameters(); 999 if (exceptionParameters != null) { 1000 for (Exception exception : exceptionParameters) { 1001 message += "; " + exception.getMessage(); // + " \t(" + 1002 // exception.getClass().getName() + ")"; 1003 // for (StackTraceElement item : exception.getStackTrace()) { 1004 // message += "\n\t" + item; 1005 // } 1006 } 1007 } 1008 return message.replace('\t', ' '); 1009 } 1010 setMessage(String message)1011 public CheckStatus setMessage(String message) { 1012 if (cause == null) { 1013 throw new IllegalArgumentException("Must have cause set."); 1014 } 1015 if (message == null) { 1016 throw new IllegalArgumentException("Message cannot be null."); 1017 } 1018 this.messageFormat = message; 1019 this.parameters = null; 1020 return this; 1021 } 1022 setMessage(String message, Object... messageArguments)1023 public CheckStatus setMessage(String message, Object... messageArguments) { 1024 if (cause == null) { 1025 throw new IllegalArgumentException("Must have cause set."); 1026 } 1027 this.messageFormat = message; 1028 this.parameters = messageArguments; 1029 return this; 1030 } 1031 1032 @Override toString()1033 public String toString() { 1034 return getType() + ": " + getMessage(); 1035 } 1036 1037 /** Warning: don't change the contents of the parameters after retrieving. */ getParameters()1038 public Object[] getParameters() { 1039 return parameters; 1040 } 1041 1042 /** 1043 * Returns any Exception parameters in the status, or null if there are none. 1044 * 1045 * @return 1046 */ getExceptionParameters()1047 public Exception[] getExceptionParameters() { 1048 if (parameters == null) { 1049 return null; 1050 } 1051 1052 List<Exception> errors = new ArrayList<>(); 1053 for (Object o : parameters) { 1054 if (o instanceof Exception) { 1055 errors.add((Exception) o); 1056 } 1057 } 1058 if (errors.size() == 0) { 1059 return null; 1060 } 1061 return errors.toArray(new Exception[errors.size()]); 1062 } 1063 1064 /** Warning: don't change the contents of the parameters after passing in. */ setParameters(Object[] parameters)1065 public CheckStatus setParameters(Object[] parameters) { 1066 if (cause == null) { 1067 throw new IllegalArgumentException("Must have cause set."); 1068 } 1069 this.parameters = parameters; 1070 return this; 1071 } 1072 getDemo()1073 public SimpleDemo getDemo() { 1074 return null; 1075 } 1076 getCause()1077 public CheckCLDR getCause() { 1078 return cause instanceof CheckCLDR ? (CheckCLDR) cause : null; 1079 } 1080 setCause(CheckAccessor cause)1081 public CheckStatus setCause(CheckAccessor cause) { 1082 this.cause = cause; 1083 return this; 1084 } 1085 getSubtype()1086 public Subtype getSubtype() { 1087 return subtype; 1088 } 1089 setSubtype(Subtype subtype)1090 public CheckStatus setSubtype(Subtype subtype) { 1091 this.subtype = subtype; 1092 return this; 1093 } 1094 1095 /** 1096 * Convenience function: return true if any items in this list are of errorType 1097 * 1098 * @param result the list to check (could be null for empty) 1099 * @return true if any items in result are of errorType 1100 */ hasError(List<CheckStatus> result)1101 public static final boolean hasError(List<CheckStatus> result) { 1102 return hasType(result, errorType); 1103 } 1104 1105 /** 1106 * Convenience function: return true if any items in this list are of errorType 1107 * 1108 * @param result the list to check (could be null for empty) 1109 * @return true if any items in result are of errorType 1110 */ hasType(List<CheckStatus> result, Type type)1111 public static boolean hasType(List<CheckStatus> result, Type type) { 1112 if (result == null) return false; 1113 for (CheckStatus s : result) { 1114 if (s.getType().equals(type)) { 1115 return true; 1116 } 1117 } 1118 return false; 1119 } 1120 } 1121 1122 public abstract static class SimpleDemo { 1123 Map<String, String> internalPostArguments = new HashMap<>(); 1124 1125 /** 1126 * @param postArguments A read-write map containing post-style arguments. eg TEXTBOX=abcd, 1127 * etc. <br> 1128 * The first time this is called, the Map should be empty. 1129 * @return true if the map has been changed 1130 */ getHTML(Map<String, String> postArguments)1131 public abstract String getHTML(Map<String, String> postArguments) throws Exception; 1132 1133 /** Only here for compatibility. Use the other getHTML instead */ getHTML(String path, String fullPath, String value)1134 public final String getHTML(String path, String fullPath, String value) throws Exception { 1135 return getHTML(internalPostArguments); 1136 } 1137 1138 /** 1139 * THIS IS ONLY FOR COMPATIBILITY: you can call this, then the non-postArguments form of 1140 * getHTML; or better, call getHTML with the postArguments. 1141 * 1142 * @param postArguments A read-write map containing post-style arguments. eg TEXTBOX=abcd, 1143 * etc. 1144 * @return true if the map has been changed 1145 */ processPost(Map<String, String> postArguments)1146 public final boolean processPost(Map<String, String> postArguments) { 1147 internalPostArguments.clear(); 1148 internalPostArguments.putAll(postArguments); 1149 return true; 1150 } 1151 } 1152 1153 public abstract static class FormatDemo extends SimpleDemo { 1154 protected String currentPattern, currentInput, currentFormatted, currentReparsed; 1155 protected ParsePosition parsePosition = new ParsePosition(0); 1156 getPattern()1157 protected abstract String getPattern(); 1158 getSampleInput()1159 protected abstract String getSampleInput(); 1160 getArguments(Map<String, String> postArguments)1161 protected abstract void getArguments(Map<String, String> postArguments); 1162 1163 @Override getHTML(Map<String, String> postArguments)1164 public String getHTML(Map<String, String> postArguments) throws Exception { 1165 getArguments(postArguments); 1166 StringBuffer htmlMessage = new StringBuffer(); 1167 FormatDemo.appendTitle(htmlMessage); 1168 FormatDemo.appendLine( 1169 htmlMessage, currentPattern, currentInput, currentFormatted, currentReparsed); 1170 htmlMessage.append("</table>"); 1171 return htmlMessage.toString(); 1172 } 1173 getPlainText(Map<String, String> postArguments)1174 public String getPlainText(Map<String, String> postArguments) { 1175 getArguments(postArguments); 1176 return MessageFormat.format( 1177 "<\"\u200E{0}\u200E\", \"{1}\"> \u2192 \"\u200E{2}\u200E\" \u2192 \"{3}\"", 1178 (Object[]) 1179 new String[] { 1180 currentPattern, currentInput, currentFormatted, currentReparsed 1181 }); 1182 } 1183 1184 /** 1185 * @param htmlMessage 1186 * @param pattern 1187 * @param input 1188 * @param formatted 1189 * @param reparsed 1190 */ appendLine( StringBuffer htmlMessage, String pattern, String input, String formatted, String reparsed)1191 public static void appendLine( 1192 StringBuffer htmlMessage, 1193 String pattern, 1194 String input, 1195 String formatted, 1196 String reparsed) { 1197 htmlMessage 1198 .append("<tr><td><input type='text' name='pattern' value='") 1199 .append(TransliteratorUtilities.toXML.transliterate(pattern)) 1200 .append("'></td><td><input type='text' name='input' value='") 1201 .append(TransliteratorUtilities.toXML.transliterate(input)) 1202 .append("'></td><td>") 1203 .append("<input type='submit' value='Test' name='Test'>") 1204 .append("</td><td>" + "<input type='text' name='formatted' value='") 1205 .append(TransliteratorUtilities.toXML.transliterate(formatted)) 1206 .append("'></td><td>" + "<input type='text' name='reparsed' value='") 1207 .append(TransliteratorUtilities.toXML.transliterate(reparsed)) 1208 .append("'></td></tr>"); 1209 } 1210 1211 /** 1212 * @param htmlMessage 1213 */ appendTitle(StringBuffer htmlMessage)1214 public static void appendTitle(StringBuffer htmlMessage) { 1215 htmlMessage.append( 1216 "<table border='1' cellspacing='0' cellpadding='2'" 1217 + 1218 // " style='border-collapse: collapse' style='width: 100%'" + 1219 "><tr>" 1220 + "<th>Pattern</th>" 1221 + "<th>Unlocalized Input</th>" 1222 + "<th></th>" 1223 + "<th>Localized Format</th>" 1224 + "<th>Re-Parsed</th>" 1225 + "</tr>"); 1226 } 1227 } 1228 1229 /** 1230 * Checks the path/value in the cldrFileToCheck for correctness, according to some criterion. If 1231 * the path is relevant to the check, there is an alert or warning, then a CheckStatus is added 1232 * to List. 1233 * 1234 * @param path Must be a distinguished path, such as what comes out of CLDRFile.iterator() 1235 * @param fullPath Must be the full path 1236 * @param value the value associated with the path 1237 * @param result 1238 */ check( String path, String fullPath, String value, Options options, List<CheckStatus> result)1239 public final CheckCLDR check( 1240 String path, String fullPath, String value, Options options, List<CheckStatus> result) { 1241 if (cldrFileToCheck == null) { 1242 throw new InternalCldrException("CheckCLDR problem: cldrFileToCheck must not be null"); 1243 } 1244 if (path == null) { 1245 throw new InternalCldrException("CheckCLDR problem: path must not be null"); 1246 } 1247 // if (fullPath == null) { 1248 // throw new InternalError("CheckCLDR problem: fullPath must not be null"); 1249 // } 1250 // if (value == null) { 1251 // throw new InternalError("CheckCLDR problem: value must not be null"); 1252 // } 1253 result.clear(); 1254 1255 /* 1256 * If the item is non-winning, and either inherited or it is code-fallback, then don't run 1257 * any tests on this item. See http://unicode.org/cldr/trac/ticket/7574 1258 * 1259 * The following conditional formerly used "value == ..." and "value != ...", which in Java doesn't 1260 * mean what it does in some other languages. The condition has been changed to use the equals() method. 1261 * Since value can be null, check for that first. 1262 */ 1263 // if (value == cldrFileToCheck.getBaileyValue(path, null, null) && value != 1264 // cldrFileToCheck.getWinningValue(path)) { 1265 if (value != null 1266 && !value.equals(cldrFileToCheck.getWinningValue(path)) 1267 && cldrFileToCheck.getUnresolved().getStringValue(path) == null) { 1268 return this; 1269 } 1270 1271 // If we're being asked to run tests for an inheritance marker, then we need to change it 1272 // to the "real" value first before running tests. Testing the value 1273 // CldrUtility.INHERITANCE_MARKER ("↑↑↑") doesn't make sense. 1274 if (CldrUtility.INHERITANCE_MARKER.equals(value)) { 1275 value = cldrFileToCheck.getBaileyValue(path, null, null); 1276 // If it hasn't changed, then don't run any tests. 1277 if (CldrUtility.INHERITANCE_MARKER.equals(value)) { 1278 return this; 1279 } 1280 } 1281 CheckCLDR instance = handleCheck(path, fullPath, value, options, result); 1282 Iterator<CheckStatus> iterator = result.iterator(); 1283 // Filter out any errors/warnings that match the filter list in CheckCLDR-exceptions.txt. 1284 while (iterator.hasNext()) { 1285 CheckStatus status = iterator.next(); 1286 if (shouldExcludeStatus(fullPath, status)) { 1287 iterator.remove(); 1288 } 1289 } 1290 return instance; 1291 } 1292 1293 /** 1294 * Returns any examples in the result parameter. Both examples and demos can be returned. A demo 1295 * will have getType() == CheckStatus.demoType. In that case, there will be no getMessage 1296 * available; instead, call getDemo() to get the demo, then call getHTML() to get the initial 1297 * HTML. 1298 */ getExamples( String path, String fullPath, String value, Options options, List<CheckStatus> result)1299 public final CheckCLDR getExamples( 1300 String path, String fullPath, String value, Options options, List<CheckStatus> result) { 1301 result.clear(); 1302 return handleGetExamples(path, fullPath, value, options, result); 1303 } 1304 1305 @SuppressWarnings("unused") handleGetExamples( String path, String fullPath, String value, Options options2, List<CheckStatus> result)1306 protected CheckCLDR handleGetExamples( 1307 String path, 1308 String fullPath, 1309 String value, 1310 Options options2, 1311 List<CheckStatus> result) { 1312 return this; // NOOP unless overridden 1313 } 1314 1315 /** 1316 * This is what the subclasses override. 1317 * 1318 * <p>If a path is not applicable, exit early with <code>return this;</code> Once a path is 1319 * applicable, call <code>accept(result);</code> to add deferred possible problems. 1320 * 1321 * <p>If something is found, a CheckStatus is added to result. This can be done multiple times 1322 * in one call, if multiple errors or warnings are found. The CheckStatus may return warnings, 1323 * errors, examples, or demos. We may expand that in the future. 1324 * 1325 * <p>The code to add the CheckStatus will look something like:: 1326 * 1327 * <pre> 1328 * result.add(new CheckStatus() 1329 * .setType(CheckStatus.errorType) 1330 * .setMessage("Value should be {0}", new Object[] { pattern })); 1331 * </pre> 1332 */ handleCheck( String path, String fullPath, String value, Options options, List<CheckStatus> result)1333 public abstract CheckCLDR handleCheck( 1334 String path, String fullPath, String value, Options options, List<CheckStatus> result); 1335 1336 /** Only for use in ConsoleCheck, for debugging */ handleFinish()1337 public void handleFinish() {} 1338 1339 /** 1340 * Internal class used to bundle up a number of Checks. 1341 * 1342 * @author davis 1343 */ 1344 static class CompoundCheckCLDR extends CheckCLDR { 1345 private Matcher filter; 1346 private List<CheckCLDR> checkList = new ArrayList<>(); 1347 private List<CheckCLDR> filteredCheckList = new ArrayList<>(); 1348 add(CheckCLDR item)1349 public CompoundCheckCLDR add(CheckCLDR item) { 1350 checkList.add(item); 1351 if (filter == null) { 1352 filteredCheckList.add(item); 1353 } else { 1354 final String className = item.getClass().getName(); 1355 if (filter.reset(className).find()) { 1356 filteredCheckList.add(item); 1357 } 1358 } 1359 return this; 1360 } 1361 1362 @Override handleCheck( String path, String fullPath, String value, Options options, List<CheckStatus> result)1363 public CheckCLDR handleCheck( 1364 String path, 1365 String fullPath, 1366 String value, 1367 Options options, 1368 List<CheckStatus> result) { 1369 result.clear(); 1370 1371 if (!accept(result)) return this; 1372 1373 // If we're being asked to run tests for an inheritance marker, then we need to change 1374 // it 1375 // to the "real" value first before running tests. Testing the value 1376 // CldrUtility.INHERITANCE_MARKER ("↑↑↑") doesn't make sense. 1377 if (CldrUtility.INHERITANCE_MARKER.equals(value)) { 1378 value = getCldrFileToCheck().getBaileyValue(path, null, null); 1379 } 1380 for (Iterator<CheckCLDR> it = filteredCheckList.iterator(); it.hasNext(); ) { 1381 CheckCLDR item = it.next(); 1382 // skip proposed items in final testing. 1383 if (Phase.FINAL_TESTING == item.getPhase()) { 1384 if (path.contains("proposed") && path.contains("[@alt=")) { 1385 continue; 1386 } 1387 } 1388 try { 1389 if (!item.isSkipTest()) { 1390 item.handleCheck(path, fullPath, value, options, result); 1391 } 1392 } catch (Exception e) { 1393 addError(result, item, e); 1394 return this; 1395 } 1396 } 1397 return this; 1398 } 1399 1400 @Override handleFinish()1401 public void handleFinish() { 1402 for (Iterator<CheckCLDR> it = filteredCheckList.iterator(); it.hasNext(); ) { 1403 CheckCLDR item = it.next(); 1404 item.handleFinish(); 1405 } 1406 } 1407 1408 @Override handleGetExamples( String path, String fullPath, String value, Options options, List<CheckStatus> result)1409 protected CheckCLDR handleGetExamples( 1410 String path, 1411 String fullPath, 1412 String value, 1413 Options options, 1414 List<CheckStatus> result) { 1415 result.clear(); 1416 for (Iterator<CheckCLDR> it = filteredCheckList.iterator(); it.hasNext(); ) { 1417 CheckCLDR item = it.next(); 1418 try { 1419 item.handleGetExamples(path, fullPath, value, options, result); 1420 } catch (Exception e) { 1421 addError(result, item, e); 1422 return this; 1423 } 1424 } 1425 return this; 1426 } 1427 addError(List<CheckStatus> result, CheckCLDR item, Exception e)1428 private void addError(List<CheckStatus> result, CheckCLDR item, Exception e) { 1429 // send to java.util.logging, useful for servers 1430 logger.log( 1431 java.util.logging.Level.SEVERE, 1432 e, 1433 () -> { 1434 String locale = "(unknown)"; 1435 if (item.cldrFileToCheck != null) { 1436 locale = item.cldrFileToCheck.getLocaleID(); 1437 } 1438 return String.format( 1439 "Internal error: %s in %s", item.getClass().getName(), locale); 1440 }); 1441 // also add as a check 1442 result.add( 1443 new CheckStatus() 1444 .setCause(this) 1445 .setMainType(CheckStatus.errorType) 1446 .setSubtype(Subtype.internalError) 1447 .setMessage( 1448 "Internal error in {0}. Exception: {1}, Message: {2}, Trace: {3}", 1449 new Object[] { 1450 item.getClass().getName(), 1451 e.getClass().getName(), 1452 e, 1453 Arrays.asList(e.getStackTrace()) 1454 })); 1455 } 1456 1457 @Override handleCheckPossibleErrors( CLDRFile cldrFileToCheck, Options options, List<CheckStatus> possibleErrors)1458 public void handleCheckPossibleErrors( 1459 CLDRFile cldrFileToCheck, Options options, List<CheckStatus> possibleErrors) { 1460 ElapsedTimer testTime = null, testOverallTime = null; 1461 if (cldrFileToCheck == null) return; 1462 boolean SHOW_TIMES = options.contains(Options.Option.SHOW_TIMES); 1463 setPhase(Phase.forString(options.get(Options.Option.phase))); 1464 if (SHOW_TIMES) 1465 testOverallTime = new ElapsedTimer("Test setup time for setCldrFileToCheck: {0}"); 1466 super.handleCheckPossibleErrors(cldrFileToCheck, options, possibleErrors); 1467 possibleErrors.clear(); 1468 1469 for (Iterator<CheckCLDR> it = filteredCheckList.iterator(); it.hasNext(); ) { 1470 CheckCLDR item = it.next(); 1471 if (SHOW_TIMES) 1472 testTime = 1473 new ElapsedTimer( 1474 "Test setup time for " + item.getClass().toString() + ": {0}"); 1475 try { 1476 item.setPhase(getPhase()); 1477 item.setCldrFileToCheck(cldrFileToCheck, options, possibleErrors); 1478 if (SHOW_TIMES) { 1479 if (item.isSkipTest()) { 1480 System.out.println("Disabled : " + testTime); 1481 } else { 1482 System.out.println("OK : " + testTime); 1483 } 1484 } 1485 } catch (RuntimeException e) { 1486 addError(possibleErrors, item, e); 1487 if (SHOW_TIMES) System.out.println("ERR: " + testTime + " - " + e.toString()); 1488 } 1489 } 1490 if (SHOW_TIMES) System.out.println("Overall: " + testOverallTime + ": {0}"); 1491 } 1492 getFilter()1493 public Matcher getFilter() { 1494 return filter; 1495 } 1496 setFilter(Matcher filter)1497 public CompoundCheckCLDR setFilter(Matcher filter) { 1498 this.filter = filter; 1499 filteredCheckList.clear(); 1500 for (Iterator<CheckCLDR> it = checkList.iterator(); it.hasNext(); ) { 1501 CheckCLDR item = it.next(); 1502 if (filter == null || filter.reset(item.getClass().getName()).matches()) { 1503 filteredCheckList.add(item); 1504 item.handleSetCldrFileToCheck(getCldrFileToCheck(), (Options) null, null); 1505 } 1506 } 1507 return this; 1508 } 1509 getFilteredTests()1510 public String getFilteredTests() { 1511 return filteredCheckList.toString(); 1512 } 1513 getFilteredTestList()1514 public List<CheckCLDR> getFilteredTestList() { 1515 return filteredCheckList; 1516 } 1517 } 1518 1519 @Override getPhase()1520 public Phase getPhase() { 1521 return phase; 1522 } 1523 setPhase(Phase phase)1524 public void setPhase(Phase phase) { 1525 this.phase = phase; 1526 } 1527 1528 /** A map of error/warning types to their filters. */ 1529 private static List<R3<Pattern, Subtype, Pattern>> allFilters; 1530 1531 /** Loads the set of filters used for CheckCLDR results. */ loadFilters()1532 private void loadFilters() { 1533 if (allFilters != null) return; 1534 allFilters = new ArrayList<>(); 1535 RegexFileParser fileParser = new RegexFileParser(); 1536 fileParser.setLineParser( 1537 new RegexLineParser() { 1538 @Override 1539 public void parse(String line) { 1540 String[] fields = line.split("\\s*;\\s*"); 1541 Subtype subtype = Subtype.valueOf(fields[0]); 1542 Pattern locale = PatternCache.get(fields[1]); 1543 Pattern xpathRegex = 1544 PatternCache.get(fields[2].replaceAll("\\[@", "\\\\[@")); 1545 allFilters.add(new R3<>(locale, subtype, xpathRegex)); 1546 } 1547 }); 1548 fileParser.parse(CheckCLDR.class, "/org/unicode/cldr/util/data/CheckCLDR-exceptions.txt"); 1549 } 1550 1551 /** 1552 * Checks if a status should be excluded from the list of results returned from CheckCLDR. 1553 * 1554 * @param xpath the xpath that the status belongs to 1555 * @param status the status 1556 * @return true if the status should be included 1557 */ shouldExcludeStatus(String xpath, CheckStatus status)1558 private boolean shouldExcludeStatus(String xpath, CheckStatus status) { 1559 List<Pattern> xpathPatterns = filtersForLocale.get(status.getSubtype()); 1560 if (xpathPatterns == null) { 1561 return false; 1562 } 1563 for (Pattern xpathPattern : xpathPatterns) { 1564 if (xpathPattern.matcher(xpath).matches()) { 1565 return true; 1566 } 1567 } 1568 return false; 1569 } 1570 getEnglishFile()1571 public CLDRFile getEnglishFile() { 1572 return englishFile; 1573 } 1574 setEnglishFile(CLDRFile englishFile)1575 public void setEnglishFile(CLDRFile englishFile) { 1576 this.englishFile = englishFile; 1577 } 1578 fixedValueIfInherited(String value, String path)1579 public CharSequence fixedValueIfInherited(String value, String path) { 1580 return !CldrUtility.INHERITANCE_MARKER.equals(value) 1581 ? value 1582 : getCldrFileToCheck().getStringValueWithBailey(path); 1583 } 1584 } 1585