1 package org.unicode.cldr.test; 2 3 import com.google.common.base.Joiner; 4 import com.ibm.icu.impl.Relation; 5 import com.ibm.icu.text.BreakIterator; 6 import com.ibm.icu.text.DateTimePatternGenerator; 7 import com.ibm.icu.text.DateTimePatternGenerator.VariableField; 8 import com.ibm.icu.text.MessageFormat; 9 import com.ibm.icu.text.NumberFormat; 10 import com.ibm.icu.text.SimpleDateFormat; 11 import com.ibm.icu.text.UnicodeSet; 12 import com.ibm.icu.util.Output; 13 import com.ibm.icu.util.ULocale; 14 import java.text.ParseException; 15 import java.util.ArrayList; 16 import java.util.Arrays; 17 import java.util.Calendar; 18 import java.util.Collection; 19 import java.util.Date; 20 import java.util.EnumMap; 21 import java.util.HashMap; 22 import java.util.HashSet; 23 import java.util.Iterator; 24 import java.util.LinkedHashSet; 25 import java.util.List; 26 import java.util.Locale; 27 import java.util.Map; 28 import java.util.Random; 29 import java.util.Set; 30 import java.util.TreeSet; 31 import java.util.regex.Matcher; 32 import java.util.regex.Pattern; 33 import org.unicode.cldr.test.CheckCLDR.CheckStatus.Subtype; 34 import org.unicode.cldr.util.ApproximateWidth; 35 import org.unicode.cldr.util.CLDRFile; 36 import org.unicode.cldr.util.CLDRFile.Status; 37 import org.unicode.cldr.util.CLDRLocale; 38 import org.unicode.cldr.util.CLDRURLS; 39 import org.unicode.cldr.util.CldrUtility; 40 import org.unicode.cldr.util.DateTimeCanonicalizer.DateTimePatternType; 41 import org.unicode.cldr.util.DayPeriodInfo; 42 import org.unicode.cldr.util.DayPeriodInfo.DayPeriod; 43 import org.unicode.cldr.util.DayPeriodInfo.Type; 44 import org.unicode.cldr.util.Factory; 45 import org.unicode.cldr.util.ICUServiceBuilder; 46 import org.unicode.cldr.util.Level; 47 import org.unicode.cldr.util.LocaleIDParser; 48 import org.unicode.cldr.util.LogicalGrouping; 49 import org.unicode.cldr.util.PathHeader; 50 import org.unicode.cldr.util.PathStarrer; 51 import org.unicode.cldr.util.PatternCache; 52 import org.unicode.cldr.util.PreferredAndAllowedHour; 53 import org.unicode.cldr.util.RegexUtilities; 54 import org.unicode.cldr.util.SupplementalDataInfo; 55 import org.unicode.cldr.util.XPathParts; 56 import org.unicode.cldr.util.props.UnicodeProperty.PatternMatcher; 57 58 public class CheckDates extends FactoryCheckCLDR { 59 static boolean GREGORIAN_ONLY = CldrUtility.getProperty("GREGORIAN", false); 60 61 ICUServiceBuilder icuServiceBuilder = new ICUServiceBuilder(); 62 NumberFormat english = NumberFormat.getNumberInstance(ULocale.ENGLISH); 63 PatternMatcher m; 64 DateTimePatternGenerator.FormatParser formatParser = 65 new DateTimePatternGenerator.FormatParser(); 66 DateTimePatternGenerator dateTimePatternGenerator = DateTimePatternGenerator.getEmptyInstance(); 67 private CoverageLevel2 coverageLevel; 68 private SupplementalDataInfo sdi = SupplementalDataInfo.getInstance(); 69 // Ordered list of this CLDRFile and parent CLDRFiles up to root 70 List<CLDRFile> parentCLDRFiles = new ArrayList<>(); 71 // Map from calendar type (i.e. "gregorian", "generic", "chinese") to DateTimePatternGenerator 72 // instance for that type 73 Map<String, DateTimePatternGenerator> dtpgForType = new HashMap<>(); 74 75 // Use the width of the character "0" as the basic unit for checking widths 76 // It's not perfect, but I'm not sure that anything can be. This helps us 77 // weed out some false positives in width checking, like 10月 vs. 十月 78 // in Chinese, which although technically longer, shouldn't trigger an 79 // error. 80 private static final int REFCHAR = ApproximateWidth.getWidth("0"); 81 82 private Level requiredLevel; 83 private String language; 84 private String territory; 85 86 private DayPeriodInfo dateFormatInfoFormat; 87 88 static String[] samples = { 89 // "AD 1970-01-01T00:00:00Z", 90 // "BC 4004-10-23T07:00:00Z", // try a BC date: creation according to Ussher & Lightfoot. 91 // Assuming garden of 92 // eden 2 hours ahead of UTC 93 "2005-12-02 12:15:16", 94 // "AD 2100-07-11T10:15:16Z", 95 }; // keep aligned with following 96 static String SampleList = "{0}" 97 // + Utility.LINE_SEPARATOR + "\t\u200E{1}\u200E" + Utility.LINE_SEPARATOR + 98 // "\t\u200E{2}\u200E" + 99 // Utility.LINE_SEPARATOR + "\t\u200E{3}\u200E" 100 ; // keep aligned with previous 101 102 private static final String DECIMAL_XPATH = 103 "//ldml/numbers/symbols[@numberSystem='latn']/decimal"; 104 private static final Pattern HOUR_SYMBOL = PatternCache.get("H{1,2}"); 105 private static final Pattern MINUTE_SYMBOL = PatternCache.get("mm"); 106 private static final Pattern YEAR_FIELDS = PatternCache.get("(y|Y|u|U|r){1,5}"); 107 108 private static String CALENDAR_ID_PREFIX = "/calendar[@type=\""; 109 110 static String[] calTypePathsToCheck = { 111 "//ldml/dates/calendars/calendar[@type=\"buddhist\"]", 112 "//ldml/dates/calendars/calendar[@type=\"gregorian\"]", 113 "//ldml/dates/calendars/calendar[@type=\"hebrew\"]", 114 "//ldml/dates/calendars/calendar[@type=\"islamic\"]", 115 "//ldml/dates/calendars/calendar[@type=\"japanese\"]", 116 "//ldml/dates/calendars/calendar[@type=\"roc\"]", 117 }; 118 static String[] calSymbolPathsWhichNeedDistinctValues = { 119 // === for months, days, quarters - format wide & abbrev sets must have distinct values === 120 "/months/monthContext[@type=\"format\"]/monthWidth[@type=\"abbreviated\"]/month", 121 "/months/monthContext[@type=\"format\"]/monthWidth[@type=\"wide\"]/month", 122 "/days/dayContext[@type=\"format\"]/dayWidth[@type=\"abbreviated\"]/day", 123 "/days/dayContext[@type=\"format\"]/dayWidth[@type=\"short\"]/day", 124 "/days/dayContext[@type=\"format\"]/dayWidth[@type=\"wide\"]/day", 125 "/quarters/quarterContext[@type=\"format\"]/quarterWidth[@type=\"abbreviated\"]/quarter", 126 "/quarters/quarterContext[@type=\"format\"]/quarterWidth[@type=\"wide\"]/quarter", 127 // === for dayPeriods - all values for a given context/width must be distinct === 128 "/dayPeriods/dayPeriodContext[@type=\"format\"]/dayPeriodWidth[@type=\"abbreviated\"]/dayPeriod", 129 "/dayPeriods/dayPeriodContext[@type=\"format\"]/dayPeriodWidth[@type=\"narrow\"]/dayPeriod", 130 "/dayPeriods/dayPeriodContext[@type=\"format\"]/dayPeriodWidth[@type=\"wide\"]/dayPeriod", 131 "/dayPeriods/dayPeriodContext[@type=\"stand-alone\"]/dayPeriodWidth[@type=\"abbreviated\"]/dayPeriod", 132 "/dayPeriods/dayPeriodContext[@type=\"stand-alone\"]/dayPeriodWidth[@type=\"narrow\"]/dayPeriod", 133 "/dayPeriods/dayPeriodContext[@type=\"stand-alone\"]/dayPeriodWidth[@type=\"wide\"]/dayPeriod", 134 // === for eras - all values for a given context/width should be distinct (warning) === 135 "/eras/eraNames/era", 136 "/eras/eraAbbr/era", // Hmm, root eraAbbr for japanese has many dups, should we change them 137 // or drop this test? 138 "/eras/eraNarrow/era", // We may need to allow dups here too 139 }; 140 141 // The following calendar symbol sets need not have distinct values 142 // "/months/monthContext[@type=\"format\"]/monthWidth[@type=\"narrow\"]/month", 143 // "/months/monthContext[@type=\"stand-alone\"]/monthWidth[@type=\"abbreviated\"]/month", 144 // "/months/monthContext[@type=\"stand-alone\"]/monthWidth[@type=\"narrow\"]/month", 145 // "/months/monthContext[@type=\"stand-alone\"]/monthWidth[@type=\"wide\"]/month", 146 // "/days/dayContext[@type=\"format\"]/dayWidth[@type=\"narrow\"]/day", 147 // "/days/dayContext[@type=\"stand-alone\"]/dayWidth[@type=\"abbreviated\"]/day", 148 // "/days/dayContext[@type=\"stand-alone\"]/dayWidth[@type=\"narrow\"]/day", 149 // "/days/dayContext[@type=\"stand-alone\"]/dayWidth[@type=\"wide\"]/day", 150 // "/quarters/quarterContext[@type=\"format\"]/quarterWidth[@type=\"narrow\"]/quarter", 151 // "/quarters/quarterContext[@type=\"stand-alone\"]/quarterWidth[@type=\"abbreviated\"]/quarter", 152 // "/quarters/quarterContext[@type=\"stand-alone\"]/quarterWidth[@type=\"narrow\"]/quarter", 153 // "/quarters/quarterContext[@type=\"stand-alone\"]/quarterWidth[@type=\"wide\"]/quarter", 154 155 // The above are followed by trailing pieces such as 156 // "[@type=\"am\"]", 157 // "[@type=\"sun\"]", 158 // "[@type=\"0\"]", 159 // "[@type=\"1\"]", 160 // "[@type=\"12\"]", 161 162 // Map<String, Set<String>> calPathsToSymbolSets; 163 // Map<String, Map<String, String>> calPathsToSymbolMaps = new HashMap<String, Map<String, 164 // String>>(); 165 CheckDates(Factory factory)166 public CheckDates(Factory factory) { 167 super(factory); 168 } 169 170 @Override handleSetCldrFileToCheck( CLDRFile cldrFileToCheck, Options options, List<CheckStatus> possibleErrors)171 public CheckCLDR handleSetCldrFileToCheck( 172 CLDRFile cldrFileToCheck, Options options, List<CheckStatus> possibleErrors) { 173 if (cldrFileToCheck == null) return this; 174 super.handleSetCldrFileToCheck(cldrFileToCheck, options, possibleErrors); 175 176 icuServiceBuilder.setCldrFile(getResolvedCldrFileToCheck()); 177 // the following is a hack to work around a bug in ICU4J (the snapshot, not the released 178 // version). 179 try { 180 bi = BreakIterator.getCharacterInstance(new ULocale(cldrFileToCheck.getLocaleID())); 181 } catch (RuntimeException e) { 182 bi = BreakIterator.getCharacterInstance(new ULocale("")); 183 } 184 CLDRFile resolved = getResolvedCldrFileToCheck(); 185 flexInfo = new FlexibleDateFromCLDR(); // ought to just clear(), but not available. 186 flexInfo.set(resolved); 187 188 // load decimal path specially 189 String decimal = resolved.getWinningValue(DECIMAL_XPATH); 190 if (decimal != null) { 191 flexInfo.checkFlexibles(DECIMAL_XPATH, decimal, DECIMAL_XPATH); 192 } 193 194 String localeID = cldrFileToCheck.getLocaleID(); 195 LocaleIDParser lp = new LocaleIDParser(); 196 territory = lp.set(localeID).getRegion(); 197 language = lp.getLanguage(); 198 if (territory == null || territory.length() == 0) { 199 if (language.equals("root")) { 200 territory = "001"; 201 } else { 202 CLDRLocale loc = CLDRLocale.getInstance(localeID); 203 CLDRLocale defContent = sdi.getDefaultContentFromBase(loc); 204 if (defContent == null) { 205 territory = "001"; 206 } else { 207 territory = defContent.getCountry(); 208 } 209 // Set territory for 12/24 hour clock to Egypt (12 hr) for ar_001 210 // instead of 24 hour (exception). 211 if (territory.equals("001") && language.equals("ar")) { 212 territory = "EG"; 213 } 214 } 215 } 216 coverageLevel = CoverageLevel2.getInstance(sdi, localeID); 217 requiredLevel = options.getRequiredLevel(localeID); 218 219 // load gregorian appendItems 220 for (Iterator<String> it = 221 resolved.iterator("//ldml/dates/calendars/calendar[@type=\"gregorian\"]"); 222 it.hasNext(); ) { 223 String path = it.next(); 224 String value = resolved.getWinningValue(path); 225 String fullPath = resolved.getFullXPath(path); 226 try { 227 flexInfo.checkFlexibles(path, value, fullPath); 228 } catch (Exception e) { 229 final String message = e.getMessage(); 230 CheckStatus item = 231 new CheckStatus() 232 .setCause(this) 233 .setMainType(CheckStatus.errorType) 234 .setSubtype( 235 message.contains("Conflicting fields") 236 ? Subtype.dateSymbolCollision 237 : Subtype.internalError) 238 .setMessage(message); 239 possibleErrors.add(item); 240 } 241 // possibleErrors.add(flexInfo.getFailurePath(path)); 242 } 243 redundants.clear(); 244 /* 245 * TODO: NullPointerException may be thrown in ICU here during cldr-unittest TestAll 246 */ 247 flexInfo.getRedundants(redundants); 248 // Set baseSkeletons = flexInfo.gen.getBaseSkeletons(new TreeSet()); 249 // Set notCovered = new TreeSet(neededFormats); 250 // if (flexInfo.preferred12Hour()) { 251 // notCovered.addAll(neededHours12); 252 // } else { 253 // notCovered.addAll(neededHours24); 254 // } 255 // notCovered.removeAll(baseSkeletons); 256 // if (notCovered.size() != 0) { 257 // possibleErrors.add(new CheckStatus().setCause(this).setType(CheckCLDR.finalErrorType) 258 // .setCheckOnSubmit(false) 259 // .setMessage("Missing availableFormats: {0}", new Object[]{notCovered.toString()})); 260 // } 261 pathsWithConflictingOrder2sample = 262 DateOrder.getOrderingInfo(cldrFileToCheck, resolved, flexInfo.fp); 263 if (pathsWithConflictingOrder2sample == null) { 264 CheckStatus item = 265 new CheckStatus() 266 .setCause(this) 267 .setMainType(CheckStatus.errorType) 268 .setSubtype(Subtype.internalError) 269 .setMessage("DateOrder.getOrderingInfo fails"); 270 possibleErrors.add(item); 271 } 272 273 // calPathsToSymbolMaps.clear(); 274 // for (String calTypePath: calTypePathsToCheck) { 275 // for (String calSymbolPath: calSymbolPathsWhichNeedDistinctValues) { 276 // calPathsToSymbolMaps.put(calTypePath.concat(calSymbolPath), null); 277 // } 278 // } 279 280 dateFormatInfoFormat = sdi.getDayPeriods(Type.format, cldrFileToCheck.getLocaleID()); 281 282 // Make new list of parent CLDRFiles 283 parentCLDRFiles.clear(); 284 parentCLDRFiles.add(cldrFileToCheck); 285 while ((localeID = LocaleIDParser.getParent(localeID)) != null) { 286 CLDRFile resolvedParentCLDRFile = getFactory().make(localeID, true); 287 parentCLDRFiles.add(resolvedParentCLDRFile); 288 } 289 // Clear out map of DateTimePatternGenerators for calendarType 290 dtpgForType.clear(); 291 292 return this; 293 } 294 295 Map<String, Map<DateOrder, String>> pathsWithConflictingOrder2sample; 296 297 // Set neededFormats = new TreeSet(Arrays.asList(new String[]{ 298 // "yM", "yMMM", "yMd", "yMMMd", "Md", "MMMd","yQ" 299 // })); 300 // Set neededHours12 = new TreeSet(Arrays.asList(new String[]{ 301 // "hm", "hms" 302 // })); 303 // Set neededHours24 = new TreeSet(Arrays.asList(new String[]{ 304 // "Hm", "Hms" 305 // })); 306 /** 307 * hour+minute, hour+minute+second (12 & 24) year+month, year+month+day (numeric & string) 308 * month+day (numeric & string) year+quarter 309 */ 310 BreakIterator bi; 311 312 FlexibleDateFromCLDR flexInfo; 313 Collection<String> redundants = new HashSet<>(); 314 Status status = new Status(); 315 PathStarrer pathStarrer = new PathStarrer(); 316 stripPrefix(String s)317 private String stripPrefix(String s) { 318 if (s != null) { 319 int prefEnd = s.lastIndexOf(" "); 320 if (prefEnd < 0 || prefEnd >= 3) { 321 prefEnd = s.lastIndexOf("\u2019"); // as in d’ 322 } 323 if (prefEnd >= 0 && prefEnd < 3) { 324 return s.substring(prefEnd + 1); 325 } 326 } 327 return s; 328 } 329 330 @Override handleCheck( String path, String fullPath, String value, Options options, List<CheckStatus> result)331 public CheckCLDR handleCheck( 332 String path, String fullPath, String value, Options options, List<CheckStatus> result) { 333 334 if (fullPath == null) { 335 return this; // skip paths that we don't have 336 } 337 338 if (path.indexOf("/dates") < 0 || path.endsWith("/default") || path.endsWith("/alias")) { 339 return this; 340 } 341 342 if (!accept(result)) return this; 343 344 String sourceLocale = getCldrFileToCheck().getSourceLocaleID(path, status); 345 346 if (!path.equals(status.pathWhereFound) 347 || !sourceLocale.equals(getCldrFileToCheck().getLocaleID())) { 348 return this; 349 } 350 351 if (value == null) { 352 return this; 353 } 354 355 if (pathsWithConflictingOrder2sample != null) { 356 Map<DateOrder, String> problem = pathsWithConflictingOrder2sample.get(path); 357 if (problem != null) { 358 CheckStatus item = 359 new CheckStatus() 360 .setCause(this) 361 .setMainType(CheckStatus.warningType) 362 .setSubtype(Subtype.incorrectDatePattern) 363 .setMessage( 364 "The ordering of date fields is inconsistent with others: {0}", 365 getValues(getResolvedCldrFileToCheck(), problem.values())); 366 result.add(item); 367 } 368 } 369 370 try { 371 if (path.indexOf("[@type=\"abbreviated\"]") >= 0) { 372 String pathToWide = path.replace("[@type=\"abbreviated\"]", "[@type=\"wide\"]"); 373 String wideValue = getCldrFileToCheck().getWinningValueWithBailey(pathToWide); 374 if (wideValue != null && isTooMuchWiderThan(value, wideValue)) { 375 CheckStatus item = 376 new CheckStatus() 377 .setCause(this) 378 .setMainType(CheckStatus.errorType) 379 .setSubtype(Subtype.abbreviatedDateFieldTooWide) 380 .setMessage( 381 "Abbreviated value \"{0}\" can't be longer than the corresponding wide value \"{1}\"", 382 value, wideValue); 383 result.add(item); 384 } 385 Set<String> grouping = LogicalGrouping.getPaths(getCldrFileToCheck(), path); 386 if (grouping != null) { 387 for (String lgPath : grouping) { 388 String lgPathValue = getCldrFileToCheck().getWinningValueWithBailey(lgPath); 389 if (lgPathValue == null) { 390 continue; 391 } 392 String lgPathToWide = 393 lgPath.replace("[@type=\"abbreviated\"]", "[@type=\"wide\"]"); 394 String lgPathWideValue = 395 getCldrFileToCheck().getWinningValueWithBailey(lgPathToWide); 396 // This helps us get around things like "de març" vs. "març" in Catalan 397 String thisValueStripped = stripPrefix(value); 398 String wideValueStripped = stripPrefix(wideValue); 399 String lgPathValueStripped = stripPrefix(lgPathValue); 400 String lgPathWideValueStripped = stripPrefix(lgPathWideValue); 401 boolean thisPathHasPeriod = value.contains("."); 402 boolean lgPathHasPeriod = lgPathValue.contains("."); 403 if (!thisValueStripped.equalsIgnoreCase(wideValueStripped) 404 && !lgPathValueStripped.equalsIgnoreCase(lgPathWideValueStripped) 405 && thisPathHasPeriod != lgPathHasPeriod) { 406 CheckStatus.Type et = CheckStatus.errorType; 407 if (path.contains("dayPeriod")) { 408 et = CheckStatus.warningType; 409 } 410 CheckStatus item = 411 new CheckStatus() 412 .setCause(this) 413 .setMainType(et) 414 .setSubtype(Subtype.inconsistentPeriods) 415 .setMessage( 416 "Inconsistent use of periods in abbreviations for this section."); 417 result.add(item); 418 break; 419 } 420 } 421 } 422 } else if (path.indexOf("[@type=\"narrow\"]") >= 0) { 423 String pathToAbbr = path.replace("[@type=\"narrow\"]", "[@type=\"abbreviated\"]"); 424 String abbrValue = getCldrFileToCheck().getWinningValueWithBailey(pathToAbbr); 425 if (abbrValue != null && isTooMuchWiderThan(value, abbrValue)) { 426 CheckStatus item = 427 new CheckStatus() 428 .setCause(this) 429 .setMainType( 430 CheckStatus.warningType) // Making this just a warning, 431 // because there are some oddball 432 // cases. 433 .setSubtype(Subtype.narrowDateFieldTooWide) 434 .setMessage( 435 "Narrow value \"{0}\" shouldn't be longer than the corresponding abbreviated value \"{1}\"", 436 value, abbrValue); 437 result.add(item); 438 } 439 } else if (path.indexOf("/eraNarrow") >= 0) { 440 String pathToAbbr = path.replace("/eraNarrow", "/eraAbbr"); 441 String abbrValue = getCldrFileToCheck().getWinningValueWithBailey(pathToAbbr); 442 if (abbrValue != null && isTooMuchWiderThan(value, abbrValue)) { 443 CheckStatus item = 444 new CheckStatus() 445 .setCause(this) 446 .setMainType(CheckStatus.errorType) 447 .setSubtype(Subtype.narrowDateFieldTooWide) 448 .setMessage( 449 "Narrow value \"{0}\" can't be longer than the corresponding abbreviated value \"{1}\"", 450 value, abbrValue); 451 result.add(item); 452 } 453 } else if (path.indexOf("/eraAbbr") >= 0) { 454 String pathToWide = path.replace("/eraAbbr", "/eraNames"); 455 String wideValue = getCldrFileToCheck().getWinningValueWithBailey(pathToWide); 456 if (wideValue != null && isTooMuchWiderThan(value, wideValue)) { 457 CheckStatus item = 458 new CheckStatus() 459 .setCause(this) 460 .setMainType(CheckStatus.errorType) 461 .setSubtype(Subtype.abbreviatedDateFieldTooWide) 462 .setMessage( 463 "Abbreviated value \"{0}\" can't be longer than the corresponding wide value \"{1}\"", 464 value, wideValue); 465 result.add(item); 466 } 467 } 468 469 String failure = flexInfo.checkValueAgainstSkeleton(path, value); 470 if (failure != null) { 471 result.add( 472 new CheckStatus() 473 .setCause(this) 474 .setMainType(CheckStatus.errorType) 475 .setSubtype(Subtype.illegalDatePattern) 476 .setMessage(failure)); 477 } 478 479 final String collisionPrefix = "//ldml/dates/calendars/calendar"; 480 main: 481 if (path.startsWith(collisionPrefix)) { 482 int pos = path.indexOf("\"]"); // end of first type 483 if (pos < 0 || skipPath(path)) { // skip narrow, no-calendar 484 break main; 485 } 486 pos += 2; 487 String myType = getLastType(path); 488 if (myType == null) { 489 break main; 490 } 491 String myMainType = getMainType(path); 492 493 String calendarPrefix = path.substring(0, pos); 494 boolean endsWithDisplayName = 495 path.endsWith("displayName"); // special hack, these shouldn't be in 496 // calendar. 497 498 Set<String> retrievedPaths = new HashSet<>(); 499 getResolvedCldrFileToCheck() 500 .getPathsWithValue(value, calendarPrefix, null, retrievedPaths); 501 if (retrievedPaths.size() < 2) { 502 break main; 503 } 504 // ldml/dates/calendars/calendar[@type="gregorian"]/eras/eraAbbr/era[@type="0"], 505 // ldml/dates/calendars/calendar[@type="gregorian"]/eras/eraNames/era[@type="0"], 506 // ldml/dates/calendars/calendar[@type="gregorian"]/eras/eraNarrow/era[@type="0"]] 507 Type type = null; 508 DayPeriod dayPeriod = null; 509 final boolean isDayPeriod = path.contains("dayPeriod"); 510 if (isDayPeriod) { 511 XPathParts parts = XPathParts.getFrozenInstance(fullPath); 512 type = Type.fromString(parts.getAttributeValue(5, "type")); 513 dayPeriod = DayPeriod.valueOf(parts.getAttributeValue(-1, "type")); 514 } 515 516 // TODO redo above and below in terms of parts instead of searching strings 517 518 Set<String> filteredPaths = new HashSet<>(); 519 Output<Integer> sampleError = new Output<>(); 520 521 for (String item : retrievedPaths) { 522 XPathParts itemParts = XPathParts.getFrozenInstance(item); 523 if (item.equals(path) 524 || skipPath(item) 525 || endsWithDisplayName != item.endsWith("displayName") 526 || itemParts.containsElement("alias")) { 527 continue; 528 } 529 String otherType = getLastType(item); 530 if (myType.equals( 531 otherType)) { // we don't care about items with the same type value 532 continue; 533 } 534 String mainType = getMainType(item); 535 if (!myMainType.equals( 536 mainType)) { // we *only* care about items with the same type value 537 continue; 538 } 539 if (isDayPeriod) { 540 // ldml/dates/calendars/calendar[@type="gregorian"]/dayPeriods/dayPeriodContext[@type="format"]/dayPeriodWidth[@type="wide"]/dayPeriod[@type="am"] 541 Type itemType = Type.fromString(itemParts.getAttributeValue(5, "type")); 542 DayPeriod itemDayPeriod = 543 DayPeriod.valueOf(itemParts.getAttributeValue(-1, "type")); 544 545 if (!dateFormatInfoFormat.collisionIsError( 546 type, dayPeriod, itemType, itemDayPeriod, sampleError)) { 547 continue; 548 } 549 } 550 filteredPaths.add(item); 551 } 552 if (filteredPaths.size() == 0) { 553 break main; 554 } 555 Set<String> others = new TreeSet<>(); 556 for (String path2 : filteredPaths) { 557 PathHeader pathHeader = getPathHeaderFactory().fromPath(path2); 558 others.add(pathHeader.getHeaderCode()); 559 } 560 CheckStatus.Type statusType = 561 getPhase() == Phase.SUBMISSION || getPhase() == Phase.BUILD 562 ? CheckStatus.warningType 563 : CheckStatus.errorType; 564 final CheckStatus checkStatus = 565 new CheckStatus() 566 .setCause(this) 567 .setMainType(statusType) 568 .setSubtype(Subtype.dateSymbolCollision); 569 if (sampleError.value == null) { 570 checkStatus.setMessage( 571 "The date value “{0}” is the same as what is used for a different item: {1}", 572 value, others.toString()); 573 } else { 574 checkStatus.setMessage( 575 "The date value “{0}” is the same as what is used for a different item: {1}. Sample problem: {2}", 576 value, others.toString(), sampleError.value / DayPeriodInfo.HOUR); 577 } 578 result.add(checkStatus); 579 } 580 DateTimePatternType dateTypePatternType = DateTimePatternType.fromPath(path); 581 if (DateTimePatternType.STOCK_AVAILABLE_INTERVAL_PATTERNS.contains( 582 dateTypePatternType)) { 583 boolean patternBasicallyOk = false; 584 try { 585 formatParser.set(value); 586 patternBasicallyOk = true; 587 } catch (RuntimeException e) { 588 String message = e.getMessage(); 589 if (message.contains("Illegal datetime field:")) { 590 CheckStatus item = 591 new CheckStatus() 592 .setCause(this) 593 .setMainType(CheckStatus.errorType) 594 .setSubtype(Subtype.illegalDatePattern) 595 .setMessage(message); 596 result.add(item); 597 } else { 598 CheckStatus item = 599 new CheckStatus() 600 .setCause(this) 601 .setMainType(CheckStatus.errorType) 602 .setSubtype(Subtype.illegalDatePattern) 603 .setMessage( 604 "Illegal date format pattern {0}", 605 new Object[] {e}); 606 result.add(item); 607 } 608 } 609 if (patternBasicallyOk) { 610 checkPattern(dateTypePatternType, path, fullPath, value, result); 611 } 612 } else if (path.contains("datetimeSkeleton") 613 && !path.contains("[@alt=")) { // cannot test any alt skeletons 614 // Get calendar type from //ldml/dates/calendars/calendar[@type="..."]/ 615 int startIndex = path.indexOf(CALENDAR_ID_PREFIX); 616 if (startIndex > 0) { 617 startIndex += CALENDAR_ID_PREFIX.length(); 618 int endIndex = path.indexOf("\"]", startIndex); 619 String calendarType = path.substring(startIndex, endIndex); 620 // Get pattern generated from datetimeSkeleton 621 DateTimePatternGenerator dtpg = getDTPGForCalendarType(calendarType); 622 String patternFromSkeleton = dtpg.getBestPattern(value); 623 // Get actual stock pattern 624 String patternPath = 625 path.replace("/datetimeSkeleton", "/pattern[@type=\"standard\"]"); 626 String patternStock = getCldrFileToCheck().getWinningValue(patternPath); 627 // Compare and flag error if mismatch 628 if (!patternFromSkeleton.equals(patternStock)) { 629 CheckStatus item = 630 new CheckStatus() 631 .setCause(this) 632 .setMainType(CheckStatus.warningType) 633 .setSubtype(Subtype.inconsistentDatePattern) 634 .setMessage( 635 "Pattern \"{0}\" from datetimeSkeleton should match corresponding standard pattern \"{1}\", adjust availableFormats to fix.", 636 patternFromSkeleton, patternStock); 637 result.add(item); 638 } 639 } 640 } else if (path.contains("hourFormat")) { 641 int semicolonPos = value.indexOf(';'); 642 if (semicolonPos < 0) { 643 CheckStatus item = 644 new CheckStatus() 645 .setCause(this) 646 .setMainType(CheckStatus.errorType) 647 .setSubtype(Subtype.illegalDatePattern) 648 .setMessage( 649 "Value should contain a positive hour format and a negative hour format separated by a semicolon."); 650 result.add(item); 651 } else { 652 String[] formats = value.split(";"); 653 if (formats[0].equals(formats[1])) { 654 CheckStatus item = 655 new CheckStatus() 656 .setCause(this) 657 .setMainType(CheckStatus.errorType) 658 .setSubtype(Subtype.illegalDatePattern) 659 .setMessage("The hour formats should not be the same."); 660 result.add(item); 661 } else { 662 checkHasHourMinuteSymbols(formats[0], result); 663 checkHasHourMinuteSymbols(formats[1], result); 664 } 665 } 666 } 667 } catch (ParseException e) { 668 CheckStatus item = 669 new CheckStatus() 670 .setCause(this) 671 .setMainType(CheckStatus.errorType) 672 .setSubtype(Subtype.illegalDatePattern) 673 .setMessage( 674 "ParseException in creating date format {0}", new Object[] {e}); 675 result.add(item); 676 } catch (Exception e) { 677 // e.printStackTrace(); 678 // HACK 679 String msg = e.getMessage(); 680 if (msg == null || !HACK_CONFLICTING.matcher(msg).find()) { 681 CheckStatus item = 682 new CheckStatus() 683 .setCause(this) 684 .setMainType(CheckStatus.errorType) 685 .setSubtype(Subtype.illegalDatePattern) 686 .setMessage("Error in creating date format {0}", new Object[] {e}); 687 result.add(item); 688 } 689 } 690 return this; 691 } 692 isTooMuchWiderThan(String shortString, String longString)693 private boolean isTooMuchWiderThan(String shortString, String longString) { 694 // We all 1/3 the width of the reference character as a "fudge factor" in determining the 695 // allowable width 696 return ApproximateWidth.getWidth(shortString) 697 > ApproximateWidth.getWidth(longString) + REFCHAR / 3; 698 } 699 700 /** 701 * Check for the presence of hour and minute symbols. 702 * 703 * @param value the value to be checked 704 * @param result the list to add any errors to. 705 */ checkHasHourMinuteSymbols(String value, List<CheckStatus> result)706 private void checkHasHourMinuteSymbols(String value, List<CheckStatus> result) { 707 boolean hasHourSymbol = HOUR_SYMBOL.matcher(value).find(); 708 boolean hasMinuteSymbol = MINUTE_SYMBOL.matcher(value).find(); 709 if (!hasHourSymbol && !hasMinuteSymbol) { 710 result.add( 711 createErrorCheckStatus() 712 .setMessage( 713 "The hour and minute symbols are missing from {0}.", value)); 714 } else if (!hasHourSymbol) { 715 result.add( 716 createErrorCheckStatus() 717 .setMessage( 718 "The hour symbol (H or HH) should be present in {0}.", value)); 719 } else if (!hasMinuteSymbol) { 720 result.add( 721 createErrorCheckStatus() 722 .setMessage("The minute symbol (mm) should be present in {0}.", value)); 723 } 724 } 725 726 /** 727 * Convenience method for creating errors. 728 * 729 * @return 730 */ createErrorCheckStatus()731 private CheckStatus createErrorCheckStatus() { 732 return new CheckStatus() 733 .setCause(this) 734 .setMainType(CheckStatus.errorType) 735 .setSubtype(Subtype.illegalDatePattern); 736 } 737 skipPath(String path)738 public boolean skipPath(String path) { 739 return path.contains("arrow") 740 || path.contains("/availableFormats") 741 || path.contains("/interval") 742 || path.contains("/dateTimeFormat") 743 // || path.contains("/dayPeriod[") 744 // && !path.endsWith("=\"pm\"]") 745 // && !path.endsWith("=\"am\"]") 746 ; 747 } 748 getLastType(String path)749 public String getLastType(String path) { 750 int secondType = path.lastIndexOf("[@type=\""); 751 if (secondType < 0) { 752 return null; 753 } 754 secondType += 8; 755 int secondEnd = path.indexOf("\"]", secondType); 756 if (secondEnd < 0) { 757 return null; 758 } 759 return path.substring(secondType, secondEnd); 760 } 761 getMainType(String path)762 public String getMainType(String path) { 763 int secondType = path.indexOf("\"]/"); 764 if (secondType < 0) { 765 return null; 766 } 767 secondType += 3; 768 int secondEnd = path.indexOf("/", secondType); 769 if (secondEnd < 0) { 770 return null; 771 } 772 return path.substring(secondType, secondEnd); 773 } 774 getValues(CLDRFile resolvedCldrFileToCheck, Collection<String> values)775 private String getValues(CLDRFile resolvedCldrFileToCheck, Collection<String> values) { 776 Set<String> results = new TreeSet<>(); 777 for (String path : values) { 778 final String stringValue = resolvedCldrFileToCheck.getStringValue(path); 779 if (stringValue != null) { 780 results.add(stringValue); 781 } 782 } 783 return "{" + Joiner.on("},{").join(results) + "}"; 784 } 785 786 static final Pattern HACK_CONFLICTING = PatternCache.get("Conflicting fields:\\s+M+,\\s+l"); 787 788 @Override handleGetExamples( String path, String fullPath, String value, Options options, List<CheckStatus> result)789 public CheckCLDR handleGetExamples( 790 String path, String fullPath, String value, Options options, List<CheckStatus> result) { 791 if (path.indexOf("/dates") < 0 || path.indexOf("gregorian") < 0) return this; 792 try { 793 if (path.indexOf("/pattern") >= 0 && path.indexOf("/dateTimeFormat") < 0 794 || path.indexOf("/dateFormatItem") >= 0) { 795 checkPattern2(path, value, result); 796 } 797 } catch (Exception e) { 798 // don't worry about errors 799 } 800 return this; 801 } 802 803 static final SimpleDateFormat neutralFormat = 804 new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", ULocale.ENGLISH); 805 806 static { 807 neutralFormat.setTimeZone(ExampleGenerator.ZONE_SAMPLE); 808 } 809 810 // Get Date-Time in milliseconds getDateTimeinMillis( int year, int month, int date, int hourOfDay, int minute, int second)811 private static long getDateTimeinMillis( 812 int year, int month, int date, int hourOfDay, int minute, int second) { 813 Calendar cal = Calendar.getInstance(); 814 cal.set(year, month, date, hourOfDay, minute, second); 815 return cal.getTimeInMillis(); 816 } 817 818 static long date1950 = getDateTimeinMillis(1950, 0, 1, 0, 0, 0); 819 static long date2010 = getDateTimeinMillis(2010, 0, 1, 0, 0, 0); 820 static long date4004BC = getDateTimeinMillis(-4004, 9, 23, 2, 0, 0); 821 static Random random = new Random(0); 822 checkPattern( DateTimePatternType dateTypePatternType, String path, String fullPath, String value, List<CheckStatus> result)823 private void checkPattern( 824 DateTimePatternType dateTypePatternType, 825 String path, 826 String fullPath, 827 String value, 828 List<CheckStatus> result) 829 throws ParseException { 830 String skeleton = dateTimePatternGenerator.getSkeletonAllowingDuplicates(value); 831 String skeletonCanonical = 832 dateTimePatternGenerator.getCanonicalSkeletonAllowingDuplicates(value); 833 834 if (value.contains("MMM.") 835 || value.contains("LLL.") 836 || value.contains("E.") 837 || value.contains("eee.") 838 || value.contains("ccc.") 839 || value.contains("QQQ.") 840 || value.contains("qqq.")) { 841 result.add( 842 new CheckStatus() 843 .setCause(this) 844 .setMainType(CheckStatus.warningType) 845 .setSubtype(Subtype.incorrectDatePattern) 846 .setMessage( 847 "Your pattern ({0}) is probably incorrect; abbreviated month/weekday/quarter names that need a period should include it in the name, rather than adding it to the pattern.", 848 value)); 849 } 850 XPathParts pathParts = XPathParts.getFrozenInstance(path); 851 String calendar = pathParts.findAttributeValue("calendar", "type"); 852 String id; 853 switch (dateTypePatternType) { 854 case AVAILABLE: 855 id = pathParts.getAttributeValue(-1, "id"); 856 break; 857 case INTERVAL: 858 id = pathParts.getAttributeValue(-2, "id"); 859 break; 860 case STOCK: 861 id = pathParts.getAttributeValue(-3, "type"); 862 break; 863 default: 864 throw new IllegalArgumentException(); 865 } 866 867 if (dateTypePatternType == DateTimePatternType.AVAILABLE 868 || dateTypePatternType == DateTimePatternType.INTERVAL) { 869 String idCanonical = 870 dateTimePatternGenerator.getCanonicalSkeletonAllowingDuplicates(id); 871 if (skeleton.isEmpty()) { 872 result.add( 873 new CheckStatus() 874 .setCause(this) 875 .setMainType(CheckStatus.errorType) 876 .setSubtype(Subtype.incorrectDatePattern) 877 // "Internal ID ({0}) doesn't match generated ID ({1}) for pattern 878 // ({2}). " + 879 .setMessage( 880 "Your pattern ({1}) is incorrect for ID ({0}). " 881 + "You need to supply a pattern according to " 882 + CLDRURLS.DATE_TIME_PATTERNS_URL 883 + ".", 884 id, 885 value)); 886 } else if (!dateTimePatternGenerator.skeletonsAreSimilar( 887 idCanonical, skeletonCanonical)) { 888 String fixedValue = dateTimePatternGenerator.replaceFieldTypes(value, id); 889 result.add( 890 new CheckStatus() 891 .setCause(this) 892 .setMainType(CheckStatus.errorType) 893 .setSubtype(Subtype.incorrectDatePattern) 894 // "Internal ID ({0}) doesn't match generated ID ({1}) for pattern 895 // ({2}). " + 896 .setMessage( 897 "Your pattern ({2}) doesn't correspond to what is asked for. Yours would be right for an ID ({1}) but not for the ID ({0}). " 898 + "Please change your pattern to match what was asked, such as ({3}), with the right punctuation and/or ordering for your language. See " 899 + CLDRURLS.DATE_TIME_PATTERNS_URL 900 + ".", 901 id, 902 skeletonCanonical, 903 value, 904 fixedValue)); 905 } 906 if (dateTypePatternType == DateTimePatternType.AVAILABLE) { 907 // index y+w+ must correpond to pattern containing only Y+ and w+ 908 if (idCanonical.matches("y+w+") 909 && !(skeleton.matches("Y+w+") || skeleton.matches("w+Y+"))) { 910 result.add( 911 new CheckStatus() 912 .setCause(this) 913 .setMainType(CheckStatus.errorType) 914 .setSubtype(Subtype.incorrectDatePattern) 915 .setMessage( 916 "For id {0}, the pattern ({1}) must contain fields Y and w, and no others.", 917 id, value)); 918 } 919 // index M+W msut correspond to pattern containing only M+/L+ and W 920 if (idCanonical.matches("M+W") 921 && !(skeletonCanonical.matches("M+W") 922 || skeletonCanonical.matches("WM+"))) { 923 result.add( 924 new CheckStatus() 925 .setCause(this) 926 .setMainType(CheckStatus.errorType) 927 .setSubtype(Subtype.incorrectDatePattern) 928 .setMessage( 929 "For id {0}, the pattern ({1}) must contain fields M or L, plus W, and no others.", 930 id, value)); 931 } 932 } 933 String failureMessage = (String) flexInfo.getFailurePath(path); 934 if (failureMessage != null) { 935 result.add( 936 new CheckStatus() 937 .setCause(this) 938 .setMainType(CheckStatus.errorType) 939 .setSubtype(Subtype.illegalDatePattern) 940 .setMessage("{0}", new Object[] {failureMessage})); 941 } 942 } 943 if (dateTypePatternType == DateTimePatternType.STOCK) { 944 int style = 0; 945 String len = pathParts.findAttributeValue("timeFormatLength", "type"); 946 DateOrTime dateOrTime = DateOrTime.time; 947 if (len == null) { 948 dateOrTime = DateOrTime.date; 949 style += 4; 950 len = pathParts.findAttributeValue("dateFormatLength", "type"); 951 if (len == null) { 952 len = pathParts.findAttributeValue("dateTimeFormatLength", "type"); 953 dateOrTime = DateOrTime.dateTime; 954 } 955 } 956 957 DateTimeLengths dateTimeLength = 958 DateTimeLengths.valueOf(len.toUpperCase(Locale.ENGLISH)); 959 960 if (calendar.equals("gregorian") 961 && !"root".equals(getCldrFileToCheck().getLocaleID())) { 962 checkValue(dateTimeLength, dateOrTime, value, result); 963 } 964 if (dateOrTime == DateOrTime.dateTime) { 965 return; // We don't need to do the rest for date/time combo patterns. 966 } 967 style += dateTimeLength.ordinal(); 968 // do regex match with skeletonCanonical but report errors using skeleton; they have 969 // corresponding field lengths 970 if (!dateTimePatterns[style].matcher(skeletonCanonical).matches() 971 && !calendar.equals("chinese") 972 && !calendar.equals("hebrew")) { 973 int i = RegexUtilities.findMismatch(dateTimePatterns[style], skeletonCanonical); 974 String skeletonPosition = skeleton.substring(0, i) + "☹" + skeleton.substring(i); 975 result.add( 976 new CheckStatus() 977 .setCause(this) 978 .setMainType(CheckStatus.errorType) 979 .setSubtype(Subtype.missingOrExtraDateField) 980 .setMessage( 981 "Field is missing, extra, or the wrong length. Expected {0} [Internal: {1} / {2}]", 982 new Object[] { 983 dateTimeMessage[style], 984 skeletonPosition, 985 dateTimePatterns[style].pattern() 986 })); 987 } 988 } else if (dateTypePatternType == DateTimePatternType.INTERVAL) { 989 if (id.contains("y")) { 990 String greatestDifference = 991 pathParts.findAttributeValue("greatestDifference", "id"); 992 int requiredYearFieldCount = 1; 993 if ("y".equals(greatestDifference)) { 994 requiredYearFieldCount = 2; 995 } 996 int yearFieldCount = 0; 997 Matcher yearFieldMatcher = YEAR_FIELDS.matcher(value); 998 while (yearFieldMatcher.find()) { 999 yearFieldCount++; 1000 } 1001 if (yearFieldCount < requiredYearFieldCount) { 1002 result.add( 1003 new CheckStatus() 1004 .setCause(this) 1005 .setMainType(CheckStatus.errorType) 1006 .setSubtype(Subtype.missingOrExtraDateField) 1007 .setMessage( 1008 "Not enough year fields in interval pattern. Must have {0} but only found {1}", 1009 new Object[] {requiredYearFieldCount, yearFieldCount})); 1010 } 1011 } 1012 } 1013 1014 if (value.contains("G") && calendar.equals("gregorian")) { 1015 GyState actual = GyState.forPattern(value); 1016 GyState expected = getExpectedGy(getCldrFileToCheck().getLocaleID()); 1017 if (actual != expected) { 1018 result.add( 1019 new CheckStatus() 1020 .setCause(this) 1021 .setMainType(CheckStatus.warningType) 1022 .setSubtype(Subtype.unexpectedOrderOfEraYear) 1023 .setMessage( 1024 "Unexpected order of era/year. Expected {0}, but got {1} in 〈{2}〉 for {3}/{4}", 1025 expected, actual, value, calendar, id)); 1026 } 1027 } 1028 } 1029 1030 enum DateOrTime { 1031 date, 1032 time, 1033 dateTime 1034 } 1035 1036 static final Map<DateOrTime, Relation<DateTimeLengths, String>> STOCK_PATTERNS = 1037 new EnumMap<>(DateOrTime.class); 1038 1039 // add( Map<DateOrTime, Relation<DateTimeLengths, String>> stockPatterns, DateOrTime dateOrTime, DateTimeLengths dateTimeLength, String... keys)1040 private static void add( 1041 Map<DateOrTime, Relation<DateTimeLengths, String>> stockPatterns, 1042 DateOrTime dateOrTime, 1043 DateTimeLengths dateTimeLength, 1044 String... keys) { 1045 Relation<DateTimeLengths, String> rel = STOCK_PATTERNS.get(dateOrTime); 1046 if (rel == null) { 1047 STOCK_PATTERNS.put( 1048 dateOrTime, 1049 rel = 1050 Relation.of( 1051 new EnumMap<DateTimeLengths, Set<String>>( 1052 DateTimeLengths.class), 1053 LinkedHashSet.class)); 1054 } 1055 rel.putAll(dateTimeLength, Arrays.asList(keys)); 1056 } 1057 1058 /* Ticket #4936 1059 value(short time) = value(hm) or value(Hm) 1060 value(medium time) = value(hms) or value(Hms) 1061 value(long time) = value(medium time+z) 1062 value(full time) = value(medium time+zzzz) 1063 */ 1064 static { add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.SHORT, "hm", "Hm")1065 add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.SHORT, "hm", "Hm"); add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.MEDIUM, "hms", "Hms")1066 add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.MEDIUM, "hms", "Hms"); add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.LONG, "hms*z", "Hms*z")1067 add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.LONG, "hms*z", "Hms*z"); add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.FULL, "hms*zzzz", "Hms*zzzz")1068 add(STOCK_PATTERNS, DateOrTime.time, DateTimeLengths.FULL, "hms*zzzz", "Hms*zzzz"); add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.SHORT, "yMd")1069 add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.SHORT, "yMd"); add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.MEDIUM, "yMMMd")1070 add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.MEDIUM, "yMMMd"); add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.LONG, "yMMMMd", "yMMMd")1071 add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.LONG, "yMMMMd", "yMMMd"); add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.FULL, "yMMMMEd", "yMMMEd")1072 add(STOCK_PATTERNS, DateOrTime.date, DateTimeLengths.FULL, "yMMMMEd", "yMMMEd"); 1073 } 1074 1075 static final String AVAILABLE_PREFIX = 1076 "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/availableFormats/dateFormatItem[@id=\""; 1077 static final String AVAILABLE_SUFFIX = "\"]"; 1078 static final String APPEND_TIMEZONE = 1079 "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/appendItems/appendItem[@request=\"Timezone\"]"; 1080 checkValue( DateTimeLengths dateTimeLength, DateOrTime dateOrTime, String value, List<CheckStatus> result)1081 private void checkValue( 1082 DateTimeLengths dateTimeLength, 1083 DateOrTime dateOrTime, 1084 String value, 1085 List<CheckStatus> result) { 1086 // Check consistency of the pattern vs. supplemental wrt 12 vs. 24 hour clock. 1087 if (dateOrTime == DateOrTime.time) { 1088 PreferredAndAllowedHour pref = sdi.getTimeData().get(territory); 1089 if (pref == null) { 1090 pref = sdi.getTimeData().get("001"); 1091 } 1092 String checkForHour, clockType; 1093 if (pref.preferred.equals(PreferredAndAllowedHour.HourStyle.h)) { 1094 checkForHour = "h"; 1095 clockType = "12"; 1096 } else { 1097 checkForHour = "H"; 1098 clockType = "24"; 1099 } 1100 if (!value.contains(checkForHour)) { 1101 CheckStatus.Type errType = CheckStatus.errorType; 1102 // French/Canada is strange, they use 24 hr clock while en_CA uses 12. 1103 if (language.equals("fr") && territory.equals("CA")) { 1104 errType = CheckStatus.warningType; 1105 } 1106 1107 result.add( 1108 new CheckStatus() 1109 .setCause(this) 1110 .setMainType(errType) 1111 .setSubtype(Subtype.inconsistentTimePattern) 1112 .setMessage( 1113 "Time format inconsistent with supplemental time data for territory \"" 1114 + territory 1115 + "\"." 1116 + " Use '" 1117 + checkForHour 1118 + "' for " 1119 + clockType 1120 + " hour clock.")); 1121 } 1122 } 1123 if (dateOrTime == DateOrTime.dateTime) { 1124 boolean inQuotes = false; 1125 for (int i = 0; i < value.length(); i++) { 1126 char ch = value.charAt(i); 1127 if (ch == '\'') { 1128 inQuotes = !inQuotes; 1129 } 1130 if (!inQuotes && (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z')) { 1131 result.add( 1132 new CheckStatus() 1133 .setCause(this) 1134 .setMainType(CheckStatus.errorType) 1135 .setSubtype(Subtype.patternContainsInvalidCharacters) 1136 .setMessage("Unquoted letter \"{0}\" in dateTime format.", ch)); 1137 } 1138 } 1139 } else { 1140 Set<String> keys = STOCK_PATTERNS.get(dateOrTime).get(dateTimeLength); 1141 StringBuilder b = new StringBuilder(); 1142 boolean onlyNulls = true; 1143 int countMismatches = 0; 1144 boolean errorOnMissing = false; 1145 String timezonePattern = null; 1146 Set<String> bases = new LinkedHashSet<>(); 1147 for (String key : keys) { 1148 int star = key.indexOf('*'); 1149 boolean hasStar = star >= 0; 1150 String base = !hasStar ? key : key.substring(0, star); 1151 bases.add(base); 1152 String xpath = AVAILABLE_PREFIX + base + AVAILABLE_SUFFIX; 1153 String value1 = getCldrFileToCheck().getStringValue(xpath); 1154 // String localeFound = getCldrFileToCheck().getSourceLocaleID(xpath, null); && 1155 // !localeFound.equals("root") && !localeFound.equals("code-fallback") 1156 if (value1 != null) { 1157 onlyNulls = false; 1158 if (hasStar) { 1159 String zone = key.substring(star + 1); 1160 timezonePattern = 1161 getResolvedCldrFileToCheck().getStringValue(APPEND_TIMEZONE); 1162 value1 = MessageFormat.format(timezonePattern, value1, zone); 1163 } 1164 if (equalsExceptWidth(value, value1)) { 1165 return; 1166 } 1167 } else { 1168 // Example, if the requiredLevel for the locale is moderate, 1169 // and the level for the path is modern, then we'll skip the error, 1170 // but if the level for the path is basic, then we won't 1171 Level pathLevel = coverageLevel.getLevel(xpath); 1172 if (requiredLevel.compareTo(pathLevel) >= 0) { 1173 errorOnMissing = true; 1174 } 1175 } 1176 add(b, base, value1); 1177 countMismatches++; 1178 } 1179 if (!onlyNulls) { 1180 if (timezonePattern != null) { 1181 b.append(" (with appendZonePattern: “" + timezonePattern + "”)"); 1182 } 1183 String msg = 1184 countMismatches != 1 1185 ? "{1}-{0} → “{2}” didn't match any of the corresponding flexible skeletons: [{3}]. This or the flexible patterns needs to be changed." 1186 : "{1}-{0} → “{2}” didn't match the corresponding flexible skeleton: {3}. This or the flexible pattern needs to be changed."; 1187 result.add( 1188 new CheckStatus() 1189 .setCause(this) 1190 .setMainType(CheckStatus.warningType) 1191 .setSubtype(Subtype.inconsistentDatePattern) 1192 .setMessage(msg, dateTimeLength, dateOrTime, value, b)); 1193 } else { 1194 if (errorOnMissing) { 1195 String msg = 1196 countMismatches != 1 1197 ? "{1}-{0} → “{2}” doesn't have at least one value for a corresponding flexible skeleton {3}, which needs to be added." 1198 : "{1}-{0} → “{2}” doesn't have a value for the corresponding flexible skeleton {3}, which needs to be added."; 1199 result.add( 1200 new CheckStatus() 1201 .setCause(this) 1202 .setMainType(CheckStatus.warningType) 1203 .setSubtype(Subtype.missingDatePattern) 1204 .setMessage( 1205 msg, 1206 dateTimeLength, 1207 dateOrTime, 1208 value, 1209 Joiner.on(", ").join(bases))); 1210 } 1211 } 1212 } 1213 } 1214 add(StringBuilder b, String key, String value1)1215 private void add(StringBuilder b, String key, String value1) { 1216 if (value1 == null) { 1217 return; 1218 } 1219 if (b.length() != 0) { 1220 b.append(" or "); 1221 } 1222 b.append(key + (value1 == null ? " - missing" : " → “" + value1 + "”")); 1223 } 1224 equalsExceptWidth(String value1, String value2)1225 private boolean equalsExceptWidth(String value1, String value2) { 1226 if (value1.equals(value2)) { 1227 return true; 1228 } else if (value2 == null) { 1229 return false; 1230 } 1231 1232 List<Object> items1 = new ArrayList<>(formatParser.set(value1).getItems()); // clone 1233 List<Object> items2 = formatParser.set(value2).getItems(); 1234 if (items1.size() != items2.size()) { 1235 return false; 1236 } 1237 Iterator<Object> it2 = items2.iterator(); 1238 for (Object item1 : items1) { 1239 Object item2 = it2.next(); 1240 if (item1.equals(item2)) { 1241 continue; 1242 } 1243 if (item1 instanceof VariableField && item2 instanceof VariableField) { 1244 // simple test for now, ignore widths 1245 if (item1.toString().charAt(0) == item2.toString().charAt(0)) { 1246 continue; 1247 } 1248 } 1249 return false; 1250 } 1251 return true; 1252 } 1253 1254 static final Set<String> YgLanguages = 1255 new HashSet<>( 1256 Arrays.asList( 1257 "ar", "cs", "da", "de", "en", "es", "fa", "fi", "fr", "he", "hr", "id", 1258 "it", "nl", "no", "pt", "ru", "sv", "tr")); 1259 getExpectedGy(String localeID)1260 private GyState getExpectedGy(String localeID) { 1261 // hack for now 1262 int firstBar = localeID.indexOf('_'); 1263 String lang = firstBar < 0 ? localeID : localeID.substring(0, firstBar); 1264 return YgLanguages.contains(lang) ? GyState.YEAR_ERA : GyState.ERA_YEAR; 1265 } 1266 1267 enum GyState { 1268 YEAR_ERA, 1269 ERA_YEAR, 1270 OTHER; 1271 static DateTimePatternGenerator.FormatParser formatParser = 1272 new DateTimePatternGenerator.FormatParser(); 1273 1274 static synchronized GyState forPattern(String value) { 1275 formatParser.set(value); 1276 int last = -1; 1277 for (Object x : formatParser.getItems()) { 1278 if (x instanceof VariableField) { 1279 int type = ((VariableField) x).getType(); 1280 if (type == DateTimePatternGenerator.ERA 1281 && last == DateTimePatternGenerator.YEAR) { 1282 return GyState.YEAR_ERA; 1283 } else if (type == DateTimePatternGenerator.YEAR 1284 && last == DateTimePatternGenerator.ERA) { 1285 return GyState.ERA_YEAR; 1286 } 1287 last = type; 1288 } 1289 } 1290 return GyState.OTHER; 1291 } 1292 } 1293 1294 enum DateTimeLengths { 1295 SHORT, 1296 MEDIUM, 1297 LONG, 1298 FULL 1299 } 1300 1301 // The patterns below should only use the *canonical* characters for each field type: 1302 // y (not Y, u, U) 1303 // Q (not q) 1304 // M (not L) 1305 // E (not e, c) 1306 // a (not b, B) 1307 // H or h (not k or K) 1308 // v (not z, Z, V) 1309 static final Pattern[] dateTimePatterns = { 1310 PatternCache.get("a*(h|hh|H|HH)(m|mm)"), // time-short 1311 PatternCache.get("a*(h|hh|H|HH)(m|mm)(s|ss)"), // time-medium 1312 PatternCache.get("a*(h|hh|H|HH)(m|mm)(s|ss)(v+)"), // time-long 1313 PatternCache.get("a*(h|hh|H|HH)(m|mm)(s|ss)(v+)"), // time-full 1314 PatternCache.get("G*y{1,4}M{1,2}(d|dd)"), // date-short; allow yyy for Minguo/ROC calendar 1315 PatternCache.get("G*y(yyy)?M{1,3}(d|dd)"), // date-medium 1316 PatternCache.get("G*y(yyy)?M{1,4}(d|dd)"), // date-long 1317 PatternCache.get("G*y(yyy)?M{1,4}E*(d|dd)"), // date-full 1318 PatternCache.get(".*"), // datetime-short 1319 PatternCache.get(".*"), // datetime-medium 1320 PatternCache.get(".*"), // datetime-long 1321 PatternCache.get(".*"), // datetime-full 1322 }; 1323 1324 static final String[] dateTimeMessage = { 1325 "hours (H, HH, h, or hh), and minutes (m or mm)", // time-short 1326 "hours (H, HH, h, or hh), minutes (m or mm), and seconds (s or ss)", // time-medium 1327 "hours (H, HH, h, or hh), minutes (m or mm), and seconds (s or ss); optionally timezone (z, zzzz, v, vvvv)", // time-long 1328 "hours (H, HH, h, or hh), minutes (m or mm), seconds (s or ss), and timezone (z, zzzz, v, vvvv)", // time-full 1329 "year (y, yy, yyyy), month (M or MM), and day (d or dd); optionally era (G)", // date-short 1330 "year (y), month (M, MM, or MMM), and day (d or dd); optionally era (G)", // date-medium 1331 "year (y), month (M, ... MMMM), and day (d or dd); optionally era (G)", // date-long 1332 "year (y), month (M, ... MMMM), and day (d or dd); optionally day of week (EEEE or cccc) or era (G)", // date-full 1333 }; 1334 1335 public String toString(DateTimePatternGenerator.FormatParser formatParser) { 1336 StringBuffer result = new StringBuffer(); 1337 for (Object x : formatParser.getItems()) { 1338 if (x instanceof DateTimePatternGenerator.VariableField) { 1339 result.append(x.toString()); 1340 } else { 1341 result.append(formatParser.quoteLiteral(x.toString())); 1342 } 1343 } 1344 return result.toString(); 1345 } 1346 1347 private void checkPattern2(String path, String value, List<CheckStatus> result) 1348 throws ParseException { 1349 XPathParts pathParts = XPathParts.getFrozenInstance(path); 1350 String calendar = pathParts.findAttributeValue("calendar", "type"); 1351 SimpleDateFormat x = icuServiceBuilder.getDateFormat(calendar, value); 1352 x.setTimeZone(ExampleGenerator.ZONE_SAMPLE); 1353 result.add( 1354 new MyCheckStatus().setFormat(x).setCause(this).setMainType(CheckStatus.demoType)); 1355 } 1356 1357 private DateTimePatternGenerator getDTPGForCalendarType(String calendarType) { 1358 DateTimePatternGenerator dtpg = dtpgForType.get(calendarType); 1359 if (dtpg == null) { 1360 dtpg = flexInfo.getDTPGForCalendarType(calendarType, parentCLDRFiles); 1361 dtpgForType.put(calendarType, dtpg); 1362 } 1363 return dtpg; 1364 } 1365 1366 static final UnicodeSet XGRAPHEME = 1367 new UnicodeSet("[[:mark:][:grapheme_extend:][:punctuation:]]"); 1368 static final UnicodeSet DIGIT = new UnicodeSet("[:decimal_number:]"); 1369 1370 public static class MyCheckStatus extends CheckStatus { 1371 private SimpleDateFormat df; 1372 1373 public MyCheckStatus setFormat(SimpleDateFormat df) { 1374 this.df = df; 1375 return this; 1376 } 1377 1378 @Override 1379 public SimpleDemo getDemo() { 1380 return new MyDemo().setFormat(df); 1381 } 1382 } 1383 1384 static class MyDemo extends FormatDemo { 1385 private SimpleDateFormat df; 1386 1387 @Override 1388 protected String getPattern() { 1389 return df.toPattern(); 1390 } 1391 1392 @Override 1393 protected String getSampleInput() { 1394 return neutralFormat.format(ExampleGenerator.DATE_SAMPLE); 1395 } 1396 1397 public MyDemo setFormat(SimpleDateFormat df) { 1398 this.df = df; 1399 return this; 1400 } 1401 1402 @Override 1403 protected void getArguments(Map<String, String> inout) { 1404 currentPattern = currentInput = currentFormatted = currentReparsed = "?"; 1405 Date d; 1406 try { 1407 currentPattern = inout.get("pattern"); 1408 if (currentPattern != null) df.applyPattern(currentPattern); 1409 else currentPattern = getPattern(); 1410 } catch (Exception e) { 1411 currentPattern = "Use format like: ##,###.##"; 1412 return; 1413 } 1414 try { 1415 currentInput = inout.get("input"); 1416 if (currentInput == null) { 1417 currentInput = getSampleInput(); 1418 } 1419 d = neutralFormat.parse(currentInput); 1420 } catch (Exception e) { 1421 currentInput = "Use neutral format like: 1993-11-31 13:49:02"; 1422 return; 1423 } 1424 try { 1425 currentFormatted = df.format(d); 1426 } catch (Exception e) { 1427 currentFormatted = "Can't format: " + e.getMessage(); 1428 return; 1429 } 1430 try { 1431 parsePosition.setIndex(0); 1432 Date n = df.parse(currentFormatted, parsePosition); 1433 if (parsePosition.getIndex() != currentFormatted.length()) { 1434 currentReparsed = 1435 "Couldn't parse past: " 1436 + "\u200E" 1437 + currentFormatted.substring(0, parsePosition.getIndex()) 1438 + "\u200E"; 1439 } else { 1440 currentReparsed = neutralFormat.format(n); 1441 } 1442 } catch (Exception e) { 1443 currentReparsed = "Can't parse: " + e.getMessage(); 1444 } 1445 } 1446 } 1447 } 1448