1 package org.unicode.cldr.unittest; 2 3 import com.google.common.base.CharMatcher; 4 import com.google.common.base.Joiner; 5 import com.google.common.base.Splitter; 6 import com.google.common.collect.ImmutableSet; 7 import com.google.common.collect.ImmutableSortedSet; 8 import com.google.common.collect.Multimap; 9 import com.google.common.collect.TreeMultimap; 10 import com.ibm.icu.dev.util.UnicodeMap; 11 import com.ibm.icu.impl.Row; 12 import com.ibm.icu.impl.Row.R3; 13 import com.ibm.icu.impl.Row.R4; 14 import com.ibm.icu.impl.Utility; 15 import com.ibm.icu.text.Collator; 16 import com.ibm.icu.text.UnicodeSet; 17 import java.io.File; 18 import java.util.Arrays; 19 import java.util.Collection; 20 import java.util.Collections; 21 import java.util.HashMap; 22 import java.util.HashSet; 23 import java.util.Iterator; 24 import java.util.LinkedHashSet; 25 import java.util.Locale; 26 import java.util.Map; 27 import java.util.Map.Entry; 28 import java.util.Set; 29 import java.util.TreeMap; 30 import java.util.TreeSet; 31 import java.util.regex.Pattern; 32 import org.unicode.cldr.test.CoverageLevel2; 33 import org.unicode.cldr.util.Annotations; 34 import org.unicode.cldr.util.Annotations.AnnotationSet; 35 import org.unicode.cldr.util.CLDRConfig; 36 import org.unicode.cldr.util.CLDRFile; 37 import org.unicode.cldr.util.CLDRPaths; 38 import org.unicode.cldr.util.CldrUtility; 39 import org.unicode.cldr.util.Emoji; 40 import org.unicode.cldr.util.Factory; 41 import org.unicode.cldr.util.Level; 42 import org.unicode.cldr.util.Pair; 43 import org.unicode.cldr.util.PathHeader; 44 import org.unicode.cldr.util.PathHeader.PageId; 45 import org.unicode.cldr.util.SimpleFactory; 46 import org.unicode.cldr.util.SupplementalDataInfo; 47 import org.unicode.cldr.util.XListFormatter; 48 import org.unicode.cldr.util.XListFormatter.ListTypeLength; 49 import org.unicode.cldr.util.XPathParts; 50 51 public class TestAnnotations extends TestFmwkPlus { 52 private static final String APPS_EMOJI_DIRECTORY = 53 CLDRPaths.BASE_DIRECTORY + "/tools/cldr-apps/src/main/webapp/images/emoji"; 54 private static final boolean DEBUG = false; 55 main(String[] args)56 public static void main(String[] args) { 57 new TestAnnotations().run(args); 58 } 59 60 enum Containment { 61 contains, 62 empty, 63 not_contains 64 } 65 TestBasic()66 public void TestBasic() { 67 String[][] tests = { 68 {"en", "[\u2650]", "contains", "sagitarius", "zodiac"}, 69 {"en", "[\u0020]", "empty"}, 70 {"en", "[\u2651]", "not_contains", "foobar"}, 71 }; 72 for (String[] test : tests) { 73 UnicodeMap<Annotations> data = Annotations.getData(test[0]); 74 UnicodeSet us = new UnicodeSet(test[1]); 75 Set<String> annotations = new LinkedHashSet<>(); 76 Containment contains = Containment.valueOf(test[2]); 77 for (int i = 3; i < test.length; ++i) { 78 annotations.add(test[i]); 79 } 80 for (String s : us) { 81 Set<String> set = data.get(s).getKeywords(); 82 if (set == null) { 83 set = Collections.emptySet(); 84 } 85 switch (contains) { 86 case contains: 87 if (Collections.disjoint(set, annotations)) { 88 LinkedHashSet<String> temp = new LinkedHashSet<>(annotations); 89 temp.removeAll(set); 90 assertEquals("Missing items", Collections.EMPTY_SET, temp); 91 } 92 break; 93 case not_contains: 94 if (!Collections.disjoint(set, annotations)) { 95 LinkedHashSet<String> temp = new LinkedHashSet<>(annotations); 96 temp.retainAll(set); 97 assertEquals("Extra items", Collections.EMPTY_SET, temp); 98 } 99 break; 100 case empty: 101 assertEquals("mismatch", Collections.emptySet(), set); 102 break; 103 } 104 } 105 } 106 } 107 108 final AnnotationSet eng = Annotations.getDataSet("en"); 109 TestNames()110 public void TestNames() { 111 String[][] tests = { // the expected value for keywords can use , as well as |. 112 {"����", "man: light skin tone", "adult | man | light skin tone"}, 113 {"��♂️", "man: blond hair", "blond, blond-haired man, hair, man, man: blond hair"}, 114 { 115 "����♂️", 116 "man: light skin tone, blond hair", 117 "blond, blond-haired man, hair, man, man: blond hair, light skin tone, blond hair" 118 }, 119 {"����", "man: red hair", "adult | man | red hair"}, 120 { 121 "������", 122 "man: light skin tone, red hair", 123 "adult | man | light skin tone| red hair" 124 }, 125 {"����", "flag: European Union", "flag"}, 126 {"#️⃣", "keycap: #", "keycap"}, 127 {"9️⃣", "keycap: 9", "keycap"}, 128 {"��", "kiss", "couple | kiss"}, 129 {"��❤️����", "kiss: woman, woman", "couple | kiss | woman"}, 130 {"��", "couple with heart", "couple | couple with heart | love"}, 131 { 132 "��❤️��", 133 "couple with heart: woman, woman", 134 "couple | couple with heart | love | woman" 135 }, 136 {"��", "family", "family"}, 137 {"������", "family: woman, woman, girl", "family | woman | girl"}, 138 {"����", "boy: light skin tone", "boy | young | light skin tone"}, 139 {"����", "woman: dark skin tone", "adult | woman | dark skin tone"}, 140 {"��⚖", "man judge", "judge | justice | law | man | scales"}, 141 { 142 "����⚖", 143 "man judge: dark skin tone", 144 "judge | justice | law | man | scales | dark skin tone" 145 }, 146 {"��⚖", "woman judge", "judge | justice | law | scales | woman"}, 147 { 148 "����⚖", 149 "woman judge: medium-light skin tone", 150 "judge | justice | law | scales | woman | medium-light skin tone" 151 }, 152 {"��", "police officer", "cop | officer | police"}, 153 {"����", "police officer: dark skin tone", "cop | officer | police | dark skin tone"}, 154 {"��♂️", "man police officer", "cop | man | officer | police"}, 155 { 156 "����♂️", 157 "man police officer: medium-light skin tone", 158 "cop | man | officer | police | medium-light skin tone" 159 }, 160 {"��♀️", "woman police officer", "cop | officer | police | woman"}, 161 { 162 "����♀️", 163 "woman police officer: dark skin tone", 164 "cop | officer | police | woman | dark skin tone" 165 }, 166 {"��", "person biking", "bicycle | biking | cyclist | person biking"}, 167 { 168 "����", 169 "person biking: dark skin tone", 170 "bicycle | biking | cyclist | person biking | dark skin tone" 171 }, 172 {"��♂️", "man biking", "bicycle | biking | cyclist | man"}, 173 { 174 "����♂️", 175 "man biking: dark skin tone", 176 "bicycle | biking | cyclist | man | dark skin tone" 177 }, 178 {"��♀️", "woman biking", "bicycle | biking | cyclist | woman"}, 179 { 180 "����♀️", 181 "woman biking: dark skin tone", 182 "bicycle | biking | cyclist | woman | dark skin tone" 183 }, 184 }; 185 186 Splitter BAR = Splitter.on(CharMatcher.anyOf("|,")).trimResults(); 187 boolean ok = true; 188 for (String[] test : tests) { 189 String emoji = test[0]; 190 String expectedName = test[1]; 191 Set<String> expectedKeywords = new HashSet<>(BAR.splitToList(test[2])); 192 final String shortName = eng.getShortName(emoji); 193 final Set<String> keywords = eng.getKeywords(emoji); 194 ok &= assertEquals("short name for " + emoji, expectedName, shortName); 195 ok &= assertEquals("keywords for " + emoji, expectedKeywords, keywords); 196 } 197 if (!ok) { 198 System.out.println("Possible replacement, but check"); 199 for (String[] test : tests) { 200 String emoji = test[0]; 201 final String shortName = eng.getShortName(emoji); 202 final Set<String> keywords = eng.getKeywords(emoji); 203 System.out.println( 204 "{\"" 205 + emoji 206 + "\",\"" 207 + shortName 208 + "\",\"" 209 + Joiner.on(" | ").join(keywords) 210 + "\"},"); 211 } 212 } 213 } 214 215 static final UnicodeSet symbols = 216 new UnicodeSet(Emoji.EXTRA_SYMBOL_MINOR_CATEGORIES.keySet()).freeze(); 217 /** The English name should line up with the emoji-test.txt file */ TestNamesVsEmojiData()218 public void TestNamesVsEmojiData() { 219 for (Entry<String, Annotations> s : eng.getExplicitValues().entrySet()) { 220 String emoji = s.getKey(); 221 Annotations annotations = s.getValue(); 222 String name = Emoji.getName(emoji); 223 String annotationName = annotations.getShortName(); 224 if (!symbols.contains(emoji) && !emoji.contains("��")) { 225 assertEquals(emoji + " (en.xml vs. emoji-test.txt)", name, annotationName); 226 } 227 } 228 } 229 TestCategories()230 public void TestCategories() { 231 if (DEBUG) System.out.println(); 232 233 TreeSet<R4<PageId, Long, String, R3<String, String, String>>> sorted = new TreeSet<>(); 234 for (Entry<String, Annotations> s : eng.getExplicitValues().entrySet()) { 235 String emoji = s.getKey(); 236 Annotations annotations = s.getValue(); 237 final String rawCategory = Emoji.getMajorCategory(emoji); 238 PageId majorCategory = PageId.forString(rawCategory); 239 if (majorCategory == PageId.Symbols) { 240 majorCategory = PageId.EmojiSymbols; 241 } 242 String minorCategory = Emoji.getMinorCategory(emoji); 243 long emojiOrder = Emoji.getEmojiToOrder(emoji); 244 R3<String, String, String> row2 = 245 Row.of( 246 emoji, 247 annotations.getShortName(), 248 Joiner.on(" | ").join(annotations.getKeywords())); 249 R4<PageId, Long, String, R3<String, String, String>> row = 250 Row.of(majorCategory, emojiOrder, minorCategory, row2); 251 sorted.add(row); 252 } 253 for (R4<PageId, Long, String, R3<String, String, String>> row : sorted) { 254 PageId majorCategory = row.get0(); 255 Long emojiOrder = row.get1(); 256 String minorCategory = row.get2(); 257 R3<String, String, String> row2 = row.get3(); 258 String emoji = row2.get0(); 259 String shortName = row2.get1(); 260 String keywords = row2.get2(); 261 if (DEBUG) 262 System.out.println( 263 majorCategory 264 + "\t" 265 + emojiOrder 266 + "\t" 267 + minorCategory 268 + "\t" 269 + emoji 270 + "\t" 271 + shortName 272 + "\t" 273 + keywords); 274 } 275 } 276 TestUniqueness()277 public void TestUniqueness() { 278 if (logKnownIssue( 279 "CLDR-16947", "skip duplicate TestUniqueness in favor of CheckDisplayCollisions")) { 280 return; 281 } 282 Set<String> locales = new TreeSet<>(); 283 locales.add("en"); 284 locales.addAll(Annotations.getAvailable()); 285 locales.remove("root"); 286 /* 287 * Note: "problems" here is a work-around for what appears to be a deficiency 288 * in the function sourceLocation, involving the call stack. Seemingly sourceLocation 289 * can't handle the "->" notation used for parallelStream().forEach() if 290 * uniquePerLocale calls errln directly. 291 */ 292 Set<String> problems = new HashSet<>(); 293 locales.parallelStream().forEach(locale -> uniquePerLocale(locale, problems)); 294 if (!problems.isEmpty()) { 295 problems.forEach(s -> errln(s)); 296 } 297 } 298 uniquePerLocale(String locale, Set<String> problems)299 private void uniquePerLocale(String locale, Set<String> problems) { 300 logln("uniqueness: " + locale); 301 Multimap<String, String> nameToEmoji = TreeMultimap.create(); 302 AnnotationSet data = Annotations.getDataSet(locale); 303 for (String emoji : Emoji.getAllRgi()) { 304 String name = data.getShortName(emoji); 305 if (name == null) { 306 continue; 307 } 308 if (name.contains(CldrUtility.INHERITANCE_MARKER)) { 309 throw new IllegalArgumentException( 310 CldrUtility.INHERITANCE_MARKER + " in name of " + emoji + " in " + locale); 311 } 312 nameToEmoji.put(name, emoji); 313 } 314 Multimap<String, String> duplicateNameToEmoji = null; 315 for (Entry<String, Collection<String>> entry : nameToEmoji.asMap().entrySet()) { 316 String name = entry.getKey(); 317 Collection<String> emojis = entry.getValue(); 318 if (emojis.size() > 1) { 319 synchronized (problems) { 320 if (problems.add( 321 "Duplicate name in " 322 + locale 323 + ": “" 324 + name 325 + "” for " 326 + Joiner.on(" & ").join(emojis))) { 327 int debug = 0; 328 } 329 } 330 if (duplicateNameToEmoji == null) { 331 duplicateNameToEmoji = TreeMultimap.create(); 332 } 333 duplicateNameToEmoji.putAll(name, emojis); 334 } 335 } 336 if (isVerbose() && duplicateNameToEmoji != null && !duplicateNameToEmoji.isEmpty()) { 337 System.out.println("\nCollisions"); 338 for (Entry<String, String> entry : duplicateNameToEmoji.entries()) { 339 String emoji = entry.getValue(); 340 System.out.println(locale + "\t" + eng.getShortName(emoji) + "\t" + emoji); 341 } 342 } 343 } 344 testAnnotationPaths()345 public void testAnnotationPaths() { 346 assertTrue("", Emoji.getNonConstructed().contains("®")); 347 Factory factoryAnnotations = SimpleFactory.make(CLDRPaths.ANNOTATIONS_DIRECTORY, ".*"); 348 for (String locale : Arrays.asList("en", "root")) { 349 CLDRFile enAnnotations = factoryAnnotations.make(locale, false); 350 // //ldml/annotations/annotation[@cp="��"][@type="tts"] 351 Set<String> annotationPaths = 352 enAnnotations.getPaths( 353 "//ldml/anno", 354 Pattern.compile("//ldml/annotations/annotation.*tts.*").matcher(""), 355 new TreeSet<>()); 356 Set<String> annotationPathsExpected = Emoji.getNamePaths(); 357 if (!checkAMinusBIsC( 358 "(" + locale + ".xml - Emoji.getNamePaths)", 359 annotationPaths, 360 annotationPathsExpected, 361 Collections.<String>emptySet())) { 362 System.out.println("Check Emoji.SPECIALS"); 363 } 364 checkAMinusBIsC( 365 "(Emoji.getNamePaths - " + locale + ".xml)", 366 annotationPathsExpected, 367 annotationPaths, 368 Collections.<String>emptySet()); 369 } 370 } 371 testEmojiImages()372 public void testEmojiImages() { 373 if (CLDRPaths.ANNOTATIONS_DIRECTORY.contains("cldr-staging/production/")) { 374 return; // don't bother checking production for this: the images are only in main, not 375 // production 376 } 377 Factory factoryAnnotations = SimpleFactory.make(CLDRPaths.ANNOTATIONS_DIRECTORY, ".*"); 378 CLDRFile enAnnotations = factoryAnnotations.make("en", false); 379 380 String emojiImageDir = APPS_EMOJI_DIRECTORY; 381 for (String emoji : Emoji.getNonConstructed()) { 382 String noVs = emoji.replace(Emoji.EMOJI_VARIANT, ""); 383 384 // example: emoji_1f1e7_1f1ec.png 385 String fileName = 386 "emoji_" + Utility.hex(noVs, 4, "_").toLowerCase(Locale.ENGLISH) + ".png"; 387 File file = new File(emojiImageDir, fileName); 388 389 if (!file.exists() && !fileName.endsWith("_200d_27a1.png")) { 390 String name = 391 enAnnotations.getStringValue( 392 "//ldml/annotations/annotation[@cp=\"" 393 + noVs 394 + "\"][@type=\"tts\"]"); 395 errln(fileName + " missing; " + name); 396 } 397 } 398 } 399 400 /** Check that the order info, categories, and collation are consistent. */ testEmojiOrdering()401 public void testEmojiOrdering() { 402 // load an array for sorting 403 // and test that every order value maps to exactly one emoji 404 Map<String, String> minorToMajor = new HashMap<>(); 405 Map<Long, String> orderToEmoji = new TreeMap<>(); 406 Collator col = CLDRConfig.getInstance().getCollatorRoot(); 407 408 for (String emoji : Emoji.getNonConstructed()) { 409 Long emojiOrder = Emoji.getEmojiToOrder(emoji); 410 if (DEBUG) { 411 String minor = Emoji.getMinorCategory(emoji); 412 System.out.println(emojiOrder + "\t" + emoji + "\t" + minor); 413 } 414 String oldEmoji = orderToEmoji.get(emojiOrder); 415 if (oldEmoji == null) { 416 orderToEmoji.put(emojiOrder, emoji); 417 } else { 418 errln("single order value with different emoji" + emoji + " ≠ " + oldEmoji); 419 } 420 } 421 Set<String> majorsSoFar = new TreeSet<>(); 422 String lastMajor = ""; 423 Set<String> minorsSoFar = new TreeSet<>(); 424 String lastMinor = ""; 425 Set<String> lastMajorGroup = new LinkedHashSet<>(); 426 Set<String> lastMinorGroup = new LinkedHashSet<>(); 427 String lastEmoji = ""; 428 long lastEmojiOrdering = -1L; 429 for (Entry<Long, String> entry : orderToEmoji.entrySet()) { 430 String emoji = entry.getValue(); 431 Long emojiOrdering = entry.getKey(); 432 // check against collation 433 if (col.compare(emoji, lastEmoji) <= 0) { 434 String name = eng.getShortName(emoji); 435 String lastName = eng.getShortName(lastEmoji); 436 int errorType = ERR; 437 if (logKnownIssue("CLDR-16394", "slightly out of order")) { 438 errorType = WARN; 439 } 440 msg( 441 "Out of order: " 442 + lastEmoji 443 + " (" 444 + lastEmojiOrdering 445 + ") " 446 + lastName 447 + " > " 448 + emoji 449 + " (" 450 + emojiOrdering 451 + ") " 452 + name, 453 errorType, 454 true, 455 true); 456 } 457 458 String major = Emoji.getMajorCategory(emoji); 459 String minor = Emoji.getMinorCategory(emoji); 460 if (isVerbose()) { 461 System.out.println(major + "\t" + minor + "\t" + emoji); 462 } 463 String oldMajor = minorToMajor.get(minor); 464 // never get major1:minor1 and major2:minor1 465 if (oldMajor == null) { 466 minorToMajor.put(minor, major); 467 } else { 468 assertEquals( 469 minor + " maps to different majors for " + Utility.hex(emoji), 470 oldMajor, 471 major); 472 } 473 // never get major1 < major2 < major1 474 if (!major.equals(lastMajor)) { 475 // System.out.println(lastMajor + "\t" + lastMajorGroup); 476 477 // if (majorsSoFar.contains(major)) { 478 // errln("Non-contiguous majors: " + major + " <… " + lastMajor + 479 // " < " + major); 480 // } 481 majorsSoFar.add(major); 482 lastMajor = major; 483 lastMajorGroup.clear(); 484 lastMajorGroup.add(emoji); // add emoji with different cat 485 } else { 486 lastMajorGroup.add(emoji); 487 } 488 // never get minor1 < minor2 < minor1 489 if (!minor.equals(lastMinor)) { 490 if (DEBUG) System.out.println(lastMinor + "\t" + lastMinorGroup); 491 if (minorsSoFar.contains(minor)) { 492 errln("Non-contiguous minors: " + minor + " <… " + lastMinor + " < " + minor); 493 } 494 minorsSoFar.add(minor); 495 lastMinor = minor; 496 lastMinorGroup.clear(); 497 lastMinorGroup.add(emoji); // add emoji with different cat 498 } else { 499 lastMinorGroup.add(emoji); 500 } 501 lastEmoji = emoji; 502 lastEmojiOrdering = emojiOrdering; 503 } 504 if (DEBUG) System.out.println(lastMinor + "\t" + lastMinorGroup); 505 } 506 testSuperfluousAnnotationPaths()507 public void testSuperfluousAnnotationPaths() { 508 if (CLDRPaths.ANNOTATIONS_DIRECTORY.contains("cldr-staging/production/")) { 509 return; // don't bother checking production for this: root is empty 510 } 511 Factory factoryAnnotations = SimpleFactory.make(CLDRPaths.ANNOTATIONS_DIRECTORY, ".*"); 512 ImmutableSet<String> rootPaths = 513 ImmutableSortedSet.copyOf( 514 factoryAnnotations.make("root", false).iterator("//ldml/annotations/")); 515 516 CLDRFile englishAnnotations = factoryAnnotations.make("en", false); 517 ImmutableSet<String> englishPaths = 518 ImmutableSortedSet.copyOf(englishAnnotations.iterator("//ldml/annotations/")); 519 520 Set<String> superfluous2 = setDifference(rootPaths, englishPaths); 521 assertTrue("en contains root", superfluous2.isEmpty()); 522 if (!superfluous2.isEmpty()) { 523 for (String path : superfluous2) { 524 // XPathParts parts = XPathParts.getFrozenInstance(path); 525 // String emoji = parts.getAttributeValue(-1, "cp"); 526 System.out.println("locale=en; action=add; path=" + path + "; value=XXX"); 527 } 528 } 529 530 Set<String> allSuperfluous = new TreeSet<>(); 531 for (String locale : factoryAnnotations.getAvailable()) { 532 ImmutableSet<String> currentPaths = 533 ImmutableSortedSet.copyOf( 534 factoryAnnotations.make(locale, false).iterator("//ldml/annotations/")); 535 Set<String> superfluous = setDifference(currentPaths, rootPaths); 536 if (!assertTrue("root contains " + locale, superfluous.isEmpty())) { 537 int debug = 0; 538 } 539 allSuperfluous.addAll(superfluous); 540 for (String s : currentPaths) { 541 if (s.contains("\uFE0F")) { 542 errln("Contains FE0F: " + s); 543 break; 544 } 545 } 546 } 547 // get items to fix 548 if (!allSuperfluous.isEmpty()) { 549 for (String path : allSuperfluous) { 550 // XPathParts parts = XPathParts.getFrozenInstance(path); 551 // String emoji = parts.getAttributeValue(-1, "cp"); 552 System.out.println("locale=/.*/; action=delete; path=" + path); 553 } 554 } 555 } 556 setDifference(ImmutableSet<String> a, ImmutableSet<String> b)557 private Set<String> setDifference(ImmutableSet<String> a, ImmutableSet<String> b) { 558 Set<String> superfluous = new LinkedHashSet<>(a); 559 superfluous.removeAll(b); 560 return superfluous; 561 } 562 checkAMinusBIsC(String title, Set<String> a, Set<String> b, Set<String> c)563 private boolean checkAMinusBIsC(String title, Set<String> a, Set<String> b, Set<String> c) { 564 Set<String> aMb = new TreeSet<>(a); 565 aMb.removeAll(b); 566 for (Iterator<String> it = aMb.iterator(); it.hasNext(); ) { 567 String item = it.next(); 568 if (symbols.containsSome(item)) { 569 it.remove(); 570 } 571 } 572 return assertEquals(title + " (" + aMb.size() + ")", c, aMb); 573 } 574 testListFormatter()575 public void testListFormatter() { 576 Object[][] tests = { 577 {"en", ListTypeLength.NORMAL, "ABC", "A, B, and C"}, 578 {"en", ListTypeLength.AND_SHORT, "ABC", "A, B, & C"}, 579 {"en", ListTypeLength.AND_NARROW, "ABC", "A, B, C"}, 580 {"en", ListTypeLength.OR_WIDE, "ABC", "A, B, or C"} 581 }; 582 Factory factory = CLDRConfig.getInstance().getCldrFactory(); 583 for (Object[] test : tests) { 584 CLDRFile cldrFile = factory.make((String) (test[0]), true); 585 ListTypeLength listTypeLength = (ListTypeLength) (test[1]); 586 String expected = (String) test[3]; 587 XListFormatter xlistFormatter = new XListFormatter(cldrFile, listTypeLength); 588 String source = (String) test[2]; 589 String actual = xlistFormatter.formatCodePoints(source); 590 assertEquals(test[0] + ", " + listTypeLength + ", " + source, expected, actual); 591 } 592 } 593 testCoverage()594 public void testCoverage() { 595 UnicodeMap<Level> levels = new UnicodeMap<>(); 596 for (String minorCategory : Emoji.getMinorCategoriesWithExtras()) { 597 for (String s : Emoji.getEmojiInMinorCategoriesWithExtras(minorCategory)) { 598 if (s.contentEquals("‾")) { 599 int debug = 0; 600 } 601 CoverageLevel2 coverageLevel = 602 CoverageLevel2.getInstance(SupplementalDataInfo.getInstance(), "en"); 603 final String pathKeyword = "//ldml/annotations/annotation[@cp=\"" + s + "\"]"; 604 final String pathName = pathKeyword + "[@type=\"tts\"]"; 605 Level levelKeyword = coverageLevel.getLevel(pathKeyword); 606 Level levelName = coverageLevel.getLevel(pathName); 607 assertEquals(s, levelName, levelKeyword); 608 levels.put(s, levelName); 609 } 610 } 611 for (Level level : Level.values()) { 612 UnicodeSet us = levels.getSet(level); 613 getLogger().fine(level + "\t" + us.size()); 614 switch (level) { 615 case MODERN: 616 assertNotEquals(level.toString(), 0, us.size()); 617 break; 618 default: 619 assertEquals(level.toString(), 0, us.size()); 620 break; 621 } 622 } 623 } 624 625 static final UnicodeSet allRgiNoES = Emoji.getAllRgiNoES(); 626 static final UnicodeSet punctuation = new UnicodeSet("[:P:]").freeze(); 627 static final UnicodeSet mathSymbols = new UnicodeSet("[:Sm:]").freeze(); 628 static final UnicodeSet otherSymbols = new UnicodeSet("[^[:Sm:][:P:]]").freeze(); 629 testSymbols()630 public void testSymbols() { 631 CLDRFile root = CLDRConfig.getInstance().getAnnotationsFactory().make("root", false); 632 UnicodeMap<String> expectedMap = 633 new UnicodeMap<String>() 634 .putAll(punctuation, "Punctuation") 635 .putAll(mathSymbols, "Math Symbols") 636 .putAll(otherSymbols, "Other Symbols") 637 .freeze(); 638 Set<String> nonEmojiPages = expectedMap.values(); 639 UnicodeMap<Pair<String, String>> failures = new UnicodeMap<>(); 640 PathHeader.Factory phf = PathHeader.getFactory(); 641 for (String path : root) { 642 XPathParts parts = XPathParts.getFrozenInstance(path); 643 String cp = parts.getAttributeValue(-1, "cp"); 644 if (cp == null) { 645 continue; // non-annotation line 646 } 647 PathHeader ph = phf.fromPath(path); 648 PathHeader.SectionId sectionId = ph.getSectionId(); 649 assertEquals("Section for " + cp, PathHeader.SectionId.Characters, sectionId); 650 PageId pageId = ph.getPageId(); 651 final String actual = pageId.toString(); 652 653 // collect all the failures rather than having a long list of errors 654 655 if (allRgiNoES.contains(cp)) { // check emoji 656 if (nonEmojiPages.contains(actual)) { 657 failures.put(cp, Pair.of("«Emoji-Page»", actual)); 658 } else if (actual.equals("Symbols2")) { 659 failures.put(cp, Pair.of("Emoji Symbols", actual)); 660 } 661 } else { 662 String expected = expectedMap.get(cp); 663 if (!actual.equals(expected)) { 664 failures.put(cp, Pair.of(expected, actual)); 665 } 666 } 667 } 668 if (!failures.isEmpty()) { 669 for (Pair<String, String> value : ImmutableSortedSet.copyOf(failures.values())) { 670 UnicodeSet uset = failures.getSet(value); 671 errln( 672 "Mismatch in " 673 + uset.size() 674 + " cases: expected=" 675 + value.getFirst() 676 + " actual=" 677 + value.getSecond() 678 + "\n" 679 + uset.toPattern(false)); 680 } 681 } 682 } 683 } 684