1 package org.unicode.cldr.unittest; 2 /* 3 * TODO: rename this file and class to avoid confusion with org.unicode.cldr.util TestUtilities.java 4 * When Eclipse console shows an error such as 5 * Error: (TestUtilities.java:1154) : 8 value: expected "old-value", got null 6 * the link wrongly opens the wrong file named TestUtilities.java. The two files are: 7 * cldr/tools/cldr-code/src/main/java/org/unicode/cldr/util/TestUtilities.java 8 * cldr/tools/cldr-code/src/test/java/org/unicode/cldr/unittest/TestUtilities.java 9 */ 10 11 import com.google.common.collect.ImmutableMap; 12 import com.google.common.collect.Ordering; 13 import com.ibm.icu.dev.util.UnicodeMap; 14 import com.ibm.icu.impl.Relation; 15 import com.ibm.icu.impl.Utility; 16 import com.ibm.icu.lang.UCharacter; 17 import com.ibm.icu.lang.UProperty; 18 import com.ibm.icu.text.Collator; 19 import com.ibm.icu.text.UnicodeSet; 20 import com.ibm.icu.util.ULocale; 21 import java.io.StringWriter; 22 import java.text.NumberFormat; 23 import java.util.ArrayList; 24 import java.util.Arrays; 25 import java.util.Collections; 26 import java.util.Comparator; 27 import java.util.HashMap; 28 import java.util.HashSet; 29 import java.util.LinkedHashSet; 30 import java.util.List; 31 import java.util.Map; 32 import java.util.Map.Entry; 33 import java.util.Random; 34 import java.util.Set; 35 import java.util.TreeMap; 36 import java.util.TreeSet; 37 import java.util.regex.Matcher; 38 import org.unicode.cldr.test.SubmissionLocales; 39 import org.unicode.cldr.tool.ConvertLanguageData.InverseComparator; 40 import org.unicode.cldr.util.*; 41 import org.unicode.cldr.util.PathHeader.PageId; 42 import org.unicode.cldr.util.SupplementalDataInfo.PluralInfo.Count; 43 import org.unicode.cldr.util.VettingViewer.MissingStatus; 44 import org.unicode.cldr.util.VoteResolver.Level; 45 import org.unicode.cldr.util.VoteResolver.Status; 46 import org.unicode.cldr.util.VoteResolver.VoteStatus; 47 import org.unicode.cldr.util.VoteResolver.VoterInfo; 48 import org.unicode.cldr.util.props.ICUPropertyFactory; 49 50 public class TestUtilities extends TestFmwkPlus { 51 public static boolean DEBUG = true; 52 53 private static final UnicodeSet DIGITS = new UnicodeSet("[0-9]"); 54 static CLDRConfig testInfo = CLDRConfig.getInstance(); 55 private static final SupplementalDataInfo SUPPLEMENTAL_DATA_INFO = 56 testInfo.getSupplementalDataInfo(); 57 private static final int STRING_ID_TEST_COUNT = 1024 * 16; 58 59 final int ONE_VETTER_BAR = Level.vetter.getVotes(Organization.unaffiliated); 60 final int TWO_VETTER_BAR = VoteResolver.LOWER_BAR; 61 main(String[] args)62 public static void main(String[] args) { 63 new TestUtilities().run(args); 64 } 65 TestPluralSamples()66 public void TestPluralSamples() { 67 checkPluralSamples("en"); 68 checkPluralSamples("cs"); 69 checkPluralSamples("ar"); 70 } 71 checkPluralSamples(String locale)72 private void checkPluralSamples(String locale) { 73 PluralSamples pluralSamples = PluralSamples.getInstance(locale); 74 Set<Count> counts = SUPPLEMENTAL_DATA_INFO.getPlurals(locale).getCounts(); 75 for (int i = 1; i < 5; ++i) { 76 Map<Count, Double> samplesForDigits = pluralSamples.getSamples(i); 77 if (!counts.containsAll(samplesForDigits.keySet())) { 78 errln( 79 locale 80 + ": mismatch in samples, expected " 81 + counts 82 + ", got: " 83 + samplesForDigits); 84 } else if (samplesForDigits.size() == 0) { 85 errln(locale + ": no sample for digit " + i); 86 } else { 87 logln(locale + " plural samples: " + samplesForDigits); 88 } 89 } 90 } 91 92 public static class StringIdException extends RuntimeException { 93 private static final long serialVersionUID = 1L; 94 } 95 96 public class StringIdThread extends Thread { 97 private final Random r = new Random(); 98 private final int id; 99 StringIdThread(int i)100 StringIdThread(int i) { 101 super("Demo Thread"); 102 id = i; 103 } 104 105 @Override run()106 public void run() { 107 logln("Starting thread: " + this); 108 for (int i = 0; i < STRING_ID_TEST_COUNT; ++i) { 109 String s = String.valueOf(r.nextInt()); 110 long l = StringId.getId(s); 111 String s2 = StringId.getStringFromId(l); 112 if (!s.equals(s2)) { 113 throw new StringIdException(); 114 } 115 } 116 logln("Ending thread: " + this); 117 } 118 119 @Override toString()120 public String toString() { 121 return "StringIdThread " + id; 122 } 123 } 124 TestStringId()125 public void TestStringId() { 126 ArrayList<StringIdThread> threads = new ArrayList<>(); 127 128 for (int i = 0; i < 8; i++) { 129 StringIdThread thread = new StringIdThread(i); 130 threads.add(thread); 131 thread.start(); 132 } 133 for (StringIdThread thread : threads) { 134 try { 135 thread.join(); 136 } catch (InterruptedException e) { 137 errln(e.toString()); 138 } 139 } 140 } 141 TestUrlEscape()142 public void TestUrlEscape() { 143 Matcher byte1 = PatternCache.get("%[A-Za-z0-9]{2}").matcher(""); 144 Matcher byte2 = PatternCache.get("%[A-Za-z0-9]{2}%[A-Za-z0-9]{2}").matcher(""); 145 Matcher byte3 = 146 PatternCache.get("%[A-Za-z0-9]{2}%[A-Za-z0-9]{2}%[A-Za-z0-9]{2}").matcher(""); 147 Matcher byte4 = 148 PatternCache.get("%[A-Za-z0-9]{2}%[A-Za-z0-9]{2}%[A-Za-z0-9]{2}%[A-Za-z0-9]{2}") 149 .matcher(""); 150 for (int i = 1; i <= 0x10FFFF; i = i * 3 / 2 + 1) { 151 String escaped = 152 EscapingUtilities.urlEscape(new StringBuilder().appendCodePoint(i).toString()); 153 logln(Integer.toHexString(i) + " => " + escaped); 154 if (EscapingUtilities.OK_TO_NOT_QUOTE.contains(i)) { 155 assertTrue("Should be unquoted", escaped.length() == 1); 156 } else if (i < 0x80) { 157 assertTrue("Should be %xx", byte1.reset(escaped).matches()); 158 } else if (i < 0x800) { 159 assertTrue("Should be %xx%xx", byte2.reset(escaped).matches()); 160 } else if (i < 0x10000) { 161 assertTrue("Should be %xx%xx%xx", byte3.reset(escaped).matches()); 162 } else { 163 assertTrue("Should be %xx%xx%xx%xx", byte4.reset(escaped).matches()); 164 } 165 } 166 } 167 TestDelegatingIterator()168 public void TestDelegatingIterator() { 169 Set<String> s = new TreeSet<>(Arrays.asList(new String[] {"a", "b", "c"})); 170 Set<String> t = new LinkedHashSet<>(Arrays.asList(new String[] {"f", "d", "e"})); 171 StringBuilder result = new StringBuilder(); 172 173 for (String u : DelegatingIterator.iterable(s, t)) { 174 result.append(u); 175 } 176 assertEquals("Iterator", "abcfde", result.toString()); 177 178 result.setLength(0); 179 for (String u : DelegatingIterator.array("s", "t", "u")) { 180 result.append(u); 181 } 182 assertEquals("Iterator", "stu", result.toString()); 183 184 int count = 0; 185 result.setLength(0); 186 for (int u : DelegatingIterator.array(1, 3, 5)) { 187 count += u; 188 } 189 assertEquals("Iterator", 9, count); 190 191 result.setLength(0); 192 for (Object u : DelegatingIterator.array(1, "t", "u", new UnicodeSet("[a-z]"))) { 193 result.append(u); 194 } 195 assertEquals("Iterator", "1tu[a-z]", result.toString()); 196 } 197 TestUntimedCounter()198 public void TestUntimedCounter() { 199 // simulates how Counter is used in VettingViewer 200 Counter<NotificationCategory> problemCounter = new Counter<>(); 201 problemCounter.increment(NotificationCategory.error); 202 problemCounter.increment(NotificationCategory.error); 203 problemCounter.increment(NotificationCategory.warning); 204 205 assertEquals("problemCounter error", 2, problemCounter.get(NotificationCategory.error)); 206 assertEquals("problemCounter warning", 1, problemCounter.get(NotificationCategory.warning)); 207 assertEquals("problemCounter weLost", 0, problemCounter.get(NotificationCategory.weLost)); 208 209 Counter<NotificationCategory> otherCounter = new Counter<>(); 210 otherCounter.addAll(problemCounter); 211 otherCounter.increment(NotificationCategory.error); 212 213 assertEquals("otherCounter error", 3, otherCounter.get(NotificationCategory.error)); 214 assertEquals("otherCounter warning", 1, otherCounter.get(NotificationCategory.warning)); 215 assertEquals("otherCounter weLost", 0, otherCounter.get(NotificationCategory.weLost)); 216 } 217 TestCounter()218 public void TestCounter() { 219 Counter<String> counter = new Counter<>(true); 220 Comparator<String> uca = 221 new Comparator<>() { 222 Collator col = Collator.getInstance(ULocale.ENGLISH); 223 224 @Override 225 public int compare(String o1, String o2) { 226 return col.compare(o1, o2); 227 } 228 }; 229 InverseComparator ucaDown = new InverseComparator(uca); 230 231 counter.add("c", 95); 232 counter.add("b", 50); 233 counter.add("b", 101); 234 counter.add("a", 100); 235 counter.add("a", -5); 236 counter.add("d", -3); 237 assertEquals("getCount(b)", counter.getCount("b"), 151); 238 assertEquals("getCount(a)", counter.getCount("a"), 95); 239 assertEquals("getCount(a)", counter.getTotal(), 338); 240 assertEquals("getItemCount", counter.getItemCount(), 4); 241 242 assertEquals("getMap", "{a=95, b=151, c=95, d=-3}", counter.toString()); 243 244 assertEquals( 245 "getKeysetSortedByKey", 246 Arrays.asList("a", "b", "c", "d"), 247 new ArrayList<>(counter.getKeysetSortedByKey())); 248 249 assertEquals( 250 "getKeysetSortedByCount(true, ucaDown)", 251 Arrays.asList("d", "c", "a", "b"), 252 new ArrayList<String>(counter.getKeysetSortedByCount(true, ucaDown))); 253 254 assertEquals( 255 "getKeysetSortedByCount(true, null), value", 256 Arrays.asList("d", "a", "c", "b"), 257 new ArrayList<>(counter.getKeysetSortedByCount(true, uca))); 258 259 assertEquals( 260 "getKeysetSortedByCount(false, ucaDown), descending", 261 Arrays.asList("b", "c", "a", "d"), 262 new ArrayList<String>(counter.getKeysetSortedByCount(false, ucaDown))); 263 264 assertEquals( 265 "getKeysetSortedByCount(false, null), descending, value", 266 Arrays.asList("b", "a", "c", "d"), 267 new ArrayList<>(counter.getKeysetSortedByCount(false, uca))); 268 } 269 TestOrganizationOrder()270 public void TestOrganizationOrder() { 271 Map<String, Organization> stringToOrg = new TreeMap<>(); 272 for (Organization org : Organization.values()) { 273 stringToOrg.put(org.toString(), org); 274 } 275 List<Organization> reordered = new ArrayList<>(stringToOrg.values()); 276 List<Organization> plain = Arrays.asList(Organization.values()); 277 for (int i = 0; i < reordered.size(); ++i) { 278 assertEquals("Items not in alphabetical order", reordered.get(i), plain.get(i)); 279 } 280 } 281 TestOrganizationNames()282 public void TestOrganizationNames() { 283 UnicodeSet uppercase = new UnicodeSet("[:uppercase:]"); 284 for (Organization org : Organization.values()) { 285 if (!uppercase.contains(org.getDisplayName().codePointAt(0))) { 286 errln("Organization name isn't titlecased: " + org + ", " + org.getDisplayName()); 287 } 288 assertEquals( 289 "Organization from enum name", org, Organization.fromString(org.toString())); 290 assertEquals( 291 "Organization from display name", 292 org, 293 Organization.fromString(org.getDisplayName())); 294 } 295 } 296 297 static final boolean SHOW_DETAILS = CldrUtility.getProperty("showdetails", false); 298 private static final CharSequence DEBUG_COMMENT = 299 "set up a case of conflict within organization"; 300 301 static class PathValueInfo { 302 private static Map<Integer, String> voteInfo; 303 private CLDRFile file; 304 PathValueInfo(Factory factory, String locale)305 public PathValueInfo(Factory factory, String locale) { 306 this.file = factory.make(locale, false); 307 } 308 getRealValue(int id)309 public String getRealValue(int id) { 310 return file.getStringValue(getRealPath(id)); 311 } 312 getRealPath(int id)313 public String getRealPath(int id) { 314 return voteInfo.get(id); 315 } 316 } 317 318 /** Test user data. Restructured to be easier to read, more typesafe */ 319 public enum TestUser { 320 unaffiliatedS(801, Organization.unaffiliated, Level.guest), 321 gnomeS(701, Organization.gnome, Level.guest), 322 gnomeV(702, Organization.gnome, Level.vetter), 323 googleV(404, Organization.google, Level.vetter), 324 googleS(411, Organization.google, Level.guest), 325 googleV2(424, Organization.google, Level.vetter), 326 appleV(304, Organization.apple, Level.vetter), 327 adobeE(204, Organization.adobe, Level.manager), 328 adobeV(209, Organization.adobe, Level.vetter), 329 ibmS(101, Organization.ibm, Level.guest), 330 microsoftV(134, Organization.microsoft, Level.vetter), 331 ibmE(114, Organization.ibm, Level.manager), 332 ibmT(129, Organization.ibm, Level.tc), 333 unaffiliatedS2(802, Organization.unaffiliated, Level.guest); 334 335 public static final Map<Integer, VoterInfo> TEST_USERS; 336 public final Integer voterId; 337 public final VoterInfo voterInfo; 338 TestUser(int intVoterId, Organization organization, Level level)339 TestUser(int intVoterId, Organization organization, Level level) { 340 voterId = intVoterId; 341 voterInfo = new VoterInfo(organization, level, name()); 342 } 343 344 static { 345 ImmutableMap.Builder<Integer, VoterInfo> temp = ImmutableMap.builder(); 346 for (TestUser testUser : values()) { temp.put(testUser.voterId, testUser.voterInfo)347 temp.put(testUser.voterId, testUser.voterInfo); 348 } 349 TEST_USERS = temp.build(); 350 } 351 } 352 353 public static final Map<Integer, VoterInfo> testdata = TestUser.TEST_USERS; 354 toVoterId(String s)355 private int toVoterId(String s) { 356 return TestUser.valueOf(s).voterId; 357 } 358 359 /** Public to use from other tests */ getTestVoterInfoList()360 public static VoterInfoList getTestVoterInfoList() { 361 return new VoterInfoList().setVoterToInfo(testdata); 362 } 363 TestTrunkStatus()364 public void TestTrunkStatus() { 365 VoteResolver<String> resolver = new VoteResolver<>(getTestVoterInfoList()); 366 resolver.setLocale(CLDRLocale.getInstance("de"), null); 367 resolver.setBaileyValue("bailey"); 368 resolver.setBaseline("new-item", Status.approved); 369 assertEquals("", "new-item", resolver.getWinningValue()); 370 371 /* 372 * Formerly last-release would win over trunk in a 2nd scenario here, due to 373 * the difference in status. Now last-release plays no role, that test is obsolete. 374 * Reference: https://unicode.org/cldr/trac/ticket/11916 375 */ 376 } 377 TestVoteResolverNgombaTrunkStatus()378 public void TestVoteResolverNgombaTrunkStatus() { 379 VoteResolver<String> resolver = new VoteResolver<>(getTestVoterInfoList()); 380 resolver.setBaileyValue("bailey"); 381 resolver.setLocale(CLDRLocale.getInstance("jgo"), null); 382 final String jgo22trunk = 383 "\uA78C"; // "[a á â ǎ b c d ɛ {ɛ́} {ɛ̂} {ɛ̌} {ɛ̀} {ɛ̄} f ɡ h i í î ǐ j k l m ḿ {m̀} 384 // {m̄} n ń ǹ {n̄} ŋ {ŋ́} {ŋ̀} {ŋ̄} ɔ {ɔ́} {ɔ̂} {ɔ̌} p {pf} s {sh} t {ts} 385 // u ú û ǔ ʉ {ʉ́} {ʉ̂} {ʉ̌} {ʉ̈} v w ẅ y z ꞌ]"; 386 resolver.setBaseline(jgo22trunk, Status.approved); // seed/jgo.xml from 22 387 // trunk 388 logln("SVN: " + jgo22trunk); 389 logln(resolver.toString()); 390 assertEquals("Winning Value", jgo22trunk, resolver.getWinningValue()); 391 } 392 TestVoteStatus()393 public void TestVoteStatus() { 394 VoteResolver<String> resolver = new VoteResolver<>(getTestVoterInfoList()); 395 396 resolver.setLocale(CLDRLocale.getInstance("de"), null); 397 resolver.setBaileyValue("bailey"); 398 resolver.setBaseline("foo", Status.approved); 399 resolver.add("fii", toVoterId("adobeE")); 400 resolver.add("fii", toVoterId("appleV")); 401 VoteStatus voteStatus; 402 voteStatus = resolver.getStatusForOrganization(Organization.google); 403 assertEquals("", VoteResolver.VoteStatus.ok, voteStatus); 404 voteStatus = resolver.getStatusForOrganization(Organization.apple); 405 assertEquals("", VoteResolver.VoteStatus.ok, voteStatus); 406 407 // make non-equal foo 408 String s1 = "foo"; 409 String s2 = new StringBuilder("fo").append("o").toString(); 410 if (s1 == s2) { 411 errln("Test problem"); 412 } 413 resolver.clear(); 414 resolver.setBaileyValue("bailey"); 415 resolver.setBaseline(s1, Status.approved); 416 resolver.add(s2, toVoterId("appleV")); 417 voteStatus = resolver.getStatusForOrganization(Organization.apple); 418 assertEquals("", VoteResolver.VoteStatus.ok, voteStatus); 419 } 420 TestLosingStatus()421 public void TestLosingStatus() { 422 // af 423 // losing? {baseline: {BQ, missing}, trunk: {null, null}, 424 // {orgToVotes: , totals: {}, conflicted: []}, 425 // sameVotes: [BQ], O: null, N: null, totals: {}, winning: {BQ, 426 // missing}} 427 // XPath: //ldml/localeDisplayNames/territories/territory[@type="BQ"] 428 // gcvs.openoffice_org.example.com 429 VoteResolver<String> resolver = new VoteResolver<>(getTestVoterInfoList()); 430 431 resolver.setLocale(CLDRLocale.getInstance("af"), null); 432 resolver.setBaseline("BQ", Status.missing); 433 resolver.setBaileyValue("bailey"); 434 VoteStatus status = resolver.getStatusForOrganization(Organization.openoffice_org); 435 assertEquals("", VoteResolver.VoteStatus.provisionalOrWorse, status); 436 437 // {lastRelease: {{0}: {1}, missing}, trunk: {null, null}, {orgToVotes: 438 // pakistan={{0}: {1}=8}, totals: {{0}: 439 // {1}=8}, conflicted: []}, sameVotes: [{0}: {1}], O: {0}: {1}, N: null, 440 // totals: {{0}: {1}=8}, winning: {{0}: 441 // {1}, approved}} 442 resolver.clear(); 443 resolver.setBaileyValue("bailey"); 444 // resolver.setLastRelease("{0}: {1}", Status.missing); 445 resolver.add("{0}: {1}", toVoterId("adobeE")); 446 status = resolver.getStatusForOrganization(Organization.openoffice_org); 447 assertEquals("", VoteResolver.VoteStatus.ok, status); 448 449 // {lastRelease: {Arabisch, approved}, trunk: {Arabisch, approved}, 450 // {orgToVotes: , totals: {}, conflicted: []}, 451 // sameVotes: [Arabisch], O: null, N: null, totals: {}, winning: 452 // {Arabisch, approved}} 453 resolver.clear(); 454 resolver.setBaileyValue("bailey"); 455 // resolver.setLastRelease("Arabisch", Status.approved); 456 resolver.setBaseline("Arabisch", Status.approved); 457 status = resolver.getStatusForOrganization(Organization.openoffice_org); 458 assertEquals("", VoteResolver.VoteStatus.ok_novotes, status); 459 } 460 TestTotalVotesStatus()461 public void TestTotalVotesStatus() { 462 VoteResolver<String> resolver = new VoteResolver<>(getTestVoterInfoList()); 463 464 Status oldStatus = Status.unconfirmed; 465 466 resolver.setBaileyValue("bailey"); 467 resolver.setLocale(CLDRLocale.getInstance("de"), null); 468 resolver.setBaseline("foo", oldStatus); 469 resolver.add("zebra", toVoterId("googleV")); 470 resolver.add("apple", toVoterId("appleV")); 471 472 // check that alphabetical wins when votes are equal 473 String winner = resolver.getWinningValue(); 474 Status winningStatus = resolver.getWinningStatus(); 475 assertEquals("", "apple", winner); 476 assertEquals("", Status.provisional, winningStatus); 477 478 resolver.clear(); 479 resolver.setBaileyValue("bailey"); 480 resolver.setLocale(CLDRLocale.getInstance("de"), null); 481 resolver.setBaseline("foo", oldStatus); 482 resolver.add("zebra", toVoterId("googleV")); 483 resolver.add("zebra", toVoterId("googleS")); 484 resolver.add("apple", toVoterId("appleV")); 485 486 // check that total votes over alphabetical 487 winner = resolver.getWinningValue(); 488 winningStatus = resolver.getWinningStatus(); 489 assertEquals("", "zebra", winner); 490 assertEquals("", Status.provisional, winningStatus); 491 } 492 TestVoteDowngrade()493 public void TestVoteDowngrade() { 494 VoteResolver<String> resolver = new VoteResolver<>(getTestVoterInfoList()); 495 496 Status oldStatus = Status.unconfirmed; 497 498 resolver.setBaileyValue("bailey"); 499 resolver.setLocale(CLDRLocale.getInstance("mt"), null); 500 resolver.setBaseline("foo", oldStatus); 501 resolver.add("aardvark", toVoterId("adobeE")); 502 resolver.add("zebra", toVoterId("ibmT")); 503 assertEquals("", "zebra", resolver.getWinningValue()); // TC vote of 20 504 // beats 505 // expert's 8 506 assertEquals("", Status.approved, resolver.getWinningStatus()); 507 508 resolver.clear(); 509 resolver.setBaileyValue("bailey"); 510 resolver.setLocale(CLDRLocale.getInstance("mt"), null); 511 resolver.setBaseline("foo", oldStatus); 512 resolver.add("aardvark", toVoterId("adobeE")); 513 resolver.add("zebra", toVoterId("ibmT")); 514 resolver.add("aardvark", toVoterId("ibmE")); 515 assertEquals("", "zebra", resolver.getWinningValue()); // TC vote of 20 516 // beats 517 // manager's 4 518 // and its own 519 // manager's 4 520 assertEquals("", Status.approved, resolver.getWinningStatus()); 521 522 resolver.clear(); 523 resolver.setBaileyValue("bailey"); 524 resolver.setLocale(CLDRLocale.getInstance("mt"), null); 525 resolver.setBaseline("foo", oldStatus); 526 resolver.add("aardvark", toVoterId("adobeE")); 527 resolver.add("zebra", toVoterId("ibmT"), Level.vetter.getVotes(Organization.ibm)); // NOTE: 528 // reduced 529 // votes: 530 // as 531 // vetter. 532 resolver.add("aardvark", toVoterId("ibmE")); 533 assertEquals("", "aardvark", resolver.getWinningValue()); // Now 534 // aardvark 535 // wins - 536 // managers 537 // win out as provisional 538 assertEquals("", Status.provisional, resolver.getWinningStatus()); 539 540 resolver.clear(); 541 resolver.setBaileyValue("bailey"); 542 resolver.setLocale(CLDRLocale.getInstance("mt"), null); 543 resolver.setBaseline("foo", oldStatus); 544 resolver.add("aardvark", toVoterId("adobeE")); 545 resolver.add("zebra", toVoterId("ibmT"), Level.vetter.getVotes(Organization.ibm)); // NOTE: 546 // reduced 547 // votes: 548 // as 549 // vetter. 550 assertEquals("", "aardvark", resolver.getWinningValue()); // Now 551 // aardvark 552 // wins - 553 // managers 554 // win out. 555 assertEquals("", Status.provisional, resolver.getWinningStatus()); 556 } 557 TestResolvedVoteCounts()558 public void TestResolvedVoteCounts() { 559 VoteResolver<String> resolver = new VoteResolver<>(getTestVoterInfoList()); 560 561 Status oldStatus = Status.unconfirmed; 562 563 resolver.setBaileyValue("bailey"); 564 resolver.setLocale(CLDRLocale.getInstance("de"), null); 565 resolver.setBaseline("foo", oldStatus); 566 resolver.add("zebra", toVoterId("googleV")); 567 resolver.add("apple", toVoterId("appleV")); 568 569 // check that alphabetical wins when votes are equal 570 Map<String, Long> counts = resolver.getResolvedVoteCountsIncludingIntraOrgDisputes(); 571 logln(counts.toString()); 572 assertEquals("", "foo", new ArrayList<>(counts.keySet()).get(2)); 573 574 resolver.clear(); 575 resolver.setBaileyValue("bailey"); 576 resolver.setLocale(CLDRLocale.getInstance("de"), null); 577 resolver.setBaseline("foo", Status.approved); 578 resolver.add("zebra", toVoterId("googleV")); 579 resolver.add("apple", toVoterId("appleV")); 580 counts = resolver.getResolvedVoteCountsIncludingIntraOrgDisputes(); 581 logln(counts.toString()); 582 assertEquals("", "foo", new ArrayList<>(counts.keySet()).get(0)); 583 584 resolver.clear(); 585 resolver.setBaileyValue("bailey"); 586 resolver.setLocale(CLDRLocale.getInstance("de"), null); 587 resolver.setBaseline("foo", Status.approved); 588 resolver.add("zebra", toVoterId("googleS")); 589 counts = resolver.getResolvedVoteCountsIncludingIntraOrgDisputes(); 590 logln(counts.toString()); 591 assertEquals("", "foo", new ArrayList<>(counts.keySet()).get(0)); 592 } 593 verifyRequiredVotes( VoteResolver<String> resolver, String locale, String xpath, Status baselineStatus, int required)594 private void verifyRequiredVotes( 595 VoteResolver<String> resolver, 596 String locale, 597 String xpath, 598 Status baselineStatus, 599 int required) { 600 StringBuilder sb = new StringBuilder(); 601 sb.append("Locale: " + locale); 602 resolver.clear(); 603 resolver.setBaileyValue("bailey"); 604 resolver.setBaseline("foo", baselineStatus); 605 PathHeader ph = null; 606 if (xpath != null) { 607 sb.append(" XPath: " + xpath); 608 ph = PathHeader.getFactory(testInfo.getEnglish()).fromPath(xpath); 609 } 610 resolver.setLocale(CLDRLocale.getInstance(locale), ph); 611 if (!assertEquals( 612 locale + " verifyRequiredVotes: " + ph.toString(), 613 required, 614 resolver.getRequiredVotes())) { 615 int debug = 0; 616 } 617 } 618 TestRequiredVotes()619 public void TestRequiredVotes() { 620 VoteResolver<String> resolver = new VoteResolver<>(getTestVoterInfoList()); 621 verifyRequiredVotes( 622 resolver, 623 "mt", 624 "//ldml/localeDisplayNames/languages/language[@type=\"fr_CA\"]", 625 Status.missing, 626 ONE_VETTER_BAR); 627 verifyRequiredVotes( 628 resolver, 629 "fr", 630 "//ldml/localeDisplayNames/languages/language[@type=\"fr_CA\"]", 631 Status.provisional, 632 TWO_VETTER_BAR); 633 verifyRequiredVotes( 634 resolver, 635 "es", 636 "//ldml/numbers/symbols[@numberSystem=\"latn\"]/group", 637 Status.approved, 638 VoteResolver.HIGH_BAR); 639 verifyRequiredVotes( 640 resolver, 641 "es", 642 "//ldml/numbers/symbols[@numberSystem=\"latn\"]/decimal", 643 Status.approved, 644 VoteResolver.HIGH_BAR); 645 verifyRequiredVotes( 646 resolver, 647 "hi", 648 "//ldml/numbers/symbols[@numberSystem=\"deva\"]/decimal", 649 Status.approved, 650 VoteResolver.HIGH_BAR); 651 verifyRequiredVotes( 652 resolver, 653 "hi", 654 "//ldml/numbers/symbols[@numberSystem=\"deva\"]/group", 655 Status.approved, 656 VoteResolver.HIGH_BAR); 657 verifyRequiredVotes( 658 resolver, 659 "ast", 660 "//ldml/numbers/symbols[@numberSystem=\"latn\"]/decimal", 661 Status.approved, 662 ONE_VETTER_BAR); 663 verifyRequiredVotes( 664 resolver, 665 "mt", 666 "//ldml/characters/exemplarCharacters", 667 Status.approved, 668 VoteResolver.HIGH_BAR); 669 verifyRequiredVotes( 670 resolver, 671 "mt", 672 "//ldml/characters/exemplarCharacters", 673 Status.approved, 674 VoteResolver.HIGH_BAR); 675 verifyRequiredVotes( 676 resolver, 677 "mt", 678 "//ldml/characters/exemplarCharacters[@type=\"auxiliary\"]", 679 Status.approved, 680 VoteResolver.HIGH_BAR); 681 verifyRequiredVotes( 682 resolver, 683 "mt", 684 "//ldml/characters/exemplarCharacters[@type=\"numbers\"]", 685 Status.approved, 686 VoteResolver.HIGH_BAR); 687 verifyRequiredVotes( 688 resolver, 689 "mt", 690 "//ldml/characters/exemplarCharacters[@type=\"punctuation\"]", 691 Status.approved, 692 VoteResolver.HIGH_BAR); 693 verifyRequiredVotes( 694 resolver, 695 "mt", 696 "//ldml/characters/exemplarCharacters[@type=\"index\"]", 697 Status.approved, 698 VoteResolver.HIGH_BAR); 699 verifyRequiredVotes( 700 resolver, 701 "es", 702 "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/days/dayContext[@type=\"format\"]/dayWidth[@type=\"wide\"]/day[@type=\"sun\"]", 703 Status.approved, 704 VoteResolver.HIGH_BAR); 705 verifyRequiredVotes( 706 resolver, 707 "ast", 708 "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/days/dayContext[@type=\"format\"]/dayWidth[@type=\"wide\"]/day[@type=\"sun\"]", 709 Status.approved, 710 ONE_VETTER_BAR); 711 verifyRequiredVotes( 712 resolver, 713 "es", 714 "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/months/monthContext[@type=\"format\"]/monthWidth[@type=\"wide\"]/month[@type=\"1\"]", 715 Status.provisional, 716 VoteResolver.LOWER_BAR); 717 verifyRequiredVotes( 718 resolver, 719 "ast", 720 "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/months/monthContext[@type=\"format\"]/monthWidth[@type=\"wide\"]/month[@type=\"1\"]", 721 Status.approved, 722 ONE_VETTER_BAR); 723 } 724 725 /** 726 * In sublocales, for a typical path, the required votes should be 4, except for a few specified 727 * locales. 728 */ TestSublocaleRequiredVotes()729 public void TestSublocaleRequiredVotes() { 730 final Set<String> eightVoteSublocales = 731 new HashSet<>( 732 Arrays.asList( 733 "pt_PT", 734 "zh_Hant", 735 "zh_Hant_HK", 736 "en_AU", 737 "en_GB", 738 "es_MX", 739 "fr_CA", 740 "es_419")); 741 final VoteResolver<String> resolver = new VoteResolver<>(getTestVoterInfoList()); 742 final String path = "//ldml/annotations/annotation[@cp=\"��\"][@type=\"tts\"]"; 743 for (String locale : SubmissionLocales.CLDR_OR_HIGH_LEVEL_LOCALES) { 744 if (locale.contains("_")) { 745 int expectedRequiredVotes = 746 eightVoteSublocales.contains(locale) ? TWO_VETTER_BAR : ONE_VETTER_BAR; 747 verifyRequiredVotes(resolver, locale, path, Status.approved, expectedRequiredVotes); 748 } 749 } 750 } 751 TestVoteResolver()752 public void TestVoteResolver() { 753 // to make it easier to debug failures, the first digit is an org, 754 // second is the individual in that org, and 755 // third is the voting weight. 756 VoteResolver<String> resolver = new VoteResolver<>(getTestVoterInfoList()); 757 String[] tests = { 758 "bailey=BAILEY", 759 "comment=regression case from John Emmons", 760 "locale=wae", 761 "oldValue=2802", 762 "oldStatus=approved", 763 "304=208027", // Apple vetter 764 // expected values 765 "value=208027", 766 "status=approved", 767 "sameVotes=208027", 768 "conflicts=[]", 769 "check", 770 // first test 771 "oldValue=old-value", 772 "oldStatus=provisional", 773 "comment=Check that identical values get the top overall vote, and that org is maxed (eg vetter + guest = vetter)", 774 "404=next", 775 "411=next", 776 "304=best", 777 // expected values 778 "value=next", 779 "sameVotes=next,best", 780 "conflicts=[]", 781 "status=provisional", 782 "check", 783 "comment=now give next a slight edge (5 to 4) with a different organization", 784 "404=next", 785 "304=best", 786 "801=next", 787 // expected values 788 "value=next", 789 "sameVotes=next", 790 "status=approved", 791 "check", 792 "comment=set up a case of conflict within organization", 793 "404=next", 794 "424=best", 795 // expected values 796 "value=best", // alphabetical 797 "sameVotes=best", 798 "conflicts=[google]", 799 "status=approved", 800 "check", 801 "comment=now cross-organizational conflict, also check for max value in same organization (4, 1) => 4 not 5", 802 "404=next", 803 "424=best", 804 "411=best", 805 "304=best", 806 // expected values 807 "conflicts=[google]", 808 "value=best", 809 "sameVotes=best", 810 "status=approved", 811 "check", 812 "comment=now clear winner 8 over 4", 813 "404=next", 814 // "424=best", 815 "411=best", 816 "304=best", 817 "204=primo", 818 "114=primo", 819 // expected values 820 "conflicts=[]", 821 "value=primo", 822 "sameVotes=primo", 823 "status=approved", 824 "check", 825 "comment=now not so clear, throw in a guest value. So it is 8 to 5. (used to be provisional)", 826 "404=next", 827 // "424=best", 828 "411=best", 829 "304=best", 830 "204=primo", 831 "114=primo", 832 "101=best", 833 // expected values 834 "conflicts=[]", 835 "value=primo", 836 "status=approved", 837 "check", 838 "comment=set up vote of 4 in established locale, with old provisional value", 839 "locale=fr", 840 "404=best", 841 "oldStatus=provisional", 842 // expected values 843 "value=best", 844 "sameVotes=best", 845 "status=contributed", 846 "conflicts=[]", 847 "check", 848 "comment=now set up vote of 4 in established locale, but with old contributed value", 849 "oldStatus=contributed", 850 // expected values 851 "value=old-value", 852 "sameVotes=old-value", 853 "status=contributed", 854 "conflicts=[]", 855 "check", 856 "comment=now set up vote of 1 + 1 in established locale, and with old contributed value", 857 "411=best", 858 "101=best", 859 "oldStatus=contributed", 860 // expected values 861 "value=best", 862 "sameVotes=best", 863 "status=contributed", 864 "conflicts=[]", 865 "check", 866 }; 867 String expectedValue = null; 868 String expectedConflicts = null; 869 Status expectedStatus = null; 870 String oldValue = null; 871 Status oldStatus = null; 872 String baileyValue = null; 873 List<String> sameVotes = null; 874 String locale = null; 875 Map<Integer, String> values = new TreeMap<>(); 876 int counter = -1; 877 878 for (String test : tests) { 879 String[] item = test.split("="); 880 String name = item[0]; 881 String value = item.length < 2 ? null : item[1]; 882 if (name.equalsIgnoreCase("comment")) { 883 logln("#\t" + value); 884 // System.out.println("#\t" + value); 885 if (DEBUG_COMMENT != null && value.contains(DEBUG_COMMENT)) { 886 int x = 0; 887 } 888 } else if (name.equalsIgnoreCase("locale")) { 889 locale = value; 890 } else if (name.equalsIgnoreCase("bailey")) { 891 baileyValue = value; 892 } else if (name.equalsIgnoreCase("oldValue")) { 893 oldValue = value; 894 } else if (name.equalsIgnoreCase("oldStatus")) { 895 oldStatus = Status.valueOf(value); 896 } else if (name.equalsIgnoreCase("value")) { 897 expectedValue = value; 898 } else if (name.equalsIgnoreCase("sameVotes")) { 899 sameVotes = 900 value == null ? new ArrayList<>(0) : Arrays.asList(value.split(",\\s*")); 901 } else if (name.equalsIgnoreCase("status")) { 902 expectedStatus = Status.valueOf(value); 903 } else if (name.equalsIgnoreCase("conflicts")) { 904 expectedConflicts = value; 905 } else if (DIGITS.containsAll(name)) { 906 final int voter = Integer.parseInt(name); 907 if (value == null || value.equals("null")) { 908 values.remove(voter); 909 } else { 910 values.put(voter, value); 911 } 912 } else if (name.equalsIgnoreCase("check")) { 913 counter++; 914 // load the resolver 915 resolver.setBaileyValue(baileyValue); 916 resolver.setLocale(CLDRLocale.getInstance(locale), null); 917 resolver.setBaseline(oldValue, oldStatus); 918 for (int voter : values.keySet()) { 919 resolver.add(values.get(voter), voter); 920 } 921 // print the contents 922 logln(counter + "\t" + values); 923 logln(resolver.toString()); 924 // now print the values 925 assertEquals(counter + " value", expectedValue, resolver.getWinningValue()); 926 assertEquals( 927 counter + " sameVotes", 928 sameVotes.toString(), 929 resolver.getValuesWithSameVotes().toString()); 930 assertEquals(counter + " status", expectedStatus, resolver.getWinningStatus()); 931 assertEquals( 932 counter + " conflicts", 933 expectedConflicts, 934 resolver.getConflictedOrganizations().toString()); 935 resolver.clear(); 936 resolver.setBaileyValue("bailey"); 937 values.clear(); 938 } else { 939 errln("unknown command:\t" + test); 940 } 941 } 942 } 943 944 void assertSpecialLocale(String loc, SpecialLocales.Type type) { 945 assertEquals( 946 "SpecialLocales type for " + loc, 947 type, 948 SpecialLocales.getType(CLDRLocale.getInstance(loc))); 949 } 950 951 public void TestSpecialLocales() { 952 assertSpecialLocale("sr", null); 953 assertSpecialLocale("ha_NE", SpecialLocales.Type.algorithmic); 954 assertSpecialLocale("sr_Latn", SpecialLocales.Type.algorithmic); 955 assertSpecialLocale("sr_Latn_BA", SpecialLocales.Type.algorithmic); 956 assertSpecialLocale("yue_Hans", null); // not readonly, because it is not policy DISCARD 957 assertSpecialLocale("en", SpecialLocales.Type.readonly); 958 assertSpecialLocale("en_ZZ_PROGRAMMERESE", null); // not defined 959 assertSpecialLocale(LocaleNames.UND, null); 960 assertSpecialLocale(LocaleNames.MUL, SpecialLocales.Type.scratch); 961 assertSpecialLocale("mul_ZZ", SpecialLocales.Type.scratch); 962 assertSpecialLocale("und_001", null); // not defined 963 964 CLDRLocale sr_Latn = CLDRLocale.getInstance("sr_Latn"); 965 CLDRLocale sr_Latn_BA = CLDRLocale.getInstance("sr_Latn_BA"); 966 logln("sr_Latn raw comment = " + SpecialLocales.getCommentRaw(sr_Latn)); 967 assertTrue( 968 "sr_Latn raw contains @ sign", SpecialLocales.getCommentRaw(sr_Latn).contains("@")); 969 970 logln("sr_Latn comment = " + SpecialLocales.getComment(sr_Latn)); 971 assertTrue( 972 "sr_Latn comment does NOT contain @ sign", 973 !SpecialLocales.getComment(sr_Latn).contains("@")); 974 logln("sr_Latn_BA raw comment = " + SpecialLocales.getCommentRaw(sr_Latn_BA)); 975 assertTrue( 976 "sr_Latn_BA raw contains '@sr_Latn_BA'", 977 SpecialLocales.getCommentRaw(sr_Latn_BA).contains("@sr_Latn_BA")); 978 } 979 980 public void TestCLDRURLS() { 981 final String KOREAN_LANGUAGE = "//ldml/localeDisplayNames/languages/language[@type=\"ko\"]"; 982 final String KOREAN_LANGUAGE_STRID = "821c2a2fc5c206d"; 983 final CLDRLocale maltese = CLDRLocale.getInstance("mt"); 984 assertEquals( 985 "base", "https://st.unicode.org/cldr-apps", CLDRConfig.getInstance().urls().base()); 986 assertEquals( 987 "locales list", 988 "https://st.unicode.org/cldr-apps/v#locales///", 989 CLDRConfig.getInstance().urls().forSpecial(CLDRURLS.Special.Locales)); 990 assertEquals( 991 "maltese", 992 "https://st.unicode.org/cldr-apps/v#/mt//", 993 CLDRConfig.getInstance().urls().forLocale(maltese)); 994 assertEquals( 995 "korean in maltese", 996 "https://st.unicode.org/cldr-apps/v#/mt//" + KOREAN_LANGUAGE_STRID, 997 CLDRConfig.getInstance().urls().forXpath(maltese, KOREAN_LANGUAGE)); 998 assertEquals( 999 "korean in maltese via stringid", 1000 "https://st.unicode.org/cldr-apps/v#/mt//" + KOREAN_LANGUAGE_STRID, 1001 CLDRConfig.getInstance().urls().forXpathHexId(maltese, KOREAN_LANGUAGE_STRID)); 1002 assertEquals( 1003 "south east asia in maltese", 1004 "https://st.unicode.org/cldr-apps/v#/mt/C_SEAsia/", 1005 CLDRConfig.getInstance().urls().forPage(maltese, PageId.C_SEAsia)); 1006 try { 1007 String ret = CLDRConfig.getInstance().urls().forXpathHexId(maltese, KOREAN_LANGUAGE); 1008 errln("Error- expected forXpathHexId to choke on an xpath but got " + ret); 1009 } catch (IllegalArgumentException iae) { 1010 logln("GOOD: forXpathHexId Caught expected " + iae); 1011 } 1012 try { 1013 String ret = CLDRConfig.getInstance().urls().forXpath(maltese, KOREAN_LANGUAGE_STRID); 1014 errln("Error- expected forXpath to choke on a hexid but got " + ret); 1015 } catch (IllegalArgumentException iae) { 1016 logln("GOOD: forXpath Caught expected " + iae); 1017 } 1018 1019 assertEquals( 1020 "korean in maltese - absoluteUrl", 1021 "https://st.unicode.org/cldr-apps/v#/mt//" + KOREAN_LANGUAGE_STRID, 1022 CLDRConfig.getInstance().absoluteUrls().forXpath(maltese, KOREAN_LANGUAGE)); 1023 } 1024 1025 static final UnicodeMap<String> SCRIPTS = 1026 ICUPropertyFactory.make().getProperty("script").getUnicodeMap_internal(); 1027 static final UnicodeMap<String> GC = 1028 ICUPropertyFactory.make().getProperty("general_category").getUnicodeMap_internal(); 1029 1030 public void TestUnicodeMapCompose() { 1031 logln("Getting Scripts"); 1032 1033 UnicodeMap.Composer<String> composer = 1034 new UnicodeMap.Composer<>() { 1035 @Override 1036 public String compose(int codepoint, String string, String a, String b) { 1037 return a.toString() + "_" + b.toString(); 1038 } 1039 }; 1040 1041 logln("Trying Compose"); 1042 1043 UnicodeMap<String> composed = 1044 ((UnicodeMap) SCRIPTS.cloneAsThawed()).composeWith(GC, composer); 1045 String last = ""; 1046 for (int i = 0; i < 0x10FFFF; ++i) { 1047 String comp = composed.getValue(i); 1048 String gc = GC.getValue(i); 1049 String sc = SCRIPTS.getValue(i); 1050 if (!comp.equals(composer.compose(i, null, sc, gc))) { 1051 errln("Failed compose at: " + i); 1052 break; 1053 } 1054 if (!last.equals(comp)) { 1055 logln(Utility.hex(i) + "\t" + comp); 1056 last = comp; 1057 } 1058 } 1059 } 1060 1061 private static final int SET_LIMIT = 0x10FFFF; 1062 private static final int CHECK_LIMIT = 0xFFFF; 1063 private static final NumberFormat pf = NumberFormat.getPercentInstance(); 1064 private static final NumberFormat nf = NumberFormat.getInstance(); 1065 1066 public void TestUnicodeMapTime() { 1067 boolean shortTest = getInclusion() < 10; 1068 double hashTime, umTime, icuTime, treeTime; 1069 int warmup = shortTest ? 1 : 20; 1070 umTime = checkUnicodeMapSetTime(warmup, 0); 1071 hashTime = checkUnicodeMapSetTime(warmup, 1); 1072 logln("Percentage: " + pf.format(hashTime / umTime)); 1073 treeTime = checkUnicodeMapSetTime(warmup, 3); 1074 logln("Percentage: " + pf.format(treeTime / umTime)); 1075 1076 if (shortTest) { 1077 return; 1078 } 1079 1080 umTime = checkUnicodeMapGetTime(1000, 0); 1081 hashTime = checkUnicodeMapGetTime(1000, 1); 1082 logln("Percentage: " + pf.format(hashTime / umTime)); 1083 icuTime = checkUnicodeMapGetTime(1000, 2); 1084 logln("Percentage: " + pf.format(icuTime / umTime)); 1085 treeTime = checkUnicodeMapGetTime(1000, 3); 1086 logln("Percentage: " + pf.format(treeTime / umTime)); 1087 } 1088 1089 private static final int propEnum = UProperty.GENERAL_CATEGORY; 1090 1091 private double checkUnicodeMapSetTime(int iterations, int type) { 1092 _checkUnicodeMapSetTime(1, type); 1093 double result = _checkUnicodeMapSetTime(iterations, type); 1094 logln( 1095 (type == 0 ? "UnicodeMap" : type == 1 ? "HashMap" : type == 2 ? "ICU" : "TreeMap") 1096 + "\t" 1097 + nf.format(result)); 1098 return result; 1099 } 1100 1101 private double _checkUnicodeMapSetTime(int iterations, int type) { 1102 UnicodeMap<String> map1 = SCRIPTS; 1103 Map<Integer, String> map2 = map1.putAllCodepointsInto(new HashMap<Integer, String>()); 1104 Map<Integer, String> map3 = new TreeMap<>(map2); 1105 System.gc(); 1106 double start = System.currentTimeMillis(); 1107 for (int j = 0; j < iterations; ++j) 1108 for (int cp = 0; cp <= SET_LIMIT; ++cp) { 1109 int enumValue = UCharacter.getIntPropertyValue(cp, propEnum); 1110 if (enumValue <= 0) continue; // for smaller set 1111 String value = 1112 UCharacter.getPropertyValueName( 1113 propEnum, enumValue, UProperty.NameChoice.LONG); 1114 switch (type) { 1115 case 0: 1116 map1.put(cp, value); 1117 break; 1118 case 1: 1119 map2.put(cp, value); 1120 break; 1121 case 3: 1122 map3.put(cp, value); 1123 break; 1124 } 1125 } 1126 double end = System.currentTimeMillis(); 1127 return (end - start) / 1000 / iterations; 1128 } 1129 1130 private double checkUnicodeMapGetTime(int iterations, int type) { 1131 UnicodeMap<String> map1 = new UnicodeMap<>(); 1132 Map<Integer, String> map2 = map1.putAllCodepointsInto(new HashMap<Integer, String>()); 1133 Map<Integer, String> map3 = new TreeMap<>(); 1134 _checkUnicodeMapGetTime(map1, map2, map3, 1, type); // warmup 1135 double result = _checkUnicodeMapGetTime(map1, map2, map3, iterations, type); 1136 logln( 1137 (type == 0 ? "UnicodeMap" : type == 1 ? "HashMap" : type == 2 ? "ICU" : "TreeMap") 1138 + "\t" 1139 + nf.format(result)); 1140 return result; 1141 } 1142 1143 private double _checkUnicodeMapGetTime( 1144 UnicodeMap<String> map1, 1145 Map<Integer, String> map2, 1146 Map<Integer, String> map3, 1147 int iterations, 1148 int type) { 1149 System.gc(); 1150 double start = System.currentTimeMillis(); 1151 for (int j = 0; j < iterations; ++j) 1152 for (int cp = 0; cp < CHECK_LIMIT; ++cp) { 1153 switch (type) { 1154 case 0: 1155 map1.getValue(cp); 1156 break; 1157 case 1: 1158 map2.get(cp); 1159 break; 1160 case 2: 1161 int enumValue = UCharacter.getIntPropertyValue(cp, propEnum); 1162 UCharacter.getPropertyValueName( 1163 propEnum, enumValue, UProperty.NameChoice.LONG); 1164 break; 1165 case 3: 1166 map3.get(cp); 1167 break; 1168 } 1169 } 1170 double end = System.currentTimeMillis(); 1171 return (end - start) / 1000 / iterations; 1172 } 1173 1174 public void TestStevenTest() { 1175 VoteResolver<String> resolver = new VoteResolver<>(getTestVoterInfoList()); 1176 1177 String tests[] = { 1178 "bailey=BAILEY", 1179 "comment=Steven Loomis test case tweaked by Parthinator", 1180 "locale=wae", 1181 "oldValue=_", 1182 "oldStatus=approved", 1183 "304=test", // Apple vetter 1184 // expected values 1185 "value=test", 1186 "status=approved", 1187 "sameVotes=test", 1188 "conflicts=[]", 1189 "check", 1190 1191 // test1 1192 "comment=timestamp case1", 1193 "locale=de", 1194 "oldValue=old-value", 1195 "oldStatus=provisional", 1196 "404=Foo", 1197 "424=Bar", 1198 // expected 1199 "value=Bar", 1200 "status=provisional", 1201 "sameVotes=Bar, test", 1202 "conflicts=[google]", 1203 "check", 1204 1205 // test2 1206 "comment=timestamp case2", 1207 "locale=de", 1208 "oldValue=Bar", 1209 "oldStatus=provisional", 1210 "424=Foo", 1211 "404=Bar", 1212 // expected values 1213 "value=Bar", 1214 "status=provisional", 1215 "sameVotes=Bar, test", 1216 "conflicts=[google]", 1217 "check", 1218 1219 // test 3 1220 "comment=timestamp unaffiliated case", 1221 "locale=de", 1222 "oldValue=_", 1223 "oldStatus=unconfirmed", 1224 // # // G vetter A 1225 // timestamp=1 1226 "801=Foo", 1227 // timestamp=2 1228 "802=Bar", 1229 // expected values 1230 "value=Bar", 1231 "status=contributed", 1232 "sameVotes=Bar", 1233 "conflicts=[google, unaffiliated]", 1234 "check", 1235 }; 1236 1237 String expectedValue = null; 1238 String expectedConflicts = null; 1239 Status expectedStatus = null; 1240 String oldValue = null; 1241 Status oldStatus = null; 1242 String baileyValue = null; 1243 List<String> sameVotes = null; 1244 String locale = null; 1245 int voteEntries = 0; 1246 Map<Integer, String> values = new TreeMap<>(); 1247 Map<Integer, VoteEntries> valuesMap = new TreeMap<>(); 1248 1249 int counter = -1; 1250 1251 for (String test : tests) { 1252 String[] item = test.split("="); 1253 String name = item[0]; 1254 String value = item.length < 2 ? null : item[1]; 1255 if (name.equalsIgnoreCase("comment")) { 1256 logln("#\t" + value); 1257 // System.out.println("#\t" + value); 1258 if (DEBUG_COMMENT != null && value.contains(DEBUG_COMMENT)) { 1259 int x = 0; 1260 } 1261 } else if (name.equalsIgnoreCase("locale")) { 1262 locale = value; 1263 } else if (name.equalsIgnoreCase("oldValue")) { 1264 oldValue = value; 1265 } else if (name.equalsIgnoreCase("oldStatus")) { 1266 oldStatus = Status.valueOf(value); 1267 } else if (name.equalsIgnoreCase("value")) { 1268 expectedValue = value; 1269 } else if (name.equalsIgnoreCase("bailey")) { 1270 baileyValue = value; 1271 } else if (name.equalsIgnoreCase("sameVotes")) { 1272 sameVotes = 1273 value == null ? new ArrayList<>(0) : Arrays.asList(value.split(",\\s*")); 1274 } else if (name.equalsIgnoreCase("status")) { 1275 expectedStatus = Status.valueOf(value); 1276 } else if (name.equalsIgnoreCase("conflicts")) { 1277 expectedConflicts = value; 1278 } else if (DIGITS.containsAll(name)) { 1279 final int voter = Integer.parseInt(name); 1280 if (value == null || value.equals("null")) { 1281 values.remove(voter); 1282 for (Map.Entry<Integer, VoteEntries> entry : valuesMap.entrySet()) { 1283 if (entry.getValue().getVoter() == voter) { 1284 valuesMap.remove(entry.getKey()); 1285 } 1286 } 1287 } else { 1288 values.put(voter, value); 1289 valuesMap.put(++voteEntries, new VoteEntries(voter, value)); 1290 } 1291 } else if (name.equalsIgnoreCase("check")) { 1292 counter++; 1293 // load the resolver 1294 resolver.setBaileyValue(baileyValue); 1295 resolver.setLocale(CLDRLocale.getInstance(locale), null); 1296 resolver.setBaseline(oldValue, oldStatus); 1297 for (int voteEntry : valuesMap.keySet()) { 1298 1299 resolver.add( 1300 valuesMap.get(voteEntry).getValue(), 1301 valuesMap.get(voteEntry).getVoter()); 1302 } 1303 // print the contents 1304 logln(counter + "\t" + values); 1305 logln(resolver.toString()); 1306 // now print the values 1307 assertEquals(counter + " value", expectedValue, resolver.getWinningValue()); 1308 assertEquals( 1309 counter + " sameVotes", 1310 sameVotes.toString(), 1311 resolver.getValuesWithSameVotes().toString()); 1312 assertEquals(counter + " status", expectedStatus, resolver.getWinningStatus()); 1313 assertEquals( 1314 counter + " conflicts", 1315 expectedConflicts, 1316 resolver.getConflictedOrganizations().toString()); 1317 resolver.clear(); 1318 values.clear(); 1319 } else { 1320 errln("unknown command:\t" + test); 1321 } 1322 } 1323 } 1324 1325 public void testBaileyVotes() { 1326 VoterInfoList vil = new VoterInfoList().setVoterToInfo(TestUser.TEST_USERS); 1327 VoteResolver<String> resolver = new VoteResolver<>(vil); 1328 CLDRLocale locale = CLDRLocale.getInstance("de"); 1329 PathHeader path = null; 1330 1331 /* 1332 * Simple case, all = bailey -- should get INHERITANCE_MARKER now that we have "dropped hard inheritance" 1333 */ 1334 resolver.setLocale(locale, path); 1335 resolver.setBaileyValue("bailey"); 1336 resolver.setBaseline("foo", Status.approved); 1337 1338 resolver.add("bailey", TestUser.appleV.voterId); 1339 resolver.add("bailey", TestUser.microsoftV.voterId); 1340 resolver.add("bailey", TestUser.googleV.voterId); 1341 if (VoteResolver.DROP_HARD_INHERITANCE) { 1342 assertEquals( 1343 "Simple case, all = bailey", 1344 CldrUtility.INHERITANCE_MARKER, 1345 resolver.getWinningValue()); 1346 } else { 1347 assertEquals("Simple case, all = bailey", "bailey", resolver.getWinningValue()); 1348 } 1349 1350 /* 1351 * Another simple case, all = INHERITANCE_MARKER 1352 * Added per https://unicode.org/cldr/trac/ticket/11299 1353 */ 1354 resolver.clear(); 1355 resolver.setLocale(locale, path); 1356 resolver.setBaileyValue("bailey"); 1357 resolver.setBaseline("foo", Status.approved); 1358 1359 resolver.add(CldrUtility.INHERITANCE_MARKER, TestUser.appleV.voterId); 1360 resolver.add(CldrUtility.INHERITANCE_MARKER, TestUser.microsoftV.voterId); 1361 resolver.add(CldrUtility.INHERITANCE_MARKER, TestUser.googleV.voterId); 1362 assertEquals( 1363 "Another simple case, all = INHERITANCE_MARKER", 1364 CldrUtility.INHERITANCE_MARKER, 1365 resolver.getWinningValue()); 1366 1367 /* 1368 * INHERITANCE_MARKER should win here, having more votes than bailey. 1369 * Changed per https://unicode.org/cldr/trac/ticket/11299 1370 */ 1371 resolver.clear(); 1372 resolver.setLocale(locale, path); 1373 resolver.setBaileyValue("bailey"); 1374 resolver.setBaseline("foo", Status.approved); 1375 1376 resolver.add("bailey", TestUser.appleV.voterId); 1377 resolver.add(CldrUtility.INHERITANCE_MARKER, TestUser.microsoftV.voterId); 1378 resolver.add(CldrUtility.INHERITANCE_MARKER, TestUser.googleV.voterId); 1379 assertEquals( 1380 "The bailey value and explicit value combine to win", 1381 CldrUtility.INHERITANCE_MARKER, 1382 resolver.getWinningValue()); 1383 1384 /* 1385 * INHERITANCE_MARKER should win here, having equal number of votes with bailey; 1386 * first they combine to win over other-vote. 1387 * Changed per https://unicode.org/cldr/trac/ticket/11299 1388 */ 1389 resolver.clear(); 1390 resolver.setLocale(locale, path); 1391 resolver.setBaileyValue("bailey"); 1392 resolver.setBaseline("foo", Status.approved); 1393 1394 resolver.add("bailey", TestUser.appleV.voterId); 1395 resolver.add(CldrUtility.INHERITANCE_MARKER, TestUser.microsoftV.voterId); 1396 resolver.add("other-vote", TestUser.googleV.voterId); 1397 assertEquals( 1398 "The bailey value and explicit value combine to win again", 1399 CldrUtility.INHERITANCE_MARKER, 1400 resolver.getWinningValue()); 1401 1402 /* 1403 * Split vote, no action 1404 */ 1405 resolver.clear(); 1406 resolver.setLocale(locale, path); 1407 resolver.setBaileyValue("bailey"); 1408 resolver.setBaseline("foo", Status.approved); 1409 1410 resolver.add("bailey", TestUser.appleV.voterId); 1411 resolver.add("not-bailey", TestUser.microsoftV.voterId); 1412 resolver.add("other-vote", TestUser.googleV.voterId); 1413 assertEquals("Split vote, no action", "foo", resolver.getWinningValue()); 1414 1415 /* 1416 * Bailey should lose even if it has MORE votes than INHERITANCE_MARKER, now that we 1417 * have "dropped hard inheritance" 1418 */ 1419 resolver.clear(); 1420 resolver.setLocale(locale, path); 1421 resolver.setBaileyValue("bailey"); 1422 resolver.setBaseline("foo", Status.approved); 1423 1424 resolver.add("bailey", TestUser.googleV.voterId); 1425 resolver.add("bailey", TestUser.appleV.voterId); 1426 resolver.add(CldrUtility.INHERITANCE_MARKER, TestUser.microsoftV.voterId); 1427 resolver.add("other-vote", TestUser.adobeV.voterId); 1428 resolver.add("other-vote", TestUser.gnomeV.voterId); 1429 if (VoteResolver.DROP_HARD_INHERITANCE) { 1430 assertEquals( 1431 "Bailey never beats INHERITANCE_MARKER", 1432 CldrUtility.INHERITANCE_MARKER, 1433 resolver.getWinningValue()); 1434 } else { 1435 assertEquals( 1436 "Bailey can beat INHERITANCE_MARKER if not dropped", 1437 "bailey", 1438 resolver.getWinningValue()); 1439 } 1440 } 1441 1442 /** Test XMLUploader.writeBulkInfoHtml */ 1443 public void TestBulkUploadHtml() { 1444 StringWriter out = new StringWriter(); 1445 final String bulkStage = "submit"; 1446 try { 1447 XMLUploader.writeBulkInfoHtml(bulkStage, out); 1448 } catch (Exception e) { 1449 errln("Exception for writeBulkInfoHtml in TestBulkUploadHtml: " + e); 1450 } 1451 final String expected = 1452 "<div class='bulkNextInfo'>\n<ul>\n<li class='header'>Bulk Upload:</li>\n" 1453 + "<li class='inactive'>\n<h1>1. upload</h1>\n<h2>Upload XML file</h2>\n</li>\n" 1454 + "<li class='inactive'>\n<h1>2. check</h1>\n<h2>Verify valid XML</h2>\n</li>\n" 1455 + "<li class='inactive'>\n<h1>3. test</h1>\n<h2>Test for CLDR errors</h2>\n</li>\n" 1456 + "<li class='active'>\n<h1>4. submit</h1>\n<h2>Data submitted into SurveyTool</h2>\n</li>\n" 1457 + "</ul>\n</div>\n"; 1458 assertEquals("writeBulkInfoHtml", expected, out.toString()); 1459 } 1460 1461 /** 1462 * Verify that VettingViewer.getMissingStatus returns MissingStatus.PRESENT for a typical path 1463 * in a well-populated locale 1464 * 1465 * <p>Ideally we should also test for MissingStatus.DISPUTED, etc.; that's more difficult 1466 */ 1467 public void TestMissingStatus() { 1468 final String path = 1469 "//ldml/units/unitLength[@type=\"short\"]/unit[@type=\"volume-cup\"]/displayName"; 1470 final String locale = "fr"; 1471 final CLDRFile cldrFile = testInfo.getCLDRFile(locale, true); 1472 final MissingStatus expected = MissingStatus.PRESENT; 1473 // Note: VettingViewer.getMissingStatus reports PRESENT for items with ↑↑↑ and absent if the 1474 // item 1475 // is removed to inherit from root, even though the value obtained is the same in either 1476 // case; 1477 // so for path pick an item that does not have ↑↑↑, otherwise when that item is stripped for 1478 // production data the test will fail. 1479 final MissingStatus status = 1480 VettingViewer.getMissingStatus(cldrFile, path, true /* latin */); 1481 if (status != expected) { 1482 errln( 1483 "Got getMissingStatus = " 1484 + status.toString() 1485 + "; expected " 1486 + expected.toString()); 1487 } 1488 } 1489 1490 /** Check that expected paths are Aliased, and have debugging code */ 1491 public void TestMissingGrammar() { 1492 // https://cldr-smoke.unicode.org/cldr-apps/v#/hu/Length/a4915bf505ffb49 1493 final String path = 1494 "//ldml/units/unitLength[@type=\"long\"]/unit[@type=\"length-meter\"]/unitPattern[@count=\"one\"][@case=\"accusative\"]"; 1495 checkGrammarCoverage( 1496 "hr", 1497 path, 1498 MissingStatus.PRESENT, 1499 DEBUG, 1500 1, 1501 0, 1502 0, 1503 0, 1504 0); // this isn't a very good test, since we have to adjust each time. Should create 1505 // fake cldr data instead 1506 checkGrammarCoverage("kw", path, MissingStatus.ABSENT, false, 0, 0, 1, 1, 0); 1507 checkGrammarCoverage("en_NZ", path, MissingStatus.ALIASED, DEBUG, 1, 0, 0, 0, 0); 1508 } 1509 1510 /** 1511 * Check the getMissingStatus and getStatus. Note that the values may need to be adjusted in 1512 * successive versions. The sizes are expected sizes. 1513 * 1514 * @param locale 1515 * @param path 1516 * @param statusExpected 1517 * @param debug TODO 1518 */ 1519 public void checkGrammarCoverage( 1520 final String locale, 1521 final String path, 1522 MissingStatus statusExpected, 1523 boolean debug, 1524 int... sizes) { 1525 final CLDRFile cldrFile = testInfo.getCLDRFile(locale, true); 1526 final MissingStatus expected = statusExpected; 1527 final MissingStatus status = 1528 VettingViewer.getMissingStatus(cldrFile, path, true /* latin */); 1529 if (status != expected) { 1530 errln( 1531 locale 1532 + " got getMissingStatus = " 1533 + status.toString() 1534 + "; expected " 1535 + expected.toString()); 1536 } 1537 Iterable<String> pathsToTest = Collections.singleton(path); 1538 Counter<org.unicode.cldr.util.Level> foundCounter = new Counter<>(); 1539 Counter<org.unicode.cldr.util.Level> unconfirmedCounter = new Counter<>(); 1540 Counter<org.unicode.cldr.util.Level> missingCounter = new Counter<>(); 1541 Relation<MissingStatus, String> missingPaths = 1542 new Relation( 1543 new TreeMap<MissingStatus, String>(), TreeSet.class, Ordering.natural()); 1544 Set<String> unconfirmedPaths = new TreeSet<>(); 1545 VettingViewer.getStatus( 1546 pathsToTest, 1547 cldrFile, 1548 PathHeader.getFactory(), 1549 foundCounter, 1550 unconfirmedCounter, 1551 missingCounter, 1552 missingPaths, 1553 unconfirmedPaths); 1554 assertEquals(locale + " foundCounter (0)", sizes[0], foundCounter.getTotal()); 1555 assertEquals(locale + " unconfirmedCounter (1)", sizes[1], unconfirmedCounter.getTotal()); 1556 assertEquals(locale + " missingCounter (2)", sizes[2], missingCounter.getTotal()); 1557 assertEquals(locale + " missingPaths (3)", sizes[3], missingPaths.size()); 1558 assertEquals(locale + " unconfirmedPaths (4)", sizes[4], unconfirmedPaths.size()); 1559 showStatusResults( 1560 locale, 1561 foundCounter, 1562 unconfirmedCounter, 1563 missingCounter, 1564 missingPaths, 1565 unconfirmedPaths); 1566 if (debug) { 1567 foundCounter.clear(); 1568 unconfirmedCounter.clear(); 1569 missingCounter.clear(); 1570 missingPaths.clear(); 1571 unconfirmedPaths.clear(); 1572 pathsToTest = cldrFile.fullIterable(); 1573 VettingViewer.getStatus( 1574 pathsToTest, 1575 cldrFile, 1576 PathHeader.getFactory(), 1577 foundCounter, 1578 unconfirmedCounter, 1579 missingCounter, 1580 missingPaths, 1581 unconfirmedPaths); 1582 showStatusResults( 1583 locale, 1584 foundCounter, 1585 unconfirmedCounter, 1586 missingCounter, 1587 missingPaths, 1588 unconfirmedPaths); 1589 } 1590 } 1591 1592 public void showStatusResults( 1593 final String locale, 1594 Counter<org.unicode.cldr.util.Level> foundCounter, 1595 Counter<org.unicode.cldr.util.Level> unconfirmedCounter, 1596 Counter<org.unicode.cldr.util.Level> missingCounter, 1597 Relation<MissingStatus, String> missingPaths, 1598 Set<String> unconfirmedPaths) { 1599 warnln( 1600 "\n" 1601 + locale 1602 + " foundCounter:\t" 1603 + foundCounter 1604 + "\n" 1605 + locale 1606 + " unconfirmedCounter:\t" 1607 + unconfirmedCounter 1608 + "\n" 1609 + locale 1610 + " missingCounter:\t" 1611 + missingCounter 1612 + "\n" 1613 + locale 1614 + " unconfirmedPaths:\t" 1615 + unconfirmedPaths 1616 + "\n" 1617 + locale 1618 + " missing paths (modern):"); 1619 int count = 0; 1620 for (Entry<MissingStatus, String> entry : missingPaths.entrySet()) { 1621 final MissingStatus missingStatus = entry.getKey(); 1622 final String missingPath = entry.getValue(); 1623 warnln( 1624 ++count 1625 + "\t" 1626 + locale 1627 + "\t" 1628 + missingStatus 1629 + "\t" 1630 + missingPath 1631 + "\t" 1632 + SUPPLEMENTAL_DATA_INFO.getCoverageLevel(missingPath, locale)); 1633 } 1634 } 1635 1636 /** 1637 * Test the function VoteResolver.Level.canCreateOrSetLevelTo() 1638 * 1639 * <p>Compare org.unicode.cldr.unittest.web.TestUserRegistry.TestCanSetUserLevel() 1640 */ 1641 public void TestCanCreateOrSetLevelTo() { 1642 if (Level.vetter.canCreateOrSetLevelTo(Level.guest) 1643 || Level.anonymous.canCreateOrSetLevelTo(Level.guest) 1644 || Level.guest.canCreateOrSetLevelTo(Level.locked) 1645 || Level.locked.canCreateOrSetLevelTo(Level.locked)) { 1646 errln("Only managers and above can change levels at all"); 1647 } 1648 if (Level.manager.canCreateOrSetLevelTo(Level.tc) 1649 || Level.manager.canCreateOrSetLevelTo(Level.admin) 1650 || Level.tc.canCreateOrSetLevelTo(Level.admin)) { 1651 errln("Can’t change anyone to a more privileged level than you"); 1652 } 1653 } 1654 } 1655