xref: /aosp_15_r20/external/cldr/tools/cldr-code/src/main/java/org/unicode/cldr/tool/GetChanges.java (revision 912701f9769bb47905792267661f0baf2b85bed5) 
1 package org.unicode.cldr.tool;
2 
3 import com.google.common.collect.ImmutableSet;
4 import com.google.common.collect.Multimap;
5 import com.google.common.collect.TreeMultimap;
6 import com.ibm.icu.util.Output;
7 import java.io.File;
8 import java.util.Collections;
9 import java.util.Map;
10 import java.util.Map.Entry;
11 import java.util.Objects;
12 import java.util.Set;
13 import java.util.TreeMap;
14 import java.util.regex.Matcher;
15 import java.util.regex.Pattern;
16 import org.unicode.cldr.util.CLDRConfig;
17 import org.unicode.cldr.util.CLDRFile;
18 import org.unicode.cldr.util.CLDRFile.Status;
19 import org.unicode.cldr.util.CLDRPaths;
20 import org.unicode.cldr.util.CldrUtility;
21 import org.unicode.cldr.util.Factory;
22 import org.unicode.cldr.util.LanguageTagParser;
23 import org.unicode.cldr.util.Level;
24 import org.unicode.cldr.util.Organization;
25 import org.unicode.cldr.util.PathHeader;
26 import org.unicode.cldr.util.PatternCache;
27 import org.unicode.cldr.util.StandardCodes;
28 import org.unicode.cldr.util.XMLSource;
29 
30 public class GetChanges {
31     static CLDRConfig testInfo = ToolConfig.getToolInstance();
32     static CLDRFile english = testInfo.getEnglish();
33     static boolean verbose = true;
34 
main(String[] args)35     public static void main(String[] args) {
36         System.out.println(
37                 "#Dir\tState\tTrunk Value\tmodify_config.txt Locale\tAction\tlabel\tPath Value\tlabel\tNew Value");
38         Matcher localeFilter = Pattern.compile(".*").matcher("");
39         File vxml = new File(CLDRPaths.AUX_DIRECTORY, "voting/" + "35" + "/vxml2");
40         for (File vxmlDir : vxml.listFiles()) {
41             if (!vxmlDir.isDirectory()) {
42                 continue;
43             }
44             // System.out.println("#Dir=\t" + vxmlDir);
45             // common, ...
46             File[] commonSeedFiles = vxmlDir.listFiles();
47             if (commonSeedFiles == null) {
48                 System.out.println("##No files in: " + vxmlDir);
49                 continue;
50             }
51             for (File commonSeed : commonSeedFiles) {
52                 if (!commonSeed.isDirectory() || "dtd".equals(commonSeed.getName())) {
53                     continue;
54                 }
55                 // System.out.println("#Dir=\t" + commonSeed);
56                 String subDir = vxmlDir.getName() + "/" + commonSeed.getName();
57                 String l2 = CLDRPaths.BASE_DIRECTORY + subDir;
58                 // main, annotations
59                 Factory factoryVxml = Factory.make(commonSeed.toString(), ".*");
60                 Factory factoryTrunk = Factory.make(l2, ".*");
61 
62                 for (String locale : factoryVxml.getAvailable()) {
63                     if (!localeFilter.reset(locale).matches()) {
64                         continue;
65                     }
66                     if (locale.equals("root")) {
67                         continue;
68                     }
69                     CLDRFile vxmlFile;
70                     try {
71                         vxmlFile = factoryVxml.make(locale, true);
72                     } catch (Exception e1) {
73                         System.out.println(
74                                 "##Error: dir=\t"
75                                         + commonSeed
76                                         + "\t locale=\t"
77                                         + locale
78                                         + "\tmsg=\t"
79                                         + e1.getMessage());
80                         continue;
81                     }
82                     CLDRFile trunkFile = null;
83                     try {
84                         trunkFile = factoryTrunk.make(locale, true);
85                     } catch (Exception e) {
86                     }
87                     compare(subDir, vxmlFile, trunkFile);
88                 }
89             }
90         }
91     }
92 
93     public static final Set<String> ROOT_OR_CODE_FALLBACK =
94             ImmutableSet.of(XMLSource.CODE_FALLBACK_ID, XMLSource.ROOT_ID);
95 
compare(String dir, CLDRFile vxmlFileResolved, CLDRFile trunkFileResolved)96     private static void compare(String dir, CLDRFile vxmlFileResolved, CLDRFile trunkFileResolved) {
97         String localeID = vxmlFileResolved.getLocaleID();
98         if (localeID.equals("ccp")) {
99             int debug = 0;
100         }
101         CLDRFile vxmlFileUnresolved = vxmlFileResolved.getUnresolved();
102         // System.out.println("#Dir: " + dir + ";\tLocale: " + localeID);
103         Output<String> localeWhereFound = new Output<>();
104         Output<String> pathWhereFound = new Output<>();
105         Status status = new Status();
106 
107         int countNew = 0;
108         int countChanged = 0;
109         for (String path : vxmlFileUnresolved) {
110             if (path.contains("/identity")) {
111                 continue;
112             }
113             if (path.contains("ccp")) {
114                 int debug = 0;
115             }
116 
117             // get trunk value. If not accessible or ROOT_OR_CODE_FALLBACK, then set to null
118 
119             String trunkValue = null;
120             if (trunkFileResolved != null) {
121                 trunkValue = trunkFileResolved.getStringValue(path);
122                 String foundId = trunkFileResolved.getSourceLocaleID(path, status);
123                 if (ROOT_OR_CODE_FALLBACK.contains(foundId)) {
124                     trunkValue = null;
125                 }
126             }
127             String vxmlValue = vxmlFileResolved.getStringValue(path);
128 
129             // quick test; will repeat
130 
131             if (Objects.equals(vxmlValue, trunkValue)) {
132                 continue;
133             }
134 
135             // If vxmlValue is INHERITANCE_MARKER, then get bailey
136             // if ROOT_OR_CODE_FALLBACK location, then we skip
137 
138             if (vxmlValue.equals(CldrUtility.INHERITANCE_MARKER)) {
139                 vxmlValue = vxmlFileResolved.getBaileyValue(path, pathWhereFound, localeWhereFound);
140                 if (ROOT_OR_CODE_FALLBACK.contains(localeWhereFound.value)) {
141                     continue;
142                 }
143                 //              if (!path.equals(pathWhereFound.value)) {
144                 //                  continue;
145                 //              }
146             }
147 
148             if (Objects.equals(vxmlValue, trunkValue)) {
149                 continue;
150             }
151             if (trunkValue == null) {
152                 ++countNew;
153                 if (verbose) {
154                     System.out.println(
155                             dir
156                                     + ";\tnew\t"
157                                     + "\tlocale="
158                                     + localeID
159                                     + ";"
160                                     + "\taction=add;"
161                                     + "\tpath=\t"
162                                     + path
163                                     + "\tnew_value=\t"
164                                     + vxmlValue);
165                 }
166             } else {
167                 // we do a lot of processing on the annotations, so most likely are just changes
168                 // introduced by that.
169                 // So ignore for now
170                 if (path.startsWith("//ldml/annotations")) {
171                     continue;
172                 }
173                 ++countChanged;
174                 if (verbose) {
175                     System.out.println(
176                             dir
177                                     + ";\ttrunk=\t"
178                                     + trunkValue
179                                     + "\tlocale="
180                                     + localeID
181                                     + ";"
182                                     + "\taction=add;"
183                                     + "\tpath=\t"
184                                     + path
185                                     + "\tnew_value=\t"
186                                     + vxmlValue);
187                 }
188             }
189             //            if (countNew != 0 || countChanged != 0) {
190             //                System.out.println("#Dir=\t" + dir + "\tLocale=\t" + localeID +
191             // "\tCountNew=\t" + countNew + "\tCountChanged=\t" + countChanged);
192             //            }
193         }
194     }
195 
196     static class Data {
197         final String valueLastRelease;
198         final String valueSnapshot;
199         final String valueTrunk;
200 
Data(String valueLastRelease, String valueSnapshot, String valueTrunk)201         public Data(String valueLastRelease, String valueSnapshot, String valueTrunk) {
202             super();
203             this.valueLastRelease = valueLastRelease;
204             this.valueSnapshot = valueSnapshot;
205             this.valueTrunk = valueTrunk;
206         }
207 
208         @Override
toString()209         public String toString() {
210             return "«"
211                     + valueLastRelease
212                     + "»"
213                     + (!CldrUtility.equals(valueTrunk, valueLastRelease) ? "‡" : "")
214                     + "\t"
215                     + "«"
216                     + valueSnapshot
217                     + "»"
218                     + (CldrUtility.equals(valueTrunk, valueSnapshot) ? "†" : "");
219         }
220     }
221 
old()222     private void old() {
223         boolean onlyMissing = true;
224         String release = "33.1";
225 
226         String subdir = "annotations"; // "main";
227         Factory lastReleaseFactory =
228                 Factory.make(CLDRPaths.LAST_RELEASE_DIRECTORY + "common/" + subdir, ".*");
229         Factory trunkFactory = testInfo.getAnnotationsFactory(); // CldrFactory();
230         Factory snapshotFactory =
231                 Factory.make(
232                         CLDRPaths.AUX_DIRECTORY + "voting/" + release + "/vxml/common/" + subdir,
233                         ".*");
234         PathHeader.Factory phf = PathHeader.getFactory(english);
235 
236         int totalCount = 0;
237         int localeCount = 0;
238 
239         Output<String> localeWhereFound = new Output<>();
240         Output<String> pathWhereFound = new Output<>();
241 
242         CLDRFile englishCldrFile = trunkFactory.make("en", false);
243         final Set<String> paths = ImmutableSet.copyOf(englishCldrFile.iterator());
244         System.out.println("english paths: " + paths.size());
245 
246         Multimap<String, PathHeader> missing = TreeMultimap.create();
247 
248         Set<String> locales =
249                 StandardCodes.make()
250                         .getLocaleCoverageLocales(
251                                 Organization.cldr, Collections.singleton(Level.MODERN));
252 
253         LanguageTagParser ltp = new LanguageTagParser();
254         for (String locale : locales) {
255             if (!ltp.set(locale).getRegion().isEmpty() || locale.equals("sr_Latn")) {
256                 continue; // skip region locales, they'll inherit
257             }
258             //            if (!locale.equals("sr_Latn")) {
259             //                continue;
260             //            }
261             System.out.println(locale);
262 
263             CLDRFile snapshot;
264             try {
265                 snapshot = snapshotFactory.make(locale, false);
266             } catch (Exception e) {
267                 System.out.println("##Skipping " + locale + ", no data in annotations/");
268                 continue;
269             }
270 
271             CLDRFile lastRelease = null;
272             try {
273                 lastRelease = lastReleaseFactory.make(locale, false);
274             } catch (Exception e) {
275             }
276 
277             CLDRFile trunk = null;
278             try {
279                 trunk = trunkFactory.make(locale, false);
280             } catch (Exception e) {
281             }
282 
283             Map<PathHeader, Data> results = new TreeMap<>();
284 
285             for (String xpath : paths) {
286                 if (xpath.contains("fallbackRegionFormat")
287                         || xpath.contains("exemplar")
288                         || xpath.contains("/identity")) {
289                     continue;
290                 }
291                 String newPath = fixOldPath(xpath);
292                 String valueSnapshot = snapshot.getStringValue(newPath);
293                 PathHeader ph = null;
294 
295                 if (valueSnapshot == null) {
296                     ph = phf.fromPath(newPath);
297                     missing.put(locale, ph);
298                 }
299                 String valueLastRelease =
300                         lastRelease == null ? null : lastRelease.getStringValue(xpath);
301                 if (valueSnapshot == null) {
302                     ph = phf.fromPath(newPath);
303                     missing.put(locale, ph);
304                 }
305 
306                 if (onlyMissing) {
307                     continue;
308                 }
309                 if (valueSnapshot != null && Objects.equals(valueLastRelease, valueSnapshot)) {
310                     continue;
311                 }
312                 String valueTrunk = trunk == null ? null : trunk.getStringValue(newPath);
313                 //                if (valueSnapshot == null && valueTrunk == null) { // committee
314                 // deletion
315                 //                    continue;
316                 //                }
317                 // skip inherited
318                 String baileyValue =
319                         snapshot.getBaileyValue(xpath, pathWhereFound, localeWhereFound);
320                 if (!"root".equals(localeWhereFound.value)
321                         && !"code-fallback".equals(localeWhereFound.value)
322                         && CldrUtility.equals(valueSnapshot, baileyValue)) {
323                     continue;
324                 }
325                 ph = ph != null ? ph : phf.fromPath(newPath);
326                 results.put(ph, new Data(valueLastRelease, valueSnapshot, valueTrunk));
327             }
328             if (results.isEmpty()) {
329                 continue;
330             }
331             int itemCount = 0;
332             localeCount++;
333             for (Entry<PathHeader, Data> entry : results.entrySet()) {
334                 PathHeader ph = entry.getKey();
335                 String englishValue = englishCldrFile.getStringValue(ph.getOriginalPath());
336                 System.out.println(
337                         localeCount
338                                 + "\t"
339                                 + ++itemCount
340                                 + "\t"
341                                 + locale
342                                 + "\t"
343                                 + english.getName(locale)
344                                 + "\t"
345                                 + ph
346                                 + "\t«"
347                                 + englishValue
348                                 + "»\t"
349                                 + entry.getValue());
350             }
351             totalCount += itemCount;
352         }
353         System.out.println("##Total:\t" + totalCount);
354         for (Entry<String, PathHeader> entry : missing.entries()) {
355             System.out.println(entry.getKey() + "\t" + entry.getValue());
356         }
357     }
358 
359     static Pattern OLD_PATH =
360             PatternCache.get(
361                     "//ldml/units/unit\\[@type=\"([^\"]*)\"]/unitPattern\\[@count=\"([^\"]*)\"](\\[@alt=\"([^\"]*)\"])?");
362     static Matcher OLD_PATH_MATCHER = OLD_PATH.matcher("");
363 
fixOldPath(String xpath)364     private static String fixOldPath(String xpath) {
365         // //ldml/units/unit[@type="day-future"]/unitPattern[@count="one"]
366         // to
367         // //ldml/units/unitLength[@type="long"]/unit[@type="duration-day-future"]/unitPattern[@count="one"]
368 
369         if (OLD_PATH_MATCHER.reset(xpath).matches()) {
370             String type = OLD_PATH_MATCHER.group(4);
371             return "//ldml/units/unitLength[@type=\""
372                     + (type == null ? "long" : type)
373                     + "\"]/unit[@type=\"duration-"
374                     + OLD_PATH_MATCHER.group(1)
375                     + "\"]/unitPattern[@count=\""
376                     + OLD_PATH_MATCHER.group(2)
377                     + "\"]";
378         }
379         return xpath;
380     }
381 }
382