1 package org.unicode.cldr.tool;
2 
3 import java.io.File;
4 import java.util.HashSet;
5 import java.util.Iterator;
6 import java.util.Set;
7 import java.util.TreeSet;
8 
9 import org.unicode.cldr.util.CLDRFile;
10 import org.unicode.cldr.util.CLDRPaths;
11 import org.unicode.cldr.util.CldrUtility;
12 import org.unicode.cldr.util.Factory;
13 import org.unicode.cldr.util.LocaleIDParser;
14 import org.unicode.cldr.util.PathUtilities;
15 import org.unicode.cldr.util.PrettyPath;
16 
17 import com.ibm.icu.dev.tool.UOption;
18 import com.ibm.icu.text.Collator;
19 import com.ibm.icu.text.RuleBasedCollator;
20 import com.ibm.icu.util.ULocale;
21 
22 public class CompareData {
23 
24     private static final int HELP1 = 0,
25         HELP2 = 1,
26         SOURCEDIR = 2,
27         DESTDIR = 3,
28         MATCH = 4;
29 
30     private static final UOption[] options = {
31         UOption.HELP_H(),
32         UOption.HELP_QUESTION_MARK(),
33         UOption.SOURCEDIR().setDefault(CLDRPaths.BASE_DIRECTORY),
34         UOption.DESTDIR().setDefault(CLDRPaths.BASE_DIRECTORY + "../cldr-last/"),
35         UOption.create("match", 'm', UOption.REQUIRES_ARG).setDefault(".*"),
36     };
37 
38     String[] directoryList = { "main", "collation", "segmentations" };
39 
40     static RuleBasedCollator uca = (RuleBasedCollator) Collator.getInstance(ULocale.ROOT);
41     {
42         uca.setNumericCollation(true);
43     }
44 
45     static PrettyPath prettyPathMaker = new PrettyPath();
46     static CLDRFile english;
47     static Set<String> locales;
48     static Factory cldrFactory;
49 
main(String[] args)50     public static void main(String[] args) throws Exception {
51         double deltaTime = System.currentTimeMillis();
52         try {
53             UOption.parseArgs(args, options);
54             String sourceDir = options[SOURCEDIR].value + "common/main/";
55             System.out.println(PathUtilities.getNormalizedPathString(sourceDir));
56             String compareDir = options[DESTDIR].value + "common/main/";
57             System.out.println(PathUtilities.getNormalizedPathString(compareDir));
58 
59             cldrFactory = Factory.make(sourceDir, options[MATCH].value);
60             Factory oldFactory = Factory.make(compareDir, options[MATCH].value);
61 
62             locales = new TreeSet<>(cldrFactory.getAvailable());
63             new CldrUtility.MatcherFilter(options[MATCH].value).retainAll(locales);
64             Set<String> pathsSeen = new HashSet<>();
65             int newItemsTotal = 0;
66             int replacementItemsTotal = 0;
67             int deletedItemsTotal = 0;
68             int sameItemsTotal = 0;
69 
70             for (Iterator<String> it = locales.iterator(); it.hasNext();) {
71                 int newItems = 0;
72                 int replacementItems = 0;
73                 int deletedItems = 0;
74                 int sameItems = 0;
75                 String locale = it.next();
76                 if (locale.startsWith("supplem") || locale.startsWith("character")) continue;
77                 CLDRFile file = cldrFactory.make(locale, false);
78                 try {
79                     CLDRFile oldFile = oldFactory.make(locale, false);
80                     pathsSeen.clear();
81                     for (Iterator<String> it2 = file.iterator(); it2.hasNext();) {
82                         String path = it2.next();
83                         String value = file.getStringValue(path);
84                         String oldValue = oldFile.getStringValue(path);
85                         if (oldValue == null) {
86                             newItems++;
87                         } else if (!value.equals(oldValue)) {
88                             replacementItems++;
89                         } else {
90                             sameItems++;
91                         }
92                         pathsSeen.add(path);
93                     }
94                     for (Iterator<String> it2 = oldFile.iterator(); it2.hasNext();) {
95                         String path = it2.next();
96                         if (!pathsSeen.contains(path)) {
97                             deletedItems++;
98                         }
99                     }
100                 } catch (Exception e) {
101                     newItems = size(file.iterator());
102                 }
103                 String langScript = new LocaleIDParser().set(file.getLocaleID()).getLanguageScript();
104                 System.out.println(langScript + "\t" + file.getLocaleID() + "\t" + sameItems + "\t" + newItems + "\t"
105                     + replacementItems + "\t" + deletedItems);
106                 newItemsTotal += newItems;
107                 replacementItemsTotal += replacementItems;
108                 deletedItemsTotal += deletedItems;
109                 sameItemsTotal += sameItems;
110             }
111             System.out.println("TOTAL" + "\t" + "\t" + sameItemsTotal + "\t" + newItemsTotal + "\t"
112                 + replacementItemsTotal + "\t" + deletedItemsTotal);
113         } finally {
114             deltaTime = System.currentTimeMillis() - deltaTime;
115             System.out.println("Elapsed: " + deltaTime / 1000.0 + " seconds");
116             System.out.println("Done");
117         }
118 
119     }
120 
size(Iterator iterator)121     private static int size(Iterator iterator) {
122         int count = 0;
123         for (; iterator.hasNext();) {
124             iterator.next();
125             ++count;
126         }
127         return count;
128     }
129 }
130