1 /*
2  * Copyright 2011 Red Hat, Inc.
3  *
4  * This file is made available under the terms of the Common Public License
5  * v1.0 which accompanies this distribution, and is available at
6  * http://www.eclipse.org/legal/cpl-v10.html
7  */
8 
9 import java.io.BufferedWriter;
10 import java.io.File;
11 
12 import java.io.FileOutputStream;
13 import java.io.IOException;
14 import java.io.OutputStreamWriter;
15 import java.lang.reflect.Method;
16 import java.text.DecimalFormat;
17 import java.text.NumberFormat;
18 import java.util.Date;
19 import java.util.HashMap;
20 import java.util.Map;
21 import java.util.Map.Entry;
22 import java.util.Set;
23 import java.util.Arrays;
24 import java.util.concurrent.TimeUnit;
25 import net.sourceforge.jnlp.annotations.Bug;
26 import net.sourceforge.jnlp.annotations.KnownToFail;
27 import net.sourceforge.jnlp.annotations.Remote;
28 import net.sourceforge.jnlp.browsertesting.Browsers;
29 
30 
31 import org.junit.internal.JUnitSystem;
32 import org.junit.runner.Description;
33 import org.junit.runner.Result;
34 import org.junit.runner.notification.Failure;
35 import org.junit.runner.notification.RunListener;
36 
37 /**
38  * This class listens for events in junit testsuite and wrote output to xml.
39  * Xml tryes to follow ant-tests schema, and is enriched for by-class statistics
40  * stdout and err elements are added, but must be filled from elsewhere (eg tee
41  * in make) as junit suite and listener run from our executer have no access to
42  * them.
43  *
44  */
45 public class JunitLikeXmlOutputListener extends RunListener {
46 
47     private BufferedWriter writer;
48     private Failure testFailed = null;
49     private static final String ROOT = "testsuite";
50     private static final String DATE_ELEMENT = "date";
51     private static final String TEST_ELEMENT = "testcase";
52     private static final String BUGS = "bugs";
53     private static final String BUG = "bug";
54     private static final String K2F = "known-to-fail";
55     private static final String REMOTE = "remote";
56     private static final String TEST_NAME_ATTRIBUTE = "name";
57     private static final String TEST_TIME_ATTRIBUTE = "time";
58     private static final String TEST_IGNORED_ATTRIBUTE = "ignored";
59     private static final String TEST_ERROR_ELEMENT = "error";
60     private static final String TEST_CLASS_ATTRIBUTE = "classname";
61     private static final String ERROR_MESSAGE_ATTRIBUTE = "message";
62     private static final String ERROR_TYPE_ATTRIBUTE = "type";
63     private static final String SOUT_ELEMENT = "system-out";
64     private static final String SERR_ELEMENT = "system-err";
65     private static final String CDATA_START = "<![CDATA[";
66     private static final String CDATA_END = "]]>";
67     private static final String TEST_CLASS_ELEMENT = "class";
68     private static final String STATS_ELEMENT = "stats";
69     private static final String CLASSES_ELEMENT = "classes";
70     private static final String SUMMARY_ELEMENT = "summary";
71     private static final String SUMMARY_TOTAL_ELEMENT = "total";
72     private static final String SUMMARY_PASSED_ELEMENT = "passed";
73     private static final String SUMMARY_FAILED_ELEMENT = "failed";
74     private static final String SUMMARY_IGNORED_ELEMENT = "ignored";
75     private long testStart;
76 
77     private int  failedK2F=0;
78     private int  passedK2F=0;
79     private int  ignoredK2F=0;
80 
81     private class ClassStat {
82 
83         Class<?> c;
84         int total;
85         int failed;
86         int passed;
87         int ignored;
88         long time = 0;
89         int  totalK2F=0;
90         int  failedK2F=0;
91         int  passedK2F=0;
92         int  ignoredK2F=0;
93     }
94     Map<String, ClassStat> classStats = new HashMap<String, ClassStat>();
95 
JunitLikeXmlOutputListener(JUnitSystem system, File f)96     public JunitLikeXmlOutputListener(JUnitSystem system, File f) {
97         try {
98             writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(f), "UTF-8"));
99         } catch (Exception ex) {
100             throw new RuntimeException(ex);
101         }
102     }
103 
104     @Override
testRunStarted(Description description)105     public void testRunStarted(Description description) throws Exception {
106         openElement(ROOT);
107         writeElement(DATE_ELEMENT, new Date().toString());
108     }
109 
openElement(String name)110     private void openElement(String name) throws IOException {
111         openElement(name, null);
112     }
113 
openElement(String name, Map<String, String> atts)114     private void openElement(String name, Map<String, String> atts) throws IOException {
115         StringBuilder attString = new StringBuilder();
116         if (atts != null) {
117             attString.append(" ");
118             Set<Entry<String, String>> entries = atts.entrySet();
119             for (Entry<String, String> entry : entries) {
120                 String k = entry.getKey();
121                 String v = entry.getValue();
122                 if (v == null) {
123                     v = "null";
124                 }
125                 attString.append(k).append("=\"").append(attributize(v)).append("\"");
126                 attString.append(" ");
127             }
128         }
129         writer.write("<" + name + attString.toString() + ">");
130         writer.newLine();
131     }
132 
attributize(String s)133     private static String attributize(String s) {
134         return s.replace("&", "&amp;").replace("<", "&lt;").replace("\"", "&quot;");
135     }
136 
closeElement(String name)137     private void closeElement(String name) throws IOException {
138         writer.newLine();
139         writer.write("</" + name + ">");
140         writer.newLine();
141     }
142 
writeContent(String content)143     private void writeContent(String content) throws IOException {
144         writer.write(CDATA_START + content + CDATA_END);
145     }
146 
writeElement(String name, String content)147     private void writeElement(String name, String content) throws IOException {
148         writeElement(name, content, null);
149     }
150 
writeElement(String name, String content, Map<String, String> atts)151     private void writeElement(String name, String content, Map<String, String> atts) throws IOException {
152         openElement(name, atts);
153         writeContent(content);
154         closeElement(name);
155     }
156 
157     @Override
testStarted(Description description)158     public void testStarted(Description description) throws Exception {
159         testFailed = null;
160         testStart = System.nanoTime();
161     }
162 
163     @Override
testFailure(Failure failure)164     public void testFailure(Failure failure) throws IOException {
165         testFailed = failure;
166     }
167 
168     @Override
testIgnored(Description description)169     public void testIgnored(Description description) throws Exception {
170         testDone(description, 0, 0, true);
171     }
172 
173     @Override
testFinished(org.junit.runner.Description description)174     public void testFinished(org.junit.runner.Description description) throws Exception {
175         long testTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - testStart);
176         double testTimeSeconds = ((double) testTime) / 1000d;
177         testDone(description, testTime, testTimeSeconds, false);
178     }
179 
180 
181     @SuppressWarnings("unchecked")
testDone(Description description, long testTime, double testTimeSeconds, boolean ignored)182     private void testDone(Description description, long testTime, double testTimeSeconds, boolean ignored) throws Exception {
183         Class<?> testClass = null;
184         Method testMethod = null;
185         try {
186             testClass = description.getTestClass();
187             String qs = description.getMethodName();
188             //handling @Browser'bugsIds marking of used browser
189             if (qs.contains(" - ")) {
190                 qs = qs.replaceAll(" - .*", "");
191             }
192             testMethod = testClass.getMethod(qs);
193         } catch (Exception ex) {
194             ex.printStackTrace();
195         }
196         Map<String, String> testcaseAtts = new HashMap<String, String>(4);
197         NumberFormat formatter = new DecimalFormat("#0.0000");
198         String stringedTime = formatter.format(testTimeSeconds);
199         stringedTime.replace(",", ".");
200         testcaseAtts.put(TEST_TIME_ATTRIBUTE, stringedTime);
201         testcaseAtts.put(TEST_CLASS_ATTRIBUTE, description.getClassName());
202         testcaseAtts.put(TEST_NAME_ATTRIBUTE, description.getMethodName());
203         if (ignored){
204             testcaseAtts.put(TEST_IGNORED_ATTRIBUTE, Boolean.TRUE.toString());
205         }
206         KnownToFail k2f = LessVerboseTextListener.getAnnotation(testClass, testMethod.getName(), KnownToFail.class);
207         boolean thisTestIsK2F = false;
208         Remote remote =  LessVerboseTextListener.getAnnotation(testClass, testMethod.getName(), Remote.class);
209         if (k2f != null) {
210             //determine if k2f in the current browser
211             //??
212             Browsers[] br = k2f.failsIn();
213             if(0 == br.length){//the KnownToFail annotation without optional parameter
214                 thisTestIsK2F = true;
215             }else{
216                 for(Browsers b : br){
217                     if(description.toString().contains(b.toString())){
218                         thisTestIsK2F = true;
219                     }
220                 }
221             }
222         }
223         if( thisTestIsK2F ) testcaseAtts.put(K2F, Boolean.TRUE.toString());
224         if (remote != null) {
225             testcaseAtts.put(REMOTE, Boolean.TRUE.toString());
226 
227         }
228         openElement(TEST_ELEMENT, testcaseAtts);
229         if (testFailed != null) {
230             if (thisTestIsK2F) {
231                 failedK2F++;
232             }
233             Map<String, String> errorAtts = new HashMap<String, String>(3);
234 
235             errorAtts.put(ERROR_MESSAGE_ATTRIBUTE, testFailed.getMessage());
236             int i = testFailed.getTrace().indexOf(":");
237             if (i >= 0) {
238                 errorAtts.put(ERROR_TYPE_ATTRIBUTE, testFailed.getTrace().substring(0, i));
239             } else {
240                 errorAtts.put(ERROR_TYPE_ATTRIBUTE, "?");
241             }
242 
243             writeElement(TEST_ERROR_ELEMENT, testFailed.getTrace(), errorAtts);
244         } else {
245             if (thisTestIsK2F) {
246                 if (ignored) {
247                     ignoredK2F++;
248                 } else {
249                     passedK2F++;
250 
251                 }
252             }
253         }
254         try {
255             if (testClass != null && testMethod != null) {
256                 Bug bug = testMethod.getAnnotation(Bug.class);
257                 if (bug != null) {
258                     openElement(BUGS);
259                     String[] bugsIds = bug.id();
260                     for (String bugId : bugsIds) {
261                         String idAndUrl[] = createBug(bugId);
262                         Map<String, String> visibleNameAtt = new HashMap<String, String>(1);
263                         visibleNameAtt.put("visibleName", idAndUrl[0]);
264                         openElement(BUG, visibleNameAtt);
265                         writer.write(idAndUrl[1]);
266                         closeElement(BUG);
267                     }
268                     closeElement(BUGS);
269                 }
270             }
271         } catch (Exception ex) {
272             ex.printStackTrace();
273         }
274         closeElement(TEST_ELEMENT);
275         writer.flush();
276 
277         ClassStat classStat = classStats.get(description.getClassName());
278         if (classStat == null) {
279             classStat = new ClassStat();
280             classStat.c = description.getTestClass();
281             classStats.put(description.getClassName(), classStat);
282         }
283         classStat.total++;
284         if (thisTestIsK2F) {
285             classStat.totalK2F++;
286         }
287         classStat.time += testTime;
288         if (testFailed == null) {
289             if (ignored) {
290                 classStat.ignored++;
291                 if (thisTestIsK2F) {
292                     classStat.ignoredK2F++;
293                 }
294             } else {
295                 classStat.passed++;
296                 if (thisTestIsK2F) {
297                     classStat.passedK2F++;
298                 }
299             }
300         } else {
301             classStat.failed++;
302             if (thisTestIsK2F) {
303                 classStat.failedK2F++;
304             }
305         }
306     }
307 
308     @Override
309     @SuppressWarnings("unchecked")
testRunFinished(Result result)310     public void testRunFinished(Result result) throws Exception {
311 
312         writeElement(SOUT_ELEMENT, "@sout@");
313         writeElement(SERR_ELEMENT, "@serr@");
314         openElement(STATS_ELEMENT);
315         openElement(SUMMARY_ELEMENT);
316         int passed = result.getRunCount() - result.getFailureCount() - result.getIgnoreCount();
317         int failed = result.getFailureCount();
318         int ignored = result.getIgnoreCount();
319         writeElement(SUMMARY_TOTAL_ELEMENT, String.valueOf(result.getRunCount()),createKnownToFailSumamryAttribute(failedK2F+passedK2F+ignoredK2F));
320         writeElement(SUMMARY_FAILED_ELEMENT, String.valueOf(failed),createKnownToFailSumamryAttribute(failedK2F));
321         writeElement(SUMMARY_IGNORED_ELEMENT, String.valueOf(ignored),createKnownToFailSumamryAttribute(ignoredK2F));
322         writeElement(SUMMARY_PASSED_ELEMENT, String.valueOf(passed),createKnownToFailSumamryAttribute(passedK2F));
323         closeElement(SUMMARY_ELEMENT);
324         openElement(CLASSES_ELEMENT);
325         Set<Entry<String, ClassStat>> e = classStats.entrySet();
326         for (Entry<String, ClassStat> entry : e) {
327 
328             Map<String, String> testcaseAtts = new HashMap<String, String>(3);
329             testcaseAtts.put(TEST_NAME_ATTRIBUTE, entry.getKey());
330             testcaseAtts.put(TEST_TIME_ATTRIBUTE, String.valueOf(entry.getValue().time));
331 
332             openElement(TEST_CLASS_ELEMENT, testcaseAtts);
333             writeElement(SUMMARY_PASSED_ELEMENT, String.valueOf(entry.getValue().passed),createKnownToFailSumamryAttribute(entry.getValue().passedK2F));
334             writeElement(SUMMARY_FAILED_ELEMENT, String.valueOf(entry.getValue().failed),createKnownToFailSumamryAttribute(entry.getValue().failedK2F));
335             writeElement(SUMMARY_IGNORED_ELEMENT, String.valueOf(entry.getValue().ignored),createKnownToFailSumamryAttribute(entry.getValue().ignoredK2F));
336             writeElement(SUMMARY_TOTAL_ELEMENT, String.valueOf(entry.getValue().total),createKnownToFailSumamryAttribute(entry.getValue().totalK2F));
337             try {
338                 Bug b = null;
339                 if (entry.getValue().c != null) {
340                     b = entry.getValue().c.getAnnotation(Bug.class);
341                 }
342                 if (b != null) {
343                     openElement(BUGS);
344                     String[] s = b.id();
345                     for (String string : s) {
346                         String ss[] = createBug(string);
347                         Map<String, String> visibleNameAtt = new HashMap<String, String>(1);
348                         visibleNameAtt.put("visibleName", ss[0]);
349                         openElement(BUG, visibleNameAtt);
350                         writer.write(ss[1]);
351                         closeElement(BUG);
352                     }
353                     closeElement(BUGS);
354                 }
355             } catch (Exception ex) {
356                 ex.printStackTrace();
357             }
358             closeElement(TEST_CLASS_ELEMENT);
359         }
360         closeElement(CLASSES_ELEMENT);
361         closeElement(STATS_ELEMENT);
362 
363         closeElement(ROOT);
364         writer.flush();
365         writer.close();
366 
367     }
368 
createKnownToFailSumamryAttribute(int count)369     public Map<String, String> createKnownToFailSumamryAttribute(int count) {
370         Map<String, String> atts = new HashMap<String, String>(1);
371         atts.put(K2F, String.valueOf(count));
372         return atts;
373     }
374 
375     /**
376      * When declare for suite class or for Test-marked method,
377      * should be interpreted by report generating tool to links.
378      * Known shortcuts are
379      * SX  - http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=X
380      * PRX - http://icedtea.classpath.org/bugzilla/show_bug.cgi?id=X
381      * RHX - https://bugzilla.redhat.com/show_bug.cgi?id=X
382      * DX  - http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=X
383      * GX  - http://bugs.gentoo.org/show_bug.cgi?id=X
384      * CAX - http://server.complang.tuwien.ac.at/cgi-bin/bugzilla/show_bug.cgi?id=X
385      * LPX - https://bugs.launchpad.net/bugs/X
386      *
387      * http://mail.openjdk.java.net/pipermail/distro-pkg-dev/
388      * and http://mail.openjdk.java.net/pipermail/ are proceed differently
389      *
390      * You just put eg @Bug(id="RH12345",id="http:/my.bukpage.com/terribleNew")
391      * and  RH12345 will be transalated as
392      * <a href="https://bugzilla.redhat.com/show_bug.cgi?id=123456">123456<a> or
393      * similar, the url will be inclueded as is. Both added to proper tests or suites
394      *
395      * @return Strng[2]{nameToBeShown, hrefValue}
396      */
createBug(String string)397     public static String[] createBug(String string) {
398         String[] r = {"ex", string};
399         String[] prefixes = {
400             "S",
401             "PR",
402             "RH",
403             "D",
404             "G",
405             "CA",
406             "LP",};
407         String[] urls = {
408             "http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=",
409             "http://icedtea.classpath.org/bugzilla/show_bug.cgi?id=",
410             "https://bugzilla.redhat.com/show_bug.cgi?id=",
411             "http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=",
412             "http://bugs.gentoo.org/show_bug.cgi?id=",
413             "http://server.complang.tuwien.ac.at/cgi-bin/bugzilla/show_bug.cgi?id",
414             "https://bugs.launchpad.net/bugs/",};
415 
416         for (int i = 0; i < urls.length; i++) {
417             if (string.startsWith(prefixes[i])) {
418                 r[0] = string;
419                 r[1] = urls[i] + string.substring(prefixes[i].length());
420                 return r;
421             }
422 
423         }
424 
425         String distro = "http://mail.openjdk.java.net/pipermail/distro-pkg-dev/";
426         String openjdk = "http://mail.openjdk.java.net/pipermail/";
427         String pushHead = "http://icedtea.classpath.org/hg/";
428         String pushBranch = "http://icedtea.classpath.org/hg/release/";
429         if (string.startsWith(distro)) {
430             r[0] = "distro-pkg";
431             return r;
432         }
433         if (string.startsWith(openjdk)) {
434             r[0] = "openjdk";
435             return r;
436         }
437         if (string.startsWith(pushBranch)) {
438             r[0] = "push (branch)";
439             return r;
440         }
441         if (string.startsWith(pushHead)) {
442             r[0] = "push (head)";
443             return r;
444         }
445         return r;
446 
447     }
448 
main(String[] args)449     public static void main(String[] args) {
450         String[] q = createBug("PR608");
451         System.out.println(q[0] + " : " + q[1]);
452         q = createBug("S4854");
453         System.out.println(q[0] + " : " + q[1]);
454         q = createBug("RH649423");
455         System.out.println(q[0] + " : " + q[1]);
456         q = createBug("D464");
457         System.out.println(q[0] + " : " + q[1]);
458         q = createBug("G6554");
459         System.out.println(q[0] + " : " + q[1]);
460         q = createBug("CA1654");
461         System.out.println(q[0] + " : " + q[1]);
462         q = createBug("LP5445");
463         System.out.println(q[0] + " : " + q[1]);
464 
465         q = createBug("http://mail.openjdk.java.net/pipermail/distro-pkg-dev/2011-November/016178.html");
466         System.out.println(q[0] + " : " + q[1]);
467         q = createBug("http://mail.openjdk.java.net/pipermail/awt-dev/2012-March/002324.html");
468         System.out.println(q[0] + " : " + q[1]);
469 
470         q = createBug("http://lists.fedoraproject.org/pipermail/chinese/2012-January/008868.html");
471         System.out.println(q[0] + " : " + q[1]);
472 
473         q = createBug("http://icedtea.classpath.org/hg/icedtea-web/rev/22b7becd48a7");
474         System.out.println(q[0] + " : " + q[1]);
475 
476         q = createBug("http://icedtea.classpath.org/hg/release/icedtea-web-1.6/rev/0d9faf51357d");
477         System.out.println(q[0] + " : " + q[1]);
478     }
479 }
480