1 /* 2 * Copyright 2011 Red Hat, Inc. 3 * 4 * This file is made available under the terms of the Common Public License 5 * v1.0 which accompanies this distribution, and is available at 6 * http://www.eclipse.org/legal/cpl-v10.html 7 */ 8 9 import java.io.BufferedWriter; 10 import java.io.File; 11 12 import java.io.FileOutputStream; 13 import java.io.IOException; 14 import java.io.OutputStreamWriter; 15 import java.lang.reflect.Method; 16 import java.text.DecimalFormat; 17 import java.text.NumberFormat; 18 import java.util.Date; 19 import java.util.HashMap; 20 import java.util.Map; 21 import java.util.Map.Entry; 22 import java.util.Set; 23 import java.util.Arrays; 24 import java.util.concurrent.TimeUnit; 25 import net.sourceforge.jnlp.annotations.Bug; 26 import net.sourceforge.jnlp.annotations.KnownToFail; 27 import net.sourceforge.jnlp.annotations.Remote; 28 import net.sourceforge.jnlp.browsertesting.Browsers; 29 30 31 import org.junit.internal.JUnitSystem; 32 import org.junit.runner.Description; 33 import org.junit.runner.Result; 34 import org.junit.runner.notification.Failure; 35 import org.junit.runner.notification.RunListener; 36 37 /** 38 * This class listens for events in junit testsuite and wrote output to xml. 39 * Xml tryes to follow ant-tests schema, and is enriched for by-class statistics 40 * stdout and err elements are added, but must be filled from elsewhere (eg tee 41 * in make) as junit suite and listener run from our executer have no access to 42 * them. 43 * 44 */ 45 public class JunitLikeXmlOutputListener extends RunListener { 46 47 private BufferedWriter writer; 48 private Failure testFailed = null; 49 private static final String ROOT = "testsuite"; 50 private static final String DATE_ELEMENT = "date"; 51 private static final String TEST_ELEMENT = "testcase"; 52 private static final String BUGS = "bugs"; 53 private static final String BUG = "bug"; 54 private static final String K2F = "known-to-fail"; 55 private static final String REMOTE = "remote"; 56 private static final String TEST_NAME_ATTRIBUTE = "name"; 57 private static final String TEST_TIME_ATTRIBUTE = "time"; 58 private static final String TEST_IGNORED_ATTRIBUTE = "ignored"; 59 private static final String TEST_ERROR_ELEMENT = "error"; 60 private static final String TEST_CLASS_ATTRIBUTE = "classname"; 61 private static final String ERROR_MESSAGE_ATTRIBUTE = "message"; 62 private static final String ERROR_TYPE_ATTRIBUTE = "type"; 63 private static final String SOUT_ELEMENT = "system-out"; 64 private static final String SERR_ELEMENT = "system-err"; 65 private static final String CDATA_START = "<![CDATA["; 66 private static final String CDATA_END = "]]>"; 67 private static final String TEST_CLASS_ELEMENT = "class"; 68 private static final String STATS_ELEMENT = "stats"; 69 private static final String CLASSES_ELEMENT = "classes"; 70 private static final String SUMMARY_ELEMENT = "summary"; 71 private static final String SUMMARY_TOTAL_ELEMENT = "total"; 72 private static final String SUMMARY_PASSED_ELEMENT = "passed"; 73 private static final String SUMMARY_FAILED_ELEMENT = "failed"; 74 private static final String SUMMARY_IGNORED_ELEMENT = "ignored"; 75 private long testStart; 76 77 private int failedK2F=0; 78 private int passedK2F=0; 79 private int ignoredK2F=0; 80 81 private class ClassStat { 82 83 Class<?> c; 84 int total; 85 int failed; 86 int passed; 87 int ignored; 88 long time = 0; 89 int totalK2F=0; 90 int failedK2F=0; 91 int passedK2F=0; 92 int ignoredK2F=0; 93 } 94 Map<String, ClassStat> classStats = new HashMap<String, ClassStat>(); 95 JunitLikeXmlOutputListener(JUnitSystem system, File f)96 public JunitLikeXmlOutputListener(JUnitSystem system, File f) { 97 try { 98 writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(f), "UTF-8")); 99 } catch (Exception ex) { 100 throw new RuntimeException(ex); 101 } 102 } 103 104 @Override testRunStarted(Description description)105 public void testRunStarted(Description description) throws Exception { 106 openElement(ROOT); 107 writeElement(DATE_ELEMENT, new Date().toString()); 108 } 109 openElement(String name)110 private void openElement(String name) throws IOException { 111 openElement(name, null); 112 } 113 openElement(String name, Map<String, String> atts)114 private void openElement(String name, Map<String, String> atts) throws IOException { 115 StringBuilder attString = new StringBuilder(); 116 if (atts != null) { 117 attString.append(" "); 118 Set<Entry<String, String>> entries = atts.entrySet(); 119 for (Entry<String, String> entry : entries) { 120 String k = entry.getKey(); 121 String v = entry.getValue(); 122 if (v == null) { 123 v = "null"; 124 } 125 attString.append(k).append("=\"").append(attributize(v)).append("\""); 126 attString.append(" "); 127 } 128 } 129 writer.write("<" + name + attString.toString() + ">"); 130 writer.newLine(); 131 } 132 attributize(String s)133 private static String attributize(String s) { 134 return s.replace("&", "&").replace("<", "<").replace("\"", """); 135 } 136 closeElement(String name)137 private void closeElement(String name) throws IOException { 138 writer.newLine(); 139 writer.write("</" + name + ">"); 140 writer.newLine(); 141 } 142 writeContent(String content)143 private void writeContent(String content) throws IOException { 144 writer.write(CDATA_START + content + CDATA_END); 145 } 146 writeElement(String name, String content)147 private void writeElement(String name, String content) throws IOException { 148 writeElement(name, content, null); 149 } 150 writeElement(String name, String content, Map<String, String> atts)151 private void writeElement(String name, String content, Map<String, String> atts) throws IOException { 152 openElement(name, atts); 153 writeContent(content); 154 closeElement(name); 155 } 156 157 @Override testStarted(Description description)158 public void testStarted(Description description) throws Exception { 159 testFailed = null; 160 testStart = System.nanoTime(); 161 } 162 163 @Override testFailure(Failure failure)164 public void testFailure(Failure failure) throws IOException { 165 testFailed = failure; 166 } 167 168 @Override testIgnored(Description description)169 public void testIgnored(Description description) throws Exception { 170 testDone(description, 0, 0, true); 171 } 172 173 @Override testFinished(org.junit.runner.Description description)174 public void testFinished(org.junit.runner.Description description) throws Exception { 175 long testTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - testStart); 176 double testTimeSeconds = ((double) testTime) / 1000d; 177 testDone(description, testTime, testTimeSeconds, false); 178 } 179 180 181 @SuppressWarnings("unchecked") testDone(Description description, long testTime, double testTimeSeconds, boolean ignored)182 private void testDone(Description description, long testTime, double testTimeSeconds, boolean ignored) throws Exception { 183 Class<?> testClass = null; 184 Method testMethod = null; 185 try { 186 testClass = description.getTestClass(); 187 String qs = description.getMethodName(); 188 //handling @Browser'bugsIds marking of used browser 189 if (qs.contains(" - ")) { 190 qs = qs.replaceAll(" - .*", ""); 191 } 192 testMethod = testClass.getMethod(qs); 193 } catch (Exception ex) { 194 ex.printStackTrace(); 195 } 196 Map<String, String> testcaseAtts = new HashMap<String, String>(4); 197 NumberFormat formatter = new DecimalFormat("#0.0000"); 198 String stringedTime = formatter.format(testTimeSeconds); 199 stringedTime.replace(",", "."); 200 testcaseAtts.put(TEST_TIME_ATTRIBUTE, stringedTime); 201 testcaseAtts.put(TEST_CLASS_ATTRIBUTE, description.getClassName()); 202 testcaseAtts.put(TEST_NAME_ATTRIBUTE, description.getMethodName()); 203 if (ignored){ 204 testcaseAtts.put(TEST_IGNORED_ATTRIBUTE, Boolean.TRUE.toString()); 205 } 206 KnownToFail k2f = LessVerboseTextListener.getAnnotation(testClass, testMethod.getName(), KnownToFail.class); 207 boolean thisTestIsK2F = false; 208 Remote remote = LessVerboseTextListener.getAnnotation(testClass, testMethod.getName(), Remote.class); 209 if (k2f != null) { 210 //determine if k2f in the current browser 211 //?? 212 Browsers[] br = k2f.failsIn(); 213 if(0 == br.length){//the KnownToFail annotation without optional parameter 214 thisTestIsK2F = true; 215 }else{ 216 for(Browsers b : br){ 217 if(description.toString().contains(b.toString())){ 218 thisTestIsK2F = true; 219 } 220 } 221 } 222 } 223 if( thisTestIsK2F ) testcaseAtts.put(K2F, Boolean.TRUE.toString()); 224 if (remote != null) { 225 testcaseAtts.put(REMOTE, Boolean.TRUE.toString()); 226 227 } 228 openElement(TEST_ELEMENT, testcaseAtts); 229 if (testFailed != null) { 230 if (thisTestIsK2F) { 231 failedK2F++; 232 } 233 Map<String, String> errorAtts = new HashMap<String, String>(3); 234 235 errorAtts.put(ERROR_MESSAGE_ATTRIBUTE, testFailed.getMessage()); 236 int i = testFailed.getTrace().indexOf(":"); 237 if (i >= 0) { 238 errorAtts.put(ERROR_TYPE_ATTRIBUTE, testFailed.getTrace().substring(0, i)); 239 } else { 240 errorAtts.put(ERROR_TYPE_ATTRIBUTE, "?"); 241 } 242 243 writeElement(TEST_ERROR_ELEMENT, testFailed.getTrace(), errorAtts); 244 } else { 245 if (thisTestIsK2F) { 246 if (ignored) { 247 ignoredK2F++; 248 } else { 249 passedK2F++; 250 251 } 252 } 253 } 254 try { 255 if (testClass != null && testMethod != null) { 256 Bug bug = testMethod.getAnnotation(Bug.class); 257 if (bug != null) { 258 openElement(BUGS); 259 String[] bugsIds = bug.id(); 260 for (String bugId : bugsIds) { 261 String idAndUrl[] = createBug(bugId); 262 Map<String, String> visibleNameAtt = new HashMap<String, String>(1); 263 visibleNameAtt.put("visibleName", idAndUrl[0]); 264 openElement(BUG, visibleNameAtt); 265 writer.write(idAndUrl[1]); 266 closeElement(BUG); 267 } 268 closeElement(BUGS); 269 } 270 } 271 } catch (Exception ex) { 272 ex.printStackTrace(); 273 } 274 closeElement(TEST_ELEMENT); 275 writer.flush(); 276 277 ClassStat classStat = classStats.get(description.getClassName()); 278 if (classStat == null) { 279 classStat = new ClassStat(); 280 classStat.c = description.getTestClass(); 281 classStats.put(description.getClassName(), classStat); 282 } 283 classStat.total++; 284 if (thisTestIsK2F) { 285 classStat.totalK2F++; 286 } 287 classStat.time += testTime; 288 if (testFailed == null) { 289 if (ignored) { 290 classStat.ignored++; 291 if (thisTestIsK2F) { 292 classStat.ignoredK2F++; 293 } 294 } else { 295 classStat.passed++; 296 if (thisTestIsK2F) { 297 classStat.passedK2F++; 298 } 299 } 300 } else { 301 classStat.failed++; 302 if (thisTestIsK2F) { 303 classStat.failedK2F++; 304 } 305 } 306 } 307 308 @Override 309 @SuppressWarnings("unchecked") testRunFinished(Result result)310 public void testRunFinished(Result result) throws Exception { 311 312 writeElement(SOUT_ELEMENT, "@sout@"); 313 writeElement(SERR_ELEMENT, "@serr@"); 314 openElement(STATS_ELEMENT); 315 openElement(SUMMARY_ELEMENT); 316 int passed = result.getRunCount() - result.getFailureCount() - result.getIgnoreCount(); 317 int failed = result.getFailureCount(); 318 int ignored = result.getIgnoreCount(); 319 writeElement(SUMMARY_TOTAL_ELEMENT, String.valueOf(result.getRunCount()),createKnownToFailSumamryAttribute(failedK2F+passedK2F+ignoredK2F)); 320 writeElement(SUMMARY_FAILED_ELEMENT, String.valueOf(failed),createKnownToFailSumamryAttribute(failedK2F)); 321 writeElement(SUMMARY_IGNORED_ELEMENT, String.valueOf(ignored),createKnownToFailSumamryAttribute(ignoredK2F)); 322 writeElement(SUMMARY_PASSED_ELEMENT, String.valueOf(passed),createKnownToFailSumamryAttribute(passedK2F)); 323 closeElement(SUMMARY_ELEMENT); 324 openElement(CLASSES_ELEMENT); 325 Set<Entry<String, ClassStat>> e = classStats.entrySet(); 326 for (Entry<String, ClassStat> entry : e) { 327 328 Map<String, String> testcaseAtts = new HashMap<String, String>(3); 329 testcaseAtts.put(TEST_NAME_ATTRIBUTE, entry.getKey()); 330 testcaseAtts.put(TEST_TIME_ATTRIBUTE, String.valueOf(entry.getValue().time)); 331 332 openElement(TEST_CLASS_ELEMENT, testcaseAtts); 333 writeElement(SUMMARY_PASSED_ELEMENT, String.valueOf(entry.getValue().passed),createKnownToFailSumamryAttribute(entry.getValue().passedK2F)); 334 writeElement(SUMMARY_FAILED_ELEMENT, String.valueOf(entry.getValue().failed),createKnownToFailSumamryAttribute(entry.getValue().failedK2F)); 335 writeElement(SUMMARY_IGNORED_ELEMENT, String.valueOf(entry.getValue().ignored),createKnownToFailSumamryAttribute(entry.getValue().ignoredK2F)); 336 writeElement(SUMMARY_TOTAL_ELEMENT, String.valueOf(entry.getValue().total),createKnownToFailSumamryAttribute(entry.getValue().totalK2F)); 337 try { 338 Bug b = null; 339 if (entry.getValue().c != null) { 340 b = entry.getValue().c.getAnnotation(Bug.class); 341 } 342 if (b != null) { 343 openElement(BUGS); 344 String[] s = b.id(); 345 for (String string : s) { 346 String ss[] = createBug(string); 347 Map<String, String> visibleNameAtt = new HashMap<String, String>(1); 348 visibleNameAtt.put("visibleName", ss[0]); 349 openElement(BUG, visibleNameAtt); 350 writer.write(ss[1]); 351 closeElement(BUG); 352 } 353 closeElement(BUGS); 354 } 355 } catch (Exception ex) { 356 ex.printStackTrace(); 357 } 358 closeElement(TEST_CLASS_ELEMENT); 359 } 360 closeElement(CLASSES_ELEMENT); 361 closeElement(STATS_ELEMENT); 362 363 closeElement(ROOT); 364 writer.flush(); 365 writer.close(); 366 367 } 368 createKnownToFailSumamryAttribute(int count)369 public Map<String, String> createKnownToFailSumamryAttribute(int count) { 370 Map<String, String> atts = new HashMap<String, String>(1); 371 atts.put(K2F, String.valueOf(count)); 372 return atts; 373 } 374 375 /** 376 * When declare for suite class or for Test-marked method, 377 * should be interpreted by report generating tool to links. 378 * Known shortcuts are 379 * SX - http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=X 380 * PRX - http://icedtea.classpath.org/bugzilla/show_bug.cgi?id=X 381 * RHX - https://bugzilla.redhat.com/show_bug.cgi?id=X 382 * DX - http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=X 383 * GX - http://bugs.gentoo.org/show_bug.cgi?id=X 384 * CAX - http://server.complang.tuwien.ac.at/cgi-bin/bugzilla/show_bug.cgi?id=X 385 * LPX - https://bugs.launchpad.net/bugs/X 386 * 387 * http://mail.openjdk.java.net/pipermail/distro-pkg-dev/ 388 * and http://mail.openjdk.java.net/pipermail/ are proceed differently 389 * 390 * You just put eg @Bug(id="RH12345",id="http:/my.bukpage.com/terribleNew") 391 * and RH12345 will be transalated as 392 * <a href="https://bugzilla.redhat.com/show_bug.cgi?id=123456">123456<a> or 393 * similar, the url will be inclueded as is. Both added to proper tests or suites 394 * 395 * @return Strng[2]{nameToBeShown, hrefValue} 396 */ createBug(String string)397 public static String[] createBug(String string) { 398 String[] r = {"ex", string}; 399 String[] prefixes = { 400 "S", 401 "PR", 402 "RH", 403 "D", 404 "G", 405 "CA", 406 "LP",}; 407 String[] urls = { 408 "http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=", 409 "http://icedtea.classpath.org/bugzilla/show_bug.cgi?id=", 410 "https://bugzilla.redhat.com/show_bug.cgi?id=", 411 "http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=", 412 "http://bugs.gentoo.org/show_bug.cgi?id=", 413 "http://server.complang.tuwien.ac.at/cgi-bin/bugzilla/show_bug.cgi?id", 414 "https://bugs.launchpad.net/bugs/",}; 415 416 for (int i = 0; i < urls.length; i++) { 417 if (string.startsWith(prefixes[i])) { 418 r[0] = string; 419 r[1] = urls[i] + string.substring(prefixes[i].length()); 420 return r; 421 } 422 423 } 424 425 String distro = "http://mail.openjdk.java.net/pipermail/distro-pkg-dev/"; 426 String openjdk = "http://mail.openjdk.java.net/pipermail/"; 427 String pushHead = "http://icedtea.classpath.org/hg/"; 428 String pushBranch = "http://icedtea.classpath.org/hg/release/"; 429 if (string.startsWith(distro)) { 430 r[0] = "distro-pkg"; 431 return r; 432 } 433 if (string.startsWith(openjdk)) { 434 r[0] = "openjdk"; 435 return r; 436 } 437 if (string.startsWith(pushBranch)) { 438 r[0] = "push (branch)"; 439 return r; 440 } 441 if (string.startsWith(pushHead)) { 442 r[0] = "push (head)"; 443 return r; 444 } 445 return r; 446 447 } 448 main(String[] args)449 public static void main(String[] args) { 450 String[] q = createBug("PR608"); 451 System.out.println(q[0] + " : " + q[1]); 452 q = createBug("S4854"); 453 System.out.println(q[0] + " : " + q[1]); 454 q = createBug("RH649423"); 455 System.out.println(q[0] + " : " + q[1]); 456 q = createBug("D464"); 457 System.out.println(q[0] + " : " + q[1]); 458 q = createBug("G6554"); 459 System.out.println(q[0] + " : " + q[1]); 460 q = createBug("CA1654"); 461 System.out.println(q[0] + " : " + q[1]); 462 q = createBug("LP5445"); 463 System.out.println(q[0] + " : " + q[1]); 464 465 q = createBug("http://mail.openjdk.java.net/pipermail/distro-pkg-dev/2011-November/016178.html"); 466 System.out.println(q[0] + " : " + q[1]); 467 q = createBug("http://mail.openjdk.java.net/pipermail/awt-dev/2012-March/002324.html"); 468 System.out.println(q[0] + " : " + q[1]); 469 470 q = createBug("http://lists.fedoraproject.org/pipermail/chinese/2012-January/008868.html"); 471 System.out.println(q[0] + " : " + q[1]); 472 473 q = createBug("http://icedtea.classpath.org/hg/icedtea-web/rev/22b7becd48a7"); 474 System.out.println(q[0] + " : " + q[1]); 475 476 q = createBug("http://icedtea.classpath.org/hg/release/icedtea-web-1.6/rev/0d9faf51357d"); 477 System.out.println(q[0] + " : " + q[1]); 478 } 479 } 480