1 /** 2 * Licensed to the Apache Software Foundation (ASF) under one 3 * or more contributor license agreements. See the NOTICE file 4 * distributed with this work for additional information 5 * regarding copyright ownership. The ASF licenses this file 6 * to you under the Apache License, Version 2.0 (the 7 * "License"); you may not use this file except in compliance 8 * with the License. You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 */ 18 19 package org.apache.hadoop.mapreduce.v2.hs.webapp; 20 21 import static org.apache.hadoop.yarn.util.StringHelper.join; 22 import static org.apache.hadoop.yarn.util.StringHelper.ujoin; 23 import static org.junit.Assert.assertEquals; 24 import static org.junit.Assert.assertNotNull; 25 import static org.junit.Assert.assertTrue; 26 import static org.junit.Assert.fail; 27 import static org.mockito.Mockito.mock; 28 import static org.mockito.Mockito.when; 29 30 import java.io.StringReader; 31 import java.util.Map; 32 33 import javax.ws.rs.core.MediaType; 34 import javax.xml.parsers.DocumentBuilder; 35 import javax.xml.parsers.DocumentBuilderFactory; 36 37 import org.apache.hadoop.conf.Configuration; 38 import org.apache.hadoop.mapreduce.v2.api.records.AMInfo; 39 import org.apache.hadoop.mapreduce.v2.api.records.JobId; 40 import org.apache.hadoop.mapreduce.v2.app.AppContext; 41 import org.apache.hadoop.mapreduce.v2.app.job.Job; 42 import org.apache.hadoop.mapreduce.v2.hs.HistoryContext; 43 import org.apache.hadoop.mapreduce.v2.hs.MockHistoryContext; 44 import org.apache.hadoop.mapreduce.v2.util.MRApps; 45 import org.apache.hadoop.yarn.api.records.NodeId; 46 import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; 47 import org.apache.hadoop.yarn.webapp.WebApp; 48 import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; 49 import org.codehaus.jettison.json.JSONArray; 50 import org.codehaus.jettison.json.JSONException; 51 import org.codehaus.jettison.json.JSONObject; 52 import org.junit.Before; 53 import org.junit.Test; 54 import org.w3c.dom.Document; 55 import org.w3c.dom.Element; 56 import org.w3c.dom.NodeList; 57 import org.xml.sax.InputSource; 58 59 import com.google.inject.Guice; 60 import com.google.inject.Injector; 61 import com.google.inject.servlet.GuiceServletContextListener; 62 import com.google.inject.servlet.ServletModule; 63 import com.sun.jersey.api.client.ClientResponse; 64 import com.sun.jersey.api.client.ClientResponse.Status; 65 import com.sun.jersey.api.client.UniformInterfaceException; 66 import com.sun.jersey.api.client.WebResource; 67 import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; 68 import com.sun.jersey.test.framework.JerseyTest; 69 import com.sun.jersey.test.framework.WebAppDescriptor; 70 71 /** 72 * Test the history server Rest API for getting jobs, a specific job, job 73 * counters, and job attempts. 74 * 75 * /ws/v1/history/mapreduce/jobs /ws/v1/history/mapreduce/jobs/{jobid} 76 * /ws/v1/history/mapreduce/jobs/{jobid}/counters 77 * /ws/v1/history/mapreduce/jobs/{jobid}/jobattempts 78 */ 79 public class TestHsWebServicesJobs extends JerseyTest { 80 81 private static Configuration conf = new Configuration(); 82 private static MockHistoryContext appContext; 83 private static HsWebApp webApp; 84 85 private Injector injector = Guice.createInjector(new ServletModule() { 86 @Override 87 protected void configureServlets() { 88 89 appContext = new MockHistoryContext(0, 1, 2, 1, false); 90 webApp = mock(HsWebApp.class); 91 when(webApp.name()).thenReturn("hsmockwebapp"); 92 93 bind(JAXBContextResolver.class); 94 bind(HsWebServices.class); 95 bind(GenericExceptionHandler.class); 96 bind(WebApp.class).toInstance(webApp); 97 bind(AppContext.class).toInstance(appContext); 98 bind(HistoryContext.class).toInstance(appContext); 99 bind(Configuration.class).toInstance(conf); 100 101 serve("/*").with(GuiceContainer.class); 102 } 103 }); 104 105 public class GuiceServletConfig extends GuiceServletContextListener { 106 107 @Override getInjector()108 protected Injector getInjector() { 109 return injector; 110 } 111 } 112 113 @Before 114 @Override setUp()115 public void setUp() throws Exception { 116 super.setUp(); 117 118 } 119 TestHsWebServicesJobs()120 public TestHsWebServicesJobs() { 121 super(new WebAppDescriptor.Builder( 122 "org.apache.hadoop.mapreduce.v2.hs.webapp") 123 .contextListenerClass(GuiceServletConfig.class) 124 .filterClass(com.google.inject.servlet.GuiceFilter.class) 125 .contextPath("jersey-guice-filter").servletPath("/").build()); 126 } 127 128 @Test testJobs()129 public void testJobs() throws JSONException, Exception { 130 WebResource r = resource(); 131 ClientResponse response = r.path("ws").path("v1").path("history") 132 .path("mapreduce").path("jobs").accept(MediaType.APPLICATION_JSON) 133 .get(ClientResponse.class); 134 assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); 135 JSONObject json = response.getEntity(JSONObject.class); 136 assertEquals("incorrect number of elements", 1, json.length()); 137 JSONObject jobs = json.getJSONObject("jobs"); 138 JSONArray arr = jobs.getJSONArray("job"); 139 assertEquals("incorrect number of elements", 1, arr.length()); 140 JSONObject info = arr.getJSONObject(0); 141 Job job = appContext.getPartialJob(MRApps.toJobID(info.getString("id"))); 142 VerifyJobsUtils.verifyHsJobPartial(info, job); 143 144 } 145 146 @Test testJobsSlash()147 public void testJobsSlash() throws JSONException, Exception { 148 WebResource r = resource(); 149 ClientResponse response = r.path("ws").path("v1").path("history") 150 .path("mapreduce").path("jobs/").accept(MediaType.APPLICATION_JSON) 151 .get(ClientResponse.class); 152 assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); 153 JSONObject json = response.getEntity(JSONObject.class); 154 assertEquals("incorrect number of elements", 1, json.length()); 155 JSONObject jobs = json.getJSONObject("jobs"); 156 JSONArray arr = jobs.getJSONArray("job"); 157 assertEquals("incorrect number of elements", 1, arr.length()); 158 JSONObject info = arr.getJSONObject(0); 159 Job job = appContext.getPartialJob(MRApps.toJobID(info.getString("id"))); 160 VerifyJobsUtils.verifyHsJobPartial(info, job); 161 162 } 163 164 @Test testJobsDefault()165 public void testJobsDefault() throws JSONException, Exception { 166 WebResource r = resource(); 167 ClientResponse response = r.path("ws").path("v1").path("history") 168 .path("mapreduce").path("jobs").get(ClientResponse.class); 169 assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); 170 JSONObject json = response.getEntity(JSONObject.class); 171 assertEquals("incorrect number of elements", 1, json.length()); 172 JSONObject jobs = json.getJSONObject("jobs"); 173 JSONArray arr = jobs.getJSONArray("job"); 174 assertEquals("incorrect number of elements", 1, arr.length()); 175 JSONObject info = arr.getJSONObject(0); 176 Job job = appContext.getPartialJob(MRApps.toJobID(info.getString("id"))); 177 VerifyJobsUtils.verifyHsJobPartial(info, job); 178 179 } 180 181 @Test testJobsXML()182 public void testJobsXML() throws Exception { 183 WebResource r = resource(); 184 ClientResponse response = r.path("ws").path("v1").path("history") 185 .path("mapreduce").path("jobs").accept(MediaType.APPLICATION_XML) 186 .get(ClientResponse.class); 187 assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); 188 String xml = response.getEntity(String.class); 189 DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); 190 DocumentBuilder db = dbf.newDocumentBuilder(); 191 InputSource is = new InputSource(); 192 is.setCharacterStream(new StringReader(xml)); 193 Document dom = db.parse(is); 194 NodeList jobs = dom.getElementsByTagName("jobs"); 195 assertEquals("incorrect number of elements", 1, jobs.getLength()); 196 NodeList job = dom.getElementsByTagName("job"); 197 assertEquals("incorrect number of elements", 1, job.getLength()); 198 verifyHsJobPartialXML(job, appContext); 199 } 200 verifyHsJobPartialXML(NodeList nodes, MockHistoryContext appContext)201 public void verifyHsJobPartialXML(NodeList nodes, MockHistoryContext appContext) { 202 203 assertEquals("incorrect number of elements", 1, nodes.getLength()); 204 205 for (int i = 0; i < nodes.getLength(); i++) { 206 Element element = (Element) nodes.item(i); 207 208 Job job = appContext.getPartialJob(MRApps.toJobID(WebServicesTestUtils 209 .getXmlString(element, "id"))); 210 assertNotNull("Job not found - output incorrect", job); 211 212 VerifyJobsUtils.verifyHsJobGeneric(job, 213 WebServicesTestUtils.getXmlString(element, "id"), 214 WebServicesTestUtils.getXmlString(element, "user"), 215 WebServicesTestUtils.getXmlString(element, "name"), 216 WebServicesTestUtils.getXmlString(element, "state"), 217 WebServicesTestUtils.getXmlString(element, "queue"), 218 WebServicesTestUtils.getXmlLong(element, "startTime"), 219 WebServicesTestUtils.getXmlLong(element, "finishTime"), 220 WebServicesTestUtils.getXmlInt(element, "mapsTotal"), 221 WebServicesTestUtils.getXmlInt(element, "mapsCompleted"), 222 WebServicesTestUtils.getXmlInt(element, "reducesTotal"), 223 WebServicesTestUtils.getXmlInt(element, "reducesCompleted")); 224 } 225 } 226 verifyHsJobXML(NodeList nodes, AppContext appContext)227 public void verifyHsJobXML(NodeList nodes, AppContext appContext) { 228 229 assertEquals("incorrect number of elements", 1, nodes.getLength()); 230 231 for (int i = 0; i < nodes.getLength(); i++) { 232 Element element = (Element) nodes.item(i); 233 234 Job job = appContext.getJob(MRApps.toJobID(WebServicesTestUtils 235 .getXmlString(element, "id"))); 236 assertNotNull("Job not found - output incorrect", job); 237 238 VerifyJobsUtils.verifyHsJobGeneric(job, 239 WebServicesTestUtils.getXmlString(element, "id"), 240 WebServicesTestUtils.getXmlString(element, "user"), 241 WebServicesTestUtils.getXmlString(element, "name"), 242 WebServicesTestUtils.getXmlString(element, "state"), 243 WebServicesTestUtils.getXmlString(element, "queue"), 244 WebServicesTestUtils.getXmlLong(element, "startTime"), 245 WebServicesTestUtils.getXmlLong(element, "finishTime"), 246 WebServicesTestUtils.getXmlInt(element, "mapsTotal"), 247 WebServicesTestUtils.getXmlInt(element, "mapsCompleted"), 248 WebServicesTestUtils.getXmlInt(element, "reducesTotal"), 249 WebServicesTestUtils.getXmlInt(element, "reducesCompleted")); 250 251 // restricted access fields - if security and acls set 252 VerifyJobsUtils.verifyHsJobGenericSecure(job, 253 WebServicesTestUtils.getXmlBoolean(element, "uberized"), 254 WebServicesTestUtils.getXmlString(element, "diagnostics"), 255 WebServicesTestUtils.getXmlLong(element, "avgMapTime"), 256 WebServicesTestUtils.getXmlLong(element, "avgReduceTime"), 257 WebServicesTestUtils.getXmlLong(element, "avgShuffleTime"), 258 WebServicesTestUtils.getXmlLong(element, "avgMergeTime"), 259 WebServicesTestUtils.getXmlInt(element, "failedReduceAttempts"), 260 WebServicesTestUtils.getXmlInt(element, "killedReduceAttempts"), 261 WebServicesTestUtils.getXmlInt(element, "successfulReduceAttempts"), 262 WebServicesTestUtils.getXmlInt(element, "failedMapAttempts"), 263 WebServicesTestUtils.getXmlInt(element, "killedMapAttempts"), 264 WebServicesTestUtils.getXmlInt(element, "successfulMapAttempts")); 265 } 266 } 267 268 @Test testJobId()269 public void testJobId() throws JSONException, Exception { 270 WebResource r = resource(); 271 Map<JobId, Job> jobsMap = appContext.getAllJobs(); 272 for (JobId id : jobsMap.keySet()) { 273 String jobId = MRApps.toString(id); 274 275 ClientResponse response = r.path("ws").path("v1").path("history") 276 .path("mapreduce").path("jobs").path(jobId) 277 .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); 278 assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); 279 JSONObject json = response.getEntity(JSONObject.class); 280 assertEquals("incorrect number of elements", 1, json.length()); 281 JSONObject info = json.getJSONObject("job"); 282 VerifyJobsUtils.verifyHsJob(info, appContext.getJob(id)); 283 } 284 285 } 286 287 @Test testJobIdSlash()288 public void testJobIdSlash() throws JSONException, Exception { 289 WebResource r = resource(); 290 Map<JobId, Job> jobsMap = appContext.getAllJobs(); 291 for (JobId id : jobsMap.keySet()) { 292 String jobId = MRApps.toString(id); 293 294 ClientResponse response = r.path("ws").path("v1").path("history") 295 .path("mapreduce").path("jobs").path(jobId + "/") 296 .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); 297 assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); 298 JSONObject json = response.getEntity(JSONObject.class); 299 assertEquals("incorrect number of elements", 1, json.length()); 300 JSONObject info = json.getJSONObject("job"); 301 302 VerifyJobsUtils.verifyHsJob(info, appContext.getJob(id)); 303 } 304 } 305 306 @Test testJobIdDefault()307 public void testJobIdDefault() throws JSONException, Exception { 308 WebResource r = resource(); 309 Map<JobId, Job> jobsMap = appContext.getAllJobs(); 310 for (JobId id : jobsMap.keySet()) { 311 String jobId = MRApps.toString(id); 312 313 ClientResponse response = r.path("ws").path("v1").path("history") 314 .path("mapreduce").path("jobs").path(jobId).get(ClientResponse.class); 315 assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); 316 JSONObject json = response.getEntity(JSONObject.class); 317 assertEquals("incorrect number of elements", 1, json.length()); 318 JSONObject info = json.getJSONObject("job"); 319 VerifyJobsUtils.verifyHsJob(info, appContext.getJob(id)); 320 } 321 322 } 323 324 @Test testJobIdNonExist()325 public void testJobIdNonExist() throws JSONException, Exception { 326 WebResource r = resource(); 327 328 try { 329 r.path("ws").path("v1").path("history").path("mapreduce").path("jobs") 330 .path("job_0_1234").get(JSONObject.class); 331 fail("should have thrown exception on invalid uri"); 332 } catch (UniformInterfaceException ue) { 333 ClientResponse response = ue.getResponse(); 334 assertEquals(Status.NOT_FOUND, response.getClientResponseStatus()); 335 assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); 336 JSONObject msg = response.getEntity(JSONObject.class); 337 JSONObject exception = msg.getJSONObject("RemoteException"); 338 assertEquals("incorrect number of elements", 3, exception.length()); 339 String message = exception.getString("message"); 340 String type = exception.getString("exception"); 341 String classname = exception.getString("javaClassName"); 342 WebServicesTestUtils.checkStringMatch("exception message", 343 "java.lang.Exception: job, job_0_1234, is not found", message); 344 WebServicesTestUtils.checkStringMatch("exception type", 345 "NotFoundException", type); 346 WebServicesTestUtils.checkStringMatch("exception classname", 347 "org.apache.hadoop.yarn.webapp.NotFoundException", classname); 348 } 349 } 350 351 @Test testJobIdInvalid()352 public void testJobIdInvalid() throws JSONException, Exception { 353 WebResource r = resource(); 354 355 try { 356 r.path("ws").path("v1").path("history").path("mapreduce").path("jobs") 357 .path("job_foo").accept(MediaType.APPLICATION_JSON) 358 .get(JSONObject.class); 359 fail("should have thrown exception on invalid uri"); 360 } catch (UniformInterfaceException ue) { 361 ClientResponse response = ue.getResponse(); 362 assertEquals(Status.NOT_FOUND, response.getClientResponseStatus()); 363 assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); 364 JSONObject msg = response.getEntity(JSONObject.class); 365 JSONObject exception = msg.getJSONObject("RemoteException"); 366 assertEquals("incorrect number of elements", 3, exception.length()); 367 String message = exception.getString("message"); 368 String type = exception.getString("exception"); 369 String classname = exception.getString("javaClassName"); 370 verifyJobIdInvalid(message, type, classname); 371 372 } 373 } 374 375 // verify the exception output default is JSON 376 @Test testJobIdInvalidDefault()377 public void testJobIdInvalidDefault() throws JSONException, Exception { 378 WebResource r = resource(); 379 380 try { 381 r.path("ws").path("v1").path("history").path("mapreduce").path("jobs") 382 .path("job_foo").get(JSONObject.class); 383 fail("should have thrown exception on invalid uri"); 384 } catch (UniformInterfaceException ue) { 385 ClientResponse response = ue.getResponse(); 386 assertEquals(Status.NOT_FOUND, response.getClientResponseStatus()); 387 assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); 388 JSONObject msg = response.getEntity(JSONObject.class); 389 JSONObject exception = msg.getJSONObject("RemoteException"); 390 assertEquals("incorrect number of elements", 3, exception.length()); 391 String message = exception.getString("message"); 392 String type = exception.getString("exception"); 393 String classname = exception.getString("javaClassName"); 394 verifyJobIdInvalid(message, type, classname); 395 } 396 } 397 398 // test that the exception output works in XML 399 @Test testJobIdInvalidXML()400 public void testJobIdInvalidXML() throws JSONException, Exception { 401 WebResource r = resource(); 402 403 try { 404 r.path("ws").path("v1").path("history").path("mapreduce").path("jobs") 405 .path("job_foo").accept(MediaType.APPLICATION_XML) 406 .get(JSONObject.class); 407 fail("should have thrown exception on invalid uri"); 408 } catch (UniformInterfaceException ue) { 409 ClientResponse response = ue.getResponse(); 410 assertEquals(Status.NOT_FOUND, response.getClientResponseStatus()); 411 assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); 412 String msg = response.getEntity(String.class); 413 System.out.println(msg); 414 DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); 415 DocumentBuilder db = dbf.newDocumentBuilder(); 416 InputSource is = new InputSource(); 417 is.setCharacterStream(new StringReader(msg)); 418 Document dom = db.parse(is); 419 NodeList nodes = dom.getElementsByTagName("RemoteException"); 420 Element element = (Element) nodes.item(0); 421 String message = WebServicesTestUtils.getXmlString(element, "message"); 422 String type = WebServicesTestUtils.getXmlString(element, "exception"); 423 String classname = WebServicesTestUtils.getXmlString(element, 424 "javaClassName"); 425 verifyJobIdInvalid(message, type, classname); 426 } 427 } 428 verifyJobIdInvalid(String message, String type, String classname)429 private void verifyJobIdInvalid(String message, String type, String classname) { 430 WebServicesTestUtils.checkStringMatch("exception message", 431 "java.lang.Exception: JobId string : job_foo is not properly formed", 432 message); 433 WebServicesTestUtils.checkStringMatch("exception type", 434 "NotFoundException", type); 435 WebServicesTestUtils.checkStringMatch("exception classname", 436 "org.apache.hadoop.yarn.webapp.NotFoundException", classname); 437 } 438 439 @Test testJobIdInvalidBogus()440 public void testJobIdInvalidBogus() throws JSONException, Exception { 441 WebResource r = resource(); 442 443 try { 444 r.path("ws").path("v1").path("history").path("mapreduce").path("jobs") 445 .path("bogusfoo").get(JSONObject.class); 446 fail("should have thrown exception on invalid uri"); 447 } catch (UniformInterfaceException ue) { 448 ClientResponse response = ue.getResponse(); 449 assertEquals(Status.NOT_FOUND, response.getClientResponseStatus()); 450 assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); 451 JSONObject msg = response.getEntity(JSONObject.class); 452 JSONObject exception = msg.getJSONObject("RemoteException"); 453 assertEquals("incorrect number of elements", 3, exception.length()); 454 String message = exception.getString("message"); 455 String type = exception.getString("exception"); 456 String classname = exception.getString("javaClassName"); 457 WebServicesTestUtils.checkStringMatch("exception message", 458 "java.lang.Exception: JobId string : " 459 + "bogusfoo is not properly formed", message); 460 WebServicesTestUtils.checkStringMatch("exception type", 461 "NotFoundException", type); 462 WebServicesTestUtils.checkStringMatch("exception classname", 463 "org.apache.hadoop.yarn.webapp.NotFoundException", classname); 464 } 465 } 466 467 @Test testJobIdXML()468 public void testJobIdXML() throws Exception { 469 WebResource r = resource(); 470 Map<JobId, Job> jobsMap = appContext.getAllJobs(); 471 for (JobId id : jobsMap.keySet()) { 472 String jobId = MRApps.toString(id); 473 474 ClientResponse response = r.path("ws").path("v1").path("history") 475 .path("mapreduce").path("jobs").path(jobId) 476 .accept(MediaType.APPLICATION_XML).get(ClientResponse.class); 477 assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); 478 String xml = response.getEntity(String.class); 479 DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); 480 DocumentBuilder db = dbf.newDocumentBuilder(); 481 InputSource is = new InputSource(); 482 is.setCharacterStream(new StringReader(xml)); 483 Document dom = db.parse(is); 484 NodeList job = dom.getElementsByTagName("job"); 485 verifyHsJobXML(job, appContext); 486 } 487 488 } 489 490 @Test testJobCounters()491 public void testJobCounters() throws JSONException, Exception { 492 WebResource r = resource(); 493 Map<JobId, Job> jobsMap = appContext.getAllJobs(); 494 for (JobId id : jobsMap.keySet()) { 495 String jobId = MRApps.toString(id); 496 497 ClientResponse response = r.path("ws").path("v1").path("history") 498 .path("mapreduce").path("jobs").path(jobId).path("counters") 499 .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); 500 assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); 501 JSONObject json = response.getEntity(JSONObject.class); 502 assertEquals("incorrect number of elements", 1, json.length()); 503 JSONObject info = json.getJSONObject("jobCounters"); 504 verifyHsJobCounters(info, appContext.getJob(id)); 505 } 506 } 507 508 @Test testJobCountersSlash()509 public void testJobCountersSlash() throws JSONException, Exception { 510 WebResource r = resource(); 511 Map<JobId, Job> jobsMap = appContext.getAllJobs(); 512 for (JobId id : jobsMap.keySet()) { 513 String jobId = MRApps.toString(id); 514 515 ClientResponse response = r.path("ws").path("v1").path("history") 516 .path("mapreduce").path("jobs").path(jobId).path("counters/") 517 .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); 518 assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); 519 JSONObject json = response.getEntity(JSONObject.class); 520 assertEquals("incorrect number of elements", 1, json.length()); 521 JSONObject info = json.getJSONObject("jobCounters"); 522 verifyHsJobCounters(info, appContext.getJob(id)); 523 } 524 } 525 526 @Test testJobCountersForKilledJob()527 public void testJobCountersForKilledJob() throws Exception { 528 WebResource r = resource(); 529 appContext = new MockHistoryContext(0, 1, 1, 1, true); 530 injector = Guice.createInjector(new ServletModule() { 531 @Override 532 protected void configureServlets() { 533 534 webApp = mock(HsWebApp.class); 535 when(webApp.name()).thenReturn("hsmockwebapp"); 536 537 bind(JAXBContextResolver.class); 538 bind(HsWebServices.class); 539 bind(GenericExceptionHandler.class); 540 bind(WebApp.class).toInstance(webApp); 541 bind(AppContext.class).toInstance(appContext); 542 bind(HistoryContext.class).toInstance(appContext); 543 bind(Configuration.class).toInstance(conf); 544 545 serve("/*").with(GuiceContainer.class); 546 } 547 }); 548 549 Map<JobId, Job> jobsMap = appContext.getAllJobs(); 550 for (JobId id : jobsMap.keySet()) { 551 String jobId = MRApps.toString(id); 552 553 ClientResponse response = r.path("ws").path("v1").path("history") 554 .path("mapreduce").path("jobs").path(jobId).path("counters/") 555 .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); 556 assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); 557 JSONObject json = response.getEntity(JSONObject.class); 558 assertEquals("incorrect number of elements", 1, json.length()); 559 JSONObject info = json.getJSONObject("jobCounters"); 560 WebServicesTestUtils.checkStringMatch("id", MRApps.toString(id), 561 info.getString("id")); 562 assertTrue("Job shouldn't contain any counters", info.length() == 1); 563 } 564 } 565 566 @Test testJobCountersDefault()567 public void testJobCountersDefault() throws JSONException, Exception { 568 WebResource r = resource(); 569 Map<JobId, Job> jobsMap = appContext.getAllJobs(); 570 for (JobId id : jobsMap.keySet()) { 571 String jobId = MRApps.toString(id); 572 573 ClientResponse response = r.path("ws").path("v1").path("history") 574 .path("mapreduce").path("jobs").path(jobId).path("counters/") 575 .get(ClientResponse.class); 576 assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); 577 JSONObject json = response.getEntity(JSONObject.class); 578 assertEquals("incorrect number of elements", 1, json.length()); 579 JSONObject info = json.getJSONObject("jobCounters"); 580 verifyHsJobCounters(info, appContext.getJob(id)); 581 } 582 } 583 584 @Test testJobCountersXML()585 public void testJobCountersXML() throws Exception { 586 WebResource r = resource(); 587 Map<JobId, Job> jobsMap = appContext.getAllJobs(); 588 for (JobId id : jobsMap.keySet()) { 589 String jobId = MRApps.toString(id); 590 591 ClientResponse response = r.path("ws").path("v1").path("history") 592 .path("mapreduce").path("jobs").path(jobId).path("counters") 593 .accept(MediaType.APPLICATION_XML).get(ClientResponse.class); 594 assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); 595 String xml = response.getEntity(String.class); 596 DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); 597 DocumentBuilder db = dbf.newDocumentBuilder(); 598 InputSource is = new InputSource(); 599 is.setCharacterStream(new StringReader(xml)); 600 Document dom = db.parse(is); 601 NodeList info = dom.getElementsByTagName("jobCounters"); 602 verifyHsJobCountersXML(info, appContext.getJob(id)); 603 } 604 } 605 verifyHsJobCounters(JSONObject info, Job job)606 public void verifyHsJobCounters(JSONObject info, Job job) 607 throws JSONException { 608 609 assertEquals("incorrect number of elements", 2, info.length()); 610 611 WebServicesTestUtils.checkStringMatch("id", MRApps.toString(job.getID()), 612 info.getString("id")); 613 // just do simple verification of fields - not data is correct 614 // in the fields 615 JSONArray counterGroups = info.getJSONArray("counterGroup"); 616 for (int i = 0; i < counterGroups.length(); i++) { 617 JSONObject counterGroup = counterGroups.getJSONObject(i); 618 String name = counterGroup.getString("counterGroupName"); 619 assertTrue("name not set", (name != null && !name.isEmpty())); 620 JSONArray counters = counterGroup.getJSONArray("counter"); 621 for (int j = 0; j < counters.length(); j++) { 622 JSONObject counter = counters.getJSONObject(j); 623 String counterName = counter.getString("name"); 624 assertTrue("counter name not set", 625 (counterName != null && !counterName.isEmpty())); 626 627 long mapValue = counter.getLong("mapCounterValue"); 628 assertTrue("mapCounterValue >= 0", mapValue >= 0); 629 630 long reduceValue = counter.getLong("reduceCounterValue"); 631 assertTrue("reduceCounterValue >= 0", reduceValue >= 0); 632 633 long totalValue = counter.getLong("totalCounterValue"); 634 assertTrue("totalCounterValue >= 0", totalValue >= 0); 635 636 } 637 } 638 } 639 verifyHsJobCountersXML(NodeList nodes, Job job)640 public void verifyHsJobCountersXML(NodeList nodes, Job job) { 641 642 for (int i = 0; i < nodes.getLength(); i++) { 643 Element element = (Element) nodes.item(i); 644 645 assertNotNull("Job not found - output incorrect", job); 646 647 WebServicesTestUtils.checkStringMatch("id", MRApps.toString(job.getID()), 648 WebServicesTestUtils.getXmlString(element, "id")); 649 // just do simple verification of fields - not data is correct 650 // in the fields 651 NodeList groups = element.getElementsByTagName("counterGroup"); 652 653 for (int j = 0; j < groups.getLength(); j++) { 654 Element counters = (Element) groups.item(j); 655 assertNotNull("should have counters in the web service info", counters); 656 String name = WebServicesTestUtils.getXmlString(counters, 657 "counterGroupName"); 658 assertTrue("name not set", (name != null && !name.isEmpty())); 659 NodeList counterArr = counters.getElementsByTagName("counter"); 660 for (int z = 0; z < counterArr.getLength(); z++) { 661 Element counter = (Element) counterArr.item(z); 662 String counterName = WebServicesTestUtils.getXmlString(counter, 663 "name"); 664 assertTrue("counter name not set", 665 (counterName != null && !counterName.isEmpty())); 666 667 long mapValue = WebServicesTestUtils.getXmlLong(counter, 668 "mapCounterValue"); 669 assertTrue("mapCounterValue not >= 0", mapValue >= 0); 670 671 long reduceValue = WebServicesTestUtils.getXmlLong(counter, 672 "reduceCounterValue"); 673 assertTrue("reduceCounterValue >= 0", reduceValue >= 0); 674 675 long totalValue = WebServicesTestUtils.getXmlLong(counter, 676 "totalCounterValue"); 677 assertTrue("totalCounterValue >= 0", totalValue >= 0); 678 } 679 } 680 } 681 } 682 683 @Test testJobAttempts()684 public void testJobAttempts() throws JSONException, Exception { 685 WebResource r = resource(); 686 Map<JobId, Job> jobsMap = appContext.getAllJobs(); 687 for (JobId id : jobsMap.keySet()) { 688 String jobId = MRApps.toString(id); 689 690 ClientResponse response = r.path("ws").path("v1").path("history") 691 .path("mapreduce").path("jobs").path(jobId).path("jobattempts") 692 .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); 693 assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); 694 JSONObject json = response.getEntity(JSONObject.class); 695 assertEquals("incorrect number of elements", 1, json.length()); 696 JSONObject info = json.getJSONObject("jobAttempts"); 697 verifyHsJobAttempts(info, appContext.getJob(id)); 698 } 699 } 700 701 @Test testJobAttemptsSlash()702 public void testJobAttemptsSlash() throws JSONException, Exception { 703 WebResource r = resource(); 704 Map<JobId, Job> jobsMap = appContext.getAllJobs(); 705 for (JobId id : jobsMap.keySet()) { 706 String jobId = MRApps.toString(id); 707 708 ClientResponse response = r.path("ws").path("v1").path("history") 709 .path("mapreduce").path("jobs").path(jobId).path("jobattempts/") 710 .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); 711 assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); 712 JSONObject json = response.getEntity(JSONObject.class); 713 assertEquals("incorrect number of elements", 1, json.length()); 714 JSONObject info = json.getJSONObject("jobAttempts"); 715 verifyHsJobAttempts(info, appContext.getJob(id)); 716 } 717 } 718 719 @Test testJobAttemptsDefault()720 public void testJobAttemptsDefault() throws JSONException, Exception { 721 WebResource r = resource(); 722 Map<JobId, Job> jobsMap = appContext.getAllJobs(); 723 for (JobId id : jobsMap.keySet()) { 724 String jobId = MRApps.toString(id); 725 726 ClientResponse response = r.path("ws").path("v1").path("history") 727 .path("mapreduce").path("jobs").path(jobId).path("jobattempts") 728 .get(ClientResponse.class); 729 assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); 730 JSONObject json = response.getEntity(JSONObject.class); 731 assertEquals("incorrect number of elements", 1, json.length()); 732 JSONObject info = json.getJSONObject("jobAttempts"); 733 verifyHsJobAttempts(info, appContext.getJob(id)); 734 } 735 } 736 737 @Test testJobAttemptsXML()738 public void testJobAttemptsXML() throws Exception { 739 WebResource r = resource(); 740 Map<JobId, Job> jobsMap = appContext.getAllJobs(); 741 for (JobId id : jobsMap.keySet()) { 742 String jobId = MRApps.toString(id); 743 744 ClientResponse response = r.path("ws").path("v1").path("history") 745 .path("mapreduce").path("jobs").path(jobId).path("jobattempts") 746 .accept(MediaType.APPLICATION_XML).get(ClientResponse.class); 747 assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); 748 String xml = response.getEntity(String.class); 749 DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); 750 DocumentBuilder db = dbf.newDocumentBuilder(); 751 InputSource is = new InputSource(); 752 is.setCharacterStream(new StringReader(xml)); 753 Document dom = db.parse(is); 754 NodeList attempts = dom.getElementsByTagName("jobAttempts"); 755 assertEquals("incorrect number of elements", 1, attempts.getLength()); 756 NodeList info = dom.getElementsByTagName("jobAttempt"); 757 verifyHsJobAttemptsXML(info, appContext.getJob(id)); 758 } 759 } 760 verifyHsJobAttempts(JSONObject info, Job job)761 public void verifyHsJobAttempts(JSONObject info, Job job) 762 throws JSONException { 763 764 JSONArray attempts = info.getJSONArray("jobAttempt"); 765 assertEquals("incorrect number of elements", 2, attempts.length()); 766 for (int i = 0; i < attempts.length(); i++) { 767 JSONObject attempt = attempts.getJSONObject(i); 768 verifyHsJobAttemptsGeneric(job, attempt.getString("nodeHttpAddress"), 769 attempt.getString("nodeId"), attempt.getInt("id"), 770 attempt.getLong("startTime"), attempt.getString("containerId"), 771 attempt.getString("logsLink")); 772 } 773 } 774 verifyHsJobAttemptsXML(NodeList nodes, Job job)775 public void verifyHsJobAttemptsXML(NodeList nodes, Job job) { 776 777 assertEquals("incorrect number of elements", 2, nodes.getLength()); 778 for (int i = 0; i < nodes.getLength(); i++) { 779 Element element = (Element) nodes.item(i); 780 verifyHsJobAttemptsGeneric(job, 781 WebServicesTestUtils.getXmlString(element, "nodeHttpAddress"), 782 WebServicesTestUtils.getXmlString(element, "nodeId"), 783 WebServicesTestUtils.getXmlInt(element, "id"), 784 WebServicesTestUtils.getXmlLong(element, "startTime"), 785 WebServicesTestUtils.getXmlString(element, "containerId"), 786 WebServicesTestUtils.getXmlString(element, "logsLink")); 787 } 788 } 789 verifyHsJobAttemptsGeneric(Job job, String nodeHttpAddress, String nodeId, int id, long startTime, String containerId, String logsLink)790 public void verifyHsJobAttemptsGeneric(Job job, String nodeHttpAddress, 791 String nodeId, int id, long startTime, String containerId, String logsLink) { 792 boolean attemptFound = false; 793 for (AMInfo amInfo : job.getAMInfos()) { 794 if (amInfo.getAppAttemptId().getAttemptId() == id) { 795 attemptFound = true; 796 String nmHost = amInfo.getNodeManagerHost(); 797 int nmHttpPort = amInfo.getNodeManagerHttpPort(); 798 int nmPort = amInfo.getNodeManagerPort(); 799 WebServicesTestUtils.checkStringMatch("nodeHttpAddress", nmHost + ":" 800 + nmHttpPort, nodeHttpAddress); 801 WebServicesTestUtils.checkStringMatch("nodeId", 802 NodeId.newInstance(nmHost, nmPort).toString(), nodeId); 803 assertTrue("startime not greater than 0", startTime > 0); 804 WebServicesTestUtils.checkStringMatch("containerId", amInfo 805 .getContainerId().toString(), containerId); 806 807 String localLogsLink = join( 808 "hsmockwebapp", 809 ujoin("logs", nodeId, containerId, MRApps.toString(job.getID()), 810 job.getUserName())); 811 812 assertTrue("logsLink", logsLink.contains(localLogsLink)); 813 } 814 } 815 assertTrue("attempt: " + id + " was not found", attemptFound); 816 } 817 818 } 819