1/* 2 * Licensed to the Apache Software Foundation (ASF) under one or more 3 * contributor license agreements. See the NOTICE file distributed with 4 * this work for additional information regarding copyright ownership. 5 * The ASF licenses this file to You under the Apache License, Version 2.0 6 * (the "License"); you may not use this file except in compliance with 7 * the License. You may obtain a copy of the License at 8 * 9 * http://www.apache.org/licenses/LICENSE-2.0 10 * 11 * Unless required by applicable law or agreed to in writing, software 12 * distributed under the License is distributed on an "AS IS" BASIS, 13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 * See the License for the specific language governing permissions and 15 * limitations under the License. 16 */ 17 18package org.apache.spark.ui 19 20import java.net.URLDecoder 21import java.text.SimpleDateFormat 22import java.util.{Date, Locale, TimeZone} 23 24import scala.util.control.NonFatal 25import scala.xml._ 26import scala.xml.transform.{RewriteRule, RuleTransformer} 27 28import org.apache.spark.internal.Logging 29import org.apache.spark.ui.scope.RDDOperationGraph 30 31/** Utility functions for generating XML pages with spark content. */ 32private[spark] object UIUtils extends Logging { 33 val TABLE_CLASS_NOT_STRIPED = "table table-bordered table-condensed" 34 val TABLE_CLASS_STRIPED = TABLE_CLASS_NOT_STRIPED + " table-striped" 35 val TABLE_CLASS_STRIPED_SORTABLE = TABLE_CLASS_STRIPED + " sortable" 36 37 // SimpleDateFormat is not thread-safe. Don't expose it to avoid improper use. 38 private val dateFormat = new ThreadLocal[SimpleDateFormat]() { 39 override def initialValue(): SimpleDateFormat = 40 new SimpleDateFormat("yyyy/MM/dd HH:mm:ss", Locale.US) 41 } 42 43 def formatDate(date: Date): String = dateFormat.get.format(date) 44 45 def formatDate(timestamp: Long): String = dateFormat.get.format(new Date(timestamp)) 46 47 def formatDuration(milliseconds: Long): String = { 48 if (milliseconds < 100) { 49 return "%d ms".format(milliseconds) 50 } 51 val seconds = milliseconds.toDouble / 1000 52 if (seconds < 1) { 53 return "%.1f s".format(seconds) 54 } 55 if (seconds < 60) { 56 return "%.0f s".format(seconds) 57 } 58 val minutes = seconds / 60 59 if (minutes < 10) { 60 return "%.1f min".format(minutes) 61 } else if (minutes < 60) { 62 return "%.0f min".format(minutes) 63 } 64 val hours = minutes / 60 65 "%.1f h".format(hours) 66 } 67 68 /** Generate a verbose human-readable string representing a duration such as "5 second 35 ms" */ 69 def formatDurationVerbose(ms: Long): String = { 70 try { 71 val second = 1000L 72 val minute = 60 * second 73 val hour = 60 * minute 74 val day = 24 * hour 75 val week = 7 * day 76 val year = 365 * day 77 78 def toString(num: Long, unit: String): String = { 79 if (num == 0) { 80 "" 81 } else if (num == 1) { 82 s"$num $unit" 83 } else { 84 s"$num ${unit}s" 85 } 86 } 87 88 val millisecondsString = if (ms >= second && ms % second == 0) "" else s"${ms % second} ms" 89 val secondString = toString((ms % minute) / second, "second") 90 val minuteString = toString((ms % hour) / minute, "minute") 91 val hourString = toString((ms % day) / hour, "hour") 92 val dayString = toString((ms % week) / day, "day") 93 val weekString = toString((ms % year) / week, "week") 94 val yearString = toString(ms / year, "year") 95 96 Seq( 97 second -> millisecondsString, 98 minute -> s"$secondString $millisecondsString", 99 hour -> s"$minuteString $secondString", 100 day -> s"$hourString $minuteString $secondString", 101 week -> s"$dayString $hourString $minuteString", 102 year -> s"$weekString $dayString $hourString" 103 ).foreach { case (durationLimit, durationString) => 104 if (ms < durationLimit) { 105 // if time is less than the limit (upto year) 106 return durationString 107 } 108 } 109 // if time is more than a year 110 return s"$yearString $weekString $dayString" 111 } catch { 112 case e: Exception => 113 logError("Error converting time to string", e) 114 // if there is some error, return blank string 115 return "" 116 } 117 } 118 119 /** Generate a human-readable string representing a number (e.g. 100 K) */ 120 def formatNumber(records: Double): String = { 121 val trillion = 1e12 122 val billion = 1e9 123 val million = 1e6 124 val thousand = 1e3 125 126 val (value, unit) = { 127 if (records >= 2*trillion) { 128 (records / trillion, " T") 129 } else if (records >= 2*billion) { 130 (records / billion, " B") 131 } else if (records >= 2*million) { 132 (records / million, " M") 133 } else if (records >= 2*thousand) { 134 (records / thousand, " K") 135 } else { 136 (records, "") 137 } 138 } 139 if (unit.isEmpty) { 140 "%d".formatLocal(Locale.US, value.toInt) 141 } else { 142 "%.1f%s".formatLocal(Locale.US, value, unit) 143 } 144 } 145 146 // Yarn has to go through a proxy so the base uri is provided and has to be on all links 147 def uiRoot: String = { 148 // SPARK-11484 - Use the proxyBase set by the AM, if not found then use env. 149 sys.props.get("spark.ui.proxyBase") 150 .orElse(sys.env.get("APPLICATION_WEB_PROXY_BASE")) 151 .getOrElse("") 152 } 153 154 def prependBaseUri(basePath: String = "", resource: String = ""): String = { 155 uiRoot + basePath + resource 156 } 157 158 def commonHeaderNodes: Seq[Node] = { 159 <meta http-equiv="Content-type" content="text/html; charset=utf-8" /> 160 <link rel="stylesheet" href={prependBaseUri("/static/bootstrap.min.css")} type="text/css"/> 161 <link rel="stylesheet" href={prependBaseUri("/static/vis.min.css")} type="text/css"/> 162 <link rel="stylesheet" href={prependBaseUri("/static/webui.css")} type="text/css"/> 163 <link rel="stylesheet" href={prependBaseUri("/static/timeline-view.css")} type="text/css"/> 164 <script src={prependBaseUri("/static/sorttable.js")} ></script> 165 <script src={prependBaseUri("/static/jquery-1.11.1.min.js")}></script> 166 <script src={prependBaseUri("/static/vis.min.js")}></script> 167 <script src={prependBaseUri("/static/bootstrap-tooltip.js")}></script> 168 <script src={prependBaseUri("/static/initialize-tooltips.js")}></script> 169 <script src={prependBaseUri("/static/table.js")}></script> 170 <script src={prependBaseUri("/static/additional-metrics.js")}></script> 171 <script src={prependBaseUri("/static/timeline-view.js")}></script> 172 <script src={prependBaseUri("/static/log-view.js")}></script> 173 <script src={prependBaseUri("/static/webui.js")}></script> 174 <script>setUIRoot('{UIUtils.uiRoot}')</script> 175 } 176 177 def vizHeaderNodes: Seq[Node] = { 178 <link rel="stylesheet" href={prependBaseUri("/static/spark-dag-viz.css")} type="text/css" /> 179 <script src={prependBaseUri("/static/d3.min.js")}></script> 180 <script src={prependBaseUri("/static/dagre-d3.min.js")}></script> 181 <script src={prependBaseUri("/static/graphlib-dot.min.js")}></script> 182 <script src={prependBaseUri("/static/spark-dag-viz.js")}></script> 183 } 184 185 def dataTablesHeaderNodes: Seq[Node] = { 186 <link rel="stylesheet" 187 href={prependBaseUri("/static/jquery.dataTables.1.10.4.min.css")} type="text/css"/> 188 <link rel="stylesheet" 189 href={prependBaseUri("/static/dataTables.bootstrap.css")} type="text/css"/> 190 <link rel="stylesheet" href={prependBaseUri("/static/jsonFormatter.min.css")} type="text/css"/> 191 <script src={prependBaseUri("/static/jquery.dataTables.1.10.4.min.js")}></script> 192 <script src={prependBaseUri("/static/jquery.cookies.2.2.0.min.js")}></script> 193 <script src={prependBaseUri("/static/jquery.blockUI.min.js")}></script> 194 <script src={prependBaseUri("/static/dataTables.bootstrap.min.js")}></script> 195 <script src={prependBaseUri("/static/jsonFormatter.min.js")}></script> 196 <script src={prependBaseUri("/static/jquery.mustache.js")}></script> 197 } 198 199 /** Returns a spark page with correctly formatted headers */ 200 def headerSparkPage( 201 title: String, 202 content: => Seq[Node], 203 activeTab: SparkUITab, 204 refreshInterval: Option[Int] = None, 205 helpText: Option[String] = None, 206 showVisualization: Boolean = false, 207 useDataTables: Boolean = false): Seq[Node] = { 208 209 val appName = activeTab.appName 210 val shortAppName = if (appName.length < 36) appName else appName.take(32) + "..." 211 val header = activeTab.headerTabs.map { tab => 212 <li class={if (tab == activeTab) "active" else ""}> 213 <a href={prependBaseUri(activeTab.basePath, "/" + tab.prefix + "/")}>{tab.name}</a> 214 </li> 215 } 216 val helpButton: Seq[Node] = helpText.map(tooltip(_, "bottom")).getOrElse(Seq.empty) 217 218 <html> 219 <head> 220 {commonHeaderNodes} 221 {if (showVisualization) vizHeaderNodes else Seq.empty} 222 {if (useDataTables) dataTablesHeaderNodes else Seq.empty} 223 <title>{appName} - {title}</title> 224 </head> 225 <body> 226 <div class="navbar navbar-static-top"> 227 <div class="navbar-inner"> 228 <div class="brand"> 229 <a href={prependBaseUri("/")} class="brand"> 230 <img src={prependBaseUri("/static/spark-logo-77x50px-hd.png")} /> 231 <span class="version">{org.apache.spark.SPARK_VERSION}</span> 232 </a> 233 </div> 234 <p class="navbar-text pull-right"> 235 <strong title={appName}>{shortAppName}</strong> application UI 236 </p> 237 <ul class="nav">{header}</ul> 238 </div> 239 </div> 240 <div class="container-fluid"> 241 <div class="row-fluid"> 242 <div class="span12"> 243 <h3 style="vertical-align: bottom; display: inline-block;"> 244 {title} 245 {helpButton} 246 </h3> 247 </div> 248 </div> 249 {content} 250 </div> 251 </body> 252 </html> 253 } 254 255 /** Returns a page with the spark css/js and a simple format. Used for scheduler UI. */ 256 def basicSparkPage( 257 content: => Seq[Node], 258 title: String, 259 useDataTables: Boolean = false): Seq[Node] = { 260 <html> 261 <head> 262 {commonHeaderNodes} 263 {if (useDataTables) dataTablesHeaderNodes else Seq.empty} 264 <title>{title}</title> 265 </head> 266 <body> 267 <div class="container-fluid"> 268 <div class="row-fluid"> 269 <div class="span12"> 270 <h3 style="vertical-align: middle; display: inline-block;"> 271 <a style="text-decoration: none" href={prependBaseUri("/")}> 272 <img src={prependBaseUri("/static/spark-logo-77x50px-hd.png")} /> 273 <span class="version" 274 style="margin-right: 15px;">{org.apache.spark.SPARK_VERSION}</span> 275 </a> 276 {title} 277 </h3> 278 </div> 279 </div> 280 {content} 281 </div> 282 </body> 283 </html> 284 } 285 286 /** Returns an HTML table constructed by generating a row for each object in a sequence. */ 287 def listingTable[T]( 288 headers: Seq[String], 289 generateDataRow: T => Seq[Node], 290 data: Iterable[T], 291 fixedWidth: Boolean = false, 292 id: Option[String] = None, 293 headerClasses: Seq[String] = Seq.empty, 294 stripeRowsWithCss: Boolean = true, 295 sortable: Boolean = true): Seq[Node] = { 296 297 val listingTableClass = { 298 val _tableClass = if (stripeRowsWithCss) TABLE_CLASS_STRIPED else TABLE_CLASS_NOT_STRIPED 299 if (sortable) { 300 _tableClass + " sortable" 301 } else { 302 _tableClass 303 } 304 } 305 val colWidth = 100.toDouble / headers.size 306 val colWidthAttr = if (fixedWidth) colWidth + "%" else "" 307 308 def getClass(index: Int): String = { 309 if (index < headerClasses.size) { 310 headerClasses(index) 311 } else { 312 "" 313 } 314 } 315 316 val newlinesInHeader = headers.exists(_.contains("\n")) 317 def getHeaderContent(header: String): Seq[Node] = { 318 if (newlinesInHeader) { 319 <ul class="unstyled"> 320 { header.split("\n").map { case t => <li> {t} </li> } } 321 </ul> 322 } else { 323 Text(header) 324 } 325 } 326 327 val headerRow: Seq[Node] = { 328 headers.view.zipWithIndex.map { x => 329 <th width={colWidthAttr} class={getClass(x._2)}>{getHeaderContent(x._1)}</th> 330 } 331 } 332 <table class={listingTableClass} id={id.map(Text.apply)}> 333 <thead>{headerRow}</thead> 334 <tbody> 335 {data.map(r => generateDataRow(r))} 336 </tbody> 337 </table> 338 } 339 340 def makeProgressBar( 341 started: Int, 342 completed: Int, 343 failed: Int, 344 skipped: Int, 345 killed: Int, 346 total: Int): Seq[Node] = { 347 val completeWidth = "width: %s%%".format((completed.toDouble/total)*100) 348 // started + completed can be > total when there are speculative tasks 349 val boundedStarted = math.min(started, total - completed) 350 val startWidth = "width: %s%%".format((boundedStarted.toDouble/total)*100) 351 352 <div class="progress"> 353 <span style="text-align:center; position:absolute; width:100%; left:0;"> 354 {completed}/{total} 355 { if (failed > 0) s"($failed failed)" } 356 { if (skipped > 0) s"($skipped skipped)" } 357 { if (killed > 0) s"($killed killed)" } 358 </span> 359 <div class="bar bar-completed" style={completeWidth}></div> 360 <div class="bar bar-running" style={startWidth}></div> 361 </div> 362 } 363 364 /** Return a "DAG visualization" DOM element that expands into a visualization for a stage. */ 365 def showDagVizForStage(stageId: Int, graph: Option[RDDOperationGraph]): Seq[Node] = { 366 showDagViz(graph.toSeq, forJob = false) 367 } 368 369 /** Return a "DAG visualization" DOM element that expands into a visualization for a job. */ 370 def showDagVizForJob(jobId: Int, graphs: Seq[RDDOperationGraph]): Seq[Node] = { 371 showDagViz(graphs, forJob = true) 372 } 373 374 /** 375 * Return a "DAG visualization" DOM element that expands into a visualization on the UI. 376 * 377 * This populates metadata necessary for generating the visualization on the front-end in 378 * a format that is expected by spark-dag-viz.js. Any changes in the format here must be 379 * reflected there. 380 */ 381 private def showDagViz(graphs: Seq[RDDOperationGraph], forJob: Boolean): Seq[Node] = { 382 <div> 383 <span id={if (forJob) "job-dag-viz" else "stage-dag-viz"} 384 class="expand-dag-viz" onclick={s"toggleDagViz($forJob);"}> 385 <span class="expand-dag-viz-arrow arrow-closed"></span> 386 <a data-toggle="tooltip" title={if (forJob) ToolTips.JOB_DAG else ToolTips.STAGE_DAG} 387 data-placement="right"> 388 DAG Visualization 389 </a> 390 </span> 391 <div id="dag-viz-graph"></div> 392 <div id="dag-viz-metadata" style="display:none"> 393 { 394 graphs.map { g => 395 val stageId = g.rootCluster.id.replaceAll(RDDOperationGraph.STAGE_CLUSTER_PREFIX, "") 396 val skipped = g.rootCluster.name.contains("skipped").toString 397 <div class="stage-metadata" stage-id={stageId} skipped={skipped}> 398 <div class="dot-file">{RDDOperationGraph.makeDotFile(g)}</div> 399 { g.incomingEdges.map { e => <div class="incoming-edge">{e.fromId},{e.toId}</div> } } 400 { g.outgoingEdges.map { e => <div class="outgoing-edge">{e.fromId},{e.toId}</div> } } 401 { 402 g.rootCluster.getCachedNodes.map { n => 403 <div class="cached-rdd">{n.id}</div> 404 } 405 } 406 </div> 407 } 408 } 409 </div> 410 </div> 411 } 412 413 def tooltip(text: String, position: String): Seq[Node] = { 414 <sup> 415 (<a data-toggle="tooltip" data-placement={position} title={text}>?</a>) 416 </sup> 417 } 418 419 /** 420 * Returns HTML rendering of a job or stage description. It will try to parse the string as HTML 421 * and make sure that it only contains anchors with root-relative links. Otherwise, 422 * the whole string will rendered as a simple escaped text. 423 * 424 * Note: In terms of security, only anchor tags with root relative links are supported. So any 425 * attempts to embed links outside Spark UI, or other tags like {@code <script>} will cause in 426 * the whole description to be treated as plain text. 427 * 428 * @param desc the original job or stage description string, which may contain html tags. 429 * @param basePathUri with which to prepend the relative links; this is used when plainText is 430 * false. 431 * @param plainText whether to keep only plain text (i.e. remove html tags) from the original 432 * description string. 433 * @return the HTML rendering of the job or stage description, which will be a Text when plainText 434 * is true, and an Elem otherwise. 435 */ 436 def makeDescription(desc: String, basePathUri: String, plainText: Boolean = false): NodeSeq = { 437 import scala.language.postfixOps 438 439 // If the description can be parsed as HTML and has only relative links, then render 440 // as HTML, otherwise render as escaped string 441 try { 442 // Try to load the description as unescaped HTML 443 val xml = XML.loadString(s"""<span class="description-input">$desc</span>""") 444 445 // Verify that this has only anchors and span (we are wrapping in span) 446 val allowedNodeLabels = Set("a", "span") 447 val illegalNodes = xml \\ "_" filterNot { case node: Node => 448 allowedNodeLabels.contains(node.label) 449 } 450 if (illegalNodes.nonEmpty) { 451 throw new IllegalArgumentException( 452 "Only HTML anchors allowed in job descriptions\n" + 453 illegalNodes.map { n => s"${n.label} in $n"}.mkString("\n\t")) 454 } 455 456 // Verify that all links are relative links starting with "/" 457 val allLinks = 458 xml \\ "a" flatMap { _.attributes } filter { _.key == "href" } map { _.value.toString } 459 if (allLinks.exists { ! _.startsWith ("/") }) { 460 throw new IllegalArgumentException( 461 "Links in job descriptions must be root-relative:\n" + allLinks.mkString("\n\t")) 462 } 463 464 val rule = 465 if (plainText) { 466 // Remove all tags, retaining only their texts 467 new RewriteRule() { 468 override def transform(n: Node): Seq[Node] = { 469 n match { 470 case e: Elem if e.child isEmpty => Text(e.text) 471 case e: Elem if e.child nonEmpty => Text(e.child.flatMap(transform).text) 472 case _ => n 473 } 474 } 475 } 476 } 477 else { 478 // Prepend the relative links with basePathUri 479 new RewriteRule() { 480 override def transform(n: Node): Seq[Node] = { 481 n match { 482 case e: Elem if e \ "@href" nonEmpty => 483 val relativePath = e.attribute("href").get.toString 484 val fullUri = s"${basePathUri.stripSuffix("/")}/${relativePath.stripPrefix("/")}" 485 e % Attribute(null, "href", fullUri, Null) 486 case _ => n 487 } 488 } 489 } 490 } 491 new RuleTransformer(rule).transform(xml) 492 } catch { 493 case NonFatal(e) => 494 if (plainText) Text(desc) else <span class="description-input">{desc}</span> 495 } 496 } 497 498 /** 499 * Decode URLParameter if URL is encoded by YARN-WebAppProxyServlet. 500 * Due to YARN-2844: WebAppProxyServlet cannot handle urls which contain encoded characters 501 * Therefore we need to decode it until we get the real URLParameter. 502 */ 503 def decodeURLParameter(urlParam: String): String = { 504 var param = urlParam 505 var decodedParam = URLDecoder.decode(param, "UTF-8") 506 while (param != decodedParam) { 507 param = decodedParam 508 decodedParam = URLDecoder.decode(param, "UTF-8") 509 } 510 param 511 } 512 513 def getTimeZoneOffset() : Int = 514 TimeZone.getDefault().getOffset(System.currentTimeMillis()) / 1000 / 60 515 516 /** 517 * Return the correct Href after checking if master is running in the 518 * reverse proxy mode or not. 519 */ 520 def makeHref(proxy: Boolean, id: String, origHref: String): String = { 521 if (proxy) { 522 s"/proxy/$id" 523 } else { 524 origHref 525 } 526 } 527} 528