1/* 2 * Licensed to the Apache Software Foundation (ASF) under one or more 3 * contributor license agreements. See the NOTICE file distributed with 4 * this work for additional information regarding copyright ownership. 5 * The ASF licenses this file to You under the Apache License, Version 2.0 6 * (the "License"); you may not use this file except in compliance with 7 * the License. You may obtain a copy of the License at 8 * 9 * http://www.apache.org/licenses/LICENSE-2.0 10 * 11 * Unless required by applicable law or agreed to in writing, software 12 * distributed under the License is distributed on an "AS IS" BASIS, 13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 * See the License for the specific language governing permissions and 15 * limitations under the License. 16 */ 17 18package org.apache.spark.deploy.worker.ui 19 20import java.io.File 21import javax.servlet.http.HttpServletRequest 22 23import scala.xml.{Node, Unparsed} 24 25import com.google.common.cache.{CacheBuilder, CacheLoader, LoadingCache} 26 27import org.apache.spark.internal.Logging 28import org.apache.spark.ui.{UIUtils, WebUIPage} 29import org.apache.spark.util.Utils 30import org.apache.spark.util.logging.RollingFileAppender 31 32private[ui] class LogPage(parent: WorkerWebUI) extends WebUIPage("logPage") with Logging { 33 private val worker = parent.worker 34 private val workDir = new File(parent.workDir.toURI.normalize().getPath) 35 private val supportedLogTypes = Set("stderr", "stdout") 36 private val defaultBytes = 100 * 1024 37 38 def renderLog(request: HttpServletRequest): String = { 39 val appId = Option(request.getParameter("appId")) 40 val executorId = Option(request.getParameter("executorId")) 41 val driverId = Option(request.getParameter("driverId")) 42 val logType = request.getParameter("logType") 43 val offset = Option(request.getParameter("offset")).map(_.toLong) 44 val byteLength = Option(request.getParameter("byteLength")).map(_.toInt).getOrElse(defaultBytes) 45 46 val logDir = (appId, executorId, driverId) match { 47 case (Some(a), Some(e), None) => 48 s"${workDir.getPath}/$a/$e/" 49 case (None, None, Some(d)) => 50 s"${workDir.getPath}/$d/" 51 case _ => 52 throw new Exception("Request must specify either application or driver identifiers") 53 } 54 55 val (logText, startByte, endByte, logLength) = getLog(logDir, logType, offset, byteLength) 56 val pre = s"==== Bytes $startByte-$endByte of $logLength of $logDir$logType ====\n" 57 pre + logText 58 } 59 60 def render(request: HttpServletRequest): Seq[Node] = { 61 val appId = Option(request.getParameter("appId")) 62 val executorId = Option(request.getParameter("executorId")) 63 val driverId = Option(request.getParameter("driverId")) 64 val logType = request.getParameter("logType") 65 val offset = Option(request.getParameter("offset")).map(_.toLong) 66 val byteLength = Option(request.getParameter("byteLength")).map(_.toInt).getOrElse(defaultBytes) 67 68 val (logDir, params, pageName) = (appId, executorId, driverId) match { 69 case (Some(a), Some(e), None) => 70 (s"${workDir.getPath}/$a/$e/", s"appId=$a&executorId=$e", s"$a/$e") 71 case (None, None, Some(d)) => 72 (s"${workDir.getPath}/$d/", s"driverId=$d", d) 73 case _ => 74 throw new Exception("Request must specify either application or driver identifiers") 75 } 76 77 val (logText, startByte, endByte, logLength) = getLog(logDir, logType, offset, byteLength) 78 val linkToMaster = <p><a href={worker.activeMasterWebUiUrl}>Back to Master</a></p> 79 val curLogLength = endByte - startByte 80 val range = 81 <span id="log-data"> 82 Showing {curLogLength} Bytes: {startByte.toString} - {endByte.toString} of {logLength} 83 </span> 84 85 val moreButton = 86 <button type="button" onclick={"loadMore()"} class="log-more-btn btn btn-default"> 87 Load More 88 </button> 89 90 val newButton = 91 <button type="button" onclick={"loadNew()"} class="log-new-btn btn btn-default"> 92 Load New 93 </button> 94 95 val alert = 96 <div class="no-new-alert alert alert-info" style="display: none;"> 97 End of Log 98 </div> 99 100 val logParams = "?%s&logType=%s".format(params, logType) 101 val jsOnload = "window.onload = " + 102 s"initLogPage('$logParams', $curLogLength, $startByte, $endByte, $logLength, $byteLength);" 103 104 val content = 105 <div> 106 {linkToMaster} 107 {range} 108 <div class="log-content" style="height:80vh; overflow:auto; padding:5px;"> 109 <div>{moreButton}</div> 110 <pre>{logText}</pre> 111 {alert} 112 <div>{newButton}</div> 113 </div> 114 <script>{Unparsed(jsOnload)}</script> 115 </div> 116 117 UIUtils.basicSparkPage(content, logType + " log page for " + pageName) 118 } 119 120 /** Get the part of the log files given the offset and desired length of bytes */ 121 private def getLog( 122 logDirectory: String, 123 logType: String, 124 offsetOption: Option[Long], 125 byteLength: Int 126 ): (String, Long, Long, Long) = { 127 128 if (!supportedLogTypes.contains(logType)) { 129 return ("Error: Log type must be one of " + supportedLogTypes.mkString(", "), 0, 0, 0) 130 } 131 132 // Verify that the normalized path of the log directory is in the working directory 133 val normalizedUri = new File(logDirectory).toURI.normalize() 134 val normalizedLogDir = new File(normalizedUri.getPath) 135 if (!Utils.isInDirectory(workDir, normalizedLogDir)) { 136 return ("Error: invalid log directory " + logDirectory, 0, 0, 0) 137 } 138 139 try { 140 val files = RollingFileAppender.getSortedRolledOverFiles(logDirectory, logType) 141 logDebug(s"Sorted log files of type $logType in $logDirectory:\n${files.mkString("\n")}") 142 143 val fileLengths: Seq[Long] = files.map(Utils.getFileLength(_, worker.conf)) 144 val totalLength = fileLengths.sum 145 val offset = offsetOption.getOrElse(totalLength - byteLength) 146 val startIndex = { 147 if (offset < 0) { 148 0L 149 } else if (offset > totalLength) { 150 totalLength 151 } else { 152 offset 153 } 154 } 155 val endIndex = math.min(startIndex + byteLength, totalLength) 156 logDebug(s"Getting log from $startIndex to $endIndex") 157 val logText = Utils.offsetBytes(files, fileLengths, startIndex, endIndex) 158 logDebug(s"Got log of length ${logText.length} bytes") 159 (logText, startIndex, endIndex, totalLength) 160 } catch { 161 case e: Exception => 162 logError(s"Error getting $logType logs from directory $logDirectory", e) 163 ("Error getting logs due to exception: " + e.getMessage, 0, 0, 0) 164 } 165 } 166} 167