1/*
2 * Licensed to the Apache Software Foundation (ASF) under one or more
3 * contributor license agreements.  See the NOTICE file distributed with
4 * this work for additional information regarding copyright ownership.
5 * The ASF licenses this file to You under the Apache License, Version 2.0
6 * (the "License"); you may not use this file except in compliance with
7 * the License.  You may obtain a copy of the License at
8 *
9 *    http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
17
18package org.apache.spark.deploy.worker.ui
19
20import java.io.File
21import javax.servlet.http.HttpServletRequest
22
23import scala.xml.{Node, Unparsed}
24
25import com.google.common.cache.{CacheBuilder, CacheLoader, LoadingCache}
26
27import org.apache.spark.internal.Logging
28import org.apache.spark.ui.{UIUtils, WebUIPage}
29import org.apache.spark.util.Utils
30import org.apache.spark.util.logging.RollingFileAppender
31
32private[ui] class LogPage(parent: WorkerWebUI) extends WebUIPage("logPage") with Logging {
33  private val worker = parent.worker
34  private val workDir = new File(parent.workDir.toURI.normalize().getPath)
35  private val supportedLogTypes = Set("stderr", "stdout")
36  private val defaultBytes = 100 * 1024
37
38  def renderLog(request: HttpServletRequest): String = {
39    val appId = Option(request.getParameter("appId"))
40    val executorId = Option(request.getParameter("executorId"))
41    val driverId = Option(request.getParameter("driverId"))
42    val logType = request.getParameter("logType")
43    val offset = Option(request.getParameter("offset")).map(_.toLong)
44    val byteLength = Option(request.getParameter("byteLength")).map(_.toInt).getOrElse(defaultBytes)
45
46    val logDir = (appId, executorId, driverId) match {
47      case (Some(a), Some(e), None) =>
48        s"${workDir.getPath}/$a/$e/"
49      case (None, None, Some(d)) =>
50        s"${workDir.getPath}/$d/"
51      case _ =>
52        throw new Exception("Request must specify either application or driver identifiers")
53    }
54
55    val (logText, startByte, endByte, logLength) = getLog(logDir, logType, offset, byteLength)
56    val pre = s"==== Bytes $startByte-$endByte of $logLength of $logDir$logType ====\n"
57    pre + logText
58  }
59
60  def render(request: HttpServletRequest): Seq[Node] = {
61    val appId = Option(request.getParameter("appId"))
62    val executorId = Option(request.getParameter("executorId"))
63    val driverId = Option(request.getParameter("driverId"))
64    val logType = request.getParameter("logType")
65    val offset = Option(request.getParameter("offset")).map(_.toLong)
66    val byteLength = Option(request.getParameter("byteLength")).map(_.toInt).getOrElse(defaultBytes)
67
68    val (logDir, params, pageName) = (appId, executorId, driverId) match {
69      case (Some(a), Some(e), None) =>
70        (s"${workDir.getPath}/$a/$e/", s"appId=$a&executorId=$e", s"$a/$e")
71      case (None, None, Some(d)) =>
72        (s"${workDir.getPath}/$d/", s"driverId=$d", d)
73      case _ =>
74        throw new Exception("Request must specify either application or driver identifiers")
75    }
76
77    val (logText, startByte, endByte, logLength) = getLog(logDir, logType, offset, byteLength)
78    val linkToMaster = <p><a href={worker.activeMasterWebUiUrl}>Back to Master</a></p>
79    val curLogLength = endByte - startByte
80    val range =
81      <span id="log-data">
82        Showing {curLogLength} Bytes: {startByte.toString} - {endByte.toString} of {logLength}
83      </span>
84
85    val moreButton =
86      <button type="button" onclick={"loadMore()"} class="log-more-btn btn btn-default">
87        Load More
88      </button>
89
90    val newButton =
91      <button type="button" onclick={"loadNew()"} class="log-new-btn btn btn-default">
92        Load New
93      </button>
94
95    val alert =
96      <div class="no-new-alert alert alert-info" style="display: none;">
97        End of Log
98      </div>
99
100    val logParams = "?%s&logType=%s".format(params, logType)
101    val jsOnload = "window.onload = " +
102      s"initLogPage('$logParams', $curLogLength, $startByte, $endByte, $logLength, $byteLength);"
103
104    val content =
105      <div>
106        {linkToMaster}
107        {range}
108        <div class="log-content" style="height:80vh; overflow:auto; padding:5px;">
109          <div>{moreButton}</div>
110          <pre>{logText}</pre>
111          {alert}
112          <div>{newButton}</div>
113        </div>
114        <script>{Unparsed(jsOnload)}</script>
115      </div>
116
117    UIUtils.basicSparkPage(content, logType + " log page for " + pageName)
118  }
119
120  /** Get the part of the log files given the offset and desired length of bytes */
121  private def getLog(
122      logDirectory: String,
123      logType: String,
124      offsetOption: Option[Long],
125      byteLength: Int
126    ): (String, Long, Long, Long) = {
127
128    if (!supportedLogTypes.contains(logType)) {
129      return ("Error: Log type must be one of " + supportedLogTypes.mkString(", "), 0, 0, 0)
130    }
131
132    // Verify that the normalized path of the log directory is in the working directory
133    val normalizedUri = new File(logDirectory).toURI.normalize()
134    val normalizedLogDir = new File(normalizedUri.getPath)
135    if (!Utils.isInDirectory(workDir, normalizedLogDir)) {
136      return ("Error: invalid log directory " + logDirectory, 0, 0, 0)
137    }
138
139    try {
140      val files = RollingFileAppender.getSortedRolledOverFiles(logDirectory, logType)
141      logDebug(s"Sorted log files of type $logType in $logDirectory:\n${files.mkString("\n")}")
142
143      val fileLengths: Seq[Long] = files.map(Utils.getFileLength(_, worker.conf))
144      val totalLength = fileLengths.sum
145      val offset = offsetOption.getOrElse(totalLength - byteLength)
146      val startIndex = {
147        if (offset < 0) {
148          0L
149        } else if (offset > totalLength) {
150          totalLength
151        } else {
152          offset
153        }
154      }
155      val endIndex = math.min(startIndex + byteLength, totalLength)
156      logDebug(s"Getting log from $startIndex to $endIndex")
157      val logText = Utils.offsetBytes(files, fileLengths, startIndex, endIndex)
158      logDebug(s"Got log of length ${logText.length} bytes")
159      (logText, startIndex, endIndex, totalLength)
160    } catch {
161      case e: Exception =>
162        logError(s"Error getting $logType logs from directory $logDirectory", e)
163        ("Error getting logs due to exception: " + e.getMessage, 0, 0, 0)
164    }
165  }
166}
167