xref: /dports/devel/hadoop/hadoop-1.2.1/src/mapred/org/apache/hadoop/mapred/JobHistory.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.mapred;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.Map.Entry;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.mapreduce.JobACL;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.DiskChecker.DiskErrorException;

/**
 * Provides methods for writing to and reading from job history.
 * Job History works in an append mode, JobHistory and its inner classes provide methods
 * to log job events.
 *
 * JobHistory is split into multiple files, format of each file is plain text where each line
 * is of the format [type (key=value)*], where type identifies the type of the record.
 * Type maps to UID of one of the inner classes of this class.
 *
 * Job history is maintained in a master index which contains star/stop times of all jobs with
 * a few other job level properties. Apart from this each job's history is maintained in a seperate history
 * file. name of job history files follows the format jobtrackerId_jobid
 *
 * For parsing the job history it supports a listener based interface where each line is parsed
 * and passed to listener. The listener can create an object model of history or look for specific
 * events and discard rest of the history.
 *
 * CHANGE LOG :
 * Version 0 : The history has the following format :
 *             TAG KEY1="VALUE1" KEY2="VALUE2" and so on.
               TAG can be Job, Task, MapAttempt or ReduceAttempt.
               Note that a '"' is the line delimiter.
 * Version 1 : Changes the line delimiter to '.'
               Values are now escaped for unambiguous parsing.
               Added the Meta tag to store version info.
 */
public class JobHistory {

  static final long VERSION = 1L;

  static final int DONE_DIRECTORY_FORMAT_VERSION = 1;

  static final String DONE_DIRECTORY_FORMAT_DIRNAME
    = "version-" + DONE_DIRECTORY_FORMAT_VERSION;

  static final String UNDERSCORE_ESCAPE = "%5F";

  public static final Log LOG = LogFactory.getLog(JobHistory.class);
  private static final char DELIMITER = ' ';
  static final char LINE_DELIMITER_CHAR = '.';
  static final char[] charsToEscape = new char[] {'"', '=',
                                                LINE_DELIMITER_CHAR};
  static final String DIGITS = "[0-9]+";

  static final String KEY = "(\\w+)";
  // value is any character other than quote, but escaped quotes can be there
  static final String VALUE = "[^\"\\\\]*+(?:\\\\.[^\"\\\\]*+)*+";

  static final Pattern pattern = Pattern.compile(KEY + "=" + "\"" + VALUE + "\"");

  static final int MAXIMUM_DATESTRING_COUNT = 200000;

  public static final int JOB_NAME_TRIM_LENGTH = 50;
  private static String JOBTRACKER_UNIQUE_STRING = null;
  private static String LOG_DIR = null;
  private static final String SECONDARY_FILE_SUFFIX = ".recover";
  private static long jobHistoryBlockSize = 0;
  private static String jobtrackerHostname;
  private static JobHistoryFilesManager fileManager = null;
  final static FsPermission HISTORY_DIR_PERMISSION =
    FsPermission.createImmutable((short) 0755); // rwxr-xr-x
  final static FsPermission HISTORY_FILE_PERMISSION =
    FsPermission.createImmutable((short) 0744); // rwxr--r--
  private static FileSystem LOGDIR_FS; // log dir filesystem
  protected static FileSystem DONEDIR_FS; // Done dir filesystem
  private static JobConf jtConf;
  protected static Path DONE = null; // folder for completed jobs
  private static String DONE_BEFORE_SERIAL_TAIL = doneSubdirsBeforeSerialTail();
  private static String DONE_LEAF_FILES = DONE_BEFORE_SERIAL_TAIL + "/*";
  private static boolean aclsEnabled = false;

  static final String CONF_FILE_NAME_SUFFIX = "_conf.xml";

  private static final int SERIAL_NUMBER_DIRECTORY_DIGITS = 6;
  private static int SERIAL_NUMBER_LOW_DIGITS;

  private static String SERIAL_NUMBER_FORMAT;

  private static final Set<Path> existingDoneSubdirs = new HashSet<Path>();

  private static final SortedMap<Integer, String> idToDateString
    = new TreeMap<Integer, String>();

  /**
   * A filter for conf files
   */
  private static final PathFilter CONF_FILTER = new PathFilter() {
    public boolean accept(Path path) {
      return path.getName().endsWith(CONF_FILE_NAME_SUFFIX);
    }
  };

  private static final Map<JobID, MovedFileInfo> jobHistoryFileMap =
    Collections.<JobID,MovedFileInfo>synchronizedMap(
        new LinkedHashMap<JobID, MovedFileInfo>());

  private static final SortedMap<Long, String>jobToDirectoryMap
    = new TreeMap<Long, String>();

  // JobHistory filename regex
  public static final Pattern JOBHISTORY_FILENAME_REGEX =
    Pattern.compile("(" + JobID.JOBID_REGEX + ")_.+");
  // JobHistory conf-filename regex
  public static final Pattern CONF_FILENAME_REGEX =
    Pattern.compile("(" + JobID.JOBID_REGEX + ")_conf.xml");

  private static class MovedFileInfo {
    private final String historyFile;
    private final long timestamp;
    public MovedFileInfo(String historyFile, long timestamp) {
      this.historyFile = historyFile;
      this.timestamp = timestamp;
    }
  }

  /**
   * Given the job id, return the history file path from the cache
   */
  public static String getHistoryFilePath(JobID jobId) {
    MovedFileInfo info = jobHistoryFileMap.get(jobId);
    if (info == null) {
      return null;
    }
    return info.historyFile;
  }

  /**
   * A class that manages all the files related to a job. For now
   *   - writers : list of open files
   *   - job history filename
   *   - job conf filename
   */
  private static class JobHistoryFilesManager {
    // a private (virtual) folder for all the files related to a running job
    private static class FilesHolder {
      ArrayList<PrintWriter> writers = new ArrayList<PrintWriter>();
      Path historyFilename; // path of job history file
      Path confFilename; // path of job's conf
    }

    private ThreadPoolExecutor executor = null;
    private final Configuration conf;
    private final JobTracker jobTracker;

   // cache from job-key to files associated with it.
    private Map<JobID, FilesHolder> fileCache =
      new ConcurrentHashMap<JobID, FilesHolder>();

    JobHistoryFilesManager(Configuration conf, JobTracker jobTracker)
        throws IOException {
      this.conf = conf;
      this.jobTracker = jobTracker;
    }


    void start() {
      executor = new ThreadPoolExecutor(5, 5, 1,
          TimeUnit.HOURS, new LinkedBlockingQueue<Runnable>());
      // make core threads to terminate if there has been no work
      // for the keppalive period.
      executor.allowCoreThreadTimeOut(true);
    }

    private FilesHolder getFileHolder(JobID id) {
      FilesHolder holder = fileCache.get(id);
     if (holder == null) {
         holder = new FilesHolder();
         fileCache.put(id, holder);
      }
      return holder;
    }

    void addWriter(JobID id, PrintWriter writer) {
      FilesHolder holder = getFileHolder(id);
      holder.writers.add(writer);
    }

    void setHistoryFile(JobID id, Path file) {
      FilesHolder holder = getFileHolder(id);
      holder.historyFilename = file;
    }

    void setConfFile(JobID id, Path file) {
      FilesHolder holder = getFileHolder(id);
      holder.confFilename = file;
    }

    ArrayList<PrintWriter> getWriters(JobID id) {
      FilesHolder holder = fileCache.get(id);
      return holder == null ? null : holder.writers;
    }

    Path getHistoryFile(JobID id) {
      FilesHolder holder = fileCache.get(id);
      return holder == null ? null : holder.historyFilename;
    }

    Path getConfFileWriters(JobID id) {
      FilesHolder holder = fileCache.get(id);
      return holder == null ? null : holder.confFilename;
    }

    void purgeJob(JobID id) {
      fileCache.remove(id);
    }

    void moveToDone(final JobID id) {
      final List<Path> paths = new ArrayList<Path>();
      final Path historyFile = fileManager.getHistoryFile(id);
      if (historyFile == null) {
        LOG.info("No file for job-history with " + id + " found in cache!");
      } else {
        paths.add(historyFile);
      }

      final Path confPath = fileManager.getConfFileWriters(id);
      if (confPath == null) {
        LOG.info("No file for jobconf with " + id + " found in cache!");
      } else {
        paths.add(confPath);
      }

      executor.execute(new Runnable() {

        public void run() {
          long millisecondTime = System.currentTimeMillis();

          Path resultDir = canonicalHistoryLogPath(id, millisecondTime);

          //move the files to DONE canonical subfolder
          try {
            for (Path path : paths) {
              //check if path exists, in case of retries it may not exist
              if (LOGDIR_FS.exists(path)) {
                maybeMakeSubdirectory(id, millisecondTime);

                LOG.info("Moving " + path.toString() + " to " +
                    resultDir.toString());
                DONEDIR_FS.moveFromLocalFile(path, resultDir);
                DONEDIR_FS.setPermission(new Path(resultDir, path.getName()),
                    new FsPermission(HISTORY_FILE_PERMISSION));
              }
            }
          } catch (Throwable e) {
            LOG.error("Unable to move history file to DONE canonical subfolder.", e);
          }
          String historyFileDonePath = null;
          if (historyFile != null) {
            historyFileDonePath = new Path(resultDir,
                historyFile.getName()).toString();
          }

          jobHistoryFileMap.put(id, new MovedFileInfo(historyFileDonePath,
                                                      millisecondTime));
          jobTracker.historyFileCopied(id, historyFileDonePath);

          //purge the job from the cache
          fileManager.purgeJob(id);
        }

      });
    }

    void removeWriter(JobID jobId, PrintWriter writer) {
      fileManager.getWriters(jobId).remove(writer);
    }
  }

  // several methods for manipulating the subdirectories of the DONE
  // directory

  private static int jobSerialNumber(JobID id) {
    return id.getId();
  }

  private static String serialNumberDirectoryComponent(JobID id) {
    return String.format(SERIAL_NUMBER_FORMAT,
                         Integer.valueOf(jobSerialNumber(id)))
              .substring(0, SERIAL_NUMBER_DIRECTORY_DIGITS);
  }

  // directory components may contain internal slashes, but do NOT
  // contain slashes at either end.

  private static String timestampDirectoryComponent(JobID id, long millisecondTime) {
    int serialNumber = jobSerialNumber(id);
    Integer boxedSerialNumber = serialNumber;

    // don't want to do this inside the lock
    Calendar timestamp = Calendar.getInstance();
    timestamp.setTimeInMillis(millisecondTime);

    synchronized (idToDateString) {
      String dateString = idToDateString.get(boxedSerialNumber);

      if (dateString == null) {

        dateString = String.format
          ("%04d/%02d/%02d",
           timestamp.get(Calendar.YEAR),
           // months are 0-based in Calendar, but people will expect January
           // to be month #1.
            timestamp.get(Calendar.MONTH) + 1,
            timestamp.get(Calendar.DAY_OF_MONTH));

        dateString = dateString.intern();

        idToDateString.put(boxedSerialNumber, dateString);

        if (idToDateString.size() > MAXIMUM_DATESTRING_COUNT) {
          idToDateString.remove(idToDateString.firstKey());
        }
      }

      return dateString;
    }
  }

  // returns false iff the directory already existed
  private static boolean maybeMakeSubdirectory(JobID id, long millisecondTime)
          throws IOException {
    Path dir = canonicalHistoryLogPath(id, millisecondTime);

    synchronized (existingDoneSubdirs) {
      if (existingDoneSubdirs.contains(dir)) {
        if (LOG.isDebugEnabled() && !DONEDIR_FS.exists(dir)) {
          LOG.error("JobHistory.maybeMakeSubdirectory -- We believed " + dir
              + " already existed, but it didn't.");
        }

        return true;
      }

      if (!DONEDIR_FS.exists(dir)) {
        LOG.info("Creating DONE subfolder at "+ dir);

        if (!FileSystem.mkdirs(DONEDIR_FS, dir,
                               new FsPermission(HISTORY_DIR_PERMISSION))) {
          throw new IOException("Mkdirs failed to create " + dir.toString());
        }

        existingDoneSubdirs.add(dir);

        return false;
      } else {
        if (LOG.isDebugEnabled()) {
          LOG.error("JobHistory.maybeMakeSubdirectory -- We believed " + dir
              + " didn't already exist, but it did.");
        }

        return false;
      }
    }
  }

  private static Path canonicalHistoryLogPath(JobID id, long millisecondTime) {
    return new Path(DONE, historyLogSubdirectory(id, millisecondTime));
  }

  private static String historyLogSubdirectory(JobID id, long millisecondTime) {
    String result
      = (DONE_DIRECTORY_FORMAT_DIRNAME
         + "/" + jobtrackerDirectoryComponent(id));

    String serialNumberDirectory = serialNumberDirectoryComponent(id);

    result = (result
              + "/" + timestampDirectoryComponent(id, millisecondTime)
              + "/" + serialNumberDirectory
              + "/");

    return result;
  }

  private static String jobtrackerDirectoryComponent(JobID id) {
    return JOBTRACKER_UNIQUE_STRING;
  }

  private static String doneSubdirsBeforeSerialTail() {
    // job tracker ID
    String result
      = ("/" + DONE_DIRECTORY_FORMAT_DIRNAME
         + "/*");   // job tracker instance ID

    // date
    result = result + "/*/*/*";  // YYYY/MM/DD ;

    return result;
  }

  /**
   * Record types are identifiers for each line of log in history files.
   * A record type appears as the first token in a single line of log.
   */
  public static enum RecordTypes {
    Jobtracker, Job, Task, MapAttempt, ReduceAttempt, Meta
  }

  /**
   * Job history files contain key="value" pairs, where keys belong to this enum.
   * It acts as a global namespace for all keys.
   */
  public static enum Keys {
    JOBTRACKERID,
    START_TIME, FINISH_TIME, JOBID, JOBNAME, USER, JOBCONF, SUBMIT_TIME,
    LAUNCH_TIME, TOTAL_MAPS, TOTAL_REDUCES, FAILED_MAPS, FAILED_REDUCES,
    FINISHED_MAPS, FINISHED_REDUCES, JOB_STATUS, TASKID, HOSTNAME, TASK_TYPE,
    ERROR, TASK_ATTEMPT_ID, TASK_STATUS, COPY_PHASE, SORT_PHASE, REDUCE_PHASE,
    SHUFFLE_FINISHED, SORT_FINISHED, COUNTERS, SPLITS, JOB_PRIORITY, HTTP_PORT,
    TRACKER_NAME, STATE_STRING, VERSION, MAP_COUNTERS, REDUCE_COUNTERS,
    VIEW_JOB, MODIFY_JOB, JOB_QUEUE, FAIL_REASON, LOCALITY, AVATAAR,
    WORKFLOW_ID, WORKFLOW_NAME, WORKFLOW_NODE_NAME, WORKFLOW_ADJACENCIES,
    WORKFLOW_TAGS
  }

  /**
   * This enum contains some of the values commonly used by history log events.
   * since values in history can only be strings - Values.name() is used in
   * most places in history file.
   */
  public static enum Values {
    SUCCESS, FAILED, KILLED, MAP, REDUCE, CLEANUP, RUNNING, PREP, SETUP
  }

  /**
   * Initialize JobHistory files.
   * @param conf Jobconf of the job tracker.
   * @param hostname jobtracker's hostname
   * @param jobTrackerStartTime jobtracker's start time
   */
  public static void init(JobTracker jobTracker, JobConf conf,
             String hostname, long jobTrackerStartTime) throws IOException {
    initLogDir(conf);
    SERIAL_NUMBER_LOW_DIGITS = 3;
    SERIAL_NUMBER_FORMAT = ("%0"
       + (SERIAL_NUMBER_DIRECTORY_DIGITS + SERIAL_NUMBER_LOW_DIGITS)
       + "d");
    JOBTRACKER_UNIQUE_STRING = hostname + "_" +
                                  String.valueOf(jobTrackerStartTime) + "_";
    jobtrackerHostname = hostname;
    Path logDir = new Path(LOG_DIR);
    if (!LOGDIR_FS.exists(logDir)){
      if (!LOGDIR_FS.mkdirs(logDir, new FsPermission(HISTORY_DIR_PERMISSION))) {
        throw new IOException("Mkdirs failed to create " + logDir.toString());
      }
    } else { // directory exists
      checkDirectoryPermissions(LOGDIR_FS, logDir, "hadoop.job.history.location");
    }
    conf.set("hadoop.job.history.location", LOG_DIR);
    // set the job history block size (default is 3MB)
    jobHistoryBlockSize =
      conf.getLong("mapred.jobtracker.job.history.block.size",
                   3 * 1024 * 1024);
    jtConf = conf;

    // queue and job level security is enabled on the mapreduce cluster or not
    aclsEnabled = conf.getBoolean(JobConf.MR_ACLS_ENABLED, false);

    // initialize the file manager
    fileManager = new JobHistoryFilesManager(conf, jobTracker);
  }

  private static void initLogDir(JobConf conf) throws IOException {
    LOG_DIR = conf.get("hadoop.job.history.location" ,
      "file:///" + new File(
      System.getProperty("hadoop.log.dir")).getAbsolutePath()
      + File.separator + "history");
    Path logDir = new Path(LOG_DIR);
    LOGDIR_FS = logDir.getFileSystem(conf);
  }

  static void initDone(JobConf conf, FileSystem fs) throws IOException {
    initDone(conf, fs, true);
  }

  static void initDone(JobConf conf, FileSystem fs,
                                     boolean setup)
      throws IOException {
    //if completed job history location is set, use that
    String doneLocation = conf.
                     get("mapred.job.tracker.history.completed.location");
    if (doneLocation != null) {
      DONE = fs.makeQualified(new Path(doneLocation));
      DONEDIR_FS = fs;
    } else {
      if (!setup) {
        initLogDir(conf);
      }
      DONE = new Path(LOG_DIR, "done");
      DONEDIR_FS = LOGDIR_FS;
    }
    Path versionSubdir = new Path(DONE, DONE_DIRECTORY_FORMAT_DIRNAME);
    //If not already present create the done folder with appropriate
    //permission
    if (!DONEDIR_FS.exists(DONE)) {
      LOG.info("Creating DONE folder at "+ DONE);
      if (!DONEDIR_FS.mkdirs(DONE,
          new FsPermission(HISTORY_DIR_PERMISSION))) {
        throw new IOException("Mkdirs failed to create " + DONE.toString());
      }

      if (!DONEDIR_FS.exists(versionSubdir)) {
        if (!DONEDIR_FS.mkdirs(versionSubdir,
                               new FsPermission(HISTORY_DIR_PERMISSION))) {
          throw new IOException("Mkdirs failed to create " + versionSubdir);
        }
      }
    } else { // directory exists. Checks version subdirectory permissions as
      // well.
      checkDirectoryPermissions(DONEDIR_FS, DONE,
          "mapred.job.tracker.history.completed.location");
      if (DONEDIR_FS.exists(versionSubdir))
        checkDirectoryPermissions(DONEDIR_FS, versionSubdir,
            "mapred.job.tracker.history.completed.location-versionsubdir");
    }

    if (!setup) {
      return;
    }

    fileManager.start();

    HistoryCleaner.cleanupFrequency =
    	      conf.getLong("mapreduce.jobhistory.cleaner.interval-ms",
    	      HistoryCleaner.DEFAULT_CLEANUP_FREQUENCY);
    HistoryCleaner.maxAgeOfHistoryFiles =
    	      conf.getLong("mapreduce.jobhistory.max-age-ms",
    	      HistoryCleaner.DEFAULT_HISTORY_MAX_AGE);
    LOG.info(String.format("Job History MaxAge is %d ms (%.2f days), " +
    	      "Cleanup Frequency is %d ms (%.2f days)",
    	      HistoryCleaner.maxAgeOfHistoryFiles,
    	      ((float) HistoryCleaner.maxAgeOfHistoryFiles)/HistoryCleaner.ONE_DAY_IN_MS,
    	      HistoryCleaner.cleanupFrequency,
    	      ((float) HistoryCleaner.cleanupFrequency)/HistoryCleaner.ONE_DAY_IN_MS));
  }

  /**
   * @param FileSystem
   * @param Path
   * @param configKey
   * @throws IOException
   * @throws DiskErrorException
   */
  static void checkDirectoryPermissions(FileSystem fs, Path path,
      String configKey) throws IOException, DiskErrorException {
    FileStatus stat = fs.getFileStatus(path);
    FsPermission actual = stat.getPermission();
    if (!stat.isDir())
      throw new DiskErrorException(configKey + " - not a directory: "
          + path.toString());
    FsAction user = actual.getUserAction();
    if (!user.implies(FsAction.READ))
      throw new DiskErrorException("bad " + configKey
          + "- directory is not readable: " + path.toString());
    if (!user.implies(FsAction.WRITE))
      throw new DiskErrorException("bad " + configKey
          + "- directory is not writable " + path.toString());
  }

  /**
   * Manages job-history's meta information such as version etc.
   * Helps in logging version information to the job-history and recover
   * version information from the history.
   */
  static class MetaInfoManager implements Listener {
    private long version = 0L;
    private KeyValuePair pairs = new KeyValuePair();

    // Extract the version of the history that was used to write the history
    public MetaInfoManager(String line) throws IOException {
      if (null != line) {
        // Parse the line
        parseLine(line, this, false);
      }
    }

    // Get the line delimiter
    char getLineDelim() {
      if (version == 0) {
        return '"';
      } else {
        return LINE_DELIMITER_CHAR;
      }
    }

    // Checks if the values are escaped or not
    boolean isValueEscaped() {
      // Note that the values are not escaped in version 0
      return version != 0;
    }

    public void handle(RecordTypes recType, Map<Keys, String> values)
    throws IOException {
      // Check if the record is of type META
      if (RecordTypes.Meta == recType) {
        pairs.handle(values);
        version = pairs.getLong(Keys.VERSION); // defaults to 0
      }
    }

    /**
     * Logs history meta-info to the history file. This needs to be called once
     * per history file.
     * @param jobId job id, assigned by jobtracker.
     */
    static void logMetaInfo(ArrayList<PrintWriter> writers){
      if (null != writers){
         JobHistory.log(writers, RecordTypes.Meta,
             new Keys[] {Keys.VERSION},
             new String[] {String.valueOf(VERSION)});
      }
    }
  }

  /** Escapes the string especially for {@link JobHistory}
   */
  static String escapeString(String data) {
    return StringUtils.escapeString(data, StringUtils.ESCAPE_CHAR,
                                    charsToEscape);
  }

  /**
   * Parses history file and invokes Listener.handle() for
   * each line of history. It can be used for looking through history
   * files for specific items without having to keep whole history in memory.
   * @param path path to history file
   * @param l Listener for history events
   * @param fs FileSystem where history file is present
   * @throws IOException
   */
  public static void parseHistoryFromFS(String path, Listener l, FileSystem fs)
  throws IOException{
    FSDataInputStream in = fs.open(new Path(path));
    BufferedReader reader = new BufferedReader(new InputStreamReader (in));
    try {
      String line = null;
      StringBuffer buf = new StringBuffer();

      // Read the meta-info line. Note that this might a jobinfo line for files
      // written with older format
      line = reader.readLine();

      // Check if the file is empty
      if (line == null) {
        return;
      }

      // Get the information required for further processing
      MetaInfoManager mgr = new MetaInfoManager(line);
      boolean isEscaped = mgr.isValueEscaped();
      String lineDelim = String.valueOf(mgr.getLineDelim());
      String escapedLineDelim =
        StringUtils.escapeString(lineDelim, StringUtils.ESCAPE_CHAR,
                                 mgr.getLineDelim());

      do {
        buf.append(line);
        if (!line.trim().endsWith(lineDelim)
            || line.trim().endsWith(escapedLineDelim)) {
          buf.append("\n");
          continue;
        }
        parseLine(buf.toString(), l, isEscaped);
        buf = new StringBuffer();
      } while ((line = reader.readLine())!= null);
    } finally {
      try { reader.close(); } catch (IOException ex) {}
    }
  }

  /**
   * Parse a single line of history.
   * @param line
   * @param l
   * @throws IOException
   */
  private static void parseLine(String line, Listener l, boolean isEscaped)
  throws IOException{
    // extract the record type
    int idx = line.indexOf(' ');
    String recType = line.substring(0, idx);
    String data = line.substring(idx+1, line.length());

    Matcher matcher = pattern.matcher(data);
    Map<Keys,String> parseBuffer = new HashMap<Keys, String>();

    while(matcher.find()){
      String tuple = matcher.group(0);
      String []parts = StringUtils.split(tuple, StringUtils.ESCAPE_CHAR, '=');
      String value = parts[1].substring(1, parts[1].length() -1);
      if (isEscaped) {
        value = StringUtils.unEscapeString(value, StringUtils.ESCAPE_CHAR,
                                           charsToEscape);
      }
      parseBuffer.put(Keys.valueOf(parts[0]), value);
    }

    l.handle(RecordTypes.valueOf(recType), parseBuffer);

    parseBuffer.clear();
  }


  /**
   * Log a raw record type with keys and values. This is method is generally not used directly.
   * @param recordType type of log event
   * @param key key
   * @param value value
   */

  static void log(PrintWriter out, RecordTypes recordType, Keys key,
                  String value){
    value = escapeString(value);
    out.println(recordType.name() + DELIMITER + key + "=\"" + value + "\""
                + DELIMITER + LINE_DELIMITER_CHAR);
  }

  /**
   * Log a number of keys and values with record. the array length of keys and values
   * should be same.
   * @param recordType type of log event
   * @param keys type of log event
   * @param values type of log event
   */

  /**
   * Log a number of keys and values with record. the array length of keys and values
   * should be same.
   * @param recordType type of log event
   * @param keys type of log event
   * @param values type of log event
   */

  static void log(ArrayList<PrintWriter> writers, RecordTypes recordType,
                  Keys[] keys, String[] values) {
    log(writers, recordType, keys, values, null);
  }

  static class JobHistoryLogger {
    static final Log LOG = LogFactory.getLog(JobHistoryLogger.class);
  }

  /**
   * Log a number of keys and values with record. the array length of keys and values
   * should be same.
   * @param recordType type of log event
   * @param keys type of log event
   * @param values type of log event
   * @param JobID jobid of the job
   */

  static void log(ArrayList<PrintWriter> writers, RecordTypes recordType,
                  Keys[] keys, String[] values, JobID id) {

    // First up calculate the length of buffer, so that we are performant
    // enough.
    int length = recordType.name().length() + keys.length * 4 + 2;
    for (int i = 0; i < keys.length; i++) {
      values[i] = escapeString(values[i]);
      length += values[i].length() + keys[i].toString().length();
    }

    // We have the length of the buffer, now construct it.
    StringBuilder builder = new StringBuilder(length);
    builder.append(recordType.name());
    builder.append(DELIMITER);
    for(int i =0; i< keys.length; i++){
      builder.append(keys[i]);
      builder.append("=\"");
      builder.append(values[i]);
      builder.append("\"");
      builder.append(DELIMITER);
    }
    builder.append(LINE_DELIMITER_CHAR);

    String logLine = builder.toString();
    for (Iterator<PrintWriter> iter = writers.iterator(); iter.hasNext();) {
      PrintWriter out = iter.next();
      out.println(logLine);
      if (out.checkError() && id != null) {
        LOG.info("Logging failed for job " + id + "removing PrintWriter from FileManager");
        iter.remove();
      }
    }
    if (recordType != RecordTypes.Meta) {
      JobHistoryLogger.LOG.debug(logLine);
    }
  }

  /**
   * Get the history location
   */
  static Path getJobHistoryLocation() {
    return new Path(LOG_DIR);
  }

  /**
   * Get the history location for completed jobs
   */
  static Path getCompletedJobHistoryLocation() {
    return DONE;
  }

  static int serialNumberDirectoryDigits() {
    return SERIAL_NUMBER_DIRECTORY_DIGITS;
  }

  static int serialNumberTotalDigits() {
    return serialNumberDirectoryDigits() + SERIAL_NUMBER_LOW_DIGITS;
  }

  /**
   * Get the
   */

  /**
   * Base class contais utility stuff to manage types key value pairs with enums.
   */
  static class KeyValuePair{
    private Map<Keys, String> values = new HashMap<Keys, String>();

    /**
     * Get 'String' value for given key. Most of the places use Strings as
     * values so the default get' method returns 'String'.  This method never returns
     * null to ease on GUIs. if no value is found it returns empty string ""
     * @param k
     * @return if null it returns empty string - ""
     */
    public String get(Keys k){
      String s = values.get(k);
      return s == null ? "" : s;
    }
    /**
     * Convert value from history to int and return.
     * if no value is found it returns 0.
     * @param k key
     */
    public int getInt(Keys k){
      String s = values.get(k);
      if (null != s){
        return Integer.parseInt(s);
      }
      return 0;
    }
    /**
     * Convert value from history to int and return.
     * if no value is found it returns 0.
     * @param k
     */
    public long getLong(Keys k){
      String s = values.get(k);
      if (null != s){
        return Long.parseLong(s);
      }
      return 0;
    }
    /**
     * Set value for the key.
     * @param k
     * @param s
     */
    public void set(Keys k, String s){
      values.put(k, s);
    }
    /**
     * Adds all values in the Map argument to its own values.
     * @param m
     */
    public void set(Map<Keys, String> m){
      values.putAll(m);
    }
    /**
     * Reads values back from the history, input is same Map as passed to Listener by parseHistory().
     * @param values
     */
    public synchronized void handle(Map<Keys, String> values){
      set(values);
    }
    /**
     * Returns Map containing all key-values.
     */
    public Map<Keys, String> getValues(){
      return values;
    }
  }

  // hasMismatches is just used to return a second value if you want
  // one.  I would have used MutableBoxedBoolean if such had been provided.
  static Path[] filteredStat2Paths
          (FileStatus[] stats, boolean dirs, AtomicBoolean hasMismatches) {
    int resultCount = 0;

    if (hasMismatches == null) {
      hasMismatches = new AtomicBoolean(false);
    }

    for (int i = 0; i < stats.length; ++i) {
      if (stats[i].isDir() == dirs) {
        stats[resultCount++] = stats[i];
      } else {
        hasMismatches.set(true);
      }
    }

    Path[] paddedResult = FileUtil.stat2Paths(stats);

    Path[] result = new Path[resultCount];

    System.arraycopy(paddedResult, 0, result, 0, resultCount);

    return result;
  }

  static FileStatus[] localGlobber
        (FileSystem fs, Path root, String tail)
      throws IOException {
    return localGlobber(fs, root, tail, null);
  }

  static FileStatus[] localGlobber
        (FileSystem fs, Path root, String tail, PathFilter filter)
      throws IOException {
    return localGlobber(fs, root, tail, filter, null);
  }

  private static FileStatus[] nullToEmpty(FileStatus[] result) {
    return result == null ? new FileStatus[0] : result;
  }

  private static FileStatus[] listFilteredStatus
        (FileSystem fs, Path root, PathFilter filter)
     throws IOException {
    return filter == null ? fs.listStatus(root) : fs.listStatus(root, filter);
  }

  // hasMismatches is just used to return a second value if you want
  // one.  I would have used MutableBoxedBoolean if such had been provided.
  static FileStatus[] localGlobber
    (FileSystem fs, Path root, String tail, PathFilter filter,
     AtomicBoolean hasFlatFiles)
      throws IOException {
    if (tail.equals("")) {
      return nullToEmpty(listFilteredStatus(fs, root, filter));
    }

    if (tail.startsWith("/*")) {
      Path[] subdirs = filteredStat2Paths(nullToEmpty(fs.listStatus(root)),
                                          true, hasFlatFiles);

      FileStatus[][] subsubdirs = new FileStatus[subdirs.length][];

      int subsubdirCount = 0;

      if (subsubdirs.length == 0) {
        return new FileStatus[0];
      }

      String newTail = tail.substring(2);

      for (int i = 0; i < subdirs.length; ++i) {
        subsubdirs[i] = localGlobber(fs, subdirs[i], newTail, filter, null);
        subsubdirCount += subsubdirs[i].length;
      }

      FileStatus[] result = new FileStatus[subsubdirCount];

      int segmentStart = 0;

      for (int i = 0; i < subsubdirs.length; ++i) {
        System.arraycopy(subsubdirs[i], 0, result, segmentStart, subsubdirs[i].length);
        segmentStart += subsubdirs[i].length;
      }

      return result;
    }

    if (tail.startsWith("/")) {
      int split = tail.indexOf('/', 1);

      if (split < 0) {
        return nullToEmpty
          (listFilteredStatus(fs, new Path(root, tail.substring(1)), filter));
      } else {
        String thisSegment = tail.substring(1, split);
        String newTail = tail.substring(split);
        return localGlobber
          (fs, new Path(root, thisSegment), newTail, filter, hasFlatFiles);
      }
    }

    IOException e = new IOException("localGlobber: bad tail");

    throw e;
  }

  static Path confPathFromLogFilePath(Path logFile) {
    String jobId = jobIdNameFromLogFileName(logFile.getName());

    Path logDir = logFile.getParent();

    return new Path(logDir, jobId + CONF_FILE_NAME_SUFFIX);
  }

  static String jobIdNameFromLogFileName(String logFileName) {
    String[] jobDetails = logFileName.split("_");
    return jobDetails[0] + "_" + jobDetails[1] + "_" + jobDetails[2];
  }

  static String userNameFromLogFileName(String logFileName) {
    String[] jobDetails = logFileName.split("_");
    return jobDetails[3];
  }

  static String jobNameFromLogFileName(String logFileName) {
    String[] jobDetails = logFileName.split("_");
    return jobDetails[4];
  }


  // This code will be inefficient if the subject contains dozens of underscores
  static String escapeUnderscores(String escapee) {
    return replaceStringInstances(escapee, "_", UNDERSCORE_ESCAPE);
  }

  static String nonOccursString(String logFileName) {
    int adHocIndex = 0;

    String unfoundString = "q" + adHocIndex;

    while (logFileName.contains(unfoundString)) {
      unfoundString = "q" + ++adHocIndex;
    }

    return unfoundString + "q";
  }

  // I tolerate this code because I expect a low number of
  // occurrences in a relatively short string
  static String replaceStringInstances
      (String logFileName, String old, String replacement) {
    int index = logFileName.indexOf(old);

    while (index > 0) {
      logFileName = (logFileName.substring(0, index)
                     + replacement
                     + replaceStringInstances
                         (logFileName.substring(index + old.length()),
                          old, replacement));

      index = logFileName.indexOf(old);
    }

    return logFileName;
  }


  /**
   * Helper class for logging or reading back events related to job start, finish or failure.
   */
  public static class JobInfo extends KeyValuePair{

    private Map<String, Task> allTasks = new TreeMap<String, Task>();
    private Map<JobACL, AccessControlList> jobACLs =
        new HashMap<JobACL, AccessControlList>();
    private String queueName = null;// queue to which this job was submitted to

    /** Create new JobInfo */
    public JobInfo(String jobId){
      set(Keys.JOBID, jobId);
    }

    /**
     * Returns all map and reduce tasks <taskid-Task>.
     */
    public Map<String, Task> getAllTasks() { return allTasks; }

    /**
     * Get the job acls.
     *
     * @return a {@link Map} from {@link JobACL} to {@link AccessControlList}
     */
    public Map<JobACL, AccessControlList> getJobACLs() {
      return jobACLs;
    }

    @Override
    public synchronized void handle(Map<Keys, String> values) {
      if (values.containsKey(Keys.SUBMIT_TIME)) {// job submission
        // construct the job ACLs
        String viewJobACL = values.get(Keys.VIEW_JOB);
        String modifyJobACL = values.get(Keys.MODIFY_JOB);
        if (viewJobACL != null) {
          jobACLs.put(JobACL.VIEW_JOB, new AccessControlList(viewJobACL));
        }
        if (modifyJobACL != null) {
          jobACLs.put(JobACL.MODIFY_JOB, new AccessControlList(modifyJobACL));
        }
        // get the job queue name
        queueName = values.get(Keys.JOB_QUEUE);
      }
      super.handle(values);
    }

    String getJobQueue() {
      return queueName;
    }

    /**
     * Get the path of the locally stored job file
     * @param jobId id of the job
     * @return the path of the job file on the local file system
     */
    public static String getLocalJobFilePath(JobID jobId){
      return System.getProperty("hadoop.log.dir") + File.separator +
               jobId + CONF_FILE_NAME_SUFFIX;
    }


    /**
     * Helper function to encode the URL of the path of the job-history
     * log file.
     *
     * @param logFile path of the job-history file
     * @return URL encoded path
     * @throws IOException
     */
    public static String encodeJobHistoryFilePath(String logFile)
    throws IOException {
      Path rawPath = new Path(logFile);
      String encodedFileName = null;
      try {
        encodedFileName = URLEncoder.encode(rawPath.getName(), "UTF-8");
      } catch (UnsupportedEncodingException uee) {
        IOException ioe = new IOException();
        ioe.initCause(uee);
        ioe.setStackTrace(uee.getStackTrace());
        throw ioe;
      }

      Path encodedPath = new Path(rawPath.getParent(), encodedFileName);
      return encodedPath.toString();
    }

    /**
     * Helper function to encode the URL of the filename of the job-history
     * log file.
     *
     * @param logFileName file name of the job-history file
     * @return URL encoded filename
     * @throws IOException
     */
    public static String encodeJobHistoryFileName(String logFileName)
    throws IOException {
      String replacementUnderscoreEscape = null;

      if (logFileName.contains(UNDERSCORE_ESCAPE)) {
        replacementUnderscoreEscape = nonOccursString(logFileName);

        logFileName = replaceStringInstances
          (logFileName, UNDERSCORE_ESCAPE, replacementUnderscoreEscape);
      }

      String encodedFileName = null;
      try {
        encodedFileName = URLEncoder.encode(logFileName, "UTF-8");
      } catch (UnsupportedEncodingException uee) {
        IOException ioe = new IOException();
        ioe.initCause(uee);
        ioe.setStackTrace(uee.getStackTrace());
        throw ioe;
      }

      if (replacementUnderscoreEscape != null) {
        encodedFileName = replaceStringInstances
          (encodedFileName, replacementUnderscoreEscape, UNDERSCORE_ESCAPE);
      }

      return encodedFileName;
    }

    /**
     * Helper function to decode the URL of the filename of the job-history
     * log file.
     *
     * @param logFileName file name of the job-history file
     * @return URL decoded filename
     * @throws IOException
     */
    public static String decodeJobHistoryFileName(String logFileName)
    throws IOException {
      String decodedFileName = null;
      try {
        decodedFileName = URLDecoder.decode(logFileName, "UTF-8");
      } catch (UnsupportedEncodingException uee) {
        IOException ioe = new IOException();
        ioe.initCause(uee);
        ioe.setStackTrace(uee.getStackTrace());
        throw ioe;
      }
      return decodedFileName;
    }

    /**
     * Get the job name from the job conf
     */
    static String getJobName(JobConf jobConf) {
      String jobName = jobConf.getJobName();
      if (jobName == null || jobName.length() == 0) {
        jobName = "NA";
      }
      return jobName;
    }

    /**
     * Get the user name from the job conf
     */
    public static String getUserName(JobConf jobConf) {
      String user = jobConf.getUser();
      if (user == null || user.length() == 0) {
        user = "NA";
      }
      return user;
    }

    /**
     * Get the workflow adjacencies from the job conf
     * The string returned is of the form "key"="value" "key"="value" ...
     */
    public static String getWorkflowAdjacencies(Configuration conf) {
      int prefixLen = JobConf.WORKFLOW_ADJACENCY_PREFIX_STRING.length();
      Map<String,String> adjacencies =
          conf.getValByRegex(JobConf.WORKFLOW_ADJACENCY_PREFIX_PATTERN);
      if (adjacencies.isEmpty())
        return "";
      int size = 0;
      for (Entry<String,String> entry : adjacencies.entrySet()) {
        int keyLen = entry.getKey().length();
        size += keyLen - prefixLen;
        size += entry.getValue().length() + 6;
      }
      StringBuilder sb = new StringBuilder(size);
      for (Entry<String,String> entry : adjacencies.entrySet()) {
        int keyLen = entry.getKey().length();
        sb.append("\"");
        sb.append(escapeString(entry.getKey().substring(prefixLen, keyLen)));
        sb.append("\"=\"");
        sb.append(escapeString(entry.getValue()));
        sb.append("\" ");
      }
      return sb.toString();
    }

    /**
     * Get the job history file path given the history filename
     */
    public static Path getJobHistoryLogLocation(String logFileName)
    {
      return LOG_DIR == null ? null : new Path(LOG_DIR, logFileName);
    }

    /**
     * Get the user job history file path
     */
    public static Path getJobHistoryLogLocationForUser(String logFileName,
                                                       JobConf jobConf) {
      // find user log directory
      Path userLogFile = null;
      Path outputPath = FileOutputFormat.getOutputPath(jobConf);
      String userLogDir = jobConf.get("hadoop.job.history.user.location",
                                      outputPath == null
                                      ? null
                                      : outputPath.toString());
      if ("none".equals(userLogDir)) {
        userLogDir = null;
      }
      if (userLogDir != null) {
        userLogDir = userLogDir + Path.SEPARATOR + "_logs" + Path.SEPARATOR
                     + "history";
        userLogFile = new Path(userLogDir, logFileName);
      }
      return userLogFile;
    }

    /**
     * Generates the job history filename for a new job
     */
    private static String getNewJobHistoryFileName(JobConf jobConf, JobID id, long submitTime) {
      return
        id.toString() + "_"
        + submitTime + "_"
        + escapeUnderscores(getUserName(jobConf)) + "_"
        + escapeUnderscores(trimJobName(getJobName(jobConf)));
    }

    /**
     * Trims the job-name if required
     */
    private static String trimJobName(String jobName) {
      if (jobName.length() > JOB_NAME_TRIM_LENGTH) {
        jobName = jobName.substring(0, JOB_NAME_TRIM_LENGTH);
      }
      return jobName;
    }

    private static String escapeRegexChars( String string ) {
      return "\\Q"+string.replaceAll("\\\\E", "\\\\E\\\\\\\\E\\\\Q")+"\\E";
    }

    /**
     * Recover the job history filename from the history folder.
     * Uses the following pattern
     *    $jt-hostname_[0-9]*_$job-id_$user_$job-name*
     * @param jobConf the job conf
     * @param id job id
     */
    public static synchronized String getJobHistoryFileName(JobConf jobConf,
                                                            JobID id)
    throws IOException {
      return getJobHistoryFileName(jobConf, id, new Path(LOG_DIR), LOGDIR_FS);
    }

    // Returns that portion of the pathname that sits under the DONE directory
    static synchronized String getDoneJobHistoryFileName(JobConf jobConf,
        JobID id) throws IOException {
      if (DONE == null) {
        return null;
      }
      return getJobHistoryFileName(jobConf, id, DONE, DONEDIR_FS);
    }

    /**
     * @param dir The directory where to search.
     */
    private static synchronized String getJobHistoryFileName(JobConf jobConf,
                                          JobID id, Path dir, FileSystem fs)
    throws IOException {
      String user =  getUserName(jobConf);
      String jobName = trimJobName(getJobName(jobConf));
      if (LOG_DIR == null) {
        return null;
      }

      // Make the pattern matching the job's history file

      final String regexp
        = id.toString() + "_" + DIGITS + "_" + user + "_"
             + escapeRegexChars(jobName) + "+";

      final Pattern historyFilePattern = Pattern.compile(regexp);

      // a path filter that matches 4 parts of the filenames namely
      //  - jt-hostname
      //  - job-id
      //  - username
      //  - jobname
      PathFilter filter = new PathFilter() {
        public boolean accept(Path path) {
          String unescapedFileName = path.getName();
          String fileName = null;
          try {
            fileName = decodeJobHistoryFileName(unescapedFileName);
          } catch (IOException ioe) {
            LOG.info("Error while decoding history file " + fileName + "."
                     + " Ignoring file.", ioe);
            return false;
          }

          return historyFilePattern.matcher(fileName).find();
        }
      };

      FileStatus[] statuses = null;

      if (dir == DONE) {
        final String scanTail
          = (DONE_BEFORE_SERIAL_TAIL
             + "/" + serialNumberDirectoryComponent(id));

        if (LOG.isDebugEnabled()) {
          LOG.debug("JobHistory.getJobHistoryFileName DONE dir: scanning "
              + scanTail);
          if (LOG.isTraceEnabled()) {
            LOG.trace(Thread.currentThread().getStackTrace());
          }
        }

        statuses = localGlobber(fs, DONE, scanTail, filter);
      } else {
        statuses = fs.listStatus(dir, filter);
      }

      String filename = null;
      if (statuses == null || statuses.length == 0) {
        LOG.info("Nothing to recover for job " + id);
      } else {
        // return filename considering that fact the name can be a
        // secondary filename like filename.recover
        filename = getPrimaryFilename(statuses[0].getPath().getName(), jobName);
        if (dir == DONE) {
          Path parent = statuses[0].getPath().getParent();
          String parentPathName = parent.toString();
          String donePathName = DONE.toString();
          filename = (parentPathName.substring(donePathName.length() + Path.SEPARATOR.length())
                      + Path.SEPARATOR + filename);
        }

        LOG.info("Recovered job history filename for job " + id + " is "
                 + filename);
      }
      return filename;
    }

    // removes all extra extensions from a filename and returns the core/primary
    // filename
    private static String getPrimaryFilename(String filename, String jobName)
    throws IOException{
      filename = decodeJobHistoryFileName(filename);
      // Remove the '.recover' suffix if it exists
      if (filename.endsWith(jobName + SECONDARY_FILE_SUFFIX)) {
        int newLength = filename.length() - SECONDARY_FILE_SUFFIX.length();
        filename = filename.substring(0, newLength);
      }
      return encodeJobHistoryFileName(filename);
    }

    /** Since there was a restart, there should be a master file and
     * a recovery file. Once the recovery is complete, the master should be
     * deleted as an indication that the recovery file should be treated as the
     * master upon completion or next restart.
     * @param fileName the history filename that needs checkpointing
     * @param conf Job conf
     * @throws IOException
     */
    static synchronized void checkpointRecovery(String fileName, JobConf conf)
    throws IOException {
      Path logPath = JobHistory.JobInfo.getJobHistoryLogLocation(fileName);
      if (logPath != null) {
        LOG.info("Deleting job history file " + logPath.getName());
        LOGDIR_FS.delete(logPath, false);
      }
      // do the same for the user file too
      logPath = JobHistory.JobInfo.getJobHistoryLogLocationForUser(fileName,
                                                                   conf);
      if (logPath != null) {
        FileSystem fs = logPath.getFileSystem(conf);
        fs.delete(logPath, false);
      }
    }

    static String getSecondaryJobHistoryFile(String filename)
    throws IOException {
      return encodeJobHistoryFileName(
          decodeJobHistoryFileName(filename) + SECONDARY_FILE_SUFFIX);
    }

    /** Selects one of the two files generated as a part of recovery.
     * The thumb rule is that always select the oldest file.
     * This call makes sure that only one file is left in the end.
     * @param conf job conf
     * @param logFilePath Path of the log file
     * @throws IOException
     */
    public synchronized static Path recoverJobHistoryFile(JobConf conf,
                                                          Path logFilePath)
    throws IOException {
      Path ret;
      String logFileName = logFilePath.getName();
      String tmpFilename = getSecondaryJobHistoryFile(logFileName);
      Path logDir = logFilePath.getParent();
      Path tmpFilePath = new Path(logDir, tmpFilename);
      if (LOGDIR_FS.exists(logFilePath)) {
        LOG.info(logFileName + " exists!");
        if (LOGDIR_FS.exists(tmpFilePath)) {
          LOG.info("Deleting " + tmpFilename
                   + "  and using " + logFileName + " for recovery.");
          LOGDIR_FS.delete(tmpFilePath, false);
        }
        ret = tmpFilePath;
      } else {
        LOG.info(logFileName + " doesnt exist! Using "
                 + tmpFilename + " for recovery.");
        if (LOGDIR_FS.exists(tmpFilePath)) {
          LOG.info("Renaming " + tmpFilename + " to " + logFileName);
          LOGDIR_FS.rename(tmpFilePath, logFilePath);
          ret = tmpFilePath;
        } else {
          ret = logFilePath;
        }
      }

      // do the same for the user files too
      logFilePath = getJobHistoryLogLocationForUser(logFileName, conf);
      if (logFilePath != null) {
        FileSystem fs = logFilePath.getFileSystem(conf);
        logDir = logFilePath.getParent();
        tmpFilePath = new Path(logDir, tmpFilename);
        if (fs.exists(logFilePath)) {
          LOG.info(logFileName + " exists!");
          if (fs.exists(tmpFilePath)) {
            LOG.info("Deleting " + tmpFilename + "  and making " + logFileName
                     + " as the master history file for user.");
            fs.delete(tmpFilePath, false);
          }
        } else {
          LOG.info(logFileName + " doesnt exist! Using "
                   + tmpFilename + " as the master history file for user.");
          if (fs.exists(tmpFilePath)) {
            LOG.info("Renaming " + tmpFilename + " to " + logFileName
                     + " in user directory");
            fs.rename(tmpFilePath, logFilePath);
          }
        }
      }

      return ret;
    }

    /** Finalize the recovery and make one file in the end.
     * This invloves renaming the recover file to the master file.
     * Note that this api should be invoked only if recovery is involved.
     * @param id Job id
     * @param conf the job conf
     * @throws IOException
     */
    static synchronized void finalizeRecovery(JobID id, JobConf conf)
    throws IOException {
       Path tmpLogPath = fileManager.getHistoryFile(id);
       if (tmpLogPath == null) {
         if (LOG.isDebugEnabled()) {
           LOG.debug("No file for job with " + id + " found in cache!");
         }
         return;
       }
       String tmpLogFileName = tmpLogPath.getName();

       // get the primary filename from the cached filename
       String masterLogFileName =
         getPrimaryFilename(tmpLogFileName, getJobName(conf));
       Path masterLogPath = new Path(tmpLogPath.getParent(), masterLogFileName);

       // rename the tmp file to the master file. Note that this should be
       // done only when the file is closed and handles are released.
       LOG.info("Renaming " + tmpLogFileName + " to " + masterLogFileName);
       LOGDIR_FS.rename(tmpLogPath, masterLogPath);
       // update the cache
       fileManager.setHistoryFile(id, masterLogPath);

      // do the same for the user file too
      masterLogPath =
        JobHistory.JobInfo.getJobHistoryLogLocationForUser(masterLogFileName,
                                                           conf);
      tmpLogPath =
        JobHistory.JobInfo.getJobHistoryLogLocationForUser(tmpLogFileName,
                                                           conf);
      if (masterLogPath != null) {
        FileSystem fs = masterLogPath.getFileSystem(conf);
        if (fs.exists(tmpLogPath)) {
          LOG.info("Renaming " + tmpLogFileName + " to " + masterLogFileName
                   + " in user directory");
          fs.rename(tmpLogPath, masterLogPath);
        }
      }
    }

    /**
     * Deletes job data from the local disk.
     * For now just deletes the localized copy of job conf
     */
    static void cleanupJob(JobID id) {
      String localJobFilePath =  JobInfo.getLocalJobFilePath(id);
      File f = new File (localJobFilePath);
      LOG.info("Deleting localized job conf at " + f);
      if (!f.delete()) {
        if (LOG.isDebugEnabled()) {
          LOG.debug("Failed to delete file " + f);
        }
      }
    }

    /**
     * Delete job conf from the history folder.
     */
    static void deleteConfFiles() throws IOException {
      LOG.info("Cleaning up config files from the job history folder");
      FileSystem fs = new Path(LOG_DIR).getFileSystem(jtConf);
      FileStatus[] status = fs.listStatus(new Path(LOG_DIR), CONF_FILTER);
      for (FileStatus s : status) {
        LOG.info("Deleting conf file " + s.getPath());
        fs.delete(s.getPath(), false);
      }
    }

    /**
     * Move the completed job into the completed folder.
     * This assumes that the jobhistory file is closed and all operations on the
     * jobhistory file is complete.
     * This *should* be the last call to jobhistory for a given job.
     */
    static void markCompleted(JobID id) throws IOException {
      fileManager.moveToDone(id);
    }

     /**
     * Log job submitted event to history. Creates a new file in history
     * for the job. if history file creation fails, it disables history
     * for all other events.
     * @param jobId job id assigned by job tracker.
     * @param jobConf job conf of the job
     * @param jobConfPath path to job conf xml file in HDFS.
     * @param submitTime time when job tracker received the job
     * @throws IOException
     * @deprecated Use
     *     {@link #logSubmitted(JobID, JobConf, String, long, boolean)} instead.
     */
    @Deprecated
    public static void logSubmitted(JobID jobId, JobConf jobConf,
                                    String jobConfPath, long submitTime)
    throws IOException {
      logSubmitted(jobId, jobConf, jobConfPath, submitTime, true);
    }

    public static void logSubmitted(JobID jobId, JobConf jobConf,
                                    String jobConfPath, long submitTime,
                                    boolean restarted)
    throws IOException {
      FileSystem fs = null;
      String userLogDir = null;
      String jobUniqueString = jobId.toString();

      // Get the username and job name to be used in the actual log filename;
      // sanity check them too
      String jobName = getJobName(jobConf);
      String user = getUserName(jobConf);

      // get the history filename
      String logFileName = null;
      if (restarted) {
        logFileName = getJobHistoryFileName(jobConf, jobId);
        if (logFileName == null) {
          logFileName =
            encodeJobHistoryFileName(getNewJobHistoryFileName
                                     (jobConf, jobId, submitTime));
        } else {
          String parts[] = logFileName.split("_");
          //TODO this is a hack :(
          // jobtracker-hostname_jobtracker-identifier_
          String jtUniqueString = parts[0] + "_" + parts[1] + "_";
          jobUniqueString = jobId.toString();
        }
      } else {
        logFileName =
          encodeJobHistoryFileName(getNewJobHistoryFileName
                                   (jobConf, jobId, submitTime));
      }

      // setup the history log file for this job
      Path logFile = getJobHistoryLogLocation(logFileName);

      // find user log directory
      Path userLogFile =
        getJobHistoryLogLocationForUser(logFileName, jobConf);
      PrintWriter writer = null;
      try{
        FSDataOutputStream out = null;
        if (LOG_DIR != null) {
          // create output stream for logging in hadoop.job.history.location
          if (restarted) {
            logFile = recoverJobHistoryFile(jobConf, logFile);
            logFileName = logFile.getName();
          }

          int defaultBufferSize =
            LOGDIR_FS.getConf().getInt("io.file.buffer.size", 4096);
          out = LOGDIR_FS.create(logFile,
                          new FsPermission(HISTORY_FILE_PERMISSION),
                          true,
                          defaultBufferSize,
                          LOGDIR_FS.getDefaultReplication(),
                          jobHistoryBlockSize, null);
          writer = new PrintWriter(out);
          fileManager.addWriter(jobId, writer);

          // cache it ...
          fileManager.setHistoryFile(jobId, logFile);
        }
        if (userLogFile != null) {
          // Get the actual filename as recoverJobHistoryFile() might return
          // a different filename
          userLogDir = userLogFile.getParent().toString();
          userLogFile = new Path(userLogDir, logFileName);

          // create output stream for logging
          // in hadoop.job.history.user.location
          fs = userLogFile.getFileSystem(jobConf);

          out = fs.create(userLogFile, true, 4096);
          writer = new PrintWriter(out);
          fileManager.addWriter(jobId, writer);
        }

        ArrayList<PrintWriter> writers = fileManager.getWriters(jobId);
        // Log the history meta info
        JobHistory.MetaInfoManager.logMetaInfo(writers);

        String viewJobACL = "*";
        String modifyJobACL = "*";
        if (aclsEnabled) {
          viewJobACL = jobConf.get(JobACL.VIEW_JOB.getAclName(), " ");
          modifyJobACL = jobConf.get(JobACL.MODIFY_JOB.getAclName(), " ");
        }
        //add to writer as well
        JobHistory.log(writers, RecordTypes.Job,
                       new Keys[]{Keys.JOBID, Keys.JOBNAME, Keys.USER,
                                  Keys.SUBMIT_TIME, Keys.JOBCONF,
                                  Keys.VIEW_JOB, Keys.MODIFY_JOB,
                                  Keys.JOB_QUEUE, Keys.WORKFLOW_ID,
                                  Keys.WORKFLOW_NAME, Keys.WORKFLOW_NODE_NAME,
                                  Keys.WORKFLOW_ADJACENCIES,
                                  Keys.WORKFLOW_TAGS},
                       new String[]{jobId.toString(), jobName, user,
                                    String.valueOf(submitTime) , jobConfPath,
                                    viewJobACL, modifyJobACL,
                                    jobConf.getQueueName(),
                                    jobConf.get(JobConf.WORKFLOW_ID, ""),
                                    jobConf.get(JobConf.WORKFLOW_NAME, ""),
                                    jobConf.get(JobConf.WORKFLOW_NODE_NAME, ""),
                                    getWorkflowAdjacencies(jobConf),
                                    jobConf.get(JobConf.WORKFLOW_TAGS, ""),
                                    },
                                    jobId
                      );

      }catch(IOException e){
        LOG.error("Failed creating job history log file for job " + jobId, e);
        if (writer != null) {
          fileManager.removeWriter(jobId, writer);
        }
      }
      // Always store job conf on local file system
      String localJobFilePath =  JobInfo.getLocalJobFilePath(jobId);
      File localJobFile = new File(localJobFilePath);
      FileOutputStream jobOut = null;
      try {
        jobOut = new FileOutputStream(localJobFile);
        jobConf.writeXml(jobOut);
        if (LOG.isDebugEnabled()) {
          LOG.debug("Job conf for " + jobId + " stored at "
                    + localJobFile.getAbsolutePath());
        }
      } catch (IOException ioe) {
        LOG.error("Failed to store job conf on the local filesystem ", ioe);
      } finally {
        if (jobOut != null) {
          try {
            jobOut.close();
          } catch (IOException ie) {
            LOG.info("Failed to close the job configuration file "
                       + StringUtils.stringifyException(ie));
          }
        }
      }

      /* Storing the job conf on the log dir */
      Path jobFilePath = null;
      if (LOG_DIR != null) {
        jobFilePath = new Path(LOG_DIR + File.separator +
                               jobUniqueString + CONF_FILE_NAME_SUFFIX);
        fileManager.setConfFile(jobId, jobFilePath);
      }
      Path userJobFilePath = null;
      if (userLogDir != null) {
        userJobFilePath = new Path(userLogDir + File.separator +
                                   jobUniqueString + CONF_FILE_NAME_SUFFIX);
      }
      FSDataOutputStream jobFileOut = null;
      try {
        if (LOG_DIR != null) {
          int defaultBufferSize =
              LOGDIR_FS.getConf().getInt("io.file.buffer.size", 4096);
          if (!LOGDIR_FS.exists(jobFilePath)) {
            jobFileOut = LOGDIR_FS.create(jobFilePath,
                                   new FsPermission(HISTORY_FILE_PERMISSION),
                                   true,
                                   defaultBufferSize,
                                   LOGDIR_FS.getDefaultReplication(),
                                   LOGDIR_FS.getDefaultBlockSize(), null);
            jobConf.writeXml(jobFileOut);
            jobFileOut.close();
          }
        }
        if (userLogDir != null) {
          fs = new Path(userLogDir).getFileSystem(jobConf);
          jobFileOut = fs.create(userJobFilePath);
          jobConf.writeXml(jobFileOut);
        }
        if (LOG.isDebugEnabled()) {
          LOG.debug("Job conf for " + jobId + " stored at "
                    + jobFilePath + "and" + userJobFilePath );
        }
      } catch (IOException ioe) {
        LOG.error("Failed to store job conf in the log dir", ioe);
      } finally {
        if (jobFileOut != null) {
          try {
            jobFileOut.close();
          } catch (IOException ie) {
            LOG.info("Failed to close the job configuration file "
                     + StringUtils.stringifyException(ie));
          }
        }
      }
    }
    /**
     * Logs launch time of job.
     *
     * @param jobId job id, assigned by jobtracker.
     * @param startTime start time of job.
     * @param totalMaps total maps assigned by jobtracker.
     * @param totalReduces total reduces.
     */
    public static void logInited(JobID jobId, long startTime,
                                 int totalMaps, int totalReduces) {
      ArrayList<PrintWriter> writer = fileManager.getWriters(jobId);

      if (null != writer){
        JobHistory.log(writer, RecordTypes.Job,
            new Keys[] {Keys.JOBID, Keys.LAUNCH_TIME, Keys.TOTAL_MAPS,
                        Keys.TOTAL_REDUCES, Keys.JOB_STATUS},
            new String[] {jobId.toString(), String.valueOf(startTime),
                          String.valueOf(totalMaps),
                          String.valueOf(totalReduces),
                          Values.PREP.name()}, jobId);
      }
    }

   /**
     * Logs the job as RUNNING.
     *
     * @param jobId job id, assigned by jobtracker.
     * @param startTime start time of job.
     * @param totalMaps total maps assigned by jobtracker.
     * @param totalReduces total reduces.
     * @deprecated Use {@link #logInited(JobID, long, int, int)} and
     * {@link #logStarted(JobID)}
     */
    @Deprecated
    public static void logStarted(JobID jobId, long startTime,
                                  int totalMaps, int totalReduces) {
      logStarted(jobId);
    }

    /**
     * Logs job as running
     * @param jobId job id, assigned by jobtracker.
     */
    public static void logStarted(JobID jobId){
      ArrayList<PrintWriter> writer = fileManager.getWriters(jobId);

      if (null != writer){
        JobHistory.log(writer, RecordTypes.Job,
            new Keys[] {Keys.JOBID, Keys.JOB_STATUS},
            new String[] {jobId.toString(),
                          Values.RUNNING.name()}, jobId);
      }
    }

    /**
     * Log job finished. closes the job file in history.
     * @param jobId job id, assigned by jobtracker.
     * @param finishTime finish time of job in ms.
     * @param finishedMaps no of maps successfully finished.
     * @param finishedReduces no of reduces finished sucessfully.
     * @param failedMaps no of failed map tasks.
     * @param failedReduces no of failed reduce tasks.
     * @param counters the counters from the job
     */
    public static void logFinished(JobID jobId, long finishTime,
                                   int finishedMaps, int finishedReduces,
                                   int failedMaps, int failedReduces,
                                   Counters mapCounters,
                                   Counters reduceCounters,
                                   Counters counters) {
        // close job file for this job
      ArrayList<PrintWriter> writer = fileManager.getWriters(jobId);

      if (null != writer){
        JobHistory.log(writer, RecordTypes.Job,
                       new Keys[] {Keys.JOBID, Keys.FINISH_TIME,
                                   Keys.JOB_STATUS, Keys.FINISHED_MAPS,
                                   Keys.FINISHED_REDUCES,
                                   Keys.FAILED_MAPS, Keys.FAILED_REDUCES,
                                   Keys.MAP_COUNTERS, Keys.REDUCE_COUNTERS,
                                   Keys.COUNTERS},
                       new String[] {jobId.toString(),  Long.toString(finishTime),
                                     Values.SUCCESS.name(),
                                     String.valueOf(finishedMaps),
                                     String.valueOf(finishedReduces),
                                     String.valueOf(failedMaps),
                                     String.valueOf(failedReduces),
                                     mapCounters.makeEscapedCompactString(),
                                     reduceCounters.makeEscapedCompactString(),
                                     counters.makeEscapedCompactString()}, jobId);
        for (PrintWriter out : writer) {
          out.close();
        }
      }
      Thread historyCleaner  = new Thread(new HistoryCleaner());
      historyCleaner.start();
    }
    /**
     * Logs job failed event. Closes the job history log file.
     * @param jobid job id
     * @param timestamp time when job failure was detected in ms.
     * @param finishedMaps no finished map tasks.
     * @param finishedReduces no of finished reduce tasks.
     */
    public static void logFailed(JobID jobid, long timestamp, int finishedMaps, int finishedReduces, String failReason){
      ArrayList<PrintWriter> writer = fileManager.getWriters(jobid);

      if (null != writer){
        JobHistory.log(writer, RecordTypes.Job,
                       new Keys[] {Keys.JOBID, Keys.FINISH_TIME, Keys.JOB_STATUS, Keys.FINISHED_MAPS, Keys.FINISHED_REDUCES, Keys.FAIL_REASON },
                       new String[] {jobid.toString(),  String.valueOf(timestamp), Values.FAILED.name(), String.valueOf(finishedMaps),
                                     String.valueOf(finishedReduces), failReason}, jobid);
        for (PrintWriter out : writer) {
          out.close();
        }
      }
    }
    /**
     * Logs job killed event. Closes the job history log file.
     *
     * @param jobid
     *          job id
     * @param timestamp
     *          time when job killed was issued in ms.
     * @param finishedMaps
     *          no finished map tasks.
     * @param finishedReduces
     *          no of finished reduce tasks.
     */
    public static void logKilled(JobID jobid, long timestamp, int finishedMaps,
        int finishedReduces) {
      ArrayList<PrintWriter> writer = fileManager.getWriters(jobid);

      if (null != writer) {
        JobHistory.log(writer, RecordTypes.Job, new Keys[] { Keys.JOBID,
            Keys.FINISH_TIME, Keys.JOB_STATUS, Keys.FINISHED_MAPS,
            Keys.FINISHED_REDUCES }, new String[] { jobid.toString(),
            String.valueOf(timestamp), Values.KILLED.name(),
            String.valueOf(finishedMaps), String.valueOf(finishedReduces) }, jobid);
        for (PrintWriter out : writer) {
          out.close();
        }
      }
    }
    /**
     * Log job's priority.
     * @param jobid job id
     * @param priority Jobs priority
     */
    public static void logJobPriority(JobID jobid, JobPriority priority){
      ArrayList<PrintWriter> writer = fileManager.getWriters(jobid);

      if (null != writer){
        JobHistory.log(writer, RecordTypes.Job,
                       new Keys[] {Keys.JOBID, Keys.JOB_PRIORITY},
                       new String[] {jobid.toString(), priority.toString()}, jobid);
      }
    }
    /**
     * Log job's submit-time/launch-time
     * @param jobid job id
     * @param submitTime job's submit time
     * @param launchTime job's launch time
     * @param restartCount number of times the job got restarted
     * @deprecated Use {@link #logJobInfo(JobID, long, long)} instead.
     */
    @Deprecated
    public static void logJobInfo(JobID jobid, long submitTime, long launchTime,
                                  int restartCount){
      logJobInfo(jobid, submitTime, launchTime);
    }

    public static void logJobInfo(JobID jobid, long submitTime, long launchTime)
    {
      ArrayList<PrintWriter> writer = fileManager.getWriters(jobid);

      if (null != writer){
        JobHistory.log(writer, RecordTypes.Job,
                       new Keys[] {Keys.JOBID, Keys.SUBMIT_TIME,
                                   Keys.LAUNCH_TIME},
                       new String[] {jobid.toString(),
                                     String.valueOf(submitTime),
                                     String.valueOf(launchTime)}, jobid);
      }
    }
  }

  /**
   * Helper class for logging or reading back events related to Task's start, finish or failure.
   * All events logged by this class are logged in a separate file per job in
   * job tracker history. These events map to TIPs in jobtracker.
   */
  public static class Task extends KeyValuePair{
    private Map <String, TaskAttempt> taskAttempts = new TreeMap<String, TaskAttempt>();

    /**
     * Log start time of task (TIP).
     * @param taskId task id
     * @param taskType MAP or REDUCE
     * @param startTime startTime of tip.
     */
    public static void logStarted(TaskID taskId, String taskType,
                                  long startTime, String splitLocations) {
      JobID id = taskId.getJobID();
      ArrayList<PrintWriter> writer = fileManager.getWriters(id);

      if (null != writer){
        JobHistory.log(writer, RecordTypes.Task,
                       new Keys[]{Keys.TASKID, Keys.TASK_TYPE ,
                                  Keys.START_TIME, Keys.SPLITS},
                       new String[]{taskId.toString(), taskType,
                                    String.valueOf(startTime),
                                    splitLocations}, id);
      }
    }
    /**
     * Log finish time of task.
     * @param taskId task id
     * @param taskType MAP or REDUCE
     * @param finishTime finish timeof task in ms
     */
    public static void logFinished(TaskID taskId, String taskType,
                                   long finishTime, Counters counters){
      JobID id = taskId.getJobID();
      ArrayList<PrintWriter> writer = fileManager.getWriters(id);

      if (null != writer){
         JobHistory.log(writer, RecordTypes.Task,
                        new Keys[]{Keys.TASKID, Keys.TASK_TYPE,
                                   Keys.TASK_STATUS, Keys.FINISH_TIME,
                                   Keys.COUNTERS},
                        new String[]{ taskId.toString(), taskType, Values.SUCCESS.name(),
                                      String.valueOf(finishTime),
                                      counters.makeEscapedCompactString()}, id);
      }
    }

    /**
     * Update the finish time of task.
     * @param taskId task id
     * @param finishTime finish time of task in ms
     */
    public static void logUpdates(TaskID taskId, long finishTime){
      JobID id = taskId.getJobID();
      ArrayList<PrintWriter> writer = fileManager.getWriters(id);

      if (null != writer){
        JobHistory.log(writer, RecordTypes.Task,
                       new Keys[]{Keys.TASKID, Keys.FINISH_TIME},
                       new String[]{ taskId.toString(),
                                     String.valueOf(finishTime)}, id);
      }
    }

    /**
     * Log job failed event.
     * @param taskId task id
     * @param taskType MAP or REDUCE.
     * @param time timestamp when job failed detected.
     * @param error error message for failure.
     */
    public static void logFailed(TaskID taskId, String taskType, long time, String error){
      logFailed(taskId, taskType, time, error, null);
    }

    /**
     * @param failedDueToAttempt The attempt that caused the failure, if any
     */
    public static void logFailed(TaskID taskId, String taskType, long time,
                                 String error,
                                 TaskAttemptID failedDueToAttempt){
      JobID id = taskId.getJobID();
      ArrayList<PrintWriter> writer = fileManager.getWriters(id);

      if (null != writer){
        String failedAttempt = failedDueToAttempt == null
                               ? ""
                               : failedDueToAttempt.toString();
        JobHistory.log(writer, RecordTypes.Task,
                       new Keys[]{Keys.TASKID, Keys.TASK_TYPE,
                                  Keys.TASK_STATUS, Keys.FINISH_TIME,
                                  Keys.ERROR, Keys.TASK_ATTEMPT_ID},
                       new String[]{ taskId.toString(),  taskType,
                                    Values.FAILED.name(),
                                    String.valueOf(time) , error,
                                    failedAttempt}, id);
      }
    }
    /**
     * Returns all task attempts for this task. <task attempt id - TaskAttempt>
     */
    public Map<String, TaskAttempt> getTaskAttempts(){
      return this.taskAttempts;
    }
  }

  /**
   * Base class for Map and Reduce TaskAttempts.
   */
  public static class TaskAttempt extends Task{}

  /**
   * Helper class for logging or reading back events related to start, finish or failure of
   * a Map Attempt on a node.
   */
  public static class MapAttempt extends TaskAttempt{
    /**
     * Log start time of this map task attempt.
     * @param taskAttemptId task attempt id
     * @param startTime start time of task attempt as reported by task tracker.
     * @param hostName host name of the task attempt.
     * @deprecated Use
     *             {@link #logStarted(TaskAttemptID, long, String, int, String)}
     */
    @Deprecated
    public static void logStarted(TaskAttemptID taskAttemptId, long startTime, String hostName){
      logStarted(taskAttemptId, startTime, hostName, -1, Values.MAP.name());
    }

    @Deprecated
    public static void logStarted(TaskAttemptID taskAttemptId, long startTime,
        String trackerName, int httpPort, String taskType) {
      logStarted(taskAttemptId, startTime, trackerName, httpPort, taskType,
          Locality.OFF_SWITCH, Avataar.VIRGIN);
    }

    /**
     * Log start time of this map task attempt.
     *
     * @param taskAttemptId task attempt id
     * @param startTime start time of task attempt as reported by task tracker.
     * @param trackerName name of the tracker executing the task attempt.
     * @param httpPort http port of the task tracker executing the task attempt
     * @param taskType Whether the attempt is cleanup or setup or map
     * @param locality the data locality of the task attempt
     * @param Avataar the avataar of the task attempt
     */
    public static void logStarted(TaskAttemptID taskAttemptId, long startTime,
                                  String trackerName, int httpPort,
                                  String taskType,
                                  Locality locality, Avataar avataar) {
      JobID id = taskAttemptId.getJobID();
      ArrayList<PrintWriter> writer = fileManager.getWriters(id);

      if (null != writer){
        JobHistory.log(writer, RecordTypes.MapAttempt,
                       new Keys[]{ Keys.TASK_TYPE, Keys.TASKID,
                                   Keys.TASK_ATTEMPT_ID, Keys.START_TIME,
                                   Keys.TRACKER_NAME, Keys.HTTP_PORT,
                                   Keys.LOCALITY, Keys.AVATAAR},
                       new String[]{taskType,
                                    taskAttemptId.getTaskID().toString(),
                                    taskAttemptId.toString(),
                                    String.valueOf(startTime), trackerName,
                                    httpPort == -1 ? "" : String.valueOf(httpPort),
                                    locality.toString(), avataar.toString()},
                       id
                       );
      }
    }

    /**
     * Log finish time of map task attempt.
     * @param taskAttemptId task attempt id
     * @param finishTime finish time
     * @param hostName host name
     * @deprecated Use
     * {@link #logFinished(TaskAttemptID, long, String, String, String, Counters)}
     */
    @Deprecated
    public static void logFinished(TaskAttemptID taskAttemptId, long finishTime,
                                   String hostName){
      logFinished(taskAttemptId, finishTime, hostName, Values.MAP.name(), "",
                  new Counters());
    }

    /**
     * Log finish time of map task attempt.
     *
     * @param taskAttemptId task attempt id
     * @param finishTime finish time
     * @param hostName host name
     * @param taskType Whether the attempt is cleanup or setup or map
     * @param stateString state string of the task attempt
     * @param counter counters of the task attempt
     */
    public static void logFinished(TaskAttemptID taskAttemptId,
                                   long finishTime,
                                   String hostName,
                                   String taskType,
                                   String stateString,
                                   Counters counter) {
      JobID id = taskAttemptId.getJobID();
      ArrayList<PrintWriter> writer = fileManager.getWriters(id);

      if (null != writer){
        JobHistory.log(writer, RecordTypes.MapAttempt,
                       new Keys[]{ Keys.TASK_TYPE, Keys.TASKID,
                                   Keys.TASK_ATTEMPT_ID, Keys.TASK_STATUS,
                                   Keys.FINISH_TIME, Keys.HOSTNAME,
                                   Keys.STATE_STRING, Keys.COUNTERS},
                       new String[]{taskType,
                                    taskAttemptId.getTaskID().toString(),
                                    taskAttemptId.toString(),
                                    Values.SUCCESS.name(),
                                    String.valueOf(finishTime), hostName,
                                    stateString,
                                    counter.makeEscapedCompactString()}, id);
      }
    }

    /**
     * Log task attempt failed event.
     * @param taskAttemptId task attempt id
     * @param timestamp timestamp
     * @param hostName hostname of this task attempt.
     * @param error error message if any for this task attempt.
     * @deprecated Use
     * {@link #logFailed(TaskAttemptID, long, String, String, String)}
     */
    @Deprecated
    public static void logFailed(TaskAttemptID taskAttemptId,
                                 long timestamp, String hostName,
                                 String error) {
      logFailed(taskAttemptId, timestamp, hostName, error, Values.MAP.name());
    }

    /**
     * Log task attempt failed event.
     *
     * @param taskAttemptId task attempt id
     * @param timestamp timestamp
     * @param hostName hostname of this task attempt.
     * @param error error message if any for this task attempt.
     * @param taskType Whether the attempt is cleanup or setup or map
     */
    public static void logFailed(TaskAttemptID taskAttemptId,
                                 long timestamp, String hostName,
                                 String error, String taskType) {
      JobID id = taskAttemptId.getJobID();
      ArrayList<PrintWriter> writer = fileManager.getWriters(id);

      if (null != writer){
        JobHistory.log(writer, RecordTypes.MapAttempt,
                       new Keys[]{Keys.TASK_TYPE, Keys.TASKID,
                                  Keys.TASK_ATTEMPT_ID, Keys.TASK_STATUS,
                                  Keys.FINISH_TIME, Keys.HOSTNAME, Keys.ERROR},
                       new String[]{ taskType,
                                     taskAttemptId.getTaskID().toString(),
                                     taskAttemptId.toString(),
                                     Values.FAILED.name(),
                                     String.valueOf(timestamp),
                                     hostName, error}, id);
      }
    }

    /**
     * Log task attempt killed event.
     * @param taskAttemptId task attempt id
     * @param timestamp timestamp
     * @param hostName hostname of this task attempt.
     * @param error error message if any for this task attempt.
     * @deprecated Use
     * {@link #logKilled(TaskAttemptID, long, String, String, String)}
     */
    @Deprecated
    public static void logKilled(TaskAttemptID taskAttemptId,
                                 long timestamp, String hostName, String error){
      logKilled(taskAttemptId, timestamp, hostName, error, Values.MAP.name());
    }

    /**
     * Log task attempt killed event.
     *
     * @param taskAttemptId task attempt id
     * @param timestamp timestamp
     * @param hostName hostname of this task attempt.
     * @param error error message if any for this task attempt.
     * @param taskType Whether the attempt is cleanup or setup or map
     */
    public static void logKilled(TaskAttemptID taskAttemptId,
                                 long timestamp, String hostName,
                                 String error, String taskType) {
      JobID id = taskAttemptId.getJobID();
      ArrayList<PrintWriter> writer = fileManager.getWriters(id);

      if (null != writer){
        JobHistory.log(writer, RecordTypes.MapAttempt,
                       new Keys[]{Keys.TASK_TYPE, Keys.TASKID,
                                  Keys.TASK_ATTEMPT_ID, Keys.TASK_STATUS,
                                  Keys.FINISH_TIME, Keys.HOSTNAME,
                                  Keys.ERROR},
                       new String[]{ taskType,
                                     taskAttemptId.getTaskID().toString(),
                                     taskAttemptId.toString(),
                                     Values.KILLED.name(),
                                     String.valueOf(timestamp),
                                     hostName, error}, id);
      }
    }
  }
  /**
   * Helper class for logging or reading back events related to start, finish or failure of
   * a Map Attempt on a node.
   */
  public static class ReduceAttempt extends TaskAttempt{
    /**
     * Log start time of  Reduce task attempt.
     * @param taskAttemptId task attempt id
     * @param startTime start time
     * @param hostName host name
     * @deprecated Use
     * {@link #logStarted(TaskAttemptID, long, String, int, String)}
     */
    @Deprecated
    public static void logStarted(TaskAttemptID taskAttemptId,
                                  long startTime, String hostName){
      logStarted(taskAttemptId, startTime, hostName, -1, Values.REDUCE.name());
    }

    @Deprecated
    public static void logStarted(TaskAttemptID taskAttemptId,
        long startTime, String trackerName,
        int httpPort,
        String taskType) {
      logStarted(taskAttemptId, startTime, trackerName, httpPort, taskType,
          Locality.OFF_SWITCH, Avataar.VIRGIN);
    }
    /**
     * Log start time of  Reduce task attempt.
     *
     * @param taskAttemptId task attempt id
     * @param startTime start time
     * @param trackerName tracker name
     * @param httpPort the http port of the tracker executing the task attempt
     * @param taskType Whether the attempt is cleanup or setup or reduce
     * @param locality the data locality of the task attempt
     * @param Avataar the avataar of the task attempt
     */
    public static void logStarted(TaskAttemptID taskAttemptId,
                                  long startTime, String trackerName,
                                  int httpPort,
                                  String taskType,
                                  Locality locality, Avataar avataar) {
      JobID id = taskAttemptId.getJobID();
	    ArrayList<PrintWriter> writer = fileManager.getWriters(id);

      if (null != writer){
        JobHistory.log(writer, RecordTypes.ReduceAttempt,
            new Keys[] {Keys.TASK_TYPE, Keys.TASKID,
                Keys.TASK_ATTEMPT_ID, Keys.START_TIME,
                Keys.TRACKER_NAME, Keys.HTTP_PORT,
                Keys.LOCALITY, Keys.AVATAAR},
            new String[]{taskType,
                taskAttemptId.getTaskID().toString(),
                taskAttemptId.toString(),
                String.valueOf(startTime), trackerName,
                httpPort == -1 ? "" :
                String.valueOf(httpPort),
                locality.toString(), avataar.toString()},
            id);
        }
	    }

	    /**
	     * Log finished event of this task.
	     * @param taskAttemptId task attempt id
	     * @param shuffleFinished shuffle finish time
	     * @param sortFinished sort finish time
	     * @param finishTime finish time of task
	     * @param hostName host name where task attempt executed
	     * @deprecated Use
	     * {@link #logFinished(TaskAttemptID, long, long, long, String, String, String, Counters)}
	     */
	    @Deprecated
	    public static void logFinished(TaskAttemptID taskAttemptId, long shuffleFinished,
					   long sortFinished, long finishTime,
					   String hostName){
	      logFinished(taskAttemptId, shuffleFinished, sortFinished,
			  finishTime, hostName, Values.REDUCE.name(),
			  "", new Counters());
	    }

	    /**
	     * Log finished event of this task.
	     *
	     * @param taskAttemptId task attempt id
	     * @param shuffleFinished shuffle finish time
	     * @param sortFinished sort finish time
	     * @param finishTime finish time of task
	     * @param hostName host name where task attempt executed
	     * @param taskType Whether the attempt is cleanup or setup or reduce
	     * @param stateString the state string of the attempt
	     * @param counter counters of the attempt
	     */
	    public static void logFinished(TaskAttemptID taskAttemptId,
					   long shuffleFinished,
					   long sortFinished, long finishTime,
					   String hostName, String taskType,
					   String stateString, Counters counter) {
        JobID id = taskAttemptId.getJobID();
        ArrayList<PrintWriter> writer = fileManager.getWriters(id);

        if (null != writer){
          JobHistory.log(writer, RecordTypes.ReduceAttempt,
                         new Keys[]{ Keys.TASK_TYPE, Keys.TASKID,
                                     Keys.TASK_ATTEMPT_ID, Keys.TASK_STATUS,
                                     Keys.SHUFFLE_FINISHED, Keys.SORT_FINISHED,
                                     Keys.FINISH_TIME, Keys.HOSTNAME,
                                     Keys.STATE_STRING, Keys.COUNTERS},
                         new String[]{taskType,
                                      taskAttemptId.getTaskID().toString(),
                                      taskAttemptId.toString(),
                                      Values.SUCCESS.name(),
                                      String.valueOf(shuffleFinished),
                                      String.valueOf(sortFinished),
                                      String.valueOf(finishTime), hostName,
                                      stateString,
                                      counter.makeEscapedCompactString()}, id);
        }
    }

    /**
     * Log failed reduce task attempt.
     * @param taskAttemptId task attempt id
     * @param timestamp time stamp when task failed
     * @param hostName host name of the task attempt.
     * @param error error message of the task.
     * @deprecated Use
     * {@link #logFailed(TaskAttemptID, long, String, String, String)}
     */
    @Deprecated
    public static void logFailed(TaskAttemptID taskAttemptId, long timestamp,
                                 String hostName, String error){
      logFailed(taskAttemptId, timestamp, hostName, error, Values.REDUCE.name());
    }

    /**
     * Log failed reduce task attempt.
     *
     * @param taskAttemptId task attempt id
     * @param timestamp time stamp when task failed
     * @param hostName host name of the task attempt.
     * @param error error message of the task.
     * @param taskType Whether the attempt is cleanup or setup or reduce
     */
    public static void logFailed(TaskAttemptID taskAttemptId, long timestamp,
                                 String hostName, String error,
                                 String taskType) {
      JobID id = taskAttemptId.getJobID();
      ArrayList<PrintWriter> writer = fileManager.getWriters(id);

      if (null != writer){
        JobHistory.log(writer, RecordTypes.ReduceAttempt,
                       new Keys[]{  Keys.TASK_TYPE, Keys.TASKID,
                                    Keys.TASK_ATTEMPT_ID, Keys.TASK_STATUS,
                                    Keys.FINISH_TIME, Keys.HOSTNAME,
                                    Keys.ERROR },
                       new String[]{ taskType,
                                     taskAttemptId.getTaskID().toString(),
                                     taskAttemptId.toString(),
                                     Values.FAILED.name(),
                                     String.valueOf(timestamp), hostName, error }, id);
      }
    }

    /**
     * Log killed reduce task attempt.
     * @param taskAttemptId task attempt id
     * @param timestamp time stamp when task failed
     * @param hostName host name of the task attempt.
     * @param error error message of the task.
     * @deprecated Use
     * {@link #logKilled(TaskAttemptID, long, String, String, String)}
     */
    @Deprecated
    public static void logKilled(TaskAttemptID taskAttemptId, long timestamp,
                                 String hostName, String error) {
      logKilled(taskAttemptId, timestamp, hostName, error, Values.REDUCE.name());
    }

    /**
     * Log killed reduce task attempt.
     *
     * @param taskAttemptId task attempt id
     * @param timestamp time stamp when task failed
     * @param hostName host name of the task attempt.
     * @param error error message of the task.
     * @param taskType Whether the attempt is cleanup or setup or reduce
    */
    public static void logKilled(TaskAttemptID taskAttemptId, long timestamp,
                                 String hostName, String error,
                                 String taskType) {
      JobID id = taskAttemptId.getJobID();
      ArrayList<PrintWriter> writer = fileManager.getWriters(id);

      if (null != writer){
        JobHistory.log(writer, RecordTypes.ReduceAttempt,
                       new Keys[]{  Keys.TASK_TYPE, Keys.TASKID,
                                    Keys.TASK_ATTEMPT_ID, Keys.TASK_STATUS,
                                    Keys.FINISH_TIME, Keys.HOSTNAME,
                                    Keys.ERROR },
                       new String[]{ taskType,
                                     taskAttemptId.getTaskID().toString(),
                                     taskAttemptId.toString(),
                                     Values.KILLED.name(),
                                     String.valueOf(timestamp),
                                     hostName, error }, id);
      }
    }
  }

  /**
   * Callback interface for reading back log events from JobHistory. This interface
   * should be implemented and passed to JobHistory.parseHistory()
   *
   */
  public static interface Listener{
    /**
     * Callback method for history parser.
     * @param recType type of record, which is the first entry in the line.
     * @param values a map of key-value pairs as thry appear in history.
     * @throws IOException
     */
    public void handle(RecordTypes recType, Map<Keys, String> values) throws IOException;
  }

  /**
   * Returns the time in milliseconds, truncated to the day.
   */
  static long directoryTime(String year, String month, String day) {
    Calendar result = Calendar.getInstance();
    result.clear();

    result.set(Calendar.YEAR, Integer.parseInt(year));

    // months are 0-based in Calendar, but people will expect January
    // to be month #1 .  Therefore the number is bumped before we make the
    // directory name and must be debumped to seek the time.
    result.set(Calendar.MONTH, Integer.parseInt(month) - 1);

    result.set(Calendar.DAY_OF_MONTH, Integer.parseInt(day));

    // truncate to day granularity
    long timeInMillis = result.getTimeInMillis();
    return timeInMillis;
  }

  /**
   * Delete history files older than one month. Update master index and remove all
   * jobs older than one month. Also if a job tracker has no jobs in last one month
   * remove reference to the job tracker.
   *
   */
  public static class HistoryCleaner implements Runnable {
    static final long ONE_DAY_IN_MS = 24 * 60 * 60 * 1000L;
    static final long DEFAULT_HISTORY_MAX_AGE = 30 * ONE_DAY_IN_MS;
    static final long DEFAULT_CLEANUP_FREQUENCY = ONE_DAY_IN_MS;
    static long cleanupFrequency = DEFAULT_CLEANUP_FREQUENCY;
    static long maxAgeOfHistoryFiles = DEFAULT_HISTORY_MAX_AGE;
    private long now;
    private static final AtomicBoolean isRunning = new AtomicBoolean(false);
    private static long lastRan = 0;

    private static Pattern parseDirectory
      = Pattern.compile(".+/([0-9]+)/([0-9]+)/([0-9]+)/[0-9]+/?");

    /**
     * Cleans up history data.
     */
    public void run() {
      if (isRunning.getAndSet(true)) {
        return;
      }
      now = System.currentTimeMillis();
      // clean history only once a day at max
      if (lastRan != 0 && (now - lastRan) < cleanupFrequency) {
        isRunning.set(false);
        return;
      }
      lastRan = now;
      clean(now);
    }

    public void clean(long now) {
      Set<String> deletedPathnames = new HashSet<String>();

      // XXXXX debug code
      boolean printedOneDeletee = false;
      boolean printedOneMovedFile = false;

      try {
        Path[] datedDirectories
          = FileUtil.stat2Paths(localGlobber(DONEDIR_FS, DONE,
                                             DONE_BEFORE_SERIAL_TAIL, null));

        // any file with a timestamp earlier than cutoff should be deleted
        long cutoff = now - maxAgeOfHistoryFiles;
        Calendar cutoffDay = Calendar.getInstance();
        cutoffDay.setTimeInMillis(cutoff);
        cutoffDay.set(Calendar.HOUR_OF_DAY, 0);
        cutoffDay.set(Calendar.MINUTE, 0);
        cutoffDay.set(Calendar.SECOND, 0);
        cutoffDay.set(Calendar.MILLISECOND, 0);

        // find directories older than the maximum age
        for (int i = 0; i < datedDirectories.length; ++i) {
          String thisDir = datedDirectories[i].toString();
          Matcher pathMatcher = parseDirectory.matcher(thisDir);

          if (pathMatcher.matches()) {
            long dirDay = directoryTime(pathMatcher.group(1),
                                         pathMatcher.group(2),
                                         pathMatcher.group(3));

            if (LOG.isDebugEnabled()) {
              LOG.debug("HistoryCleaner.run just parsed " + thisDir
                  + " as year/month/day = " + pathMatcher.group(1) + "/"
                  + pathMatcher.group(2) + "/" + pathMatcher.group(3));
            }

            if (dirDay <= cutoffDay.getTimeInMillis()) {
              if (LOG.isDebugEnabled()) {
                Calendar nnow = Calendar.getInstance();
                nnow.setTimeInMillis(now);
                Calendar then = Calendar.getInstance();
                then.setTimeInMillis(dirDay);

                LOG.debug("HistoryCleaner.run directory: " + thisDir
                    + " because its time is " + then + " but it's now " + nnow);
              }
            }

            // if dirDay is cutoffDay, some files may be old enough and others not
            if (dirDay == cutoffDay.getTimeInMillis()) {
              // remove old enough files in the directory
              FileStatus[] possibleDeletees = DONEDIR_FS.listStatus(datedDirectories[i]);

              for (int j = 0; j < possibleDeletees.length; ++j) {
            	  if (possibleDeletees[j].getModificationTime() < now -
            	      maxAgeOfHistoryFiles) {
            	    Path deletee = possibleDeletees[j].getPath();
                  if (LOG.isDebugEnabled() && !printedOneDeletee) {
                    LOG.debug("HistoryCleaner.run deletee: "
                        + deletee.toString());
                    printedOneDeletee = true;
                  }

                  DONEDIR_FS.delete(deletee);
                  deletedPathnames.add(deletee.toString());
            	  }
              }
            }

            // if the directory is older than cutoffDay, we can flat out
            // delete it because all the files in it are old enough
            if (dirDay < cutoffDay.getTimeInMillis()) {
              synchronized (existingDoneSubdirs) {
                if (!existingDoneSubdirs.contains(datedDirectories[i])) {
                  LOG.warn("JobHistory: existingDoneSubdirs doesn't contain "
                      + datedDirectories[i] + ", but should.");
                }
                DONEDIR_FS.delete(datedDirectories[i], true);
                existingDoneSubdirs.remove(datedDirectories[i]);
              }
            }
          }
        }

        //walking over the map to purge entries from jobHistoryFileMap
        synchronized (jobHistoryFileMap) {
          Iterator<Entry<JobID, MovedFileInfo>> it =
            jobHistoryFileMap.entrySet().iterator();
          while (it.hasNext()) {
            MovedFileInfo info = it.next().getValue();

            if (LOG.isDebugEnabled() && !printedOneMovedFile) {
              LOG.debug("HistoryCleaner.run a moved file: " + info.historyFile);
              printedOneMovedFile = true;
            }

            if (deletedPathnames.contains(info.historyFile)) {
              it.remove();
            }
          }
        }
      } catch (IOException ie) {
        LOG.info("Error cleaning up history directory" +
                 StringUtils.stringifyException(ie));
      } finally {
          isRunning.set(false);
      }
    }

    static long getLastRan() {
      return lastRan;
    }
  }

  /**
   * Return the TaskLogsUrl of a particular TaskAttempt
   *
   * @param attempt
   * @return the taskLogsUrl. null if http-port or tracker-name or
   *         task-attempt-id are unavailable.
   */
  public static String getTaskLogsUrl(JobHistory.TaskAttempt attempt) {
    if (attempt.get(Keys.HTTP_PORT).equals("")
        || attempt.get(Keys.TRACKER_NAME).equals("")
        || attempt.get(Keys.TASK_ATTEMPT_ID).equals("")) {
      return null;
    }

    String taskTrackerName =
      JobInProgress.convertTrackerNameToHostName(
        attempt.get(Keys.TRACKER_NAME));
    return TaskLogServlet.getTaskLogUrl(taskTrackerName, attempt
        .get(Keys.HTTP_PORT), attempt.get(Keys.TASK_ATTEMPT_ID));
  }
}