1 /** 2 * Licensed to the Apache Software Foundation (ASF) under one 3 * or more contributor license agreements. See the NOTICE file 4 * distributed with this work for additional information 5 * regarding copyright ownership. The ASF licenses this file 6 * to you under the Apache License, Version 2.0 (the 7 * "License"); you may not use this file except in compliance 8 * with the License. You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 */ 18 package org.apache.hadoop.hdfs.tools.offlineImageViewer; 19 20 import java.io.EOFException; 21 import java.io.IOException; 22 import java.io.PrintStream; 23 import java.io.RandomAccessFile; 24 25 import org.apache.commons.cli.CommandLine; 26 import org.apache.commons.cli.CommandLineParser; 27 import org.apache.commons.cli.OptionBuilder; 28 import org.apache.commons.cli.Options; 29 import org.apache.commons.cli.ParseException; 30 import org.apache.commons.cli.PosixParser; 31 import org.apache.commons.logging.Log; 32 import org.apache.commons.logging.LogFactory; 33 import org.apache.hadoop.classification.InterfaceAudience; 34 import org.apache.hadoop.conf.Configuration; 35 import org.apache.hadoop.net.NetUtils; 36 37 /** 38 * OfflineImageViewerPB to dump the contents of an Hadoop image file to XML or 39 * the console. Main entry point into utility, either via the command line or 40 * programatically. 41 */ 42 @InterfaceAudience.Private 43 public class OfflineImageViewerPB { 44 public static final Log LOG = LogFactory.getLog(OfflineImageViewerPB.class); 45 46 private final static String usage = "Usage: bin/hdfs oiv [OPTIONS] -i INPUTFILE -o OUTPUTFILE\n" 47 + "Offline Image Viewer\n" 48 + "View a Hadoop fsimage INPUTFILE using the specified PROCESSOR,\n" 49 + "saving the results in OUTPUTFILE.\n" 50 + "\n" 51 + "The oiv utility will attempt to parse correctly formed image files\n" 52 + "and will abort fail with mal-formed image files.\n" 53 + "\n" 54 + "The tool works offline and does not require a running cluster in\n" 55 + "order to process an image file.\n" 56 + "\n" 57 + "The following image processors are available:\n" 58 + " * XML: This processor creates an XML document with all elements of\n" 59 + " the fsimage enumerated, suitable for further analysis by XML\n" 60 + " tools.\n" 61 + " * FileDistribution: This processor analyzes the file size\n" 62 + " distribution in the image.\n" 63 + " -maxSize specifies the range [0, maxSize] of file sizes to be\n" 64 + " analyzed (128GB by default).\n" 65 + " -step defines the granularity of the distribution. (2MB by default)\n" 66 + " * Web: Run a viewer to expose read-only WebHDFS API.\n" 67 + " -addr specifies the address to listen. (localhost:5978 by default)\n" 68 + " * Delimited (experimental): Generate a text file with all of the elements common\n" 69 + " to both inodes and inodes-under-construction, separated by a\n" 70 + " delimiter. The default delimiter is \\t, though this may be\n" 71 + " changed via the -delimiter argument.\n" 72 + "\n" 73 + "Required command line arguments:\n" 74 + "-i,--inputFile <arg> FSImage file to process.\n" 75 + "\n" 76 + "Optional command line arguments:\n" 77 + "-o,--outputFile <arg> Name of output file. If the specified\n" 78 + " file exists, it will be overwritten.\n" 79 + " (output to stdout by default)\n" 80 + "-p,--processor <arg> Select which type of processor to apply\n" 81 + " against image file. (XML|FileDistribution|Web|Delimited)\n" 82 + " (Web by default)\n" 83 + "-delimiter <arg> Delimiting string to use with Delimited processor. \n" 84 + "-t,--temp <arg> Use temporary dir to cache intermediate result to generate\n" 85 + " Delimited outputs. If not set, Delimited processor constructs\n" 86 + " the namespace in memory before outputting text.\n" 87 + "-h,--help Display usage information and exit\n"; 88 89 /** 90 * Build command-line options and descriptions 91 */ buildOptions()92 private static Options buildOptions() { 93 Options options = new Options(); 94 95 // Build in/output file arguments, which are required, but there is no 96 // addOption method that can specify this 97 OptionBuilder.isRequired(); 98 OptionBuilder.hasArgs(); 99 OptionBuilder.withLongOpt("inputFile"); 100 options.addOption(OptionBuilder.create("i")); 101 102 options.addOption("o", "outputFile", true, ""); 103 options.addOption("p", "processor", true, ""); 104 options.addOption("h", "help", false, ""); 105 options.addOption("maxSize", true, ""); 106 options.addOption("step", true, ""); 107 options.addOption("addr", true, ""); 108 options.addOption("delimiter", true, ""); 109 options.addOption("t", "temp", true, ""); 110 111 return options; 112 } 113 114 /** 115 * Entry point to command-line-driven operation. User may specify options and 116 * start fsimage viewer from the command line. Program will process image file 117 * and exit cleanly or, if an error is encountered, inform user and exit. 118 * 119 * @param args 120 * Command line options 121 * @throws IOException 122 */ main(String[] args)123 public static void main(String[] args) throws Exception { 124 int status = run(args); 125 System.exit(status); 126 } 127 run(String[] args)128 public static int run(String[] args) throws Exception { 129 Options options = buildOptions(); 130 if (args.length == 0) { 131 printUsage(); 132 return 0; 133 } 134 135 CommandLineParser parser = new PosixParser(); 136 CommandLine cmd; 137 138 try { 139 cmd = parser.parse(options, args); 140 } catch (ParseException e) { 141 System.out.println("Error parsing command-line options: "); 142 printUsage(); 143 return -1; 144 } 145 146 if (cmd.hasOption("h")) { // print help and exit 147 printUsage(); 148 return 0; 149 } 150 151 String inputFile = cmd.getOptionValue("i"); 152 String processor = cmd.getOptionValue("p", "Web"); 153 String outputFile = cmd.getOptionValue("o", "-"); 154 String delimiter = cmd.getOptionValue("delimiter", 155 PBImageDelimitedTextWriter.DEFAULT_DELIMITER); 156 String tempPath = cmd.getOptionValue("t", ""); 157 158 Configuration conf = new Configuration(); 159 try (PrintStream out = outputFile.equals("-") ? 160 System.out : new PrintStream(outputFile, "UTF-8")) { 161 switch (processor) { 162 case "FileDistribution": 163 long maxSize = Long.parseLong(cmd.getOptionValue("maxSize", "0")); 164 int step = Integer.parseInt(cmd.getOptionValue("step", "0")); 165 new FileDistributionCalculator(conf, maxSize, step, out).visit( 166 new RandomAccessFile(inputFile, "r")); 167 break; 168 case "XML": 169 new PBImageXmlWriter(conf, out).visit( 170 new RandomAccessFile(inputFile, "r")); 171 break; 172 case "Web": 173 String addr = cmd.getOptionValue("addr", "localhost:5978"); 174 try (WebImageViewer viewer = new WebImageViewer( 175 NetUtils.createSocketAddr(addr))) { 176 viewer.start(inputFile); 177 } 178 break; 179 case "Delimited": 180 try (PBImageDelimitedTextWriter writer = 181 new PBImageDelimitedTextWriter(out, delimiter, tempPath)) { 182 writer.visit(new RandomAccessFile(inputFile, "r")); 183 } 184 break; 185 } 186 return 0; 187 } catch (EOFException e) { 188 System.err.println("Input file ended unexpectedly. Exiting"); 189 } catch (IOException e) { 190 System.err.println("Encountered exception. Exiting: " + e.getMessage()); 191 } 192 return -1; 193 } 194 195 /** 196 * Print application usage instructions. 197 */ printUsage()198 private static void printUsage() { 199 System.out.println(usage); 200 } 201 } 202