1 package template; 2 3 import java.io.PrintStream; 4 5 import fileIO.ByteFile; 6 import fileIO.ByteFile1; 7 import fileIO.ByteFile2; 8 import fileIO.ByteStreamWriter; 9 import fileIO.FileFormat; 10 import fileIO.ReadWrite; 11 import shared.Parse; 12 import shared.Parser; 13 import shared.PreParser; 14 import shared.Shared; 15 import shared.Timer; 16 import shared.Tools; 17 import structures.ByteBuilder; 18 19 /** 20 * Reads a text file. 21 * Prints it to another text file. 22 * Filters out invalid lines and prints them to an optional third file. 23 * @author Brian Bushnell 24 * @date May 9, 2016 25 * 26 */ 27 public class A_SampleByteFile { 28 29 /*--------------------------------------------------------------*/ 30 /*---------------- Initialization ----------------*/ 31 /*--------------------------------------------------------------*/ 32 33 /** 34 * Code entrance from the command line. 35 * @param args Command line arguments 36 */ main(String[] args)37 public static void main(String[] args){ 38 //Start a timer immediately upon code entrance. 39 Timer t=new Timer(); 40 41 //Create an instance of this class 42 A_SampleByteFile x=new A_SampleByteFile(args); 43 44 //Run the object 45 x.process(t); 46 47 //Close the print stream if it was redirected 48 Shared.closeStream(x.outstream); 49 } 50 51 /** 52 * Constructor. 53 * @param args Command line arguments 54 */ A_SampleByteFile(String[] args)55 public A_SampleByteFile(String[] args){ 56 57 {//Preparse block for help, config files, and outstream 58 PreParser pp=new PreParser(args, /*getClass()*/null, false); 59 args=pp.args; 60 outstream=pp.outstream; 61 } 62 63 //Set shared static variables prior to parsing 64 ReadWrite.USE_PIGZ=ReadWrite.USE_UNPIGZ=true; 65 ReadWrite.MAX_ZIP_THREADS=Shared.threads(); 66 67 {//Parse the arguments 68 final Parser parser=parse(args); 69 overwrite=parser.overwrite; 70 append=parser.append; 71 72 in1=parser.in1; 73 74 out1=parser.out1; 75 } 76 77 fixExtensions(); //Add or remove .gz or .bz2 as needed 78 checkFileExistence(); //Ensure files can be read and written 79 checkStatics(); //Adjust file-related static fields as needed for this program 80 81 ffout1=FileFormat.testOutput(out1, FileFormat.TXT, null, true, overwrite, append, false); 82 ffoutInvalid=FileFormat.testOutput(outInvalid, FileFormat.TXT, null, true, overwrite, append, false); 83 ffin1=FileFormat.testInput(in1, FileFormat.TXT, null, true, true); 84 } 85 86 /*--------------------------------------------------------------*/ 87 /*---------------- Initialization Helpers ----------------*/ 88 /*--------------------------------------------------------------*/ 89 90 /** Parse arguments from the command line */ parse(String[] args)91 private Parser parse(String[] args){ 92 93 Parser parser=new Parser(); 94 for(int i=0; i<args.length; i++){ 95 String arg=args[i]; 96 String[] split=arg.split("="); 97 String a=split[0].toLowerCase(); 98 String b=split.length>1 ? split[1] : null; 99 if(b!=null && b.equalsIgnoreCase("null")){b=null;} 100 101 if(a.equals("invalid")){ 102 outInvalid=b; 103 }else if(a.equals("lines")){ 104 maxLines=Long.parseLong(b); 105 if(maxLines<0){maxLines=Long.MAX_VALUE;} 106 }else if(a.equals("verbose")){ 107 verbose=Parse.parseBoolean(b); 108 ByteFile1.verbose=verbose; 109 ByteFile2.verbose=verbose; 110 ReadWrite.verbose=verbose; 111 }else if(parser.parse(arg, a, b)){ 112 //do nothing 113 }else{ 114 outstream.println("Unknown parameter "+args[i]); 115 assert(false) : "Unknown parameter "+args[i]; 116 // throw new RuntimeException("Unknown parameter "+args[i]); 117 } 118 } 119 120 return parser; 121 } 122 123 /** Add or remove .gz or .bz2 as needed */ fixExtensions()124 private void fixExtensions(){ 125 in1=Tools.fixExtension(in1); 126 if(in1==null){throw new RuntimeException("Error - at least one input file is required.");} 127 } 128 129 /** Ensure files can be read and written */ checkFileExistence()130 private void checkFileExistence(){ 131 //Ensure output files can be written 132 if(!Tools.testOutputFiles(overwrite, append, false, out1)){ 133 outstream.println((out1==null)+", "+out1); 134 throw new RuntimeException("\n\noverwrite="+overwrite+"; Can't write to output file "+out1+"\n"); 135 } 136 137 //Ensure input files can be read 138 if(!Tools.testInputFiles(false, true, in1)){ 139 throw new RuntimeException("\nCan't read some input files.\n"); 140 } 141 142 //Ensure that no file was specified multiple times 143 if(!Tools.testForDuplicateFiles(true, in1, out1)){ 144 throw new RuntimeException("\nSome file names were specified multiple times.\n"); 145 } 146 } 147 148 /** Adjust file-related static fields as needed for this program */ checkStatics()149 private static void checkStatics(){ 150 //Adjust the number of threads for input file reading 151 if(!ByteFile.FORCE_MODE_BF1 && !ByteFile.FORCE_MODE_BF2 && Shared.threads()>2){ 152 ByteFile.FORCE_MODE_BF2=true; 153 } 154 155 // if(!ByteFile.FORCE_MODE_BF2){ 156 // ByteFile.FORCE_MODE_BF2=false; 157 // ByteFile.FORCE_MODE_BF1=true; 158 // } 159 } 160 161 /*--------------------------------------------------------------*/ 162 /*---------------- Outer Methods ----------------*/ 163 /*--------------------------------------------------------------*/ 164 process(Timer t)165 void process(Timer t){ 166 167 ByteFile bf=ByteFile.makeByteFile(ffin1); 168 ByteStreamWriter bsw=makeBSW(ffout1); 169 ByteStreamWriter bswInvalid=makeBSW(ffoutInvalid); 170 171 // assert(false) : "Header goes here."; 172 if(bsw!=null){ 173 // assert(false) : "Header goes here."; 174 } 175 176 processInner(bf, bsw, bswInvalid); 177 178 errorState|=bf.close(); 179 if(bsw!=null){errorState|=bsw.poisonAndWait();} 180 if(bswInvalid!=null){errorState|=bswInvalid.poisonAndWait();} 181 182 t.stop(); 183 184 outstream.println(Tools.timeLinesBytesProcessed(t, linesProcessed, bytesProcessed, 8)); 185 186 outstream.println(); 187 outstream.println("Valid Lines: \t"+linesOut); 188 outstream.println("Invalid Lines: \t"+(linesProcessed-linesOut)); 189 190 if(errorState){ 191 throw new RuntimeException(getClass().getName()+" terminated in an error state; the output may be corrupt."); 192 } 193 } 194 195 /*--------------------------------------------------------------*/ 196 /*---------------- Inner Methods ----------------*/ 197 /*--------------------------------------------------------------*/ 198 processInner(ByteFile bf, ByteStreamWriter bsw, ByteStreamWriter bswInvalid)199 private void processInner(ByteFile bf, ByteStreamWriter bsw, ByteStreamWriter bswInvalid){ 200 byte[] line=bf.nextLine(); 201 ByteBuilder bb=new ByteBuilder(); 202 203 while(line!=null){ 204 if(line.length>0){ 205 if(maxLines>0 && linesProcessed>=maxLines){break;} 206 linesProcessed++; 207 bytesProcessed+=(line.length+1); 208 209 final boolean valid=(line[0]!='#'); 210 211 if(valid){ 212 linesOut++; 213 bytesOut+=(line.length+1); 214 for(int i=0; i<line.length && line[i]!='\t'; i++){ 215 bb.append(line[i]); 216 } 217 bb.nl(); 218 bsw.print(bb.toBytes()); 219 bb.clear(); 220 }else{ 221 if(bswInvalid!=null){ 222 bswInvalid.println(line); 223 } 224 } 225 } 226 line=bf.nextLine(); 227 } 228 } 229 makeBSW(FileFormat ff)230 private static ByteStreamWriter makeBSW(FileFormat ff){ 231 if(ff==null){return null;} 232 ByteStreamWriter bsw=new ByteStreamWriter(ff); 233 bsw.start(); 234 return bsw; 235 } 236 237 /*--------------------------------------------------------------*/ 238 /*---------------- Fields ----------------*/ 239 /*--------------------------------------------------------------*/ 240 241 private String in1=null; 242 private String out1=null; 243 private String outInvalid=null; 244 245 /*--------------------------------------------------------------*/ 246 247 private long linesProcessed=0; 248 private long linesOut=0; 249 private long bytesProcessed=0; 250 private long bytesOut=0; 251 252 private long maxLines=Long.MAX_VALUE; 253 254 /*--------------------------------------------------------------*/ 255 /*---------------- Final Fields ----------------*/ 256 /*--------------------------------------------------------------*/ 257 258 private final FileFormat ffin1; 259 private final FileFormat ffout1; 260 private final FileFormat ffoutInvalid; 261 262 /*--------------------------------------------------------------*/ 263 /*---------------- Common Fields ----------------*/ 264 /*--------------------------------------------------------------*/ 265 266 private PrintStream outstream=System.err; 267 public static boolean verbose=false; 268 public boolean errorState=false; 269 private boolean overwrite=false; 270 private boolean append=false; 271 272 } 273