1 package template;
2 
3 import java.io.PrintStream;
4 
5 import fileIO.ByteFile;
6 import fileIO.ByteFile1;
7 import fileIO.ByteFile2;
8 import fileIO.ByteStreamWriter;
9 import fileIO.FileFormat;
10 import fileIO.ReadWrite;
11 import shared.Parse;
12 import shared.Parser;
13 import shared.PreParser;
14 import shared.Shared;
15 import shared.Timer;
16 import shared.Tools;
17 import structures.ByteBuilder;
18 
19 /**
20  * Reads a text file.
21  * Prints it to another text file.
22  * Filters out invalid lines and prints them to an optional third file.
23  * @author Brian Bushnell
24  * @date May 9, 2016
25  *
26  */
27 public class A_SampleByteFile {
28 
29 	/*--------------------------------------------------------------*/
30 	/*----------------        Initialization        ----------------*/
31 	/*--------------------------------------------------------------*/
32 
33 	/**
34 	 * Code entrance from the command line.
35 	 * @param args Command line arguments
36 	 */
main(String[] args)37 	public static void main(String[] args){
38 		//Start a timer immediately upon code entrance.
39 		Timer t=new Timer();
40 
41 		//Create an instance of this class
42 		A_SampleByteFile x=new A_SampleByteFile(args);
43 
44 		//Run the object
45 		x.process(t);
46 
47 		//Close the print stream if it was redirected
48 		Shared.closeStream(x.outstream);
49 	}
50 
51 	/**
52 	 * Constructor.
53 	 * @param args Command line arguments
54 	 */
A_SampleByteFile(String[] args)55 	public A_SampleByteFile(String[] args){
56 
57 		{//Preparse block for help, config files, and outstream
58 			PreParser pp=new PreParser(args, /*getClass()*/null, false);
59 			args=pp.args;
60 			outstream=pp.outstream;
61 		}
62 
63 		//Set shared static variables prior to parsing
64 		ReadWrite.USE_PIGZ=ReadWrite.USE_UNPIGZ=true;
65 		ReadWrite.MAX_ZIP_THREADS=Shared.threads();
66 
67 		{//Parse the arguments
68 			final Parser parser=parse(args);
69 			overwrite=parser.overwrite;
70 			append=parser.append;
71 
72 			in1=parser.in1;
73 
74 			out1=parser.out1;
75 		}
76 
77 		fixExtensions(); //Add or remove .gz or .bz2 as needed
78 		checkFileExistence(); //Ensure files can be read and written
79 		checkStatics(); //Adjust file-related static fields as needed for this program
80 
81 		ffout1=FileFormat.testOutput(out1, FileFormat.TXT, null, true, overwrite, append, false);
82 		ffoutInvalid=FileFormat.testOutput(outInvalid, FileFormat.TXT, null, true, overwrite, append, false);
83 		ffin1=FileFormat.testInput(in1, FileFormat.TXT, null, true, true);
84 	}
85 
86 	/*--------------------------------------------------------------*/
87 	/*----------------    Initialization Helpers    ----------------*/
88 	/*--------------------------------------------------------------*/
89 
90 	/** Parse arguments from the command line */
parse(String[] args)91 	private Parser parse(String[] args){
92 
93 		Parser parser=new Parser();
94 		for(int i=0; i<args.length; i++){
95 			String arg=args[i];
96 			String[] split=arg.split("=");
97 			String a=split[0].toLowerCase();
98 			String b=split.length>1 ? split[1] : null;
99 			if(b!=null && b.equalsIgnoreCase("null")){b=null;}
100 
101 			if(a.equals("invalid")){
102 				outInvalid=b;
103 			}else if(a.equals("lines")){
104 				maxLines=Long.parseLong(b);
105 				if(maxLines<0){maxLines=Long.MAX_VALUE;}
106 			}else if(a.equals("verbose")){
107 				verbose=Parse.parseBoolean(b);
108 				ByteFile1.verbose=verbose;
109 				ByteFile2.verbose=verbose;
110 				ReadWrite.verbose=verbose;
111 			}else if(parser.parse(arg, a, b)){
112 				//do nothing
113 			}else{
114 				outstream.println("Unknown parameter "+args[i]);
115 				assert(false) : "Unknown parameter "+args[i];
116 				//				throw new RuntimeException("Unknown parameter "+args[i]);
117 			}
118 		}
119 
120 		return parser;
121 	}
122 
123 	/** Add or remove .gz or .bz2 as needed */
fixExtensions()124 	private void fixExtensions(){
125 		in1=Tools.fixExtension(in1);
126 		if(in1==null){throw new RuntimeException("Error - at least one input file is required.");}
127 	}
128 
129 	/** Ensure files can be read and written */
checkFileExistence()130 	private void checkFileExistence(){
131 		//Ensure output files can be written
132 		if(!Tools.testOutputFiles(overwrite, append, false, out1)){
133 			outstream.println((out1==null)+", "+out1);
134 			throw new RuntimeException("\n\noverwrite="+overwrite+"; Can't write to output file "+out1+"\n");
135 		}
136 
137 		//Ensure input files can be read
138 		if(!Tools.testInputFiles(false, true, in1)){
139 			throw new RuntimeException("\nCan't read some input files.\n");
140 		}
141 
142 		//Ensure that no file was specified multiple times
143 		if(!Tools.testForDuplicateFiles(true, in1, out1)){
144 			throw new RuntimeException("\nSome file names were specified multiple times.\n");
145 		}
146 	}
147 
148 	/** Adjust file-related static fields as needed for this program */
checkStatics()149 	private static void checkStatics(){
150 		//Adjust the number of threads for input file reading
151 		if(!ByteFile.FORCE_MODE_BF1 && !ByteFile.FORCE_MODE_BF2 && Shared.threads()>2){
152 			ByteFile.FORCE_MODE_BF2=true;
153 		}
154 
155 //		if(!ByteFile.FORCE_MODE_BF2){
156 //			ByteFile.FORCE_MODE_BF2=false;
157 //			ByteFile.FORCE_MODE_BF1=true;
158 //		}
159 	}
160 
161 	/*--------------------------------------------------------------*/
162 	/*----------------         Outer Methods        ----------------*/
163 	/*--------------------------------------------------------------*/
164 
process(Timer t)165 	void process(Timer t){
166 
167 		ByteFile bf=ByteFile.makeByteFile(ffin1);
168 		ByteStreamWriter bsw=makeBSW(ffout1);
169 		ByteStreamWriter bswInvalid=makeBSW(ffoutInvalid);
170 
171 //		assert(false) : "Header goes here.";
172 		if(bsw!=null){
173 //			assert(false) : "Header goes here.";
174 		}
175 
176 		processInner(bf, bsw, bswInvalid);
177 
178 		errorState|=bf.close();
179 		if(bsw!=null){errorState|=bsw.poisonAndWait();}
180 		if(bswInvalid!=null){errorState|=bswInvalid.poisonAndWait();}
181 
182 		t.stop();
183 
184 		outstream.println(Tools.timeLinesBytesProcessed(t, linesProcessed, bytesProcessed, 8));
185 
186 		outstream.println();
187 		outstream.println("Valid Lines:       \t"+linesOut);
188 		outstream.println("Invalid Lines:     \t"+(linesProcessed-linesOut));
189 
190 		if(errorState){
191 			throw new RuntimeException(getClass().getName()+" terminated in an error state; the output may be corrupt.");
192 		}
193 	}
194 
195 	/*--------------------------------------------------------------*/
196 	/*----------------         Inner Methods        ----------------*/
197 	/*--------------------------------------------------------------*/
198 
processInner(ByteFile bf, ByteStreamWriter bsw, ByteStreamWriter bswInvalid)199 	private void processInner(ByteFile bf, ByteStreamWriter bsw, ByteStreamWriter bswInvalid){
200 		byte[] line=bf.nextLine();
201 		ByteBuilder bb=new ByteBuilder();
202 
203 		while(line!=null){
204 			if(line.length>0){
205 				if(maxLines>0 && linesProcessed>=maxLines){break;}
206 				linesProcessed++;
207 				bytesProcessed+=(line.length+1);
208 
209 				final boolean valid=(line[0]!='#');
210 
211 				if(valid){
212 					linesOut++;
213 					bytesOut+=(line.length+1);
214 					for(int i=0; i<line.length && line[i]!='\t'; i++){
215 						bb.append(line[i]);
216 					}
217 					bb.nl();
218 					bsw.print(bb.toBytes());
219 					bb.clear();
220 				}else{
221 					if(bswInvalid!=null){
222 						bswInvalid.println(line);
223 					}
224 				}
225 			}
226 			line=bf.nextLine();
227 		}
228 	}
229 
makeBSW(FileFormat ff)230 	private static ByteStreamWriter makeBSW(FileFormat ff){
231 		if(ff==null){return null;}
232 		ByteStreamWriter bsw=new ByteStreamWriter(ff);
233 		bsw.start();
234 		return bsw;
235 	}
236 
237 	/*--------------------------------------------------------------*/
238 	/*----------------            Fields            ----------------*/
239 	/*--------------------------------------------------------------*/
240 
241 	private String in1=null;
242 	private String out1=null;
243 	private String outInvalid=null;
244 
245 	/*--------------------------------------------------------------*/
246 
247 	private long linesProcessed=0;
248 	private long linesOut=0;
249 	private long bytesProcessed=0;
250 	private long bytesOut=0;
251 
252 	private long maxLines=Long.MAX_VALUE;
253 
254 	/*--------------------------------------------------------------*/
255 	/*----------------         Final Fields         ----------------*/
256 	/*--------------------------------------------------------------*/
257 
258 	private final FileFormat ffin1;
259 	private final FileFormat ffout1;
260 	private final FileFormat ffoutInvalid;
261 
262 	/*--------------------------------------------------------------*/
263 	/*----------------        Common Fields         ----------------*/
264 	/*--------------------------------------------------------------*/
265 
266 	private PrintStream outstream=System.err;
267 	public static boolean verbose=false;
268 	public boolean errorState=false;
269 	private boolean overwrite=false;
270 	private boolean append=false;
271 
272 }
273