1 package driver;
2 
3 import java.io.File;
4 import java.io.IOException;
5 import java.io.OutputStream;
6 import java.io.PrintWriter;
7 import java.util.ArrayList;
8 import java.util.concurrent.ArrayBlockingQueue;
9 
10 import fileIO.ReadWrite;
11 import fileIO.TextFile;
12 import shared.Parse;
13 import shared.Parser;
14 import shared.PreParser;
15 import shared.ReadStats;
16 import shared.Timer;
17 
18 public class ConcatenateTextFiles {
19 
20 	/** Format: infile1,infile2,...infileN,outfile */
main(String[] args)21 	public static void main(String[] args){
22 
23 		{//Preparse block for help, config files, and outstream
24 			PreParser pp=new PreParser(args, new Object() { }.getClass().getEnclosingClass(), false);
25 			args=pp.args;
26 			//outstream=pp.outstream;
27 		}
28 
29 		Timer t=new Timer();
30 
31 		if(ReadWrite.ZIPLEVEL<6){ReadWrite.ZIPLEVEL=6;}
32 
33 		for(int i=0; i<args.length; i++){
34 			final String arg=args[i];
35 			final String[] split=arg.split("=");
36 			String a=split[0].toLowerCase();
37 			String b=split.length>1 ? split[1] : null;
38 
39 			if(Parser.parseZip(arg, a, b)){
40 				//do nothing
41 			}else if(a.equals("append") || a.equals("app")){
42 				append=ReadStats.append=Parse.parseBoolean(b);
43 			}else if(a.equals("overwrite") || a.equals("ow")){
44 				overwrite=Parse.parseBoolean(b);
45 			}else{
46 				concatenate(args[i].split(","));
47 			}
48 
49 		}
50 		t.stop();
51 		System.out.println();
52 		System.out.println("Time: \t"+t);
53 	}
54 
concatenate(String[] split)55 	private static void concatenate(String[] split) {
56 		String outname=split[split.length-1];
57 		assert(overwrite || !new File(outname).exists()) : outname+" exists.";
58 
59 		WriteThread wt=new WriteThread(outname);
60 		wt.start();
61 
62 
63 		ArrayList<String>[] bufferptr=new ArrayList[] {new ArrayList<String>(LIST_SIZE)};
64 
65 		for(int i=0; i<split.length-1; i++){
66 			processTerm(split[i], bufferptr, wt);
67 		}
68 
69 		ArrayList<String> buffer=bufferptr[0];
70 		if(buffer==null){
71 			wt.add(new ArrayList<String>(1));
72 		}else if(buffer.isEmpty()){
73 			wt.add(buffer);
74 		}else{
75 			wt.add(buffer);
76 			wt.add(new ArrayList<String>(1));
77 		}
78 
79 	}
80 
processTerm(String term, ArrayList<String>[] bufferptr, WriteThread wt)81 	private static void processTerm(String term, ArrayList<String>[] bufferptr, WriteThread wt){
82 
83 		System.out.println("Processing term "+term);
84 
85 		File f=new File(term);
86 		if(!f.isDirectory()){
87 
88 			TextFile tf=new TextFile(term, false);
89 
90 			ArrayList<String> buffer=bufferptr[0];
91 
92 			String s=null;
93 			for(s=tf.nextLine(); s!=null; s=tf.nextLine()){
94 				buffer.add(s);
95 
96 				//			System.out.println("Added to buffer");
97 				if(buffer.size()>=LIST_SIZE){
98 					//				System.out.println("Sent buffer");
99 
100 //					System.out.println("****** "+term+" ******");
101 //					for(String b : buffer){
102 //						System.out.println(b);
103 //					}
104 
105 					wt.add(buffer);
106 					bufferptr[0]=buffer=new ArrayList<String>(LIST_SIZE);
107 				}
108 			}
109 			tf.close();
110 		}else{
111 			assert(f.isDirectory());
112 			File[] contents=f.listFiles();
113 			for(File c : contents){
114 				String abs=c.getAbsolutePath();
115 				if(!abs.equals(wt.fname)){
116 //					System.out.println(c+" == "+new File(wt.fname)+" : "+c.equals(new File(wt.fname)));
117 					processTerm(abs, bufferptr, wt);
118 				}
119 			}
120 		}
121 	}
122 
123 	private static class WriteThread extends Thread{
124 
WriteThread(String fname_)125 		public WriteThread(String fname_){
126 			String temp=fname_;
127 			try {
128 				temp=new File(fname_).getCanonicalPath();
129 			} catch (IOException e) {
130 				// TODO Auto-generated catch block
131 				e.printStackTrace();
132 			}
133 			fname=temp;
134 			os=ReadWrite.getOutputStream(fname, append, true, true);
135 			writer=new PrintWriter(os);
136 		}
137 
add(ArrayList<String> list)138 		public void add(ArrayList<String> list){
139 			assert(list!=null);
140 			while(list!=null){
141 //				System.out.println("Adding list to queue "+queue.size());
142 				try {
143 					queue.put(list);
144 					list=null;
145 				} catch (InterruptedException e) {
146 					// TODO Auto-generated catch block
147 					e.printStackTrace();
148 				}
149 			}
150 		}
151 
152 		@Override
run()153 		public void run(){
154 
155 			ArrayList<String> list=null;
156 			while(list==null){
157 //				System.out.println("Waiting for list...");
158 				try {
159 					list=queue.take();
160 				} catch (InterruptedException e) {
161 					// TODO Auto-generated catch block
162 					e.printStackTrace();
163 				}
164 //				System.out.println("Took list of size "+(list==null ? "null" : list.size()+""));
165 				if(list!=null){
166 					if(list.isEmpty()){
167 						ReadWrite.finishWriting(writer, os, fname, allowSubprocess);
168 						return;
169 					}
170 					for(String s : list){
171 						if(s!=null){writer.println(s);}
172 					}
173 				}
174 				list=null;
175 			}
176 		}
177 
178 		private final OutputStream os;
179 		private final PrintWriter writer;
180 		private final ArrayBlockingQueue<ArrayList<String>> queue=new ArrayBlockingQueue<ArrayList<String>>(MAX_LISTS);
181 		final String fname;
182 
183 	}
184 
185 	public static final int MAX_LISTS=8;
186 	public static final int LIST_SIZE=100;
187 	public static boolean overwrite=true;
188 	public static boolean append=false;
189 	public static boolean allowSubprocess=true;
190 
191 }
192