1 package driver; 2 3 import java.io.File; 4 import java.io.IOException; 5 import java.io.OutputStream; 6 import java.io.PrintWriter; 7 import java.util.ArrayList; 8 import java.util.concurrent.ArrayBlockingQueue; 9 10 import fileIO.ReadWrite; 11 import fileIO.TextFile; 12 import shared.Parse; 13 import shared.Parser; 14 import shared.PreParser; 15 import shared.ReadStats; 16 import shared.Timer; 17 18 public class ConcatenateTextFiles { 19 20 /** Format: infile1,infile2,...infileN,outfile */ main(String[] args)21 public static void main(String[] args){ 22 23 {//Preparse block for help, config files, and outstream 24 PreParser pp=new PreParser(args, new Object() { }.getClass().getEnclosingClass(), false); 25 args=pp.args; 26 //outstream=pp.outstream; 27 } 28 29 Timer t=new Timer(); 30 31 if(ReadWrite.ZIPLEVEL<6){ReadWrite.ZIPLEVEL=6;} 32 33 for(int i=0; i<args.length; i++){ 34 final String arg=args[i]; 35 final String[] split=arg.split("="); 36 String a=split[0].toLowerCase(); 37 String b=split.length>1 ? split[1] : null; 38 39 if(Parser.parseZip(arg, a, b)){ 40 //do nothing 41 }else if(a.equals("append") || a.equals("app")){ 42 append=ReadStats.append=Parse.parseBoolean(b); 43 }else if(a.equals("overwrite") || a.equals("ow")){ 44 overwrite=Parse.parseBoolean(b); 45 }else{ 46 concatenate(args[i].split(",")); 47 } 48 49 } 50 t.stop(); 51 System.out.println(); 52 System.out.println("Time: \t"+t); 53 } 54 concatenate(String[] split)55 private static void concatenate(String[] split) { 56 String outname=split[split.length-1]; 57 assert(overwrite || !new File(outname).exists()) : outname+" exists."; 58 59 WriteThread wt=new WriteThread(outname); 60 wt.start(); 61 62 63 ArrayList<String>[] bufferptr=new ArrayList[] {new ArrayList<String>(LIST_SIZE)}; 64 65 for(int i=0; i<split.length-1; i++){ 66 processTerm(split[i], bufferptr, wt); 67 } 68 69 ArrayList<String> buffer=bufferptr[0]; 70 if(buffer==null){ 71 wt.add(new ArrayList<String>(1)); 72 }else if(buffer.isEmpty()){ 73 wt.add(buffer); 74 }else{ 75 wt.add(buffer); 76 wt.add(new ArrayList<String>(1)); 77 } 78 79 } 80 processTerm(String term, ArrayList<String>[] bufferptr, WriteThread wt)81 private static void processTerm(String term, ArrayList<String>[] bufferptr, WriteThread wt){ 82 83 System.out.println("Processing term "+term); 84 85 File f=new File(term); 86 if(!f.isDirectory()){ 87 88 TextFile tf=new TextFile(term, false); 89 90 ArrayList<String> buffer=bufferptr[0]; 91 92 String s=null; 93 for(s=tf.nextLine(); s!=null; s=tf.nextLine()){ 94 buffer.add(s); 95 96 // System.out.println("Added to buffer"); 97 if(buffer.size()>=LIST_SIZE){ 98 // System.out.println("Sent buffer"); 99 100 // System.out.println("****** "+term+" ******"); 101 // for(String b : buffer){ 102 // System.out.println(b); 103 // } 104 105 wt.add(buffer); 106 bufferptr[0]=buffer=new ArrayList<String>(LIST_SIZE); 107 } 108 } 109 tf.close(); 110 }else{ 111 assert(f.isDirectory()); 112 File[] contents=f.listFiles(); 113 for(File c : contents){ 114 String abs=c.getAbsolutePath(); 115 if(!abs.equals(wt.fname)){ 116 // System.out.println(c+" == "+new File(wt.fname)+" : "+c.equals(new File(wt.fname))); 117 processTerm(abs, bufferptr, wt); 118 } 119 } 120 } 121 } 122 123 private static class WriteThread extends Thread{ 124 WriteThread(String fname_)125 public WriteThread(String fname_){ 126 String temp=fname_; 127 try { 128 temp=new File(fname_).getCanonicalPath(); 129 } catch (IOException e) { 130 // TODO Auto-generated catch block 131 e.printStackTrace(); 132 } 133 fname=temp; 134 os=ReadWrite.getOutputStream(fname, append, true, true); 135 writer=new PrintWriter(os); 136 } 137 add(ArrayList<String> list)138 public void add(ArrayList<String> list){ 139 assert(list!=null); 140 while(list!=null){ 141 // System.out.println("Adding list to queue "+queue.size()); 142 try { 143 queue.put(list); 144 list=null; 145 } catch (InterruptedException e) { 146 // TODO Auto-generated catch block 147 e.printStackTrace(); 148 } 149 } 150 } 151 152 @Override run()153 public void run(){ 154 155 ArrayList<String> list=null; 156 while(list==null){ 157 // System.out.println("Waiting for list..."); 158 try { 159 list=queue.take(); 160 } catch (InterruptedException e) { 161 // TODO Auto-generated catch block 162 e.printStackTrace(); 163 } 164 // System.out.println("Took list of size "+(list==null ? "null" : list.size()+"")); 165 if(list!=null){ 166 if(list.isEmpty()){ 167 ReadWrite.finishWriting(writer, os, fname, allowSubprocess); 168 return; 169 } 170 for(String s : list){ 171 if(s!=null){writer.println(s);} 172 } 173 } 174 list=null; 175 } 176 } 177 178 private final OutputStream os; 179 private final PrintWriter writer; 180 private final ArrayBlockingQueue<ArrayList<String>> queue=new ArrayBlockingQueue<ArrayList<String>>(MAX_LISTS); 181 final String fname; 182 183 } 184 185 public static final int MAX_LISTS=8; 186 public static final int LIST_SIZE=100; 187 public static boolean overwrite=true; 188 public static boolean append=false; 189 public static boolean allowSubprocess=true; 190 191 } 192